int
usdf_cm_msg_accept(struct fid_ep *fep, const void *param, size_t paramlen)
{
struct usdf_ep *ep;
struct usdf_rx *rx;
struct usdf_domain *udp;
struct usdf_fabric *fp;
struct usdf_connreq *crp;
struct usdf_connreq_msg *reqp;
struct usd_qp_impl *qp;
int ret;
int n;
USDF_TRACE_SYS(EP_CTRL, "\n");
if (paramlen > USDF_MAX_CONN_DATA)
return -FI_EINVAL;
ep = ep_ftou(fep);
udp = ep->ep_domain;
fp = udp->dom_fabric;
crp = ep->e.msg.ep_connreq;
if (crp == NULL) {
return -FI_ENOTCONN;
}
if (ep->ep_eq == NULL) {
return -FI_ENOEQ;
}
crp->cr_ep = ep;
reqp = (struct usdf_connreq_msg *)crp->cr_data;
ep->e.msg.ep_lcl_peer_id = ntohs(reqp->creq_peer_id);
/* start creating the dest early */
ret = usd_create_dest_with_mac(udp->dom_dev, reqp->creq_ipaddr,
reqp->creq_port, reqp->creq_mac,
&ep->e.msg.ep_dest);
if (ret != 0) {
goto fail;
}
ret = usdf_ep_msg_get_queues(ep);
if (ret != 0) {
goto fail;
}
rx = ep->ep_rx;
qp = to_qpi(rx->rx_qp);
/* allocate a peer ID */
ep->e.msg.ep_rem_peer_id = udp->dom_next_peer;
udp->dom_peer_tab[udp->dom_next_peer] = ep;
++udp->dom_next_peer;
crp->cr_ptr = crp->cr_data;
crp->cr_resid = sizeof(*reqp) + paramlen;
reqp->creq_peer_id = htons(ep->e.msg.ep_rem_peer_id);
reqp->creq_ipaddr = fp->fab_dev_attrs->uda_ipaddr_be;
reqp->creq_port =
qp->uq_attrs.uqa_local_addr.ul_addr.ul_udp.u_addr.sin_port;
memcpy(reqp->creq_mac, fp->fab_dev_attrs->uda_mac_addr, ETH_ALEN);
reqp->creq_result = htonl(0);
reqp->creq_datalen = htonl(paramlen);
memcpy(reqp->creq_data, param, paramlen);
n = write(crp->cr_sockfd, crp->cr_ptr, crp->cr_resid);
if (n == -1) {
usdf_cm_msg_connreq_cleanup(crp);
ret = -errno;
goto fail;
}
crp->cr_resid -= n;
if (crp->cr_resid == 0) {
usdf_cm_msg_accept_complete(crp);
} else {
// XXX set up epoll junk to send rest
}
return 0;
fail:
free(ep->e.msg.ep_dest);
/* XXX release queues */
return ret;
}
int
usdf_cm_msg_connect(struct fid_ep *fep, const void *addr,
const void *param, size_t paramlen)
{
struct usdf_connreq *crp;
struct usdf_ep *ep;
struct usdf_rx *rx;
struct usdf_domain *udp;
const struct sockaddr_in *sin;
struct epoll_event ev;
struct usdf_fabric *fp;
struct usdf_connreq_msg *reqp;
struct usd_qp_impl *qp;
int ret;
USDF_TRACE_SYS(EP_CTRL, "\n");
if (paramlen > USDF_MAX_CONN_DATA)
return -FI_EINVAL;
ep = ep_ftou(fep);
udp = ep->ep_domain;
fp = udp->dom_fabric;
sin = addr;
crp = NULL;
crp = calloc(1, sizeof(*crp) + sizeof(struct usdf_connreq_msg) +
paramlen);
if (crp == NULL) {
ret = -errno;
goto fail;
}
crp->handle.fclass = FI_CLASS_CONNREQ;
crp->cr_sockfd = socket(AF_INET, SOCK_STREAM, 0);
if (crp->cr_sockfd == -1) {
ret = -errno;
goto fail;
}
ret = fcntl(crp->cr_sockfd, F_GETFL, 0);
if (ret == -1) {
ret = -errno;
goto fail;
}
ret = fcntl(crp->cr_sockfd, F_SETFL, ret | O_NONBLOCK);
if (ret == -1) {
ret = -errno;
goto fail;
}
ret = usdf_ep_msg_get_queues(ep);
if (ret != 0) {
goto fail;
}
rx = ep->ep_rx;
qp = to_qpi(rx->rx_qp);
ret = connect(crp->cr_sockfd, (struct sockaddr *)sin, sizeof(*sin));
if (ret != 0 && errno != EINPROGRESS) {
ret = -errno;
goto fail;
}
/* register for notification when connect completes */
crp->cr_pollitem.pi_rtn = usdf_cm_msg_connect_cb_wr;
crp->cr_pollitem.pi_context = crp;
ev.events = EPOLLOUT;
ev.data.ptr = &crp->cr_pollitem;
ret = epoll_ctl(fp->fab_epollfd, EPOLL_CTL_ADD, crp->cr_sockfd, &ev);
if (ret != 0) {
crp->cr_pollitem.pi_rtn = NULL;
ret = -errno;
goto fail;
}
/* allocate remote peer ID */
ep->e.msg.ep_rem_peer_id = udp->dom_next_peer;
udp->dom_peer_tab[udp->dom_next_peer] = ep;
++udp->dom_next_peer;
crp->cr_ep = ep;
reqp = (struct usdf_connreq_msg *)crp->cr_data;
crp->cr_ptr = crp->cr_data;
crp->cr_resid = sizeof(*reqp) + paramlen;
reqp->creq_peer_id = htons(ep->e.msg.ep_rem_peer_id);
reqp->creq_ipaddr = fp->fab_dev_attrs->uda_ipaddr_be;
reqp->creq_port =
qp->uq_attrs.uqa_local_addr.ul_addr.ul_udp.u_addr.sin_port;
memcpy(reqp->creq_mac, fp->fab_dev_attrs->uda_mac_addr, ETH_ALEN);
reqp->creq_datalen = htonl(paramlen);
memcpy(reqp->creq_data, param, paramlen);
return 0;
fail:
if (crp != NULL) {
if (crp->cr_sockfd != -1) {
close(crp->cr_sockfd);
}
free(crp);
}
usdf_ep_msg_release_queues(ep);
return ret;
}
int
usdf_cm_msg_connect(struct fid_ep *fep, const void *addr,
const void *param, size_t paramlen)
{
struct usdf_connreq *crp;
struct usdf_ep *ep;
struct usdf_rx *rx;
struct usdf_domain *udp;
const struct sockaddr_in *sin;
struct epoll_event ev;
struct usdf_fabric *fp;
struct usdf_connreq_msg *reqp;
struct usd_qp_impl *qp;
size_t request_size;
int ret;
USDF_TRACE_SYS(EP_CTRL, "\n");
if (paramlen > USDF_MAX_CONN_DATA)
return -FI_EINVAL;
ep = ep_ftou(fep);
udp = ep->ep_domain;
fp = udp->dom_fabric;
sin = addr;
/* Although paramlen may be less than USDF_MAX_CONN_DATA, the same crp
* struct is used for receiving the accept and reject payload. The
* structure has to be prepared to receive the maximum allowable amount
* of data per transfer. The maximum size includes the connection
* request structure, the connection request message, and the maximum
* amount of data per connection request message.
*/
request_size = sizeof(*crp) + sizeof(*reqp) + USDF_MAX_CONN_DATA;
crp = calloc(1, request_size);
if (crp == NULL) {
ret = -errno;
goto fail;
}
ep->e.msg.ep_connreq = crp;
crp->handle.fclass = FI_CLASS_CONNREQ;
if (ep->e.msg.ep_cm_sock == -1) {
crp->cr_sockfd = socket(AF_INET, SOCK_STREAM, 0);
if (crp->cr_sockfd == -1) {
ret = -errno;
goto fail;
}
} else {
crp->cr_sockfd = ep->e.msg.ep_cm_sock;
ep->e.msg.ep_cm_sock = -1;
}
ret = fi_fd_nonblock(crp->cr_sockfd);
if (ret) {
ret = -errno;
goto fail;
}
ret = usdf_ep_msg_get_queues(ep);
if (ret != 0) {
goto fail;
}
rx = ep->ep_rx;
qp = to_qpi(rx->rx_qp);
ret = connect(crp->cr_sockfd, (struct sockaddr *)sin, sizeof(*sin));
if (ret != 0 && errno != EINPROGRESS) {
ret = -errno;
goto fail;
}
/* If cr_sockfd was previously unbound, connect(2) will do a a bind(2)
* for us. Update our snapshot of the locally bound address. */
ret = usdf_msg_upd_lcl_addr(ep);
if (ret)
goto fail;
/* allocate remote peer ID */
ep->e.msg.ep_rem_peer_id = udp->dom_next_peer;
udp->dom_peer_tab[udp->dom_next_peer] = ep;
++udp->dom_next_peer;
crp->cr_ep = ep;
reqp = (struct usdf_connreq_msg *)crp->cr_data;
crp->cr_ptr = crp->cr_data;
crp->cr_resid = sizeof(*reqp) + paramlen;
reqp->creq_peer_id = htons(ep->e.msg.ep_rem_peer_id);
reqp->creq_ipaddr = fp->fab_dev_attrs->uda_ipaddr_be;
reqp->creq_port =
qp->uq_attrs.uqa_local_addr.ul_addr.ul_udp.u_addr.sin_port;
reqp->creq_datalen = htonl(paramlen);
memcpy(reqp->creq_data, param, paramlen);
/* register for notification when connect completes */
crp->cr_pollitem.pi_rtn = usdf_cm_msg_connect_cb_wr;
crp->cr_pollitem.pi_context = crp;
ev.events = EPOLLOUT;
ev.data.ptr = &crp->cr_pollitem;
ret = epoll_ctl(fp->fab_epollfd, EPOLL_CTL_ADD, crp->cr_sockfd, &ev);
if (ret != 0) {
crp->cr_pollitem.pi_rtn = NULL;
ret = -errno;
goto fail;
}
return 0;
fail:
if (crp != NULL) {
if (crp->cr_sockfd != -1) {
close(crp->cr_sockfd);
}
free(crp);
ep->e.msg.ep_connreq = NULL;
}
usdf_ep_msg_release_queues(ep);
return ret;
}
