/* Register as a callback triggered by the socket becoming writeable. Write as
* much data as can be written in a single write, and keep track of how much
* data is left. If the data is not fully written, it will finish getting
* written in another iteration of the progression.
*/
static int usdf_pep_reject_async(void *vreq)
{
struct usdf_connreq *crp;
int ret;
crp = vreq;
do {
ret = write(crp->cr_sockfd, crp->cr_ptr, crp->cr_resid);
} while ((ret < 0) && (errno == EINTR));
if ((ret <= 0) && (errno != EAGAIN)) {
USDF_DBG_SYS(EP_CTRL, "write failed: %s\n",
strerror(errno));
usdf_cm_msg_connreq_failed(crp, -errno);
return -errno;
}
crp->cr_resid -= ret;
crp->cr_ptr += ret;
if (crp->cr_resid == 0)
usdf_cm_msg_connreq_cleanup(crp);
return FI_SUCCESS;
}
static int
usdf_pep_listen_cb(void *v)
{
struct usdf_pep *pep;
struct sockaddr_in sin;
struct usdf_connreq *crp;
struct epoll_event ev;
socklen_t socklen;
int ret;
int s;
pep = v;
socklen = sizeof(sin);
s = accept(pep->pep_sock, &sin, &socklen);
if (s == -1) {
/* ignore early failure */
return 0;
}
crp = NULL;
pthread_spin_lock(&pep->pep_cr_lock);
if (!TAILQ_EMPTY(&pep->pep_cr_free)) {
crp = TAILQ_FIRST(&pep->pep_cr_free);
TAILQ_REMOVE_MARK(&pep->pep_cr_free, crp, cr_link);
TAILQ_NEXT(crp, cr_link) = NULL;
}
pthread_spin_unlock(&pep->pep_cr_lock);
/* no room for request, just drop it */
if (crp == NULL) {
/* XXX send response? */
close(s);
return 0;
}
crp->cr_sockfd = s;
crp->cr_pep = pep;
crp->cr_ptr = crp->cr_data;
crp->cr_resid = sizeof(struct usdf_connreq_msg);
crp->cr_pollitem.pi_rtn = usdf_pep_read_connreq;
crp->cr_pollitem.pi_context = crp;
ev.events = EPOLLIN;
ev.data.ptr = &crp->cr_pollitem;
ret = epoll_ctl(pep->pep_fabric->fab_epollfd, EPOLL_CTL_ADD,
crp->cr_sockfd, &ev);
if (ret == -1) {
crp->cr_pollitem.pi_rtn = NULL;
usdf_cm_msg_connreq_failed(crp, -errno);
return 0;
}
TAILQ_INSERT_TAIL(&pep->pep_cr_pending, crp, cr_link);
return 0;
}
/*
* Write connection request data to the listener
* Once everything is written, switch over into listening mode to
* capture the listener response.
*/
static int
usdf_cm_msg_connect_cb_wr(void *v)
{
struct usdf_connreq *crp;
struct usdf_ep *ep;
struct usdf_fabric *fp;
struct epoll_event ev;
int ret;
crp = v;
ep = crp->cr_ep;
fp = ep->ep_domain->dom_fabric;
ret = write(crp->cr_sockfd, crp->cr_ptr, crp->cr_resid);
if (ret == -1) {
usdf_cm_msg_connreq_failed(crp, -errno);
return 0;
}
crp->cr_resid -= ret;
if (crp->cr_resid == 0) {
crp->cr_pollitem.pi_rtn = usdf_cm_msg_connect_cb_rd;
ev.events = EPOLLIN;
ev.data.ptr = &crp->cr_pollitem;
ret = epoll_ctl(fp->fab_epollfd, EPOLL_CTL_MOD,
crp->cr_sockfd, &ev);
if (ret != 0) {
usdf_cm_msg_connreq_failed(crp, -errno);
return 0;
}
crp->cr_ptr = crp->cr_data;
crp->cr_resid = sizeof(struct usdf_connreq_msg);
}
return 0;
}
static int
usdf_cm_msg_accept_complete(struct usdf_connreq *crp)
{
struct usdf_ep *ep;
struct fi_eq_cm_entry entry;
int ret;
ep = crp->cr_ep;
/* post EQ entry */
entry.fid = ep_utofid(ep);
entry.info = NULL;
ret = usdf_eq_write_internal(ep->ep_eq, FI_CONNECTED, &entry,
sizeof(entry), 0);
if (ret != sizeof(entry)) {
usdf_cm_msg_connreq_failed(crp, ret);
return 0;
}
usdf_cm_msg_connreq_cleanup(crp);
return 0;
}
/*
* read connection request response from the listener
*/
static int
usdf_cm_msg_connect_cb_rd(void *v)
{
struct usdf_connreq *crp;
struct usdf_ep *ep;
struct usdf_fabric *fp;
struct usdf_domain *udp;
struct usdf_connreq_msg *reqp;
struct fi_eq_cm_entry *entry;
size_t entry_len;
int ret;
crp = v;
ep = crp->cr_ep;
fp = ep->ep_domain->dom_fabric;
ret = read(crp->cr_sockfd, crp->cr_ptr, crp->cr_resid);
if (ret == -1) {
usdf_cm_msg_connreq_failed(crp, -errno);
return 0;
}
crp->cr_resid -= ret;
reqp = (struct usdf_connreq_msg *)crp->cr_data;
if (crp->cr_resid == 0 && crp->cr_ptr == crp->cr_data + sizeof(*reqp)) {
reqp->creq_datalen = ntohl(reqp->creq_datalen);
crp->cr_resid = reqp->creq_datalen;
}
/* if resid is 0 now, completely done */
if (crp->cr_resid == 0) {
ret = epoll_ctl(fp->fab_epollfd, EPOLL_CTL_DEL,
crp->cr_sockfd, NULL);
close(crp->cr_sockfd);
crp->cr_sockfd = -1;
entry_len = sizeof(*entry) + reqp->creq_datalen;
entry = malloc(entry_len);
if (entry == NULL) {
usdf_cm_msg_connreq_failed(crp, -errno);
return 0;
}
udp = ep->ep_domain;
ep->e.msg.ep_lcl_peer_id = ntohs(reqp->creq_peer_id);
ret = usd_create_dest_with_mac(udp->dom_dev, reqp->creq_ipaddr,
reqp->creq_port, reqp->creq_mac,
&ep->e.msg.ep_dest);
if (ret != 0) {
free(entry);
usdf_cm_msg_connreq_failed(crp, ret);
return 0;
}
entry->fid = ep_utofid(ep);
entry->info = NULL;
memcpy(entry->data, reqp->creq_data, reqp->creq_datalen);
ret = usdf_eq_write_internal(ep->ep_eq, FI_CONNECTED, entry,
entry_len, 0);
free(entry);
if (ret != entry_len) {
free(ep->e.msg.ep_dest);
ep->e.msg.ep_dest = NULL;
usdf_cm_msg_connreq_failed(crp, ret);
return 0;
}
usdf_cm_msg_connreq_cleanup(crp);
}
return 0;
}
static int
usdf_pep_read_connreq(void *v)
{
struct usdf_connreq *crp;
struct usdf_pep *pep;
struct usdf_connreq_msg *reqp;
struct fi_eq_cm_entry *entry;
size_t entry_len;
int ret;
int n;
crp = v;
pep = crp->cr_pep;
n = read(crp->cr_sockfd, crp->cr_ptr, crp->cr_resid);
if (n == -1) {
usdf_cm_msg_connreq_failed(crp, -errno);
return 0;
}
crp->cr_ptr += n;
crp->cr_resid -= n;
reqp = (struct usdf_connreq_msg *)crp->cr_data;
if (crp->cr_resid == 0 && crp->cr_ptr == crp->cr_data + sizeof(*reqp)) {
reqp->creq_datalen = ntohl(reqp->creq_datalen);
crp->cr_resid = reqp->creq_datalen;
}
/* if resid is 0 now, completely done */
if (crp->cr_resid == 0) {
ret = usdf_pep_creq_epoll_del(crp);
if (ret != 0) {
usdf_cm_msg_connreq_failed(crp, ret);
return 0;
}
/* create CONNREQ EQ entry */
entry_len = sizeof(*entry) + reqp->creq_datalen;
entry = malloc(entry_len);
if (entry == NULL) {
usdf_cm_msg_connreq_failed(crp, -errno);
return 0;
}
entry->fid = &pep->pep_fid.fid;
entry->info = usdf_pep_conn_info(crp);
if (entry->info == NULL) {
free(entry);
usdf_cm_msg_connreq_failed(crp, -FI_ENOMEM);
return 0;
}
memcpy(entry->data, reqp->creq_data, reqp->creq_datalen);
ret = usdf_eq_write_internal(pep->pep_eq, FI_CONNREQ, entry,
entry_len, 0);
free(entry);
if (ret != (int)entry_len) {
usdf_cm_msg_connreq_failed(crp, ret);
return 0;
}
}
return 0;
}
