static ssize_t fi_ibv_rdm_tagged_ep_cancel(fid_t fid, void *ctx)
{
struct fi_context *context = (struct fi_context *)ctx;
struct fi_ibv_rdm_ep *ep_rdm =
container_of(fid, struct fi_ibv_rdm_ep, ep_fid);
int err = 1;
if (!ep_rdm->domain)
return -EBADF;
if (!context)
return -EINVAL;
if (context->internal[0] == NULL)
return 0;
struct fi_ibv_rdm_request *request = context->internal[0];
VERBS_DBG(FI_LOG_EP_DATA,
"ep_cancel, match %p, tag 0x%llx, len %d, ctx %p\n",
request, request->minfo.tag, request->len, request->context);
struct dlist_entry *found =
dlist_find_first_match(&fi_ibv_rdm_posted_queue,
fi_ibv_rdm_req_match, request);
if (found) {
assert(container_of(found, struct fi_ibv_rdm_request,
queue_entry) == request);
fi_ibv_rdm_remove_from_posted_queue(request, ep_rdm);
VERBS_DBG(FI_LOG_EP_DATA, "\t\t-> SUCCESS, post recv %d\n",
ep_rdm->posted_recvs);
err = 0;
} else {
static ssize_t fi_ibv_rdm_tagged_ep_cancel(fid_t fid, void *ctx)
{
struct fi_ibv_rdm_ep *fid_ep;
struct fi_context *context = (struct fi_context *)ctx;
int err = 1;
fid_ep = container_of(fid, struct fi_ibv_rdm_ep, ep_fid);
if (!fid_ep->domain)
return -EBADF;
if (!context)
return -EINVAL;
if (context->internal[0] == NULL)
return 0;
struct fi_ibv_rdm_tagged_request *request = context->internal[0];
VERBS_DBG(FI_LOG_EP_DATA,
"ep_cancel, match %p, tag 0x%llx, len %d, ctx %p\n",
request, (long long unsigned)request->tag,
request->len, request->context);
struct dlist_entry *found =
dlist_find_first_match(&fi_ibv_rdm_tagged_recv_posted_queue,
fi_ibv_rdm_tagged_match_requests, request);
if (found) {
assert(container_of(found, struct fi_ibv_rdm_tagged_request,
queue_entry) == request);
fi_ibv_rdm_tagged_remove_from_posted_queue(request, fid_ep);
assert(request->send_completions_wait == 0);
FI_IBV_RDM_TAGGED_DBG_REQUEST("to_pool: ", request,
FI_LOG_DEBUG);
fi_ibv_mem_pool_return(&request->mpe,
&fi_ibv_rdm_tagged_request_pool);
VERBS_DBG(FI_LOG_EP_DATA,
"\t\t-> SUCCESS, pend recv %d\n", fid_ep->pend_recv);
err = 0;
}
return err;
}
void fi_ibv_ep_ini_conn_done(struct fi_ibv_xrc_ep *ep, uint32_t peer_srqn,
uint32_t tgt_qpn)
{
struct fi_ibv_domain *domain = fi_ibv_ep_to_domain(&ep->base_ep);
assert(ep->base_ep.id && ep->ini_conn);
fastlock_acquire(&domain->xrc.ini_mgmt_lock);
assert(ep->ini_conn->state == FI_IBV_INI_QP_CONNECTING ||
ep->ini_conn->state == FI_IBV_INI_QP_CONNECTED);
/* If this was a physical INI/TGT QP connection, remove the QP
* from control of the RDMA CM. We don't want the shared INI QP
* to be destroyed if this endpoint closes. */
if (ep->base_ep.id->qp) {
ep->ini_conn->state = FI_IBV_INI_QP_CONNECTED;
ep->ini_conn->tgt_qpn = tgt_qpn;
ep->base_ep.id->qp = NULL;
VERBS_DBG(FI_LOG_EP_CTRL,
"Set INI Conn QP %d remote TGT QP %d\n",
ep->ini_conn->ini_qp->qp_num,
ep->ini_conn->tgt_qpn);
}
fi_ibv_log_ep_conn(ep, "INI Connection Done");
fi_ibv_sched_ini_conn(ep->ini_conn);
fastlock_release(&domain->xrc.ini_mgmt_lock);
}
static ssize_t fi_ibv_cq_read(struct fid_cq *cq_fid, void *buf, size_t count)
{
struct fi_ibv_cq *cq;
struct fi_ibv_wce *wce;
struct slist_entry *entry;
struct ibv_wc wc;
ssize_t ret = 0, i;
cq = container_of(cq_fid, struct fi_ibv_cq, util_cq.cq_fid);
cq->util_cq.cq_fastlock_acquire(&cq->util_cq.cq_lock);
for (i = 0; i < count; i++) {
if (!slist_empty(&cq->wcq)) {
wce = container_of(cq->wcq.head, struct fi_ibv_wce, entry);
if (wce->wc.status) {
ret = -FI_EAVAIL;
break;
}
entry = slist_remove_head(&cq->wcq);
wce = container_of(entry, struct fi_ibv_wce, entry);
cq->read_entry(&wce->wc, (char *)buf + i * cq->entry_size);
util_buf_release(cq->wce_pool, wce);
continue;
}
ret = fi_ibv_poll_cq(cq, &wc);
if (ret <= 0)
break;
/* Insert error entry into wcq */
if (OFI_UNLIKELY(wc.status)) {
if (wc.status == IBV_WC_WR_FLUSH_ERR) {
/* Handle case when remote side destroys
* the connection, but local side isn't aware
* about that yet */
VERBS_DBG(FI_LOG_CQ,
"Ignoring WC with status "
"IBV_WC_WR_FLUSH_ERR(%d)\n",
wc.status);
i--;
continue;
}
wce = util_buf_alloc(cq->wce_pool);
if (!wce) {
cq->util_cq.cq_fastlock_release(&cq->util_cq.cq_lock);
return -FI_ENOMEM;
}
memset(wce, 0, sizeof(*wce));
memcpy(&wce->wc, &wc, sizeof wc);
slist_insert_tail(&wce->entry, &cq->wcq);
ret = -FI_EAVAIL;
break;
}
cq->read_entry(&wc, (char *)buf + i * cq->entry_size);
}
/* Builds a list of interfaces that correspond to active verbs devices */
static int fi_ibv_getifaddrs(struct dlist_entry *verbs_devs)
{
struct ifaddrs *ifaddr, *ifa;
char name[INET6_ADDRSTRLEN];
struct rdma_addrinfo *rai;
struct rdma_cm_id *id;
const char *ret_ptr;
int ret, num_verbs_ifs = 0;
char *iface = NULL;
size_t iface_len = 0;
int exact_match = 0;
ret = getifaddrs(&ifaddr);
if (ret) {
VERBS_WARN(FI_LOG_FABRIC,
"Unable to get interface addresses\n");
return ret;
}
/* select best iface name based on user's input */
if (fi_param_get_str(&fi_ibv_prov, "iface", &iface) == FI_SUCCESS) {
iface_len = strlen(iface);
if (iface_len > IFNAMSIZ) {
VERBS_INFO(FI_LOG_EP_CTRL,
"Too long iface name: %s, max: %d\n",
iface, IFNAMSIZ);
return -FI_EINVAL;
}
for (ifa = ifaddr; ifa && !exact_match; ifa = ifa->ifa_next)
exact_match = !strcmp(ifa->ifa_name, iface);
}
for (ifa = ifaddr; ifa; ifa = ifa->ifa_next) {
if (!ifa->ifa_addr || !(ifa->ifa_flags & IFF_UP) ||
!strcmp(ifa->ifa_name, "lo"))
continue;
if(iface) {
if(exact_match) {
if(strcmp(ifa->ifa_name, iface))
continue;
} else {
if(strncmp(ifa->ifa_name, iface, iface_len))
continue;
}
}
switch (ifa->ifa_addr->sa_family) {
case AF_INET:
ret_ptr = inet_ntop(AF_INET, &ofi_sin_addr(ifa->ifa_addr),
name, INET6_ADDRSTRLEN);
break;
case AF_INET6:
ret_ptr = inet_ntop(AF_INET6, &ofi_sin6_addr(ifa->ifa_addr),
name, INET6_ADDRSTRLEN);
break;
default:
continue;
}
if (!ret_ptr) {
VERBS_WARN(FI_LOG_FABRIC,
"inet_ntop failed: %s(%d)\n",
strerror(errno), errno);
goto err1;
}
ret = fi_ibv_create_ep(name, NULL, FI_NUMERICHOST | FI_SOURCE,
NULL, &rai, &id);
if (ret)
continue;
ret = fi_ibv_add_rai(verbs_devs, id, rai);
if (ret)
goto err2;
VERBS_DBG(FI_LOG_FABRIC, "Found active interface for verbs device: "
"%s with address: %s\n",
ibv_get_device_name(id->verbs->device), name);
rdma_destroy_ep(id);
num_verbs_ifs++;
}
freeifaddrs(ifaddr);
return num_verbs_ifs ? 0 : -FI_ENODATA;
err2:
rdma_destroy_ep(id);
err1:
fi_ibv_verbs_devs_free(verbs_devs);
freeifaddrs(ifaddr);
return ret;
}
ssize_t fi_ibv_rdm_conn_cleanup(struct fi_ibv_rdm_tagged_conn *conn)
{
ssize_t ret = FI_SUCCESS;
ssize_t err = FI_SUCCESS;
VERBS_DBG(FI_LOG_AV, "conn %p, exp = %lld unexp = %lld\n", conn,
conn->exp_counter, conn->unexp_counter);
errno = 0;
if (conn->id[0]) {
rdma_destroy_qp(conn->id[0]);
if (errno) {
VERBS_INFO_ERRNO(FI_LOG_AV, "rdma_destroy_qp\n", errno);
ret = -errno;
}
if (rdma_destroy_id(conn->id[0])) {
VERBS_INFO_ERRNO(FI_LOG_AV, "rdma_destroy_id\n", errno);
if (ret == FI_SUCCESS)
ret = -errno;
}
}
if (conn->id[1]) {
assert(conn->cm_role == FI_VERBS_CM_SELF);
rdma_destroy_qp(conn->id[1]);
if (errno) {
VERBS_INFO_ERRNO(FI_LOG_AV, "rdma_destroy_qp\n", errno);
if (ret == FI_SUCCESS)
ret = -errno;
}
if (rdma_destroy_id(conn->id[1])) {
VERBS_INFO_ERRNO(FI_LOG_AV, "rdma_destroy_id\n", errno);
if (ret == FI_SUCCESS)
ret = -errno;
}
}
if (conn->s_mr) {
err = fi_ibv_rdm_dereg_and_free(&conn->s_mr, &conn->sbuf_mem_reg);
if ((err != FI_SUCCESS) && (ret == FI_SUCCESS)) {
ret = err;
}
}
if (conn->r_mr) {
err = fi_ibv_rdm_dereg_and_free(&conn->r_mr, &conn->rbuf_mem_reg);
if ((err != FI_SUCCESS) && (ret == FI_SUCCESS)) {
ret = err;
}
}
if (conn->ack_mr) {
if (ibv_dereg_mr(conn->ack_mr)) {
VERBS_INFO_ERRNO(FI_LOG_AV, "ibv_dereg_mr failed\n",
errno);
if (ret == FI_SUCCESS)
ret = -errno;
}
}
if (conn->rma_mr) {
err = fi_ibv_rdm_dereg_and_free(&conn->rma_mr,
&conn->rmabuf_mem_reg);
if ((err != FI_SUCCESS) && (ret == FI_SUCCESS)) {
ret = err;
}
}
free(conn);
return ret;
}
