static void __gnix_rma_copy_chained_get_data(struct gnix_tx_descriptor *txd)
{
struct gnix_fab_req *req = txd->req;
int head_off, head_len, tail_len;
void *addr;
head_off = req->rma.rem_addr & GNI_READ_ALIGN_MASK;
head_len = GNI_READ_ALIGN - head_off;
tail_len = (req->rma.rem_addr + req->rma.len) & GNI_READ_ALIGN_MASK;
if (head_off) {
GNIX_INFO(FI_LOG_EP_DATA, "writing %d bytes to %p\n",
head_len, req->rma.loc_addr);
memcpy((void *)req->rma.loc_addr,
txd->int_buf + head_off,
head_len);
}
if (tail_len) {
addr = (void *)req->rma.loc_addr +
req->rma.len -
tail_len;
GNIX_INFO(FI_LOG_EP_DATA, "writing %d bytes to %p\n",
tail_len, addr);
memcpy((void *)addr,
txd->int_buf + GNI_READ_ALIGN,
tail_len);
}
}
static int __gnix_rma_post_err(struct gnix_tx_descriptor *txd)
{
struct gnix_fab_req *req = txd->req;
int rc;
req->tx_failures++;
if (req->tx_failures < req->gnix_ep->domain->params.max_retransmits) {
_gnix_nic_tx_free(req->gnix_ep->nic, txd);
GNIX_INFO(FI_LOG_EP_DATA,
"Requeueing failed request: %p\n", req);
return _gnix_vc_queue_work_req(req);
}
GNIX_INFO(FI_LOG_EP_DATA, "Failed %d transmits: %p\n",
req->tx_failures, req);
rc = __gnix_rma_send_err(req->vc->ep, req);
if (rc != FI_SUCCESS)
GNIX_WARN(FI_LOG_EP_DATA,
"__gnix_rma_send_err() failed: %d\n",
rc);
__gnix_rma_fr_complete(req, txd);
return FI_SUCCESS;
}
static void __gnix_msg_copy_unaligned_get_data(struct gnix_fab_req *req)
{
int head_off, head_len, tail_len;
void *addr;
head_off = req->msg.send_addr & GNI_READ_ALIGN_MASK;
head_len = head_off ? GNI_READ_ALIGN - head_off : 0;
tail_len = (req->msg.send_addr + req->msg.send_len) &
GNI_READ_ALIGN_MASK;
if (head_off) {
addr = (void *)&req->msg.rndzv_head + head_off;
GNIX_INFO(FI_LOG_EP_DATA,
"writing %d bytes to head (%p, 0x%x)\n",
head_len, req->msg.recv_addr, *(uint32_t *)addr);
memcpy((void *)req->msg.recv_addr, addr, head_len);
}
if (tail_len) {
addr = (void *)(req->msg.recv_addr +
req->msg.send_len -
tail_len);
GNIX_INFO(FI_LOG_EP_DATA,
"writing %d bytes to tail (%p, 0x%x)\n",
tail_len, addr, req->msg.rndzv_tail);
memcpy((void *)addr, &req->msg.rndzv_tail, tail_len);
}
}
static int __gnix_rndzv_req(void *arg)
{
struct gnix_fab_req *req = (struct gnix_fab_req *)arg;
struct gnix_fid_ep *ep = req->gnix_ep;
struct gnix_nic *nic = ep->nic;
struct gnix_tx_descriptor *txd, *tail_txd = NULL;
gni_return_t status;
int rc;
int use_tx_cq_blk = 0;
struct fid_mr *auto_mr = NULL;
int inject_err = _gnix_req_inject_err(req);
int head_off, head_len, tail_len;
void *tail_data = NULL;
if (!req->msg.recv_md) {
rc = gnix_mr_reg(&ep->domain->domain_fid.fid,
(void *)req->msg.recv_addr, req->msg.recv_len,
FI_READ | FI_WRITE, 0, 0, 0, &auto_mr, NULL);
if (rc != FI_SUCCESS) {
GNIX_INFO(FI_LOG_EP_DATA,
"Failed to auto-register local buffer: %d\n",
rc);
return -FI_EAGAIN;
}
req->msg.recv_flags |= FI_LOCAL_MR;
req->msg.recv_md = container_of(auto_mr,
struct gnix_fid_mem_desc,
mr_fid);
GNIX_INFO(FI_LOG_EP_DATA, "auto-reg MR: %p\n", auto_mr);
}
int
_gnix_notifier_get_event(struct gnix_mr_notifier *mrn, void* buf, size_t len)
{
int ret, ret_errno;
if ((mrn == NULL) || (buf == NULL) || (len <= 0)) {
GNIX_INFO(FI_LOG_MR,
"Invalid argument to _gnix_notifier_get_event\n");
return -FI_EINVAL;
}
if (*(mrn->cntr) > 0) {
GNIX_DEBUG(FI_LOG_MR, "reading kdreg event\n");
ret = read(mrn->fd, buf, len);
if (ret >= 0) {
return ret;
} else {
ret_errno = errno;
if (ret_errno != EAGAIN) {
GNIX_INFO(FI_LOG_MR,
"kdreg event read failed: %s\n",
strerror(ret_errno));
}
// Not all of these map to fi_errno values
return -ret_errno;
}
} else {
GNIX_DEBUG(FI_LOG_MR, "nothing to read from kdreg :(\n");
return -FI_EAGAIN;
}
}
static int __gnix_rma_send_data_req(void *arg)
{
struct gnix_fab_req *req = (struct gnix_fab_req *)arg;
struct gnix_fid_ep *ep = req->gnix_ep;
struct gnix_nic *nic = ep->nic;
struct gnix_tx_descriptor *txd;
gni_return_t status;
int rc;
int inject_err = _gnix_req_inject_err(req);
rc = _gnix_nic_tx_alloc(nic, &txd);
if (rc) {
GNIX_INFO(FI_LOG_EP_DATA,
"_gnix_nic_tx_alloc() failed: %d\n",
rc);
return -FI_ENOSPC;
}
txd->req = req;
txd->completer_fn = __gnix_rma_txd_data_complete;
txd->rma_data_hdr.flags = FI_RMA | FI_REMOTE_CQ_DATA;
if (req->type == GNIX_FAB_RQ_RDMA_WRITE) {
txd->rma_data_hdr.flags |= FI_REMOTE_WRITE;
} else {
txd->rma_data_hdr.flags |= FI_REMOTE_READ;
}
txd->rma_data_hdr.data = req->rma.imm;
fastlock_acquire(&nic->lock);
if (inject_err) {
_gnix_nic_txd_err_inject(nic, txd);
status = GNI_RC_SUCCESS;
} else {
status = GNI_SmsgSendWTag(req->vc->gni_ep,
&txd->rma_data_hdr,
sizeof(txd->rma_data_hdr),
NULL, 0, txd->id,
GNIX_SMSG_T_RMA_DATA);
}
fastlock_release(&nic->lock);
if (status == GNI_RC_NOT_DONE) {
_gnix_nic_tx_free(nic, txd);
GNIX_INFO(FI_LOG_EP_DATA,
"GNI_SmsgSendWTag returned %s\n",
gni_err_str[status]);
} else if (status != GNI_RC_SUCCESS) {
_gnix_nic_tx_free(nic, txd);
GNIX_WARN(FI_LOG_EP_DATA,
"GNI_SmsgSendWTag returned %s\n",
gni_err_str[status]);
} else {
GNIX_INFO(FI_LOG_EP_DATA, "Sent RMA CQ data, req: %p\n", req);
}
return gnixu_to_fi_errno(status);
}
static int gnix_domain_close(fid_t fid)
{
int ret = FI_SUCCESS, references_held;
struct gnix_fid_domain *domain;
GNIX_TRACE(FI_LOG_DOMAIN, "\n");
domain = container_of(fid, struct gnix_fid_domain, domain_fid.fid);
if (domain->domain_fid.fid.fclass != FI_CLASS_DOMAIN) {
ret = -FI_EINVAL;
goto err;
}
/* before checking the refcnt, flush the memory registration cache */
if (domain->mr_cache_ro) {
fastlock_acquire(&domain->mr_cache_lock);
ret = _gnix_mr_cache_flush(domain->mr_cache_ro);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_DOMAIN,
"failed to flush memory cache on domain close\n");
fastlock_release(&domain->mr_cache_lock);
goto err;
}
fastlock_release(&domain->mr_cache_lock);
}
if (domain->mr_cache_rw) {
fastlock_acquire(&domain->mr_cache_lock);
ret = _gnix_mr_cache_flush(domain->mr_cache_rw);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_DOMAIN,
"failed to flush memory cache on domain close\n");
fastlock_release(&domain->mr_cache_lock);
goto err;
}
fastlock_release(&domain->mr_cache_lock);
}
/*
* if non-zero refcnt, there are eps, mrs, and/or an eq associated
* with this domain which have not been closed.
*/
references_held = _gnix_ref_put(domain);
if (references_held) {
GNIX_INFO(FI_LOG_DOMAIN, "failed to fully close domain due to "
"lingering references. references=%i dom=%p\n",
references_held, domain);
}
GNIX_INFO(FI_LOG_DOMAIN, "gnix_domain_close invoked returning %d\n",
ret);
err:
return ret;
}
static int __gnix_amo_send_cntr_req(void *arg)
{
struct gnix_fab_req *req = (struct gnix_fab_req *)arg;
struct gnix_fid_ep *ep = req->gnix_ep;
struct gnix_nic *nic = ep->nic;
struct gnix_tx_descriptor *txd;
gni_return_t status;
int rc;
int inject_err = _gnix_req_inject_err(req);
rc = _gnix_nic_tx_alloc(nic, &txd);
if (rc) {
GNIX_INFO(FI_LOG_EP_DATA,
"_gnix_nic_tx_alloc() failed: %d\n",
rc);
return -FI_ENOSPC;
}
txd->req = req;
txd->completer_fn = __gnix_amo_txd_cntr_complete;
if (req->type == GNIX_FAB_RQ_AMO) {
txd->amo_cntr_hdr.flags = FI_REMOTE_WRITE;
} else {
txd->amo_cntr_hdr.flags = FI_REMOTE_READ;
}
COND_ACQUIRE(nic->requires_lock, &nic->lock);
if (inject_err) {
_gnix_nic_txd_err_inject(nic, txd);
status = GNI_RC_SUCCESS;
} else {
status = GNI_SmsgSendWTag(req->vc->gni_ep,
&txd->amo_cntr_hdr,
sizeof(txd->amo_cntr_hdr),
NULL, 0, txd->id,
GNIX_SMSG_T_AMO_CNTR);
}
COND_RELEASE(nic->requires_lock, &nic->lock);
if (status == GNI_RC_NOT_DONE) {
_gnix_nic_tx_free(nic, txd);
GNIX_INFO(FI_LOG_EP_DATA,
"GNI_SmsgSendWTag returned %s\n",
gni_err_str[status]);
} else if (status != GNI_RC_SUCCESS) {
_gnix_nic_tx_free(nic, txd);
GNIX_WARN(FI_LOG_EP_DATA,
"GNI_SmsgSendWTag returned %s\n",
gni_err_str[status]);
} else {
GNIX_INFO(FI_LOG_EP_DATA, "Sent RMA CQ data, req: %p\n", req);
}
return gnixu_to_fi_errno(status);
}
static int __gnix_rndzv_req_send_fin(void *arg)
{
struct gnix_fab_req *req = (struct gnix_fab_req *)arg;
struct gnix_nic *nic;
struct gnix_fid_ep *ep;
struct gnix_tx_descriptor *txd;
gni_return_t status;
int rc;
GNIX_TRACE(FI_LOG_EP_DATA, "\n");
ep = req->gnix_ep;
assert(ep != NULL);
nic = ep->nic;
assert(nic != NULL);
rc = _gnix_nic_tx_alloc(nic, &txd);
if (rc) {
GNIX_INFO(FI_LOG_EP_DATA,
"_gnix_nic_tx_alloc() failed: %d\n",
rc);
return -FI_ENOSPC;
}
txd->rndzv_fin_hdr.req_addr = req->msg.rma_id;
txd->req = req;
txd->completer_fn = gnix_ep_smsg_completers[GNIX_SMSG_T_RNDZV_FIN];
fastlock_acquire(&nic->lock);
status = GNI_SmsgSendWTag(req->vc->gni_ep,
&txd->rndzv_fin_hdr, sizeof(txd->rndzv_fin_hdr),
NULL, 0, txd->id, GNIX_SMSG_T_RNDZV_FIN);
if ((status == GNI_RC_SUCCESS) &&
(ep->domain->data_progress == FI_PROGRESS_AUTO))
_gnix_rma_post_irq(req->vc);
fastlock_release(&nic->lock);
if (status == GNI_RC_NOT_DONE) {
_gnix_nic_tx_free(nic, txd);
GNIX_INFO(FI_LOG_EP_DATA,
"GNI_SmsgSendWTag returned %s\n",
gni_err_str[status]);
} else if (status != GNI_RC_SUCCESS) {
_gnix_nic_tx_free(nic, txd);
GNIX_WARN(FI_LOG_EP_DATA,
"GNI_SmsgSendWTag returned %s\n",
gni_err_str[status]);
}
GNIX_INFO(FI_LOG_EP_DATA, "Initiated RNDZV_FIN, req: %p\n", req);
return gnixu_to_fi_errno(status);
}
int
_gnix_notifier_open(struct gnix_mr_notifier **mrn)
{
int ret = FI_SUCCESS;
int kdreg_fd, ret_errno;
kdreg_get_user_delta_args_t get_user_delta_args;
fastlock_acquire(&global_mr_not.lock);
if (!global_mr_not.ref_cnt) {
kdreg_fd = open(KDREG_DEV, O_RDWR | O_NONBLOCK);
if (kdreg_fd < 0) {
ret_errno = errno;
if (ret_errno != FI_EBUSY) {
GNIX_INFO(FI_LOG_MR,
"kdreg device open failed: %s\n",
strerror(ret_errno));
}
/* Not all of these map to fi_errno values */
ret = -ret_errno;
goto err_exit;
}
memset(&get_user_delta_args, 0, sizeof(get_user_delta_args));
if (ioctl(kdreg_fd, KDREG_IOC_GET_USER_DELTA,
&get_user_delta_args) < 0) {
ret_errno = errno;
GNIX_INFO(FI_LOG_MR,
"kdreg get_user_delta failed: %s\n",
strerror(ret_errno));
close(kdreg_fd);
/* Not all of these map to fi_errno values */
ret = -ret_errno;
goto err_exit;
}
if (get_user_delta_args.user_delta == NULL) {
GNIX_INFO(FI_LOG_MR, "kdreg get_user_delta is NULL\n");
ret = -FI_ENODATA;
goto err_exit;
}
global_mr_not.fd = kdreg_fd;
global_mr_not.cntr = (kdreg_user_delta_t *)
get_user_delta_args.user_delta;
}
global_mr_not.ref_cnt++;
*mrn = &global_mr_not;
err_exit:
fastlock_release(&global_mr_not.lock);
return ret;
}
int
_gnix_notifier_open(struct gnix_mr_notifier *mrn)
{
int ret = FI_SUCCESS;
int kdreg_fd, ret_errno;
kdreg_get_user_delta_args_t get_user_delta_args;
if ((mrn->fd != 0) || (mrn->cntr != NULL)) {
GNIX_INFO(FI_LOG_MR, "mr notifier already open\n");
return -FI_EBUSY;
}
fastlock_acquire(&mrn->lock);
kdreg_fd = open(KDREG_DEV, O_RDWR | O_NONBLOCK);
if (kdreg_fd < 0) {
ret_errno = errno;
if (ret_errno != FI_EBUSY) {
GNIX_INFO(FI_LOG_MR, "kdreg device open failed: %s\n",
strerror(ret_errno));
}
// Not all of these map to fi_errno values
ret = -ret_errno;
goto err_exit;
}
(void) memset(&get_user_delta_args,0,sizeof(get_user_delta_args));
if (ioctl(kdreg_fd, KDREG_IOC_GET_USER_DELTA,
&get_user_delta_args) < 0) {
ret_errno = errno;
GNIX_INFO(FI_LOG_MR, "kdreg get_user_delta failed: %s\n",
strerror(ret_errno));
close(kdreg_fd);
// Not all of these map to fi_errno values
ret = -ret_errno;
goto err_exit;
}
if (get_user_delta_args.user_delta == NULL) {
GNIX_INFO(FI_LOG_MR, "kdreg get_user_delta is NULL\n");
ret = -FI_ENODATA;
goto err_exit;
}
mrn->fd = kdreg_fd;
mrn->cntr = (kdreg_user_delta_t *) get_user_delta_args.user_delta;
err_exit:
fastlock_release(&mrn->lock);
return ret;
}
static void __gnix_rma_fill_pd_chained_get(struct gnix_fab_req *req,
struct gnix_tx_descriptor *txd,
gni_mem_handle_t *rem_mdh)
{
int head_off, head_len, tail_len, desc_idx = 0;
/* Copy head and tail through intermediate buffer. Copy
* aligned data directly to user buffer. */
head_off = req->rma.rem_addr & GNI_READ_ALIGN_MASK;
head_len = head_off ? GNI_READ_ALIGN - head_off : 0;
tail_len = (req->rma.rem_addr + req->rma.len) & GNI_READ_ALIGN_MASK;
/* Use full post descriptor for aligned data */
txd->gni_desc.local_addr = (uint64_t)req->rma.loc_addr + head_len;
txd->gni_desc.remote_addr = (uint64_t)req->rma.rem_addr + head_len;
txd->gni_desc.length = req->rma.len - head_len - tail_len;
assert(txd->gni_desc.length);
txd->gni_desc.next_descr = &txd->gni_ct_descs[0];
if (head_off) {
txd->gni_ct_descs[0].ep_hndl = req->vc->gni_ep;
txd->gni_ct_descs[0].length = GNI_READ_ALIGN;
txd->gni_ct_descs[0].remote_addr =
req->rma.rem_addr & ~GNI_READ_ALIGN_MASK;
txd->gni_ct_descs[0].remote_mem_hndl = *rem_mdh;
txd->gni_ct_descs[0].local_addr =
(uint64_t)txd->int_buf;
txd->gni_ct_descs[0].local_mem_hndl =
req->vc->ep->nic->int_bufs_mdh;
if (tail_len)
txd->gni_ct_descs[0].next_descr =
&txd->gni_ct_descs[1];
else
txd->gni_ct_descs[0].next_descr = NULL;
desc_idx++;
}
if (tail_len) {
txd->gni_ct_descs[desc_idx].ep_hndl = req->vc->gni_ep;
txd->gni_ct_descs[desc_idx].length = GNI_READ_ALIGN;
txd->gni_ct_descs[desc_idx].remote_addr =
(req->rma.rem_addr +
req->rma.len) & ~GNI_READ_ALIGN_MASK;
txd->gni_ct_descs[desc_idx].remote_mem_hndl = *rem_mdh;
txd->gni_ct_descs[desc_idx].local_addr =
(uint64_t)txd->int_buf + GNI_READ_ALIGN;
txd->gni_ct_descs[desc_idx].local_mem_hndl =
req->vc->ep->nic->int_bufs_mdh;
txd->gni_ct_descs[desc_idx].next_descr = NULL;
}
GNIX_INFO(FI_LOG_EP_DATA,
"ct_rem_addr[0] = %p %p, ct_rem_addr[1] = %p %p\n",
txd->gni_ct_descs[0].remote_addr,
txd->gni_ct_descs[0].local_addr,
txd->gni_ct_descs[1].remote_addr,
txd->gni_ct_descs[1].local_addr);
}
int
_gnix_notifier_close(struct gnix_mr_notifier *mrn)
{
int ret = FI_SUCCESS;
int ret_errno;
ret = notifier_verify_stuff(mrn);
if (ret == 0) {
fastlock_acquire(&mrn->lock);
if (close(mrn->fd) != 0) {
ret_errno = errno;
GNIX_INFO(FI_LOG_MR, "error closing kdreg device: %s\n",
strerror(ret_errno));
// Not all of these map to fi_errno values
ret = -ret_errno;
goto err_exit;
}
mrn->cntr = NULL;
err_exit:
fastlock_release(&mrn->lock);
}
return ret;
}
/**
* Closes and deallocates a libfabric memory registration in the internal cache
*
* @param[in] fid libfabric memory registration fid
*
* @return FI_SUCCESS on success
* -FI_EINVAL on invalid fid
* -FI_NOENT when there isn't a matching registration for the
* provided fid
* Otherwise, GNI_RC_* ret codes converted to FI_* err codes
*/
static int fi_gnix_mr_close(fid_t fid)
{
struct gnix_fid_mem_desc *mr;
gni_return_t ret;
struct gnix_fid_domain *domain;
GNIX_TRACE(FI_LOG_MR, "\n");
if (unlikely(fid->fclass != FI_CLASS_MR))
return -FI_EINVAL;
mr = container_of(fid, struct gnix_fid_mem_desc, mr_fid.fid);
domain = mr->domain;
/* call cache deregister op */
fastlock_acquire(&domain->mr_cache_lock);
ret = domain->mr_ops->dereg_mr(domain, mr);
fastlock_release(&domain->mr_cache_lock);
/* check retcode */
if (likely(ret == FI_SUCCESS)) {
/* release references to the domain and nic */
_gnix_ref_put(domain);
} else {
GNIX_INFO(FI_LOG_MR, "failed to deregister memory, "
"ret=%i\n", ret);
}
return ret;
}
DIRECT_FN STATIC int gnix_setname(fid_t fid, void *addr, size_t addrlen)
{
struct gnix_fid_ep *ep = NULL;
struct gnix_fid_sep *sep = NULL;
struct gnix_fid_pep *pep = NULL;
if (addrlen != sizeof(struct gnix_ep_name))
return -FI_EINVAL;
switch (fid->fclass) {
case FI_CLASS_EP:
ep = container_of(fid, struct gnix_fid_ep, ep_fid.fid);
memcpy(&ep->src_addr, addr, sizeof(struct gnix_ep_name));
break;
case FI_CLASS_SEP:
sep = container_of(fid, struct gnix_fid_sep, ep_fid);
memcpy(&sep->my_name, addr, sizeof(struct gnix_ep_name));
break;
case FI_CLASS_PEP:
pep = container_of(fid, struct gnix_fid_pep, pep_fid.fid);
/* TODO: make sure we're unconnected. */
pep->bound = 1;
memcpy(&pep->src_addr, addr, sizeof(struct gnix_ep_name));
break;
default:
GNIX_INFO(FI_LOG_EP_CTRL, "Invalid fid class: %d\n",
fid->fclass);
return -FI_EINVAL;
}
return 0;
}
/**
* Retrieve the local endpoint address.
*
* addrlen: Should indicate the size of the addr buffer. On output will contain
* the size necessary to copy the proper address structure.
*
* addr: Pointer to memory that will conatin the address structure. Should be
* allocated and of size addrlen. If addrlen is less than necessary to copy
* the proper address structure then addr will contain a truncated address.
*
* return: FI_SUCCESS or negative error value.
*/
DIRECT_FN STATIC int gnix_getname(fid_t fid, void *addr, size_t *addrlen)
{
struct gnix_fid_ep *ep = NULL;
struct gnix_fid_sep *sep = NULL;
struct gnix_fid_pep *pep = NULL;
size_t len;
if (!addr) {
*addrlen = sizeof(struct gnix_ep_name);
return -FI_ETOOSMALL;
}
len = MIN(*addrlen, sizeof(struct gnix_ep_name));
switch (fid->fclass) {
case FI_CLASS_EP:
ep = container_of(fid, struct gnix_fid_ep, ep_fid.fid);
memcpy(addr, &ep->src_addr, len);
break;
case FI_CLASS_SEP:
sep = container_of(fid, struct gnix_fid_sep, ep_fid);
memcpy(addr, &sep->my_name, len);
break;
case FI_CLASS_PEP:
pep = container_of(fid, struct gnix_fid_pep, pep_fid.fid);
memcpy(addr, &pep->src_addr, len);
break;
default:
GNIX_INFO(FI_LOG_EP_CTRL, "Invalid fid class: %d\n",
fid->fclass);
return -FI_EINVAL;
}
*addrlen = sizeof(struct gnix_ep_name);
return (len == sizeof(struct gnix_ep_name)) ? 0 : -FI_ETOOSMALL;
}
int
_gnix_notifier_close(struct gnix_mr_notifier *mrn)
{
int ret = FI_SUCCESS;
int ret_errno;
fastlock_acquire(&mrn->lock);
ret = notifier_verify_stuff(mrn);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_MR, "Invalid MR notifier\n");
goto err_exit;
}
assert(mrn->ref_cnt > 0);
if (--mrn->ref_cnt) {
goto err_exit;
}
if (close(mrn->fd) != 0) {
ret_errno = errno;
GNIX_INFO(FI_LOG_MR, "error closing kdreg device: %s\n",
strerror(ret_errno));
/* Not all of these map to fi_errno values */
ret = -ret_errno;
goto err_exit;
}
mrn->fd = -1;
mrn->cntr = NULL;
err_exit:
fastlock_release(&mrn->lock);
return ret;
}
static int __gnix_deregister_region(
void *handle,
void *context)
{
struct gnix_fid_mem_desc *mr = (struct gnix_fid_mem_desc *) handle;
gni_return_t ret;
struct gnix_fid_domain *domain;
struct gnix_nic *nic;
domain = mr->domain;
nic = mr->nic;
COND_ACQUIRE(nic->requires_lock, &nic->lock);
ret = GNI_MemDeregister(nic->gni_nic_hndl, &mr->mem_hndl);
COND_RELEASE(nic->requires_lock, &nic->lock);
if (ret == GNI_RC_SUCCESS) {
/* release reference to domain */
_gnix_ref_put(domain);
/* release reference to nic */
_gnix_ref_put(nic);
} else {
GNIX_INFO(FI_LOG_MR, "failed to deregister memory"
" region, entry=%p ret=%i\n", handle, ret);
}
return ret;
}
static int __gnix_cm_nic_intra_progress_fn(void *data, int *complete_ptr)
{
struct gnix_datagram *dgram;
struct gnix_cm_nic *cm_nic;
int ret;
GNIX_INFO(FI_LOG_EP_CTRL, "\n");
dgram = (struct gnix_datagram *)data;
cm_nic = (struct gnix_cm_nic *)dgram->cache;
ret = __process_datagram(dgram,
cm_nic->my_name.gnix_addr,
GNI_POST_COMPLETED);
if (ret == FI_SUCCESS) {
GNIX_INFO(FI_LOG_EP_CTRL, "Intra-CM NIC dgram completed\n");
*complete_ptr = 1;
}
return FI_SUCCESS;
}
/* Schedule the VC for RX progress. */
int _gnix_vc_rx_schedule(struct gnix_vc *vc)
{
struct gnix_nic *nic = vc->ep->nic;
if (!_gnix_test_and_set_bit(&vc->flags, GNIX_VC_FLAG_RX_SCHEDULED)) {
fastlock_acquire(&nic->rx_vc_lock);
dlist_insert_tail(&vc->rx_list, &nic->rx_vcs);
fastlock_release(&nic->rx_vc_lock);
GNIX_INFO(FI_LOG_EP_CTRL, "Scheduled RX VC (%p)\n", vc);
}
return FI_SUCCESS;
}
int get_ccm_ptag_cookie(char *filename)
{
int rc, fd;
ccm_alps_info_t info;
GNIX_INFO(FI_LOG_FABRIC, "Reading job info file %s", filename);
fd = open(filename, O_RDONLY);
if (fd < 0)
return -FI_EIO;
rc = read(fd, &info, sizeof(ccm_alps_info_t));
if (rc != sizeof(ccm_alps_info_t))
return -FI_EIO;
gnix_app_ptag = info.ptag;
gnix_app_cookie = info.cookie;
close(fd);
GNIX_INFO(FI_LOG_FABRIC, "Ptag=0x%x, cookie=0x%x",
gnix_app_ptag, gnix_app_cookie);
return FI_SUCCESS;
}
int
_gnix_notifier_init(struct gnix_mr_notifier *mrn)
{
if (mrn == NULL) {
GNIX_INFO(FI_LOG_MR, "mr notifier NULL\n");
return -FI_EINVAL;
}
mrn->fd = 0;
mrn->cntr = NULL;
fastlock_init(&mrn->lock);
return FI_SUCCESS;
}
static int gnix_fabric_close(fid_t fid)
{
struct gnix_fid_fabric *fab;
int references_held;
fab = container_of(fid, struct gnix_fid_fabric, fab_fid);
references_held = _gnix_ref_put(fab);
if (references_held)
GNIX_INFO(FI_LOG_FABRIC, "failed to fully close fabric due "
"to lingering references. references=%i fabric=%p\n",
references_held, fab);
return FI_SUCCESS;
}
static inline int
kdreg_write(struct gnix_mr_notifier *mrn, void *buf, size_t len) {
int ret;
ret = write(mrn->fd, buf, len);
if ((ret < 0) || (ret != len)) {
// Not all of these map to fi_errno values
ret = -errno;
GNIX_INFO(FI_LOG_MR, "kdreg_write failed: %s\n",
strerror(errno));
return ret;
}
return FI_SUCCESS;
}
static int gnix_pep_close(fid_t fid)
{
int ret = FI_SUCCESS;
struct gnix_fid_pep *pep;
int references_held;
pep = container_of(fid, struct gnix_fid_pep, pep_fid.fid);
references_held = _gnix_ref_put(pep);
if (references_held)
GNIX_INFO(FI_LOG_EP_CTRL, "failed to fully close pep due "
"to lingering references. references=%i pep=%p\n",
references_held, pep);
return ret;
}
static void __gnix_amo_fr_complete(struct gnix_fab_req *req,
struct gnix_tx_descriptor *txd)
{
if (req->flags & FI_LOCAL_MR) {
GNIX_INFO(FI_LOG_EP_DATA, "freeing auto-reg MR: %p\n",
req->amo.loc_md);
fi_close(&req->amo.loc_md->mr_fid.fid);
}
atomic_dec(&req->vc->outstanding_tx_reqs);
_gnix_nic_tx_free(req->vc->ep->nic, txd);
/* Schedule VC TX queue in case the VC is 'fenced'. */
_gnix_vc_tx_schedule(req->vc);
_gnix_fr_free(req->vc->ep, req);
}
static struct gnix_vc *__gnix_nic_next_pending_rx_vc(struct gnix_nic *nic)
{
struct gnix_vc *vc = NULL;
fastlock_acquire(&nic->rx_vc_lock);
vc = dlist_first_entry(&nic->rx_vcs, struct gnix_vc, rx_list);
if (vc)
dlist_remove_init(&vc->rx_list);
fastlock_release(&nic->rx_vc_lock);
if (vc) {
GNIX_INFO(FI_LOG_EP_CTRL, "Dequeued RX VC (%p)\n", vc);
_gnix_clear_bit(&vc->flags, GNIX_VC_FLAG_RX_SCHEDULED);
}
return vc;
}
/* Schedule the VC for work progress. */
static int __gnix_vc_work_schedule(struct gnix_vc *vc)
{
struct gnix_nic *nic = vc->ep->nic;
/* Don't bother scheduling if there's no work to do. */
if (slist_empty(&vc->work_queue))
return FI_SUCCESS;
if (!_gnix_test_and_set_bit(&vc->flags, GNIX_VC_FLAG_WORK_SCHEDULED)) {
fastlock_acquire(&nic->work_vc_lock);
dlist_insert_tail(&vc->work_list, &nic->work_vcs);
fastlock_release(&nic->work_vc_lock);
GNIX_INFO(FI_LOG_EP_CTRL, "Scheduled work VC (%p)\n", vc);
}
return FI_SUCCESS;
}
static ssize_t gnix_ep_tx_size_left(struct fid_ep *ep)
{
if (!ep)
return -FI_EINVAL;
switch (ep->fid.fclass) {
case FI_CLASS_EP:
break;
case FI_CLASS_TX_CTX:
break;
default:
GNIX_INFO(FI_LOG_EP_CTRL, "Invalid EP type\n");
return -FI_EINVAL;
}
/* We can queue TXs indefinitely, return an arbitrary low water mark. */
return 64;
}
/* Process deferred request work on the VC. */
static int __gnix_vc_push_work_reqs(struct gnix_vc *vc)
{
int ret, fi_rc = FI_SUCCESS;
struct slist_entry *item;
struct gnix_fab_req *req;
try_again:
fastlock_acquire(&vc->work_queue_lock);
item = slist_remove_head(&vc->work_queue);
fastlock_release(&vc->work_queue_lock);
if (item != NULL) {
req = (struct gnix_fab_req *)container_of(item,
struct gnix_fab_req,
slist);
ret = req->work_fn(req);
if (ret == FI_SUCCESS) {
GNIX_INFO(FI_LOG_EP_DATA,
"Request processed: %p\n", req);
goto try_again;
} else {
/* Work failed. Reschedule to put this VC back on the
* end of the list. */
__gnix_vc_work_schedule(vc);
fastlock_acquire(&vc->work_queue_lock);
slist_insert_tail(item, &vc->work_queue);
fastlock_release(&vc->work_queue_lock);
/* FI_ENOSPC is reserved to indicate a lack of TXDs,
* which are shared by all VCs on the NIC. Return
* error to stall processing of VCs in this case. The
* other likely error is a lack of SMSG credits, which
* only halts this VC. */
if (ret == -FI_ENOSPC) {
fi_rc = -FI_EAGAIN;
} else if (ret != -FI_EAGAIN) {
/* TODO report error? */
GNIX_WARN(FI_LOG_EP_DATA,
"Failed to push request %p: %s\n",
req, fi_strerror(-ret));
} /* else return success to keep processing TX VCs */
}
}