/* SMSG callback for AMO remote counter control message. */ int __smsg_amo_cntr(void *data, void *msg) { int ret = FI_SUCCESS; struct gnix_vc *vc = (struct gnix_vc *)data; struct gnix_smsg_amo_cntr_hdr *hdr = (struct gnix_smsg_amo_cntr_hdr *)msg; struct gnix_fid_ep *ep = vc->ep; gni_return_t status; if (hdr->flags & FI_REMOTE_WRITE && ep->rwrite_cntr) { ret = _gnix_cntr_inc(ep->rwrite_cntr); if (ret != FI_SUCCESS) GNIX_WARN(FI_LOG_EP_DATA, "_gnix_cntr_inc() failed: %d\n", ret); } if (hdr->flags & FI_REMOTE_READ && ep->rread_cntr) { ret = _gnix_cntr_inc(ep->rread_cntr); if (ret != FI_SUCCESS) GNIX_WARN(FI_LOG_EP_DATA, "_gnix_cntr_inc() failed: %d\n", ret); } status = GNI_SmsgRelease(vc->gni_ep); if (OFI_UNLIKELY(status != GNI_RC_SUCCESS)) { GNIX_WARN(FI_LOG_EP_DATA, "GNI_SmsgRelease returned %s\n", gni_err_str[status]); ret = gnixu_to_fi_errno(status); } return ret; }
int _gnix_cm_nic_free(struct gnix_cm_nic *cm_nic) { int ret = FI_SUCCESS; gni_return_t status; GNIX_TRACE(FI_LOG_EP_CTRL, "\n"); if (cm_nic == NULL) return -FI_EINVAL; if (cm_nic->dgram_hndl != NULL) { ret = _gnix_dgram_hndl_free(cm_nic->dgram_hndl); if (ret != FI_SUCCESS) GNIX_WARN(FI_LOG_EP_CTRL, "gnix_dgram_hndl_free returned %d\n", ret); } if (cm_nic->gni_cdm_hndl != NULL) { status = GNI_CdmDestroy(cm_nic->gni_cdm_hndl); if (status != GNI_RC_SUCCESS) { GNIX_WARN(FI_LOG_EP_CTRL, "cdm destroy failed - %s\n", gni_err_str[status]); ret = gnixu_to_fi_errno(status); } } free(cm_nic); return ret; }
static int __gnix_rma_send_data_req(void *arg) { struct gnix_fab_req *req = (struct gnix_fab_req *)arg; struct gnix_fid_ep *ep = req->gnix_ep; struct gnix_nic *nic = ep->nic; struct gnix_tx_descriptor *txd; gni_return_t status; int rc; int inject_err = _gnix_req_inject_err(req); rc = _gnix_nic_tx_alloc(nic, &txd); if (rc) { GNIX_INFO(FI_LOG_EP_DATA, "_gnix_nic_tx_alloc() failed: %d\n", rc); return -FI_ENOSPC; } txd->req = req; txd->completer_fn = __gnix_rma_txd_data_complete; txd->rma_data_hdr.flags = FI_RMA | FI_REMOTE_CQ_DATA; if (req->type == GNIX_FAB_RQ_RDMA_WRITE) { txd->rma_data_hdr.flags |= FI_REMOTE_WRITE; } else { txd->rma_data_hdr.flags |= FI_REMOTE_READ; } txd->rma_data_hdr.data = req->rma.imm; fastlock_acquire(&nic->lock); if (inject_err) { _gnix_nic_txd_err_inject(nic, txd); status = GNI_RC_SUCCESS; } else { status = GNI_SmsgSendWTag(req->vc->gni_ep, &txd->rma_data_hdr, sizeof(txd->rma_data_hdr), NULL, 0, txd->id, GNIX_SMSG_T_RMA_DATA); } fastlock_release(&nic->lock); if (status == GNI_RC_NOT_DONE) { _gnix_nic_tx_free(nic, txd); GNIX_INFO(FI_LOG_EP_DATA, "GNI_SmsgSendWTag returned %s\n", gni_err_str[status]); } else if (status != GNI_RC_SUCCESS) { _gnix_nic_tx_free(nic, txd); GNIX_WARN(FI_LOG_EP_DATA, "GNI_SmsgSendWTag returned %s\n", gni_err_str[status]); } else { GNIX_INFO(FI_LOG_EP_DATA, "Sent RMA CQ data, req: %p\n", req); } return gnixu_to_fi_errno(status); }
static int __gnix_amo_send_cntr_req(void *arg) { struct gnix_fab_req *req = (struct gnix_fab_req *)arg; struct gnix_fid_ep *ep = req->gnix_ep; struct gnix_nic *nic = ep->nic; struct gnix_tx_descriptor *txd; gni_return_t status; int rc; int inject_err = _gnix_req_inject_err(req); rc = _gnix_nic_tx_alloc(nic, &txd); if (rc) { GNIX_INFO(FI_LOG_EP_DATA, "_gnix_nic_tx_alloc() failed: %d\n", rc); return -FI_ENOSPC; } txd->req = req; txd->completer_fn = __gnix_amo_txd_cntr_complete; if (req->type == GNIX_FAB_RQ_AMO) { txd->amo_cntr_hdr.flags = FI_REMOTE_WRITE; } else { txd->amo_cntr_hdr.flags = FI_REMOTE_READ; } COND_ACQUIRE(nic->requires_lock, &nic->lock); if (inject_err) { _gnix_nic_txd_err_inject(nic, txd); status = GNI_RC_SUCCESS; } else { status = GNI_SmsgSendWTag(req->vc->gni_ep, &txd->amo_cntr_hdr, sizeof(txd->amo_cntr_hdr), NULL, 0, txd->id, GNIX_SMSG_T_AMO_CNTR); } COND_RELEASE(nic->requires_lock, &nic->lock); if (status == GNI_RC_NOT_DONE) { _gnix_nic_tx_free(nic, txd); GNIX_INFO(FI_LOG_EP_DATA, "GNI_SmsgSendWTag returned %s\n", gni_err_str[status]); } else if (status != GNI_RC_SUCCESS) { _gnix_nic_tx_free(nic, txd); GNIX_WARN(FI_LOG_EP_DATA, "GNI_SmsgSendWTag returned %s\n", gni_err_str[status]); } else { GNIX_INFO(FI_LOG_EP_DATA, "Sent RMA CQ data, req: %p\n", req); } return gnixu_to_fi_errno(status); }
static int __gnix_rndzv_req_send_fin(void *arg) { struct gnix_fab_req *req = (struct gnix_fab_req *)arg; struct gnix_nic *nic; struct gnix_fid_ep *ep; struct gnix_tx_descriptor *txd; gni_return_t status; int rc; GNIX_TRACE(FI_LOG_EP_DATA, "\n"); ep = req->gnix_ep; assert(ep != NULL); nic = ep->nic; assert(nic != NULL); rc = _gnix_nic_tx_alloc(nic, &txd); if (rc) { GNIX_INFO(FI_LOG_EP_DATA, "_gnix_nic_tx_alloc() failed: %d\n", rc); return -FI_ENOSPC; } txd->rndzv_fin_hdr.req_addr = req->msg.rma_id; txd->req = req; txd->completer_fn = gnix_ep_smsg_completers[GNIX_SMSG_T_RNDZV_FIN]; fastlock_acquire(&nic->lock); status = GNI_SmsgSendWTag(req->vc->gni_ep, &txd->rndzv_fin_hdr, sizeof(txd->rndzv_fin_hdr), NULL, 0, txd->id, GNIX_SMSG_T_RNDZV_FIN); if ((status == GNI_RC_SUCCESS) && (ep->domain->data_progress == FI_PROGRESS_AUTO)) _gnix_rma_post_irq(req->vc); fastlock_release(&nic->lock); if (status == GNI_RC_NOT_DONE) { _gnix_nic_tx_free(nic, txd); GNIX_INFO(FI_LOG_EP_DATA, "GNI_SmsgSendWTag returned %s\n", gni_err_str[status]); } else if (status != GNI_RC_SUCCESS) { _gnix_nic_tx_free(nic, txd); GNIX_WARN(FI_LOG_EP_DATA, "GNI_SmsgSendWTag returned %s\n", gni_err_str[status]); } GNIX_INFO(FI_LOG_EP_DATA, "Initiated RNDZV_FIN, req: %p\n", req); return gnixu_to_fi_errno(status); }
static int __nic_setup_irq_cq(struct gnix_nic *nic) { int ret = FI_SUCCESS; size_t len; gni_return_t status; int fd = -1; void *mmap_addr; len = (size_t)sysconf(_SC_PAGESIZE); mmap_addr = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANON, fd, 0); if (mmap_addr == MAP_FAILED) { GNIX_WARN(FI_LOG_EP_CTRL, "mmap failed - %s\n", strerror(errno)); ret = -errno; goto err; } nic->irq_mmap_addr = mmap_addr; nic->irq_mmap_len = len; status = GNI_MemRegister(nic->gni_nic_hndl, (uint64_t) nic->irq_mmap_addr, len, nic->rx_cq_blk, GNI_MEM_READWRITE, -1, &nic->irq_mem_hndl); if (status != GNI_RC_SUCCESS) { ret = gnixu_to_fi_errno(status); GNIX_WARN(FI_LOG_EP_CTRL, "GNI_MemRegister returned %s\n", gni_err_str[status]); goto err_w_mmap; } #if 0 fprintf(stderr,"registered ireq memhndl 0x%016lx 0x%016lx\n", nic->irq_mem_hndl.qword1, nic->irq_mem_hndl.qword2); #endif return ret; err_w_mmap: munmap(mmap_addr, len); err: return ret; }
static int __nic_rx_progress(struct gnix_nic *nic) { int ret = FI_SUCCESS; gni_return_t status = GNI_RC_NOT_DONE; gni_cq_entry_t cqe; status = GNI_CqTestEvent(nic->rx_cq); if (status == GNI_RC_NOT_DONE) return FI_SUCCESS; COND_ACQUIRE(nic->requires_lock, &nic->lock); do { status = GNI_CqGetEvent(nic->rx_cq, &cqe); if (unlikely(status == GNI_RC_NOT_DONE)) { ret = FI_SUCCESS; break; } if (likely(status == GNI_RC_SUCCESS)) { /* Find and schedule the associated VC. */ ret = __process_rx_cqe(nic, cqe); if (ret != FI_SUCCESS) { GNIX_WARN(FI_LOG_EP_DATA, "process_rx_cqe() failed: %d\n", ret); } } else if (status == GNI_RC_ERROR_RESOURCE) { /* The remote CQ was overrun. Events related to any VC * could have been missed. Schedule each VC to be sure * all messages are processed. */ assert(GNI_CQ_OVERRUN(cqe)); __nic_rx_overrun(nic); } else { GNIX_WARN(FI_LOG_EP_DATA, "GNI_CqGetEvent returned %s\n", gni_err_str[status]); ret = gnixu_to_fi_errno(status); break; } } while (1); COND_RELEASE(nic->requires_lock, &nic->lock); return ret; }
static int __smsg_eager_msg_w_data(void *data, void *msg) { int ret = FI_SUCCESS; gni_return_t status; struct gnix_vc *vc = (struct gnix_vc *)data; struct gnix_smsg_hdr *hdr = (struct gnix_smsg_hdr *)msg; struct gnix_fid_ep *ep; ep = vc->ep; assert(ep); /* * the msg data must be consumed either by matching * a message or allocating a buffer and putting * on unexpected queue. */ ret = _gnix_ep_eager_msg_w_data_match(ep, (void *)((char *)msg + sizeof(*hdr)), vc->peer_addr, hdr->len, hdr->imm, hdr->flags); /* * we keep on going even if we got an error back because * we need to release the pending message in the SMSG buffer * and unlock the nic lock */ if (ret != FI_SUCCESS) GNIX_WARN(FI_LOG_EP_DATA, "_gnix_ep_eager_msg_rcv returned %d\n", ret); status = GNI_SmsgRelease(vc->gni_ep); if (unlikely(status != GNI_RC_SUCCESS)) { GNIX_WARN(FI_LOG_EP_DATA, "GNI_SmsgRelease returned %s\n", gni_err_str[status]); ret = gnixu_to_fi_errno(status); goto err; } err: return ret; }
/* Process a VC's SMSG mailbox. */ int _gnix_vc_dequeue_smsg(struct gnix_vc *vc) { int ret = FI_SUCCESS; struct gnix_nic *nic; gni_return_t status; void *msg_ptr; uint8_t tag; GNIX_TRACE(FI_LOG_EP_DATA, "\n"); assert(vc->gni_ep != NULL); assert(vc->conn_state == GNIX_VC_CONNECTED); nic = vc->ep->nic; assert(nic != NULL); do { tag = GNI_SMSG_ANY_TAG; status = GNI_SmsgGetNextWTag(vc->gni_ep, &msg_ptr, &tag); if (status == GNI_RC_SUCCESS) { GNIX_INFO(FI_LOG_EP_DATA, "Found RX (%p)\n", vc); ret = nic->smsg_callbacks[tag](vc, msg_ptr); if (ret != FI_SUCCESS) { /* Stalled, reschedule */ break; } } else if (status == GNI_RC_NOT_DONE) { /* No more work. */ ret = FI_SUCCESS; break; } else { GNIX_WARN(FI_LOG_EP_DATA, "GNI_SmsgGetNextWTag returned %s\n", gni_err_str[status]); ret = gnixu_to_fi_errno(status); break; } } while (1); return ret; }
/* * release resources previously set up for remote * GNI_PostCqWrite's to target */ static int __nic_teardown_irq_cq(struct gnix_nic *nic) { int ret = FI_SUCCESS; gni_return_t status; if (nic == NULL) return ret; if (nic->irq_mmap_addr == NULL) return ret; status = GNI_MemDeregister(nic->gni_nic_hndl, &nic->irq_mem_hndl); if (status != GNI_RC_SUCCESS) { ret = gnixu_to_fi_errno(status); GNIX_WARN(FI_LOG_EP_CTRL, "GNI_MemDeregister returned %s\n", gni_err_str[status]); } munmap(nic->irq_mmap_addr, nic->irq_mmap_len); return ret; }
int _gnix_rma_post_rdma_chain_req(void *data) { struct gnix_fab_req *req = (struct gnix_fab_req *)data; struct gnix_fid_ep *ep = req->gnix_ep; struct gnix_nic *nic = ep->nic; struct gnix_tx_descriptor *bte_txd, *ct_txd; gni_mem_handle_t mdh; gni_return_t status; int rc; int inject_err = _gnix_req_inject_err(req); int head_off, head_len, tail_len; int fma_chain = 0; rc = _gnix_nic_tx_alloc(nic, &bte_txd); if (rc) { GNIX_INFO(FI_LOG_EP_DATA, "BTE _gnix_nic_tx_alloc() failed: %d\n", rc); return -FI_ENOSPC; } rc = _gnix_nic_tx_alloc(nic, &ct_txd); if (rc) { _gnix_nic_tx_free(nic, bte_txd); GNIX_INFO(FI_LOG_EP_DATA, "CT _gnix_nic_tx_alloc() failed: %d\n", rc); return -FI_ENOSPC; } _gnix_convert_key_to_mhdl( (gnix_mr_key_t *)&req->rma.rem_mr_key, &mdh); /* BTE TXD */ bte_txd->completer_fn = __gnix_rma_txd_complete; bte_txd->req = req; bte_txd->gni_desc.type = GNI_POST_RDMA_GET; bte_txd->gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT; /* check flags */ bte_txd->gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE; /* check flags */ head_off = req->rma.rem_addr & GNI_READ_ALIGN_MASK; head_len = head_off ? GNI_READ_ALIGN - head_off : 0; tail_len = (req->rma.rem_addr + req->rma.len) & GNI_READ_ALIGN_MASK; bte_txd->gni_desc.local_addr = (uint64_t)req->rma.loc_addr + head_len; bte_txd->gni_desc.remote_addr = (uint64_t)req->rma.rem_addr + head_len; bte_txd->gni_desc.length = req->rma.len - head_len - tail_len; bte_txd->gni_desc.remote_mem_hndl = mdh; bte_txd->gni_desc.rdma_mode = 0; /* check flags */ bte_txd->gni_desc.src_cq_hndl = nic->tx_cq; /* check flags */ bte_txd->gni_desc.local_mem_hndl = req->rma.loc_md->mem_hndl; /* FMA TXD */ ct_txd->completer_fn = __gnix_rma_txd_complete; ct_txd->req = req; ct_txd->gni_desc.type = GNI_POST_FMA_GET; ct_txd->gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT; /* check flags */ ct_txd->gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE; /* check flags */ ct_txd->gni_desc.remote_mem_hndl = mdh; ct_txd->gni_desc.rdma_mode = 0; /* check flags */ ct_txd->gni_desc.src_cq_hndl = nic->tx_cq; /* check flags */ ct_txd->gni_desc.local_mem_hndl = nic->int_bufs_mdh; ct_txd->gni_desc.length = GNI_READ_ALIGN; if (head_off) { ct_txd->gni_desc.remote_addr = req->rma.rem_addr & ~GNI_READ_ALIGN_MASK; ct_txd->gni_desc.local_addr = (uint64_t)ct_txd->int_buf; if (tail_len) { ct_txd->gni_desc.next_descr = &ct_txd->gni_ct_descs[0]; ct_txd->gni_ct_descs[0].ep_hndl = req->vc->gni_ep; ct_txd->gni_ct_descs[0].length = GNI_READ_ALIGN; ct_txd->gni_ct_descs[0].remote_addr = (req->rma.rem_addr + req->rma.len) & ~GNI_READ_ALIGN_MASK; ct_txd->gni_ct_descs[0].remote_mem_hndl = mdh; ct_txd->gni_ct_descs[0].local_addr = (uint64_t)ct_txd->int_buf + GNI_READ_ALIGN; ct_txd->gni_ct_descs[0].local_mem_hndl = nic->int_bufs_mdh; ct_txd->gni_ct_descs[0].next_descr = NULL; fma_chain = 1; } } else { ct_txd->gni_desc.remote_addr = (req->rma.rem_addr + req->rma.len) & ~GNI_READ_ALIGN_MASK; ct_txd->gni_desc.local_addr = (uint64_t)ct_txd->int_buf + GNI_READ_ALIGN; } fastlock_acquire(&nic->lock); if (unlikely(inject_err)) { _gnix_nic_txd_err_inject(nic, bte_txd); status = GNI_RC_SUCCESS; } else { status = GNI_PostRdma(req->vc->gni_ep, &bte_txd->gni_desc); } if (status != GNI_RC_SUCCESS) { fastlock_release(&nic->lock); _gnix_nic_tx_free(nic, ct_txd); _gnix_nic_tx_free(nic, bte_txd); GNIX_INFO(FI_LOG_EP_DATA, "GNI_Post*() failed: %s\n", gni_err_str[status]); return gnixu_to_fi_errno(status); } if (unlikely(inject_err)) { _gnix_nic_txd_err_inject(nic, ct_txd); status = GNI_RC_SUCCESS; } else if (fma_chain) { status = GNI_CtPostFma(req->vc->gni_ep, &ct_txd->gni_desc); } else { status = GNI_PostFma(req->vc->gni_ep, &ct_txd->gni_desc); } if (status != GNI_RC_SUCCESS) { fastlock_release(&nic->lock); _gnix_nic_tx_free(nic, ct_txd); /* Wait for the first TX to complete, then retransmit the * entire thing. */ atomic_set(&req->rma.outstanding_txds, 1); req->rma.status = GNI_RC_TRANSACTION_ERROR; GNIX_INFO(FI_LOG_EP_DATA, "GNI_Post*() failed: %s\n", gni_err_str[status]); return FI_SUCCESS; } fastlock_release(&nic->lock); /* Wait for both TXs to complete, then process the request. */ atomic_set(&req->rma.outstanding_txds, 2); req->rma.status = 0; return FI_SUCCESS; }
/* * gnix_resolve_name: given a node hint and a valid pointer to a gnix_ep_name * will resolve the gnix specific address of node and fill the provided * gnix_ep_name pointer with the information. * * node (IN) : Node name being resolved to gnix specific address * service (IN) : Port number being resolved to gnix specific address * resolved_addr (IN/OUT) : Pointer that must be provided to contain the * resolved address. */ int _gnix_resolve_name(IN const char *node, IN const char *service, IN uint64_t flags, INOUT struct gnix_ep_name *resolved_addr) { uint32_t pe = -1; uint32_t cpu_id = -1; struct addrinfo *result = NULL; struct addrinfo *rp = NULL; struct sockaddr_in *sa = NULL; struct sockaddr_in sin; int ret = FI_SUCCESS; gni_return_t status = GNI_RC_SUCCESS; struct addrinfo hints = { .ai_family = AF_INET, .ai_socktype = SOCK_DGRAM, }; GNIX_TRACE(FI_LOG_FABRIC, "\n"); if (flags & FI_SOURCE) hints.ai_flags |= AI_PASSIVE; if (flags & FI_NUMERICHOST) hints.ai_flags |= AI_NUMERICHOST; if (!resolved_addr) { GNIX_WARN(FI_LOG_FABRIC, "Resolved_addr must be a valid pointer.\n"); ret = -FI_EINVAL; goto err; } ret = _gnix_local_ipaddr(&sin); if (ret != FI_SUCCESS) goto err; ret = getaddrinfo(node, service, &hints, &result); if (ret != 0) { GNIX_WARN(FI_LOG_FABRIC, "Failed to get address for node provided: %s\n", gai_strerror(ret)); ret = -FI_EINVAL; goto err; } for (rp = result; rp != NULL; rp = rp->ai_next) { assert(rp->ai_addr->sa_family == AF_INET); sa = (struct sockaddr_in *) rp->ai_addr; /* * If we are trying to resolve localhost then use * CdmGetNicAddress. */ if (sa->sin_addr.s_addr == sin.sin_addr.s_addr) { status = GNI_CdmGetNicAddress(0, &pe, &cpu_id); if(status == GNI_RC_SUCCESS) { break; } else { GNIX_WARN(FI_LOG_FABRIC, "Unable to get NIC address."); ret = gnixu_to_fi_errno(status); goto err; } } else { ret = __gnix_get_pe_from_ip("ipogif0", inet_ntoa(sa->sin_addr), &pe); if (ret == 0) { break; } else { ret = __gnix_get_pe_from_ip("br0", inet_ntoa(sa->sin_addr), &pe); } if (ret == 0) break; } } /* * Make sure address is valid. */ if (pe == -1) { GNIX_WARN(FI_LOG_FABRIC, "Unable to acquire valid address for node %s\n", node); ret = -FI_EADDRNOTAVAIL; goto err; } /* * Fill the INOUT parameter resolved_addr with the address information * acquired for the provided node parameter. */ memset(resolved_addr, 0, sizeof(struct gnix_ep_name)); resolved_addr->gnix_addr.device_addr = pe; if (service) { /* use resolved service/port */ resolved_addr->gnix_addr.cdm_id = ntohs(sa->sin_port); resolved_addr->name_type = GNIX_EPN_TYPE_BOUND; resolved_addr->cm_nic_cdm_id = resolved_addr->gnix_addr.cdm_id; } else { /* generate port internally */ resolved_addr->name_type = GNIX_EPN_TYPE_UNBOUND; } GNIX_INFO(FI_LOG_FABRIC, "Resolved: %s:%s to gnix_addr: 0x%lx\n", node ?: "", service ?: "", resolved_addr->gnix_addr); err: if (result != NULL) { freeaddrinfo(result); } return ret; }
int _gnix_rma_post_req(void *data) { struct gnix_fab_req *fab_req = (struct gnix_fab_req *)data; struct gnix_fid_ep *ep = fab_req->gnix_ep; struct gnix_nic *nic = ep->nic; struct gnix_fid_mem_desc *loc_md; struct gnix_tx_descriptor *txd; gni_mem_handle_t mdh; gni_return_t status; int rc; int rdma = !!(fab_req->flags & GNIX_RMA_RDMA); rc = _gnix_nic_tx_alloc(nic, &txd); if (rc) { GNIX_INFO(FI_LOG_EP_DATA, "_gnix_nic_tx_alloc() failed: %d\n", rc); return -FI_EAGAIN; } txd->desc.completer_fn = __gnix_rma_txd_complete; txd->desc.req = fab_req; _gnix_convert_key_to_mhdl((gnix_mr_key_t *)&fab_req->rma.rem_mr_key, &mdh); loc_md = (struct gnix_fid_mem_desc *)fab_req->loc_md; //txd->desc.gni_desc.post_id = (uint64_t)fab_req; /* unused */ txd->desc.gni_desc.type = __gnix_fr_post_type(fab_req->type, rdma); txd->desc.gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT; /* check flags */ txd->desc.gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE; /* check flags */ txd->desc.gni_desc.local_addr = (uint64_t)fab_req->loc_addr; if (loc_md) { txd->desc.gni_desc.local_mem_hndl = loc_md->mem_hndl; } txd->desc.gni_desc.remote_addr = (uint64_t)fab_req->rma.rem_addr; txd->desc.gni_desc.remote_mem_hndl = mdh; txd->desc.gni_desc.length = fab_req->len; txd->desc.gni_desc.rdma_mode = 0; /* check flags */ txd->desc.gni_desc.src_cq_hndl = nic->tx_cq; /* check flags */ { gni_mem_handle_t *tl_mdh = &txd->desc.gni_desc.local_mem_hndl; gni_mem_handle_t *tr_mdh = &txd->desc.gni_desc.remote_mem_hndl; GNIX_INFO(FI_LOG_EP_DATA, "la: %llx ra: %llx len: %d\n", txd->desc.gni_desc.local_addr, txd->desc.gni_desc.remote_addr, txd->desc.gni_desc.length); GNIX_INFO(FI_LOG_EP_DATA, "lmdh: %llx:%llx rmdh: %llx:%llx key: %llx\n", *(uint64_t *)tl_mdh, *(((uint64_t *)tl_mdh) + 1), *(uint64_t *)tr_mdh, *(((uint64_t *)tr_mdh) + 1), fab_req->rma.rem_mr_key); } fastlock_acquire(&nic->lock); if (rdma) { status = GNI_PostRdma(fab_req->vc->gni_ep, &txd->desc.gni_desc); } else { status = GNI_PostFma(fab_req->vc->gni_ep, &txd->desc.gni_desc); } fastlock_release(&nic->lock); if (status != GNI_RC_SUCCESS) { _gnix_nic_tx_free(nic, txd); GNIX_INFO(FI_LOG_EP_DATA, "GNI_Post*() failed: %s\n", gni_err_str[status]); } return gnixu_to_fi_errno(status); }
int _gnix_amo_post_req(void *data) { struct gnix_fab_req *fab_req = (struct gnix_fab_req *)data; struct gnix_fid_ep *ep = fab_req->gnix_ep; struct gnix_nic *nic = ep->nic; struct gnix_fid_mem_desc *loc_md; struct gnix_tx_descriptor *txd; gni_mem_handle_t mdh; gni_return_t status; int rc; int inject_err = _gnix_req_inject_err(fab_req); if (!gnix_ops_allowed(ep, fab_req->vc->peer_caps, fab_req->flags)) { GNIX_DEBUG(FI_LOG_EP_DATA, "flags:0x%llx, %s\n", fab_req->flags, fi_tostr(&fab_req->flags, FI_TYPE_OP_FLAGS)); GNIX_DEBUG(FI_LOG_EP_DATA, "caps:0x%llx, %s\n", ep->caps, fi_tostr(&ep->caps, FI_TYPE_CAPS)); GNIX_DEBUG(FI_LOG_EP_DATA, "peer_caps:0x%llx, %s\n", fab_req->vc->peer_caps, fi_tostr(&fab_req->vc->peer_caps, FI_TYPE_OP_FLAGS)); rc = __gnix_amo_post_err(fab_req, FI_EOPNOTSUPP); if (rc != FI_SUCCESS) GNIX_WARN(FI_LOG_EP_DATA, "__gnix_amo_post_err() failed: %d\n", rc); return -FI_ECANCELED; } rc = _gnix_nic_tx_alloc(nic, &txd); if (rc) { GNIX_INFO(FI_LOG_EP_DATA, "_gnix_nic_tx_alloc() failed: %d\n", rc); return -FI_ENOSPC; } txd->completer_fn = __gnix_amo_txd_complete; txd->req = fab_req; /* Mem handle CRC is not validated during FMA operations. Skip this * costly calculation. */ _gnix_convert_key_to_mhdl_no_crc( (gnix_mr_key_t *)&fab_req->amo.rem_mr_key, &mdh); loc_md = (struct gnix_fid_mem_desc *)fab_req->amo.loc_md; txd->gni_desc.type = GNI_POST_AMO; txd->gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT; /* check flags */ txd->gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE; /* check flags */ txd->gni_desc.local_addr = (uint64_t)fab_req->amo.loc_addr; if (loc_md) { txd->gni_desc.local_mem_hndl = loc_md->mem_hndl; } txd->gni_desc.remote_addr = (uint64_t)fab_req->amo.rem_addr; txd->gni_desc.remote_mem_hndl = mdh; txd->gni_desc.length = fab_req->amo.len; txd->gni_desc.rdma_mode = 0; /* check flags */ txd->gni_desc.src_cq_hndl = nic->tx_cq; /* check flags */ txd->gni_desc.amo_cmd = _gnix_atomic_cmd(fab_req->amo.datatype, fab_req->amo.op, fab_req->type); txd->gni_desc.first_operand = fab_req->amo.first_operand; txd->gni_desc.second_operand = fab_req->amo.second_operand; GNIX_DEBUG(FI_LOG_EP_DATA, "fo:%016lx so:%016lx\n", txd->gni_desc.first_operand, txd->gni_desc.second_operand); GNIX_DEBUG(FI_LOG_EP_DATA, "amo_cmd:%x\n", txd->gni_desc.amo_cmd); GNIX_LOG_DUMP_TXD(txd); COND_ACQUIRE(nic->requires_lock, &nic->lock); if (OFI_UNLIKELY(inject_err)) { _gnix_nic_txd_err_inject(nic, txd); status = GNI_RC_SUCCESS; } else { status = GNI_PostFma(fab_req->vc->gni_ep, &txd->gni_desc); } COND_RELEASE(nic->requires_lock, &nic->lock); if (status != GNI_RC_SUCCESS) { _gnix_nic_tx_free(nic, txd); GNIX_INFO(FI_LOG_EP_DATA, "GNI_Post*() failed: %s\n", gni_err_str[status]); } return gnixu_to_fi_errno(status); }
/* * helper function to initialize an SMSG connection */ static int __gnix_vc_smsg_init(struct gnix_vc *vc, int peer_id, gni_smsg_attr_t *peer_smsg_attr) { int ret = FI_SUCCESS; struct gnix_fid_ep *ep; struct gnix_fid_domain *dom; struct gnix_mbox *mbox = NULL; gni_smsg_attr_t local_smsg_attr; gni_return_t __attribute__((unused)) status; ssize_t __attribute__((unused)) len; GNIX_TRACE(FI_LOG_EP_CTRL, "\n"); assert(vc); ep = vc->ep; assert(ep); dom = ep->domain; if (dom == NULL) return -FI_EINVAL; mbox = vc->smsg_mbox; assert (mbox); local_smsg_attr.msg_type = GNI_SMSG_TYPE_MBOX_AUTO_RETRANSMIT; local_smsg_attr.msg_buffer = mbox->base; local_smsg_attr.buff_size = vc->ep->nic->mem_per_mbox; local_smsg_attr.mem_hndl = *mbox->memory_handle; local_smsg_attr.mbox_offset = (uint64_t)mbox->offset; local_smsg_attr.mbox_maxcredit = dom->params.mbox_maxcredit; local_smsg_attr.msg_maxsize = dom->params.mbox_msg_maxsize; /* * now build the SMSG connection */ fastlock_acquire(&ep->nic->lock); status = GNI_EpCreate(ep->nic->gni_nic_hndl, ep->nic->tx_cq, &vc->gni_ep); if (status != GNI_RC_SUCCESS) { GNIX_WARN(FI_LOG_EP_CTRL, "GNI_EpCreate returned %s\n", gni_err_str[status]); ret = gnixu_to_fi_errno(status); goto err; } status = GNI_EpBind(vc->gni_ep, vc->peer_addr.device_addr, vc->peer_addr.cdm_id); if (status != GNI_RC_SUCCESS) { GNIX_WARN(FI_LOG_EP_CTRL, "GNI_EpBind returned %s\n", gni_err_str[status]); ret = gnixu_to_fi_errno(status); goto err1; } status = GNI_SmsgInit(vc->gni_ep, &local_smsg_attr, peer_smsg_attr); if (status != GNI_RC_SUCCESS) { GNIX_WARN(FI_LOG_EP_CTRL, "GNI_SmsgInit returned %s\n", gni_err_str[status]); ret = gnixu_to_fi_errno(status); goto err1; } status = GNI_EpSetEventData(vc->gni_ep, vc->vc_id, peer_id); if (status != GNI_RC_SUCCESS) { GNIX_WARN(FI_LOG_EP_CTRL, "GNI_EpSetEventData returned %s\n", gni_err_str[status]); ret = gnixu_to_fi_errno(status); goto err1; } fastlock_release(&ep->nic->lock); return ret; err1: GNI_EpDestroy(vc->gni_ep); err: fastlock_release(&ep->nic->lock); return ret; }
int _gnix_rma_post_req(void *data) { struct gnix_fab_req *fab_req = (struct gnix_fab_req *)data; struct gnix_fid_ep *ep = fab_req->gnix_ep; struct gnix_nic *nic = ep->nic; struct gnix_fid_mem_desc *loc_md; struct gnix_tx_descriptor *txd; gni_mem_handle_t mdh; gni_return_t status; int rc; int rdma = !!(fab_req->flags & GNIX_RMA_RDMA); int indirect = !!(fab_req->flags & GNIX_RMA_INDIRECT); int chained = !!(fab_req->flags & GNIX_RMA_CHAINED); int inject_err = _gnix_req_inject_err(fab_req); rc = _gnix_nic_tx_alloc(nic, &txd); if (rc) { GNIX_INFO(FI_LOG_EP_DATA, "_gnix_nic_tx_alloc() failed: %d\n", rc); return -FI_ENOSPC; } txd->completer_fn = __gnix_rma_txd_complete; txd->req = fab_req; if (rdma) { _gnix_convert_key_to_mhdl( (gnix_mr_key_t *)&fab_req->rma.rem_mr_key, &mdh); } else { /* Mem handle CRC is not validated during FMA operations. Skip * this costly calculation. */ _gnix_convert_key_to_mhdl_no_crc( (gnix_mr_key_t *)&fab_req->rma.rem_mr_key, &mdh); } txd->gni_desc.type = __gnix_fr_post_type(fab_req->type, rdma); txd->gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT; /* check flags */ txd->gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE; /* check flags */ if (unlikely(indirect)) { __gnix_rma_fill_pd_indirect_get(fab_req, txd); } else if (unlikely(chained)) { __gnix_rma_fill_pd_chained_get(fab_req, txd, &mdh); } else { txd->gni_desc.local_addr = (uint64_t)fab_req->rma.loc_addr; txd->gni_desc.length = fab_req->rma.len; txd->gni_desc.remote_addr = (uint64_t)fab_req->rma.rem_addr; loc_md = (struct gnix_fid_mem_desc *)fab_req->rma.loc_md; if (loc_md) { txd->gni_desc.local_mem_hndl = loc_md->mem_hndl; } } txd->gni_desc.remote_mem_hndl = mdh; txd->gni_desc.rdma_mode = 0; /* check flags */ txd->gni_desc.src_cq_hndl = nic->tx_cq; /* check flags */ { gni_mem_handle_t *tl_mdh = &txd->gni_desc.local_mem_hndl; gni_mem_handle_t *tr_mdh = &txd->gni_desc.remote_mem_hndl; GNIX_INFO(FI_LOG_EP_DATA, "la: %llx ra: %llx len: %d\n", txd->gni_desc.local_addr, txd->gni_desc.remote_addr, txd->gni_desc.length); GNIX_INFO(FI_LOG_EP_DATA, "lmdh: %llx:%llx rmdh: %llx:%llx key: %llx\n", *(uint64_t *)tl_mdh, *(((uint64_t *)tl_mdh) + 1), *(uint64_t *)tr_mdh, *(((uint64_t *)tr_mdh) + 1), fab_req->rma.rem_mr_key); } fastlock_acquire(&nic->lock); if (unlikely(inject_err)) { _gnix_nic_txd_err_inject(nic, txd); status = GNI_RC_SUCCESS; } else if (chained) { status = GNI_CtPostFma(fab_req->vc->gni_ep, &txd->gni_desc); } else if (rdma) { status = GNI_PostRdma(fab_req->vc->gni_ep, &txd->gni_desc); } else { status = GNI_PostFma(fab_req->vc->gni_ep, &txd->gni_desc); } fastlock_release(&nic->lock); if (status != GNI_RC_SUCCESS) { _gnix_nic_tx_free(nic, txd); GNIX_INFO(FI_LOG_EP_DATA, "GNI_Post*() failed: %s\n", gni_err_str[status]); } return gnixu_to_fi_errno(status); }
int _gnix_amo_post_req(void *data) { struct gnix_fab_req *fab_req = (struct gnix_fab_req *)data; struct gnix_fid_ep *ep = fab_req->gnix_ep; struct gnix_nic *nic = ep->nic; struct gnix_fid_mem_desc *loc_md; struct gnix_tx_descriptor *txd; gni_mem_handle_t mdh; gni_return_t status; int rc; int inject_err = _gnix_req_inject_err(fab_req); rc = _gnix_nic_tx_alloc(nic, &txd); if (rc) { GNIX_INFO(FI_LOG_EP_DATA, "_gnix_nic_tx_alloc() failed: %d\n", rc); return -FI_ENOSPC; } txd->completer_fn = __gnix_amo_txd_complete; txd->req = fab_req; /* Mem handle CRC is not validated during FMA operations. Skip this * costly calculation. */ _gnix_convert_key_to_mhdl_no_crc( (gnix_mr_key_t *)&fab_req->amo.rem_mr_key, &mdh); loc_md = (struct gnix_fid_mem_desc *)fab_req->amo.loc_md; txd->gni_desc.type = GNI_POST_AMO; txd->gni_desc.cq_mode = GNI_CQMODE_GLOBAL_EVENT; /* check flags */ txd->gni_desc.dlvr_mode = GNI_DLVMODE_PERFORMANCE; /* check flags */ txd->gni_desc.local_addr = (uint64_t)fab_req->amo.loc_addr; if (loc_md) { txd->gni_desc.local_mem_hndl = loc_md->mem_hndl; } txd->gni_desc.remote_addr = (uint64_t)fab_req->amo.rem_addr; txd->gni_desc.remote_mem_hndl = mdh; txd->gni_desc.length = fab_req->amo.len; txd->gni_desc.rdma_mode = 0; /* check flags */ txd->gni_desc.src_cq_hndl = nic->tx_cq; /* check flags */ txd->gni_desc.amo_cmd = _gnix_atomic_cmd(fab_req->amo.datatype, fab_req->amo.op, fab_req->type); txd->gni_desc.first_operand = fab_req->amo.first_operand; txd->gni_desc.second_operand = fab_req->amo.second_operand; { gni_mem_handle_t *tl_mdh = &txd->gni_desc.local_mem_hndl; gni_mem_handle_t *tr_mdh = &txd->gni_desc.remote_mem_hndl; GNIX_INFO(FI_LOG_EP_DATA, "la: %llx ra: %llx len: %d\n", txd->gni_desc.local_addr, txd->gni_desc.remote_addr, txd->gni_desc.length); GNIX_INFO(FI_LOG_EP_DATA, "lmdh: %llx:%llx rmdh: %llx:%llx key: %llx\n", *(uint64_t *)tl_mdh, *(((uint64_t *)tl_mdh) + 1), *(uint64_t *)tr_mdh, *(((uint64_t *)tr_mdh) + 1), fab_req->amo.rem_mr_key); } fastlock_acquire(&nic->lock); if (unlikely(inject_err)) { _gnix_nic_txd_err_inject(nic, txd); status = GNI_RC_SUCCESS; } else { status = GNI_PostFma(fab_req->vc->gni_ep, &txd->gni_desc); } fastlock_release(&nic->lock); if (status != GNI_RC_SUCCESS) { _gnix_nic_tx_free(nic, txd); GNIX_INFO(FI_LOG_EP_DATA, "GNI_Post*() failed: %s\n", gni_err_str[status]); } return gnixu_to_fi_errno(status); }
int _gnix_cm_nic_alloc(struct gnix_fid_domain *domain, struct gnix_cm_nic **cm_nic_ptr) { int ret = FI_SUCCESS; struct gnix_cm_nic *cm_nic = NULL; uint32_t device_addr, cdm_id; gni_return_t status; GNIX_TRACE(FI_LOG_EP_CTRL, "\n"); *cm_nic_ptr = NULL; cm_nic = (struct gnix_cm_nic *)calloc(1, sizeof(*cm_nic)); if (cm_nic == NULL) { ret = -FI_ENOMEM; goto err; } ret = _gnix_get_new_cdm_id(domain, &cdm_id); if (ret != FI_SUCCESS) goto err; GNIX_INFO(FI_LOG_EP_CTRL, "creating cm_nic for %u/0x%x/%u\n", domain->ptag, domain->cookie, cdm_id); status = GNI_CdmCreate(cdm_id, domain->ptag, domain->cookie, gnix_cdm_modes, &cm_nic->gni_cdm_hndl); if (status != GNI_RC_SUCCESS) { GNIX_ERR(FI_LOG_EP_CTRL, "GNI_CdmCreate returned %s\n", gni_err_str[status]); ret = gnixu_to_fi_errno(status); goto err; } /* * Okay, now go for the attach */ status = GNI_CdmAttach(cm_nic->gni_cdm_hndl, 0, &device_addr, &cm_nic->gni_nic_hndl); if (status != GNI_RC_SUCCESS) { GNIX_ERR(FI_LOG_EP_CTRL, "GNI_CdmAttach returned %s\n", gni_err_str[status]); ret = gnixu_to_fi_errno(status); goto err; } cm_nic->cdm_id = cdm_id; cm_nic->ptag = domain->ptag; cm_nic->cookie = domain->cookie; cm_nic->device_addr = device_addr; cm_nic->control_progress = domain->control_progress; fastlock_init(&cm_nic->lock); fastlock_init(&cm_nic->wq_lock); list_head_init(&cm_nic->cm_nic_wq); /* * prep the cm nic's dgram component */ ret = _gnix_dgram_hndl_alloc(domain->fabric, cm_nic, domain->control_progress, &cm_nic->dgram_hndl); if (ret != FI_SUCCESS) goto err; *cm_nic_ptr = cm_nic; return ret; err: if (cm_nic->dgram_hndl) _gnix_dgram_hndl_free(cm_nic->dgram_hndl); if (cm_nic->gni_cdm_hndl) GNI_CdmDestroy(cm_nic->gni_cdm_hndl); if (cm_nic != NULL) free(cm_nic); return ret; }
/** * Create a slab from a handle and append to the slab list. * * @param[in] handle Handle to the allocator being used. * * @return FI_SUCCESS On successful slab creation. * * @return -FI_ENOMEM if failure to allocate memory for slab or bitmap. * @return [Unspec] if failure in alloc_bitmap. Will return error code from * alloc_bitmap. * @return [Unspec] if failure in GNI_MemRegister. Converts gni_return_t * status code to FI_ERRNO value. */ static int __create_slab(struct gnix_mbox_alloc_handle *handle) { struct gnix_slab *slab; gni_return_t status; char error_buf[256]; char *error; size_t total_size; int ret; int vmdh_index = -1; int flags = GNI_MEM_READWRITE; struct gnix_auth_key *info; GNIX_TRACE(FI_LOG_EP_CTRL, "\n"); slab = calloc(1, sizeof(*slab)); if (!slab) { error = strerror_r(errno, error_buf, sizeof(error_buf)); GNIX_WARN(FI_LOG_EP_CTRL, "Error allocating slab: %s\n", error); ret = -FI_ENOMEM; goto err_slab_calloc; } total_size = handle->page_size * __page_count(handle); GNIX_DEBUG(FI_LOG_EP_CTRL, "total_size requested for mmap: %zu.\n", total_size); slab->used = calloc(1, sizeof(*(slab->used))); if (!slab->used) { error = strerror_r(errno, error_buf, sizeof(error_buf)); GNIX_WARN(FI_LOG_EP_CTRL, "Error allocating bitmap: %s\n", error); ret = -FI_ENOMEM; goto err_bitmap_calloc; } slab->base = mmap(0, total_size, (PROT_READ | PROT_WRITE), MAP_SHARED, handle->fd, handle->last_offset); if (slab->base == MAP_FAILED) { error = strerror_r(errno, error_buf, sizeof(error_buf)); GNIX_WARN(FI_LOG_EP_CTRL, "%s\n", error); ret = -FI_ENOMEM; goto err_mmap; } ret = _gnix_alloc_bitmap(slab->used, __mbox_count(handle), NULL); if (ret) { GNIX_WARN(FI_LOG_EP_CTRL, "Error allocating bitmap.\n"); goto err_alloc_bitmap; } COND_ACQUIRE(handle->nic_handle->requires_lock, &handle->nic_handle->lock); if (handle->nic_handle->using_vmdh) { info = _gnix_auth_key_lookup(GNIX_PROV_DEFAULT_AUTH_KEY, GNIX_PROV_DEFAULT_AUTH_KEYLEN); assert(info); if (!handle->nic_handle->mdd_resources_set) { /* check to see if the ptag registration limit was set * yet or not -- becomes read-only after success */ _gnix_auth_key_enable(info); status = GNI_SetMddResources( handle->nic_handle->gni_nic_hndl, (info->attr.prov_key_limit + info->attr.user_key_limit)); assert(status == GNI_RC_SUCCESS); handle->nic_handle->mdd_resources_set = 1; } vmdh_index = _gnix_get_next_reserved_key(info); if (vmdh_index <= 0) { GNIX_FATAL(FI_LOG_DOMAIN, "failed to get reserved key for mbox " "registration, rc=%d\n", vmdh_index); } flags |= GNI_MEM_USE_VMDH; } status = GNI_MemRegister(handle->nic_handle->gni_nic_hndl, (uint64_t) slab->base, total_size, handle->cq_handle, flags, vmdh_index, &slab->memory_handle); COND_RELEASE(handle->nic_handle->requires_lock, &handle->nic_handle->lock); if (status != GNI_RC_SUCCESS) { GNIX_WARN(FI_LOG_EP_CTRL, "GNI_MemRegister failed: %s\n", gni_err_str[status]); ret = gnixu_to_fi_errno(status); goto err_memregister; } slab->allocator = handle; gnix_slist_insert_tail(&slab->list_entry, &handle->slab_list); handle->last_offset += total_size; return ret; err_memregister: _gnix_free_bitmap(slab->used); err_alloc_bitmap: munmap(slab->base, total_size); err_mmap: free(slab->used); err_bitmap_calloc: free(slab); err_slab_calloc: return ret; }
/* * gnix_resolve_name: given a node hint and a valid pointer to a gnix_ep_name * will resolve the gnix specific address of node and fill the provided * gnix_ep_name pointer with the information. * * node (IN) : Node name being resolved to gnix specific address * resolved_addr (IN/OUT) : Pointer that must be provided to contain the * resolved address. * * TODO: consider a use for service. */ int gnix_resolve_name(IN const char *node, IN const char *service, INOUT struct gnix_ep_name *resolved_addr) { int sock = -1; uint32_t pe = -1; uint32_t cpu_id = -1; struct addrinfo *result = NULL; struct addrinfo *rp = NULL; struct ifreq ifr = {{{0}}}; struct sockaddr_in *sa = NULL; struct sockaddr_in *sin = NULL; int ret = FI_SUCCESS; gni_return_t status = GNI_RC_SUCCESS; struct addrinfo hints = { .ai_family = AF_INET, .ai_socktype = SOCK_DGRAM, .ai_flags = AI_CANONNAME }; if (!resolved_addr) { GNIX_ERR(FI_LOG_FABRIC, "Resolved_addr must be a valid pointer.\n"); ret = -FI_EINVAL; goto err; } sock = socket(AF_INET, SOCK_DGRAM, 0); if (sock == -1) { GNIX_ERR(FI_LOG_FABRIC, "Socket creation failed: %s\n", strerror(errno)); ret = -FI_EIO; goto err; } /* Get the address for the ipogif0 interface */ ifr.ifr_addr.sa_family = AF_INET; snprintf(ifr.ifr_name, sizeof(ifr.ifr_name), "%s", "ipogif0"); ret = ioctl(sock, SIOCGIFADDR, &ifr); if (ret == -1) { GNIX_ERR(FI_LOG_FABRIC, "Failed to get address for ipogif0: %s\n", strerror(errno)); ret = -FI_EIO; goto sock_cleanup; } sin = (struct sockaddr_in *) &ifr.ifr_addr; ret = getaddrinfo(node, "domain", &hints, &result); if (ret != 0) { GNIX_ERR(FI_LOG_FABRIC, "Failed to get address for node provided: %s\n", strerror(errno)); ret = -FI_EINVAL; goto sock_cleanup; } for (rp = result; rp != NULL; rp = rp->ai_next) { assert(rp->ai_addr->sa_family == AF_INET); sa = (struct sockaddr_in *) rp->ai_addr; /* * If we are trying to resolve localhost then use * CdmGetNicAddress. */ if (sa->sin_addr.s_addr == sin->sin_addr.s_addr) { status = GNI_CdmGetNicAddress(0, &pe, &cpu_id); if(status == GNI_RC_SUCCESS) { break; } else { GNIX_ERR(FI_LOG_FABRIC, "Unable to get NIC address."); ret = gnixu_to_fi_errno(status); goto sock_cleanup; } } else { ret = gnixu_get_pe_from_ip(inet_ntoa(sa->sin_addr), &pe); if (ret == 0) { break; } } } /* * Make sure address is valid. */ if (pe == -1) { GNIX_ERR(FI_LOG_FABRIC, "Unable to acquire valid address for node %s\n", node); ret = -FI_EADDRNOTAVAIL; goto sock_cleanup; } /* * Fill the INOUT parameter resolved_addr with the address information * acquired for the provided node parameter. */ memset(resolved_addr, 0, sizeof(struct gnix_ep_name)); resolved_addr->gnix_addr.device_addr = pe; /* TODO: have to write a nameserver to get this info */ resolved_addr->gnix_addr.cdm_id = 0; /* TODO: likely depend on service? */ resolved_addr->name_type = 0; sock_cleanup: if(close(sock) == -1) { GNIX_ERR(FI_LOG_FABRIC, "Unable to close socket: %s\n", strerror(errno)); } err: if (result != NULL) { freeaddrinfo(result); } return ret; }