int
_gnix_notifier_get_event(struct gnix_mr_notifier *mrn, void* buf, size_t len)
{
int ret, ret_errno;
if ((mrn == NULL) || (buf == NULL) || (len <= 0)) {
GNIX_WARN(FI_LOG_MR,
"Invalid argument to _gnix_notifier_get_event\n");
return -FI_EINVAL;
}
fastlock_acquire(&mrn->lock);
if (*(mrn->cntr) > 0) {
GNIX_DEBUG(FI_LOG_MR, "reading kdreg event\n");
ret = read(mrn->fd, buf, len);
if (ret < 0) {
ret_errno = errno;
if (ret_errno != EAGAIN) {
GNIX_WARN(FI_LOG_MR,
"kdreg event read failed: %s\n",
strerror(ret_errno));
}
/* Not all of these map to fi_errno values */
ret = -ret_errno;
}
} else {
GNIX_DEBUG(FI_LOG_MR, "nothing to read from kdreg :(\n");
ret = -FI_EAGAIN;
}
fastlock_release(&mrn->lock);
return ret;
}
int
_gnix_notifier_monitor(struct gnix_mr_notifier *mrn,
void *addr, uint64_t len, uint64_t cookie)
{
int ret;
struct registration_monitor rm;
fastlock_acquire(&mrn->lock);
ret = notifier_verify_stuff(mrn);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_MR, "Invalid MR notifier\n");
goto err_exit;
}
if (ret == 0) {
GNIX_DEBUG(FI_LOG_MR, "monitoring %p (len=%lu) cookie=%lu\n",
addr, len, cookie);
memset(&rm, 0, sizeof(rm));
rm.type = REGISTRATION_MONITOR;
rm.u.mon.addr = (uint64_t) addr;
rm.u.mon.len = len;
rm.u.mon.user_cookie = cookie;
ret = kdreg_write(mrn, &rm, sizeof(rm));
}
err_exit:
fastlock_release(&mrn->lock);
return ret;
}
int
_gnix_notifier_unmonitor(struct gnix_mr_notifier *mrn, uint64_t cookie)
{
int ret;
struct registration_monitor rm;
fastlock_acquire(&mrn->lock);
ret = notifier_verify_stuff(mrn);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_MR, "Invalid MR notifier\n");
goto err_exit;
}
GNIX_DEBUG(FI_LOG_MR, "unmonitoring cookie=%lu\n", cookie);
memset(&rm, 0, sizeof(rm));
rm.type = REGISTRATION_UNMONITOR;
rm.u.unmon.user_cookie = cookie;
ret = kdreg_write(mrn, &rm, sizeof(rm));
err_exit:
fastlock_release(&mrn->lock);
return ret;
}
/*
* callback function to process incoming messages
*/
int __gnix_vc_recv_fn(struct gnix_cm_nic *cm_nic,
char *msg_buffer,
struct gnix_address src_cm_nic_addr)
{
int ret = FI_SUCCESS;
uint8_t mtype;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
__gnix_vc_get_msg_type(msg_buffer, &mtype);
GNIX_DEBUG(FI_LOG_EP_CTRL, "got a message of type %d\n", mtype);
switch (mtype) {
case GNIX_VC_CONN_REQ:
ret = __gnix_vc_hndl_conn_req(cm_nic,
msg_buffer,
src_cm_nic_addr);
break;
case GNIX_VC_CONN_RESP:
ret = __gnix_vc_hndl_conn_resp(cm_nic,
msg_buffer,
src_cm_nic_addr);
break;
default:
GNIX_WARN(FI_LOG_EP_CTRL,
"unknown cm_nic message type %d\n", mtype);
assert(0);
}
return ret;
}
int _gnix_mbox_alloc(struct gnix_mbox_alloc_handle *alloc_handle,
struct gnix_mbox **ptr)
{
struct gnix_slab *slab;
int position;
int ret;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
if (!alloc_handle || !ptr) {
GNIX_WARN(FI_LOG_EP_CTRL, "Invalid alloc_handle or ptr.\n");
ret = -FI_EINVAL;
goto err;
}
fastlock_acquire(&alloc_handle->lock);
position = __find_free(alloc_handle, &slab);
if (position < 0) {
GNIX_DEBUG(FI_LOG_EP_CTRL, "Creating new slab.\n");
ret = __create_slab(alloc_handle);
if (ret) {
GNIX_WARN(FI_LOG_EP_CTRL, "Slab creation failed.\n");
goto err;
}
slab = container_of(alloc_handle->slab_list.tail,
struct gnix_slab, list_entry);
position = ret;
}
/* Process an incoming connection request at a listening PEP. */
static int __gnix_pep_connreq(struct gnix_fid_pep *pep, int fd)
{
int ret;
struct gnix_pep_sock_conn *conn;
struct fi_eq_cm_entry *eq_entry;
int eqe_size;
/* Create and initialize a new connection request. */
conn = calloc(1, sizeof(*conn));
if (!conn) {
GNIX_WARN(FI_LOG_EP_CTRL,
"Failed to alloc accepted socket conn\n");
return -FI_ENOMEM;
}
conn->fid.fclass = FI_CLASS_CONNREQ;
conn->fid.context = pep;
conn->sock_fd = fd;
/* Pull request data from the listening socket. */
conn->bytes_read += read(fd, &conn->req, sizeof(conn->req));
if (conn->bytes_read != sizeof(conn->req)) {
/* TODO Wait for more bytes. */
GNIX_FATAL(FI_LOG_EP_CTRL, "Unexpected read size\n");
}
conn->req.info.src_addr = &conn->req.src_addr;
conn->req.info.dest_addr = &conn->req.dest_addr;
conn->req.info.tx_attr = &conn->req.tx_attr;
conn->req.info.rx_attr = &conn->req.rx_attr;
conn->req.info.ep_attr = &conn->req.ep_attr;
conn->req.info.domain_attr = &conn->req.domain_attr;
conn->req.info.fabric_attr = &conn->req.fabric_attr;
conn->req.info.domain_attr->name = NULL;
conn->req.info.fabric_attr->name = NULL;
conn->req.info.fabric_attr->prov_name = NULL;
conn->info = &conn->req.info;
conn->info->handle = &conn->fid;
/* Tell user of a new conn req via the EQ. */
eq_entry = (struct fi_eq_cm_entry *)conn->req.eqe_buf;
eq_entry->fid = &pep->pep_fid.fid;
eq_entry->info = fi_dupinfo(conn->info);
eqe_size = sizeof(*eq_entry) + conn->req.cm_data_len;
ret = fi_eq_write(&pep->eq->eq_fid, FI_CONNREQ, eq_entry, eqe_size, 0);
if (ret != eqe_size) {
GNIX_WARN(FI_LOG_EP_CTRL, "fi_eq_write failed, err: %d\n", ret);
fi_freeinfo(conn->info);
free(conn);
return ret;
}
GNIX_DEBUG(FI_LOG_EP_CTRL, "Added FI_CONNREQ EQE: %p, %p\n",
pep->eq, pep);
return FI_SUCCESS;
}
/**
* Will attempt to find a directory in hugetlbfs using the given page size and
* create a filename to use for backing an mmap.
*
* @param[in] page_size Page size to look for in the hugetlbfs
* @param[out] filename Pointer containing filename after generation.
*
* @return FI_SUCCESS On successfully finding a huge page and generating a
* file name.
*
* @return -FI_EINVAL if an invalid parameter was given
* @return -FI_EIO if an error occurred while opening the /proc/mounts
* file. This is propagated from __find_huge_page.
* @return -FI_ENOMEM if an error occurred while allocating space for the
* filename.
*/
static int __generate_file_name(size_t page_size, char **filename)
{
static const char basename[] = "gnix_map";
char *full_filename = NULL;
char *huge_page = NULL;
char *error;
char error_buf[256];
int my_file_id;
int size;
int ret;
if (!filename) {
GNIX_WARN(FI_LOG_EP_CTRL, "filename pointer is NULL.\n");
ret = -FI_EINVAL;
goto err_invalid;
}
ret = __find_huge_page(page_size, &huge_page);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"Find huge page returned error %s\n",
fi_strerror(-ret));
goto err_invalid;
}
my_file_id = ofi_atomic_inc32(&file_id_counter);
size = snprintf(NULL, 0, "%s/%s.%d.%d", huge_page, basename, getpid(),
my_file_id);
if (size < 0) {
error = strerror_r(errno, error_buf, sizeof(error_buf));
GNIX_WARN(FI_LOG_EP_CTRL,
"Error while gathering size for snprintf: %s\n",
error);
goto err_snprintf;
}
full_filename = malloc(size + 1);
if (!full_filename) {
error = strerror_r(errno, error_buf, sizeof(error_buf));
GNIX_WARN(FI_LOG_EP_CTRL,
"Error allocating full_filename: %s\n",
error);
ret = -FI_ENOMEM;
goto err_snprintf;
}
sprintf(full_filename, "%s/%s.%d.%d", huge_page, basename, getpid(),
my_file_id);
GNIX_DEBUG(FI_LOG_EP_CTRL, "Generated filename: %s\n", full_filename);
*filename = full_filename;
err_snprintf:
free(huge_page);
err_invalid:
return ret;
}
/**
* Determine how many mboxes are in a requested allocation size.
*
* @param[in] handle Handle to the allocator being used.
*
* @return Number of mail boxes being allocated.
*/
static size_t __mbox_count(struct gnix_mbox_alloc_handle *handle)
{
size_t mbox_count = (__page_count(handle) * handle->page_size) /
handle->mbox_size;
GNIX_DEBUG(FI_LOG_EP_CTRL,
"Mbox_count: %zu.\n", mbox_count);
return mbox_count;
}
/*
* Given an address pointed to by addr, stuff a string into buf representing:
* device_addr:cdm_id:name_type:cm_nic_cdm_id:cookie
* where device_addr, cdm_id, cm_nic_cdm_id and cookie are represented in
* hexadecimal. And name_type is represented as an integer.
*/
DIRECT_FN const char *gnix_av_straddr(struct fid_av *av,
const void *addr, char *buf,
size_t *len)
{
char int_buf[GNIX_AV_MAX_STR_ADDR_LEN];
int size;
struct gnix_ep_name ep_name;
struct gnix_fid_av *av_priv;
if (!av || !addr || !buf || !len) {
GNIX_DEBUG(FI_LOG_DEBUG, "NULL parameter in gnix_av_straddr\n");
return NULL;
}
av_priv = container_of(av, struct gnix_fid_av, av_fid);
if (av_priv->domain->addr_format == FI_ADDR_STR)
_gnix_resolve_str_ep_name(addr, 0, &ep_name);
else
ep_name = ((struct gnix_ep_name *) addr)[0];
/*
* if additional information is added to this string, then
* you will need to update in gnix.h:
* GNIX_AV_STR_ADDR_VERSION, increment this value
* GNIX_AV_MAX_STR_ADDR_LEN, to be the number of characters printed
*/
size = snprintf(int_buf, sizeof(int_buf), "%04i:0x%08" PRIx32 ":0x%08"
PRIx32 ":%02i:0x%06" PRIx32 ":0x%08" PRIx32
":%02i", GNIX_AV_STR_ADDR_VERSION,
ep_name.gnix_addr.device_addr,
ep_name.gnix_addr.cdm_id,
ep_name.name_type,
ep_name.cm_nic_cdm_id,
ep_name.cookie,
ep_name.rx_ctx_cnt);
/*
* snprintf returns the number of character written
* without the terminating character.
*/
if ((size + 1) < *len) {
/*
* size needs to be all the characters plus the terminating
* character. Otherwise, we could lose information.
*/
size = size + 1;
} else {
/* Do not overwrite the buffer. */
size = *len;
}
snprintf(buf, size, "%s", int_buf);
*len = size;
return buf;
}
static int __process_rx_cqe(struct gnix_nic *nic, gni_cq_entry_t cqe)
{
int ret = FI_SUCCESS, vc_id = 0;
struct gnix_vc *vc;
vc_id = GNI_CQ_GET_INST_ID(cqe);
/*
* its possible this vc has been destroyed, so may get NULL
* back.
*/
vc = __gnix_nic_elem_by_rem_id(nic, vc_id);
if (vc != NULL) {
switch (vc->conn_state) {
case GNIX_VC_CONNECTING:
GNIX_DEBUG(FI_LOG_EP_DATA,
"Scheduling VC for RX processing (%p)\n",
vc);
ret = _gnix_vc_rx_schedule(vc);
assert(ret == FI_SUCCESS);
break;
case GNIX_VC_CONNECTED:
GNIX_DEBUG(FI_LOG_EP_DATA,
"Processing VC RX (%p)\n",
vc);
ret = _gnix_vc_dequeue_smsg(vc);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_DATA,
"_gnix_vc_dqueue_smsg returned %d\n",
ret);
}
break;
default:
break; /* VC not in a state for scheduling or
SMSG processing */
}
}
return ret;
}
/*******************************************************************************
* FI_AV_TABLE specific implementations.
******************************************************************************/
static int table_insert(struct gnix_fid_av *av_priv, const void *addr,
size_t count, fi_addr_t *fi_addr, uint64_t flags,
void *context)
{
struct gnix_ep_name ep_name;
int ret = count;
size_t index, i;
int *entry_err = context;
if (gnix_check_capacity(av_priv, count)) {
return -FI_ENOMEM;
}
assert(av_priv->table);
for (index = av_priv->count, i = 0; i < count; index++, i++) {
_gnix_get_ep_name(addr, i, &ep_name, av_priv->domain);
/* check if this ep_name fits in the av context bits */
if (ep_name.name_type & GNIX_EPN_TYPE_SEP) {
if ((1 << av_priv->rx_ctx_bits) < ep_name.rx_ctx_cnt) {
if (flags & FI_SYNC_ERR) {
entry_err[i] = -FI_EINVAL;
fi_addr[i] = FI_ADDR_NOTAVAIL;
ret = -FI_EINVAL;
continue;
}
GNIX_DEBUG(FI_LOG_AV, "ep_name doesn't fit "
"into the av context bits\n");
return -FI_EINVAL;
}
}
av_priv->table[index].gnix_addr = ep_name.gnix_addr;
av_priv->valid_entry_vec[index] = 1;
av_priv->table[index].name_type = ep_name.name_type;
av_priv->table[index].cookie = ep_name.cookie;
av_priv->table[index].rx_ctx_cnt = ep_name.rx_ctx_cnt;
av_priv->table[index].cm_nic_cdm_id =
ep_name.cm_nic_cdm_id;
av_priv->table[index].key_offset = ep_name.key_offset;
if (fi_addr)
fi_addr[i] = index;
if (flags & FI_SYNC_ERR) {
entry_err[i] = FI_SUCCESS;
}
}
av_priv->count += count;
return ret;
}
/**
* Determine how many pages need to be allocated.
*
* @param[in] handle Handle to the allocator being used.
*
* @return Number of pages that need to be allocated rounded up to the nearest
* multiple of the page size.
*/
static size_t __page_count(struct gnix_mbox_alloc_handle *handle)
{
size_t total_size = CEILING((handle->mbox_size * handle->mpmmap),
handle->page_size);
size_t page_count;
page_count = total_size / handle->page_size;
GNIX_DEBUG(FI_LOG_EP_CTRL,
"Mbox_size: %zu, mpmmap: %zu, page_size: %zu\n",
handle->mbox_size, handle->mpmmap, handle->page_size);
GNIX_DEBUG(FI_LOG_EP_CTRL,
"Total size: %zu, page_count: %zu\n", total_size,
page_count);
if (page_count <= 0) {
GNIX_WARN(FI_LOG_EP_CTRL,
"Invalid size requested, truncating to single page.\n");
page_count = 1;
}
return page_count;
}
DIRECT_FN STATIC int gnix_reject(struct fid_pep *pep, fid_t handle,
const void *param, size_t paramlen)
{
struct gnix_fid_pep *pep_priv;
struct gnix_pep_sock_conn *conn;
struct gnix_pep_sock_connresp resp;
struct fi_eq_cm_entry *eqe_ptr;
int ret;
if (!pep)
return -FI_EINVAL;
pep_priv = container_of(pep, struct gnix_fid_pep, pep_fid.fid);
fastlock_acquire(&pep_priv->lock);
conn = (struct gnix_pep_sock_conn *)handle;
if (!conn || conn->fid.fclass != FI_CLASS_CONNREQ) {
fastlock_release(&pep_priv->lock);
return -FI_EINVAL;
}
resp.cmd = GNIX_PEP_SOCK_RESP_REJECT;
resp.cm_data_len = paramlen;
if (paramlen) {
eqe_ptr = (struct fi_eq_cm_entry *)resp.eqe_buf;
memcpy(eqe_ptr->data, param, paramlen);
}
ret = write(conn->sock_fd, &resp, sizeof(resp));
if (ret != sizeof(resp)) {
fastlock_release(&pep_priv->lock);
GNIX_WARN(FI_LOG_EP_CTRL,
"Failed to send resp, errno: %d\n",
errno);
return -FI_EIO;
}
close(conn->sock_fd);
free(conn);
fastlock_release(&pep_priv->lock);
GNIX_DEBUG(FI_LOG_EP_CTRL, "Sent conn reject: %p\n", pep_priv);
return FI_SUCCESS;
}
static void __pep_destruct(void *obj)
{
struct gnix_fid_pep *pep = (struct gnix_fid_pep *)obj;
GNIX_DEBUG(FI_LOG_EP_CTRL, "Destroying PEP: %p\n", pep);
fastlock_destroy(&pep->lock);
if (pep->listen_fd >= 0)
close(pep->listen_fd);
if (pep->eq) {
_gnix_eq_poll_obj_rem(pep->eq, &pep->pep_fid.fid);
_gnix_ref_put(pep->eq);
}
free(pep);
}
/**
* Find huge page, generate filename, open huge page, and attach huge page
* descriptor to handle.
*
* @param[in] handle Handle to the allocator being used.
*
* @return FI_SUCCESS On successfully opening a huge page.
*
* @return -FI_EINVAL if an invalid parameter was given. Propagated from
* __generate_file_name.
* @return -FI_EIO if an error occurred while opening the hugepage
* @return -FI_ENOMEM if an error in space allocation occurred. Propagated
* from __generate_file_name.
*/
static int __open_huge_page(struct gnix_mbox_alloc_handle *handle)
{
char *filename = NULL;
char error_buf[256];
char *error;
int ret;
int fd;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
handle->fd = -1;
handle->filename = NULL;
ret = __generate_file_name(handle->page_size, &filename);
if (ret < 0) {
GNIX_WARN(FI_LOG_EP_CTRL, "Error in generating file name.\n");
goto err_filename;
}
fd = open(filename, O_CREAT | O_RDWR | O_EXCL, 0700);
if (fd < 0) {
error = strerror_r(errno, error_buf, sizeof(error_buf));
GNIX_WARN(FI_LOG_EP_CTRL, "IO Error: %s\n", error);
ret = -FI_EIO;
goto err_open;
}
handle->fd = fd;
handle->filename = filename;
GNIX_DEBUG(FI_LOG_EP_CTRL,
"Successfully opened: %s with handle : %d\n.",
handle->filename, handle->fd);
unlink(handle->filename);
return ret;
err_open:
free(filename);
err_filename:
return ret;
}
DIRECT_FN int gnix_pep_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
{
int ret = FI_SUCCESS;
struct gnix_fid_pep *pep;
struct gnix_fid_eq *eq;
if (!fid || !bfid)
return -FI_EINVAL;
pep = container_of(fid, struct gnix_fid_pep, pep_fid.fid);
fastlock_acquire(&pep->lock);
switch (bfid->fclass) {
case FI_CLASS_EQ:
eq = container_of(bfid, struct gnix_fid_eq, eq_fid.fid);
if (pep->fabric != eq->fabric) {
ret = -FI_EINVAL;
break;
}
if (pep->eq) {
ret = -FI_EINVAL;
break;
}
pep->eq = eq;
_gnix_eq_poll_obj_add(eq, &pep->pep_fid.fid);
_gnix_ref_get(eq);
GNIX_DEBUG(FI_LOG_EP_CTRL, "Bound EQ to PEP: %p, %p\n",
eq, pep);
break;
default:
ret = -FI_ENOSYS;
break;
}
fastlock_release(&pep->lock);
return ret;
}
DIRECT_FN int gnix_passive_ep_open(struct fid_fabric *fabric,
struct fi_info *info, struct fid_pep **pep,
void *context)
{
struct gnix_fid_fabric *fabric_priv;
struct gnix_fid_pep *pep_priv;
if (!fabric || !info || !pep)
return -FI_EINVAL;
fabric_priv = container_of(fabric, struct gnix_fid_fabric, fab_fid);
pep_priv = calloc(1, sizeof(*pep_priv));
if (!pep_priv)
return -FI_ENOMEM;
pep_priv->pep_fid.fid.fclass = FI_CLASS_PEP;
pep_priv->pep_fid.fid.context = context;
pep_priv->pep_fid.fid.ops = &gnix_pep_fi_ops;
pep_priv->pep_fid.ops = &gnix_pep_ops_ep;
pep_priv->pep_fid.cm = &gnix_pep_ops_cm;
pep_priv->listen_fd = -1;
pep_priv->backlog = 5; /* TODO set via fi_control parameter. */
pep_priv->fabric = fabric_priv;
fastlock_init(&pep_priv->lock);
if (info->src_addr) {
pep_priv->bound = 1;
memcpy(&pep_priv->src_addr, info->src_addr,
sizeof(struct sockaddr_in));
} else
pep_priv->bound = 0;
_gnix_ref_init(&pep_priv->ref_cnt, 1, __pep_destruct);
*pep = &pep_priv->pep_fid;
GNIX_DEBUG(FI_LOG_EP_CTRL, "Opened PEP: %p\n", pep_priv);
return FI_SUCCESS;
}
int
_gnix_notifier_unmonitor(struct gnix_mr_notifier *mrn, uint64_t cookie)
{
int ret;
struct registration_monitor rm;
ret = notifier_verify_stuff(mrn);
if (ret == 0) {
GNIX_DEBUG(FI_LOG_MR, "unmonitoring cookie=%lu\n", cookie);
memset(&rm, 0, sizeof(rm));
rm.type = REGISTRATION_UNMONITOR;
rm.u.unmon.user_cookie = cookie;
ret = kdreg_write(mrn, &rm, sizeof(rm));
}
return ret;
}
int
_gnix_notifier_monitor(struct gnix_mr_notifier *mrn,
void *addr, uint64_t len, uint64_t cookie)
{
int ret;
struct registration_monitor rm;
ret = notifier_verify_stuff(mrn);
if (ret == 0) {
GNIX_DEBUG(FI_LOG_MR, "monitoring %p (len=%lu) cookie=%lu\n",
addr, len, cookie);
memset(&rm, 0, sizeof(rm));
rm.type = REGISTRATION_MONITOR;
rm.u.mon.addr = (uint64_t) addr;
rm.u.mon.len = len;
rm.u.mon.user_cookie = cookie;
ret = kdreg_write(mrn, &rm, sizeof(rm));
}
return ret;
}
static int __gnix_vc_conn_ack_prog_fn(void *data, int *complete_ptr)
{
int ret = FI_SUCCESS;
int complete = 0;
struct wq_hndl_conn_req *work_req_data;
struct gnix_vc *vc;
struct gnix_mbox *mbox = NULL;
gni_smsg_attr_t smsg_mbox_attr;
struct gnix_fid_ep *ep = NULL;
struct gnix_fid_domain *dom = NULL;
struct gnix_cm_nic *cm_nic = NULL;
char sbuf[GNIX_CM_NIC_MAX_MSG_SIZE] = {0};
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
work_req_data = (struct wq_hndl_conn_req *)data;
vc = work_req_data->vc;
if (vc == NULL)
return -FI_EINVAL;
ep = vc->ep;
if (ep == NULL)
return -FI_EINVAL;
dom = ep->domain;
if (dom == NULL)
return -FI_EINVAL;
cm_nic = ep->cm_nic;
if (cm_nic == NULL)
return -FI_EINVAL;
fastlock_acquire(&ep->vc_ht_lock);
/*
* we may have already been moved to connecting or
* connected, if so early exit.
*/
if(vc->conn_state == GNIX_VC_CONNECTED) {
complete = 1;
goto exit;
}
/*
* first see if we still need a mailbox
*/
if (vc->smsg_mbox == NULL) {
ret = _gnix_mbox_alloc(ep->nic->mbox_hndl,
&mbox);
if (ret == FI_SUCCESS)
vc->smsg_mbox = mbox;
else
goto exit;
}
mbox = vc->smsg_mbox;
/*
* prep the smsg_mbox_attr
¬ */
smsg_mbox_attr.msg_type = GNI_SMSG_TYPE_MBOX_AUTO_RETRANSMIT;
smsg_mbox_attr.msg_buffer = mbox->base;
smsg_mbox_attr.buff_size = ep->nic->mem_per_mbox;
smsg_mbox_attr.mem_hndl = *mbox->memory_handle;
smsg_mbox_attr.mbox_offset = (uint64_t)mbox->offset;
smsg_mbox_attr.mbox_maxcredit = dom->params.mbox_maxcredit;
smsg_mbox_attr.msg_maxsize = dom->params.mbox_msg_maxsize;
/*
* serialize the resp message in the buffer
*/
__gnix_vc_pack_conn_resp(sbuf,
work_req_data->src_vc_ptr,
(uint64_t)vc,
vc->vc_id,
&smsg_mbox_attr);
/*
* try to send the message, if it succeeds,
* initialize mailbox and move vc to connected
* state.
*/
ret = _gnix_cm_nic_send(cm_nic,
sbuf,
GNIX_CM_NIC_MAX_MSG_SIZE,
vc->peer_cm_nic_addr);
if (ret == FI_SUCCESS) {
ret = __gnix_vc_smsg_init(vc,
work_req_data->src_vc_id,
&work_req_data->src_smsg_attr);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"_gnix_vc_smsg_init returned %s\n",
fi_strerror(-ret));
goto exit;
}
complete = 1;
vc->conn_state = GNIX_VC_CONNECTED;
GNIX_DEBUG(FI_LOG_EP_CTRL,
"moving vc %p to connected\n",vc);
} else if (ret == -FI_EAGAIN) {
ret = _gnix_vc_schedule(vc);
ret = FI_SUCCESS;
} else
assert(0);
exit:
fastlock_release(&ep->vc_ht_lock);
*complete_ptr = complete;
return ret;
}
static int _gnix_ep_getinfo(enum fi_ep_type ep_type, uint32_t version,
const char *node, const char *service,
uint64_t flags, struct fi_info *hints,
struct fi_info **info)
{
uint64_t mode = GNIX_FAB_MODES;
struct fi_info *gnix_info = NULL;
int ret = -FI_ENODATA;
GNIX_TRACE(FI_LOG_FABRIC, "\n");
if ((hints && hints->ep_attr) &&
(hints->ep_attr->type != FI_EP_UNSPEC &&
hints->ep_attr->type != ep_type)) {
return -FI_ENODATA;
}
gnix_info = _gnix_allocinfo();
if (!gnix_info)
return -FI_ENOMEM;
gnix_info->ep_attr->type = ep_type;
if (hints) {
/* TODO: Add version check when we decide on how to do it */
if (hints->addr_format == FI_ADDR_STR) {
gnix_info->addr_format = FI_ADDR_STR;
}
if (hints->ep_attr) {
/* Only support FI_PROTO_GNI protocol. */
switch (hints->ep_attr->protocol) {
case FI_PROTO_UNSPEC:
case FI_PROTO_GNI:
break;
default:
goto err;
}
if ((hints->ep_attr->tx_ctx_cnt > GNIX_SEP_MAX_CNT) &&
(hints->ep_attr->tx_ctx_cnt !=
FI_SHARED_CONTEXT)) {
goto err;
}
if (hints->ep_attr->rx_ctx_cnt > GNIX_SEP_MAX_CNT)
goto err;
if (hints->ep_attr->tx_ctx_cnt)
gnix_info->ep_attr->tx_ctx_cnt =
hints->ep_attr->tx_ctx_cnt;
if (hints->ep_attr->rx_ctx_cnt)
gnix_info->ep_attr->rx_ctx_cnt =
hints->ep_attr->rx_ctx_cnt;
if (hints->ep_attr->max_msg_size > GNIX_MAX_MSG_SIZE)
goto err;
}
GNIX_DEBUG(FI_LOG_FABRIC, "Passed EP attributes check\n");
/*
* check the mode field
*/
if (hints->mode) {
if ((hints->mode & GNIX_FAB_MODES) != GNIX_FAB_MODES) {
goto err;
}
mode = hints->mode & ~GNIX_FAB_MODES_CLEAR;
}
GNIX_DEBUG(FI_LOG_FABRIC, "Passed mode check\n");
if (hints->caps) {
/* The provider must support all requested
* capabilities. */
if ((hints->caps & GNIX_EP_CAPS_FULL) != hints->caps)
goto err;
/* The provider may silently enable secondary
* capabilities that do not introduce any overhead. */
gnix_info->caps = hints->caps | GNIX_EP_SEC_CAPS;
}
GNIX_DEBUG(FI_LOG_FABRIC, "Passed caps check gnix_info->caps = 0x%016lx\n",
gnix_info->caps);
if (hints->tx_attr) {
if ((hints->tx_attr->op_flags & GNIX_EP_OP_FLAGS) !=
hints->tx_attr->op_flags) {
goto err;
}
if (hints->tx_attr->inject_size > GNIX_INJECT_SIZE) {
goto err;
}
gnix_info->tx_attr->op_flags =
hints->tx_attr->op_flags & GNIX_EP_OP_FLAGS;
}
GNIX_DEBUG(FI_LOG_FABRIC, "Passed TX attributes check\n");
if (hints->rx_attr) {
if ((hints->rx_attr->op_flags & GNIX_EP_OP_FLAGS) !=
hints->rx_attr->op_flags) {
goto err;
}
gnix_info->rx_attr->op_flags =
hints->rx_attr->op_flags & GNIX_EP_OP_FLAGS;
}
if (hints->fabric_attr && hints->fabric_attr->name &&
strncmp(hints->fabric_attr->name, gnix_fab_name,
strlen(gnix_fab_name))) {
goto err;
}
GNIX_DEBUG(FI_LOG_FABRIC, "Passed fabric name check\n");
if (hints->domain_attr) {
if (hints->domain_attr->name &&
strncmp(hints->domain_attr->name, gnix_dom_name,
strlen(gnix_dom_name))) {
goto err;
}
if (hints->domain_attr->control_progress !=
FI_PROGRESS_UNSPEC)
gnix_info->domain_attr->control_progress =
hints->domain_attr->control_progress;
if (hints->domain_attr->data_progress !=
FI_PROGRESS_UNSPEC)
gnix_info->domain_attr->data_progress =
hints->domain_attr->data_progress;
switch (hints->domain_attr->mr_mode) {
case FI_MR_UNSPEC:
case FI_MR_BASIC:
if (FI_VERSION_GE(version, FI_VERSION(1, 5))) {
hints->domain_attr->mr_mode =
OFI_MR_BASIC_MAP;
}
break;
case FI_MR_SCALABLE:
GNIX_WARN(FI_LOG_FABRIC,
"GNI provider doesn't currently support MR_SCALABLE\n");
goto err;
}
switch (hints->domain_attr->threading) {
case FI_THREAD_COMPLETION:
gnix_info->domain_attr->threading =
hints->domain_attr->threading;
break;
default:
break;
}
if (hints->domain_attr->caps) {
if (hints->domain_attr->caps & ~GNIX_DOM_CAPS) {
GNIX_WARN(FI_LOG_FABRIC,
"Invalid domain caps\n");
goto err;
}
gnix_info->domain_attr->caps =
hints->domain_attr->caps;
}
ret = ofi_check_domain_attr(&gnix_prov, version,
gnix_info->domain_attr,
hints->domain_attr);
if (ret) {
GNIX_WARN(FI_LOG_FABRIC,
"GNI failed domain attributes check\n");
goto err;
}
GNIX_DEBUG(FI_LOG_FABRIC, "Passed the domain attributes check\n");
}
}
ret = __gnix_getinfo_resolve_node(node, service, flags, hints,
gnix_info);
if (ret != FI_SUCCESS)
goto err;
gnix_info->mode = mode;
gnix_info->fabric_attr->name = strdup(gnix_fab_name);
gnix_info->tx_attr->caps = gnix_info->caps;
gnix_info->tx_attr->mode = gnix_info->mode;
gnix_info->rx_attr->caps = gnix_info->caps;
gnix_info->rx_attr->mode = gnix_info->mode;
*info = gnix_info;
GNIX_DEBUG(FI_LOG_FABRIC, "Returning EP type: %s\n",
fi_tostr(&ep_type, FI_TYPE_EP_TYPE));
return FI_SUCCESS;
err:
fi_freeinfo(gnix_info);
return ret;
}
static int __gnix_vc_conn_req_prog_fn(void *data, int *complete_ptr)
{
int ret = FI_SUCCESS;
int complete = 0;
struct gnix_vc *vc = (struct gnix_vc *)data;
struct gnix_mbox *mbox = NULL;
gni_smsg_attr_t smsg_mbox_attr;
struct gnix_fid_ep *ep = NULL;
struct gnix_fid_domain *dom = NULL;
struct gnix_cm_nic *cm_nic = NULL;
char sbuf[GNIX_CM_NIC_MAX_MSG_SIZE] = {0};
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
ep = vc->ep;
if (ep == NULL)
return -FI_EINVAL;
dom = ep->domain;
if (dom == NULL)
return -FI_EINVAL;
cm_nic = ep->cm_nic;
if (cm_nic == NULL)
return -FI_EINVAL;
fastlock_acquire(&ep->vc_ht_lock);
if ((vc->conn_state == GNIX_VC_CONNECTING) ||
(vc->conn_state == GNIX_VC_CONNECTED)) {
complete = 1;
goto err;
}
/*
* sanity check that the vc is in the hash table
*/
if (!(vc->modes & GNIX_VC_MODE_IN_HT)) {
ret = -FI_EINVAL;
goto err;
}
/*
* first see if we still need a mailbox
*/
if (vc->smsg_mbox == NULL) {
ret = _gnix_mbox_alloc(vc->ep->nic->mbox_hndl,
&mbox);
if (ret == FI_SUCCESS)
vc->smsg_mbox = mbox;
else
goto err;
}
mbox = vc->smsg_mbox;
/*
* prep the smsg_mbox_attr
¬ */
smsg_mbox_attr.msg_type = GNI_SMSG_TYPE_MBOX_AUTO_RETRANSMIT;
smsg_mbox_attr.msg_buffer = mbox->base;
smsg_mbox_attr.buff_size = vc->ep->nic->mem_per_mbox;
smsg_mbox_attr.mem_hndl = *mbox->memory_handle;
smsg_mbox_attr.mbox_offset = (uint64_t)mbox->offset;
smsg_mbox_attr.mbox_maxcredit = dom->params.mbox_maxcredit;
smsg_mbox_attr.msg_maxsize = dom->params.mbox_msg_maxsize;
/*
* serialize the message in the buffer
*/
GNIX_DEBUG(FI_LOG_EP_CTRL,
"conn req tx: (From Aries addr 0x%x Id %d to Aries 0x%x Id %d CM NIC Id %d vc %p)\n",
ep->my_name.gnix_addr.device_addr,
ep->my_name.gnix_addr.cdm_id,
vc->peer_addr.device_addr,
vc->peer_addr.cdm_id,
vc->peer_cm_nic_addr.cdm_id,
vc);
__gnix_vc_pack_conn_req(sbuf,
&vc->peer_addr,
&ep->my_name.gnix_addr,
vc->vc_id,
(uint64_t)vc,
&smsg_mbox_attr);
/*
* try to send the message, if -FI_EAGAIN is returned, okay,
* just don't mark complete.
*/
ret = _gnix_cm_nic_send(cm_nic,
sbuf,
GNIX_CM_NIC_MAX_MSG_SIZE,
vc->peer_cm_nic_addr);
if (ret == FI_SUCCESS) {
complete = 1;
vc->conn_state = GNIX_VC_CONNECTING;
GNIX_DEBUG(FI_LOG_EP_CTRL, "moving vc %p state to connecting\n",
vc);
} else if (ret == -FI_EAGAIN) {
ret = FI_SUCCESS;
}
ret = _gnix_vc_schedule(vc);
err:
fastlock_release(&ep->vc_ht_lock);
*complete_ptr = complete;
return ret;
}
static int gnix_cq_set_wait(struct gnix_fid_cq *cq)
{
int ret = FI_SUCCESS;
GNIX_TRACE(FI_LOG_CQ, "\n");
struct fi_wait_attr requested = {
.wait_obj = cq->attr.wait_obj,
.flags = 0
};
switch (cq->attr.wait_obj) {
case FI_WAIT_UNSPEC:
case FI_WAIT_FD:
case FI_WAIT_MUTEX_COND:
ret = gnix_wait_open(&cq->domain->fabric->fab_fid,
&requested, &cq->wait);
break;
case FI_WAIT_SET:
ret = _gnix_wait_set_add(cq->attr.wait_set, &cq->cq_fid.fid);
if (!ret)
cq->wait = cq->attr.wait_set;
break;
default:
break;
}
return ret;
}
static void free_cq_entry(struct slist_entry *item)
{
struct gnix_cq_entry *entry;
entry = container_of(item, struct gnix_cq_entry, item);
free(entry->the_entry);
free(entry);
}
static struct slist_entry *alloc_cq_entry(size_t size)
{
struct gnix_cq_entry *entry = malloc(sizeof(*entry));
if (!entry) {
GNIX_DEBUG(FI_LOG_CQ, "out of memory\n");
goto err;
}
entry->the_entry = malloc(size);
if (!entry->the_entry) {
GNIX_DEBUG(FI_LOG_CQ, "out of memory\n");
goto cleanup;
}
return &entry->item;
cleanup:
free(entry);
err:
return NULL;
}
static int __gnix_cq_progress(struct gnix_fid_cq *cq)
{
return _gnix_prog_progress(&cq->pset);
}
/*******************************************************************************
* Exposed helper functions
******************************************************************************/
ssize_t _gnix_cq_add_event(struct gnix_fid_cq *cq, struct gnix_fid_ep *ep,
void *op_context, uint64_t flags, size_t len,
void *buf, uint64_t data, uint64_t tag,
fi_addr_t src_addr)
{
struct gnix_cq_entry *event;
struct slist_entry *item;
uint64_t mask;
ssize_t ret = FI_SUCCESS;
if (ep) {
if (ep->info && ep->info->mode & FI_NOTIFY_FLAGS_ONLY) {
mask = (FI_REMOTE_CQ_DATA | FI_MULTI_RECV);
if (flags & FI_RMA_EVENT) {
mask |= (FI_REMOTE_READ | FI_REMOTE_WRITE |
FI_RMA);
}
flags &= mask;
}
}
COND_ACQUIRE(cq->requires_lock, &cq->lock);
item = _gnix_queue_get_free(cq->events);
if (!item) {
GNIX_DEBUG(FI_LOG_CQ, "error creating cq_entry\n");
ret = -FI_ENOMEM;
goto err;
}
event = container_of(item, struct gnix_cq_entry, item);
assert(event->the_entry);
fill_function[cq->attr.format](event->the_entry, op_context, flags,
len, buf, data, tag);
event->src_addr = src_addr;
_gnix_queue_enqueue(cq->events, &event->item);
GNIX_DEBUG(FI_LOG_CQ, "Added event: %lx\n", op_context);
if (cq->wait)
_gnix_signal_wait_obj(cq->wait);
err:
COND_RELEASE(cq->requires_lock, &cq->lock);
return ret;
}
ssize_t _gnix_cq_add_error(struct gnix_fid_cq *cq, void *op_context,
uint64_t flags, size_t len, void *buf,
uint64_t data, uint64_t tag, size_t olen,
int err, int prov_errno, void *err_data,
size_t err_data_size)
{
struct fi_cq_err_entry *error;
struct gnix_cq_entry *event;
struct slist_entry *item;
ssize_t ret = FI_SUCCESS;
GNIX_INFO(FI_LOG_CQ, "creating error event entry\n");
COND_ACQUIRE(cq->requires_lock, &cq->lock);
item = _gnix_queue_get_free(cq->errors);
if (!item) {
GNIX_WARN(FI_LOG_CQ, "error creating error entry\n");
ret = -FI_ENOMEM;
goto err;
}
event = container_of(item, struct gnix_cq_entry, item);
error = event->the_entry;
error->op_context = op_context;
error->flags = flags;
error->len = len;
error->buf = buf;
error->data = data;
error->tag = tag;
error->olen = olen;
error->err = err;
error->prov_errno = prov_errno;
error->err_data = err_data;
error->err_data_size = err_data_size;
_gnix_queue_enqueue(cq->errors, &event->item);
if (cq->wait)
_gnix_signal_wait_obj(cq->wait);
err:
COND_RELEASE(cq->requires_lock, &cq->lock);
return ret;
}
int _gnix_cq_poll_obj_add(struct gnix_fid_cq *cq, void *obj,
int (*prog_fn)(void *data))
{
return _gnix_prog_obj_add(&cq->pset, obj, prog_fn);
}
int _gnix_cq_poll_obj_rem(struct gnix_fid_cq *cq, void *obj,
int (*prog_fn)(void *data))
{
return _gnix_prog_obj_rem(&cq->pset, obj, prog_fn);
}
static void __cq_destruct(void *obj)
{
struct gnix_fid_cq *cq = (struct gnix_fid_cq *) obj;
_gnix_ref_put(cq->domain);
switch (cq->attr.wait_obj) {
case FI_WAIT_NONE:
break;
case FI_WAIT_SET:
_gnix_wait_set_remove(cq->wait, &cq->cq_fid.fid);
break;
case FI_WAIT_UNSPEC:
case FI_WAIT_FD:
case FI_WAIT_MUTEX_COND:
assert(cq->wait);
gnix_wait_close(&cq->wait->fid);
break;
default:
GNIX_WARN(FI_LOG_CQ, "format: %d unsupported.\n",
cq->attr.wait_obj);
break;
}
_gnix_prog_fini(&cq->pset);
_gnix_queue_destroy(cq->events);
_gnix_queue_destroy(cq->errors);
fastlock_destroy(&cq->lock);
free(cq->cq_fid.ops);
free(cq->cq_fid.fid.ops);
free(cq);
}
/*******************************************************************************
* API functions.
******************************************************************************/
static int gnix_cq_close(fid_t fid)
{
struct gnix_fid_cq *cq;
int references_held;
GNIX_TRACE(FI_LOG_CQ, "\n");
cq = container_of(fid, struct gnix_fid_cq, cq_fid);
references_held = _gnix_ref_put(cq);
if (references_held) {
GNIX_INFO(FI_LOG_CQ, "failed to fully close cq due to lingering "
"references. references=%i cq=%p\n",
references_held, cq);
}
return FI_SUCCESS;
}
static ssize_t __gnix_cq_readfrom(struct fid_cq *cq, void *buf,
size_t count, fi_addr_t *src_addr)
{
struct gnix_fid_cq *cq_priv;
struct gnix_cq_entry *event;
struct slist_entry *temp;
ssize_t read_count = 0;
if (!cq || !buf || !count)
return -FI_EINVAL;
cq_priv = container_of(cq, struct gnix_fid_cq, cq_fid);
__gnix_cq_progress(cq_priv);
if (_gnix_queue_peek(cq_priv->errors))
return -FI_EAVAIL;
COND_ACQUIRE(cq_priv->requires_lock, &cq_priv->lock);
while (_gnix_queue_peek(cq_priv->events) && count--) {
temp = _gnix_queue_dequeue(cq_priv->events);
event = container_of(temp, struct gnix_cq_entry, item);
assert(event->the_entry);
memcpy(buf, event->the_entry, cq_priv->entry_size);
if (src_addr)
memcpy(&src_addr[read_count], &event->src_addr, sizeof(fi_addr_t));
_gnix_queue_enqueue_free(cq_priv->events, &event->item);
buf = (void *) ((uint8_t *) buf + cq_priv->entry_size);
read_count++;
}
COND_RELEASE(cq_priv->requires_lock, &cq_priv->lock);
return read_count ?: -FI_EAGAIN;
}
static ssize_t __gnix_cq_sreadfrom(int blocking, struct fid_cq *cq, void *buf,
size_t count, fi_addr_t *src_addr,
const void *cond, int timeout)
{
struct gnix_fid_cq *cq_priv;
cq_priv = container_of(cq, struct gnix_fid_cq, cq_fid);
if ((blocking && !cq_priv->wait) ||
(blocking && cq_priv->attr.wait_obj == FI_WAIT_SET))
return -FI_EINVAL;
if (_gnix_queue_peek(cq_priv->errors))
return -FI_EAVAIL;
if (cq_priv->wait)
gnix_wait_wait((struct fid_wait *)cq_priv->wait, timeout);
return __gnix_cq_readfrom(cq, buf, count, src_addr);
}
DIRECT_FN STATIC ssize_t gnix_cq_sreadfrom(struct fid_cq *cq, void *buf,
size_t count, fi_addr_t *src_addr,
const void *cond, int timeout)
{
return __gnix_cq_sreadfrom(1, cq, buf, count, src_addr, cond, timeout);
}
DIRECT_FN STATIC ssize_t gnix_cq_read(struct fid_cq *cq,
void *buf,
size_t count)
{
return __gnix_cq_sreadfrom(0, cq, buf, count, NULL, NULL, 0);
}
DIRECT_FN STATIC ssize_t gnix_cq_sread(struct fid_cq *cq, void *buf,
size_t count, const void *cond,
int timeout)
{
return __gnix_cq_sreadfrom(1, cq, buf, count, NULL, cond, timeout);
}
DIRECT_FN STATIC ssize_t gnix_cq_readfrom(struct fid_cq *cq, void *buf,
size_t count, fi_addr_t *src_addr)
{
return __gnix_cq_sreadfrom(0, cq, buf, count, src_addr, NULL, 0);
}
DIRECT_FN STATIC ssize_t gnix_cq_readerr(struct fid_cq *cq,
struct fi_cq_err_entry *buf,
uint64_t flags)
{
struct gnix_fid_cq *cq_priv;
struct gnix_cq_entry *event;
struct slist_entry *entry;
size_t err_data_cpylen;
struct fi_cq_err_entry *gnix_cq_err;
ssize_t read_count = 0;
if (!cq || !buf)
return -FI_EINVAL;
cq_priv = container_of(cq, struct gnix_fid_cq, cq_fid);
/*
* we need to progress cq. some apps may be only using
* cq to check for errors.
*/
_gnix_prog_progress(&cq_priv->pset);
COND_ACQUIRE(cq_priv->requires_lock, &cq_priv->lock);
entry = _gnix_queue_dequeue(cq_priv->errors);
if (!entry) {
read_count = -FI_EAGAIN;
goto err;
}
event = container_of(entry, struct gnix_cq_entry, item);
gnix_cq_err = event->the_entry;
buf->op_context = gnix_cq_err->op_context;
buf->flags = gnix_cq_err->flags;
buf->len = gnix_cq_err->len;
buf->buf = gnix_cq_err->buf;
buf->data = gnix_cq_err->data;
buf->tag = gnix_cq_err->tag;
buf->olen = gnix_cq_err->olen;
buf->err = gnix_cq_err->err;
buf->prov_errno = gnix_cq_err->prov_errno;
if (gnix_cq_err->err_data != NULL) {
/*
* Note: If the api version is >= 1.5 then copy err_data into
* buf->err_data and copy at most buf->err_data_size.
* If buf->err_data_size is zero or the api version is < 1.5,
* use the old method of allocating space in provider.
*/
if (FI_VERSION_LT(cq_priv->domain->fabric->fab_fid.api_version,
FI_VERSION(1, 5)) || buf->err_data_size == 0) {
err_data_cpylen = sizeof(cq_priv->err_data);
memcpy(cq_priv->err_data, gnix_cq_err->err_data,
err_data_cpylen);
buf->err_data = cq_priv->err_data;
} else {
if (buf->err_data == NULL)
return -FI_EINVAL;
err_data_cpylen = MIN(buf->err_data_size,
gnix_cq_err->err_data_size);
memcpy(buf->err_data, gnix_cq_err->err_data, err_data_cpylen);
buf->err_data_size = err_data_cpylen;
}
free(gnix_cq_err->err_data);
gnix_cq_err->err_data = NULL;
} else {
if (FI_VERSION_LT(cq_priv->domain->fabric->fab_fid.api_version,
FI_VERSION(1, 5))) {
buf->err_data = NULL;
} else {
buf->err_data_size = 0;
}
}
_gnix_queue_enqueue_free(cq_priv->errors, &event->item);
read_count++;
err:
COND_RELEASE(cq_priv->requires_lock, &cq_priv->lock);
return read_count;
}
DIRECT_FN STATIC const char *gnix_cq_strerror(struct fid_cq *cq, int prov_errno,
const void *prov_data, char *buf,
size_t len)
{
return NULL;
}
DIRECT_FN STATIC int gnix_cq_signal(struct fid_cq *cq)
{
struct gnix_fid_cq *cq_priv;
cq_priv = container_of(cq, struct gnix_fid_cq, cq_fid);
if (cq_priv->wait)
_gnix_signal_wait_obj(cq_priv->wait);
return FI_SUCCESS;
}
static int gnix_cq_control(struct fid *cq, int command, void *arg)
{
switch (command) {
case FI_GETWAIT:
return -FI_ENOSYS;
default:
return -FI_EINVAL;
}
}
DIRECT_FN int gnix_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
struct fid_cq **cq, void *context)
{
struct gnix_fid_domain *domain_priv;
struct gnix_fid_cq *cq_priv;
struct fi_ops_cq *cq_ops;
struct fi_ops *fi_cq_ops;
int ret = FI_SUCCESS;
GNIX_TRACE(FI_LOG_CQ, "\n");
cq_ops = calloc(1, sizeof(*cq_ops));
if (!cq_ops) {
return -FI_ENOMEM;
}
fi_cq_ops = calloc(1, sizeof(*fi_cq_ops));
if (!fi_cq_ops) {
ret = -FI_ENOMEM;
goto free_cq_ops;
}
*cq_ops = gnix_cq_ops;
*fi_cq_ops = gnix_cq_fi_ops;
ret = verify_cq_attr(attr, cq_ops, fi_cq_ops);
if (ret)
goto free_fi_cq_ops;
domain_priv = container_of(domain, struct gnix_fid_domain, domain_fid);
if (!domain_priv) {
ret = -FI_EINVAL;
goto free_fi_cq_ops;
}
cq_priv = calloc(1, sizeof(*cq_priv));
if (!cq_priv) {
ret = -FI_ENOMEM;
goto free_fi_cq_ops;
}
cq_priv->requires_lock = (domain_priv->thread_model !=
FI_THREAD_COMPLETION);
cq_priv->domain = domain_priv;
cq_priv->attr = *attr;
_gnix_ref_init(&cq_priv->ref_cnt, 1, __cq_destruct);
_gnix_ref_get(cq_priv->domain);
_gnix_prog_init(&cq_priv->pset);
cq_priv->cq_fid.fid.fclass = FI_CLASS_CQ;
cq_priv->cq_fid.fid.context = context;
cq_priv->cq_fid.fid.ops = fi_cq_ops;
cq_priv->cq_fid.ops = cq_ops;
/*
* Although we don't need to store entry_size since we're already
* storing the format, this might provide a performance benefit
* when allocating storage.
*/
cq_priv->entry_size = format_sizes[cq_priv->attr.format];
fastlock_init(&cq_priv->lock);
ret = gnix_cq_set_wait(cq_priv);
if (ret)
goto free_cq_priv;
ret = _gnix_queue_create(&cq_priv->events, alloc_cq_entry,
free_cq_entry, cq_priv->entry_size,
cq_priv->attr.size);
if (ret)
goto free_cq_priv;
ret = _gnix_queue_create(&cq_priv->errors, alloc_cq_entry,
free_cq_entry, sizeof(struct fi_cq_err_entry),
0);
if (ret)
goto free_gnix_queue;
*cq = &cq_priv->cq_fid;
return ret;
free_gnix_queue:
_gnix_queue_destroy(cq_priv->events);
free_cq_priv:
_gnix_ref_put(cq_priv->domain);
fastlock_destroy(&cq_priv->lock);
free(cq_priv);
free_fi_cq_ops:
free(fi_cq_ops);
free_cq_ops:
free(cq_ops);
return ret;
}
/*******************************************************************************
* FI_OPS_* data structures.
******************************************************************************/
static const struct fi_ops gnix_cq_fi_ops = {
.size = sizeof(struct fi_ops),
.close = gnix_cq_close,
.bind = fi_no_bind,
.control = gnix_cq_control,
.ops_open = fi_no_ops_open
};
static const struct fi_ops_cq gnix_cq_ops = {
.size = sizeof(struct fi_ops_cq),
.read = gnix_cq_read,
.readfrom = gnix_cq_readfrom,
.readerr = gnix_cq_readerr,
.sread = gnix_cq_sread,
.sreadfrom = gnix_cq_sreadfrom,
.signal = gnix_cq_signal,
.strerror = gnix_cq_strerror
};
int _gnix_pe_to_ip(const struct gnix_ep_name *ep_name,
struct sockaddr_in *saddr)
{
int ret = -FI_EIO;
FILE *arp_table;
char buf[1024];
char ip_str[128], mac_str[128];
union mac_addr mac;
union mac_addr tmp_mac = {0};
gni_return_t status;
uint32_t pe, cpu_id;
status = GNI_CdmGetNicAddress(0, &pe, &cpu_id);
if (status == GNI_RC_SUCCESS &&
ep_name->gnix_addr.device_addr == pe) {
ret = _gnix_local_ipaddr(saddr);
saddr->sin_port = ep_name->gnix_addr.cdm_id;
return ret;
}
arp_table = fopen(ARP_TABLE_FILE, "r");
if (!arp_table) {
GNIX_WARN(FI_LOG_FABRIC, "Failed to fopen(): %s\n",
ARP_TABLE_FILE);
return -FI_EIO;
}
/* Eat header line. */
if (!fgets(buf, sizeof(buf), arp_table)) {
GNIX_WARN(FI_LOG_FABRIC, "Failed to fgets(): %s\n",
ARP_TABLE_FILE);
return -FI_EIO;
}
mac.u64 = __gnix_pe_to_mac(ep_name->gnix_addr.device_addr);
while (fscanf(arp_table, ARP_TABLE_FORMAT, ip_str, mac_str) == 2) {
ret = sscanf(mac_str, "%hhx:%hhx:%hhx:%hhx:%hhx:%hhx",
&tmp_mac.octets[5], &tmp_mac.octets[4],
&tmp_mac.octets[3], &tmp_mac.octets[2],
&tmp_mac.octets[1], &tmp_mac.octets[0]);
if (ret == 6) {
GNIX_DEBUG(FI_LOG_FABRIC,
"Comparing 0x%llx, 0x%llx\n",
mac.u64, tmp_mac.u64);
if (mac.u64 == tmp_mac.u64) {
saddr->sin_family = AF_INET;
saddr->sin_port = ep_name->gnix_addr.cdm_id;
saddr->sin_addr.s_addr = inet_addr(ip_str);
ret = FI_SUCCESS;
GNIX_DEBUG(FI_LOG_FABRIC,
"Translated %s->%s\n",
ip_str, mac_str);
break;
}
} else {
GNIX_WARN(FI_LOG_FABRIC, "Parse error: %d : %s\n",
ret, mac_str);
break;
}
}
fclose(arp_table);
return ret;
}
DIRECT_FN int gnix_pep_listen(struct fid_pep *pep)
{
int ret;
struct gnix_fid_pep *pep_priv;
struct sockaddr_in saddr;
int sockopt = 1;
if (!pep)
return -FI_EINVAL;
pep_priv = container_of(pep, struct gnix_fid_pep, pep_fid.fid);
fastlock_acquire(&pep_priv->lock);
if (!pep_priv->eq) {
ret = -FI_EINVAL;
goto err_unlock;
}
pep_priv->listen_fd = socket(AF_INET, SOCK_STREAM | SOCK_NONBLOCK, 0);
if (pep_priv->listen_fd < 0) {
GNIX_WARN(FI_LOG_EP_CTRL,
"Failed to create listening socket, errno: %d\n",
errno);
ret = -FI_ENOSPC;
goto err_unlock;
}
ret = setsockopt(pep_priv->listen_fd, SOL_SOCKET, SO_REUSEADDR,
&sockopt, sizeof(sockopt));
if (ret < 0)
GNIX_WARN(FI_LOG_EP_CTRL,
"setsockopt(SO_REUSEADDR) failed, errno: %d\n",
errno);
/* Bind to the ipogif interface using resolved service number as CDM
* ID. */
ret = _gnix_local_ipaddr(&saddr);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL, "Failed to find local IP\n");
ret = -FI_ENOSPC;
goto err_sock;
}
/* If source addr was not specified, use auto assigned port. */
if (pep_priv->bound)
saddr.sin_port = pep_priv->src_addr.gnix_addr.cdm_id;
else
saddr.sin_port = 0;
ret = bind(pep_priv->listen_fd, &saddr, sizeof(struct sockaddr_in));
if (ret < 0) {
GNIX_WARN(FI_LOG_EP_CTRL,
"Failed to bind listening socket, errno: %d\n",
errno);
ret = -FI_ENOSPC;
goto err_sock;
}
ret = listen(pep_priv->listen_fd, pep_priv->backlog);
if (ret < 0) {
GNIX_WARN(FI_LOG_EP_CTRL,
"Failed to start listening socket, errno: %d\n",
errno);
ret = -FI_ENOSPC;
goto err_sock;
}
fastlock_release(&pep_priv->lock);
GNIX_DEBUG(FI_LOG_EP_CTRL,
"Configured PEP for listening: %p (%s:%d)\n",
pep, inet_ntoa(saddr.sin_addr), saddr.sin_port);
return FI_SUCCESS;
err_sock:
close(pep_priv->listen_fd);
err_unlock:
fastlock_release(&pep_priv->lock);
return ret;
}
/**
* Create a slab from a handle and append to the slab list.
*
* @param[in] handle Handle to the allocator being used.
*
* @return FI_SUCCESS On successful slab creation.
*
* @return -FI_ENOMEM if failure to allocate memory for slab or bitmap.
* @return [Unspec] if failure in alloc_bitmap. Will return error code from
* alloc_bitmap.
* @return [Unspec] if failure in GNI_MemRegister. Converts gni_return_t
* status code to FI_ERRNO value.
*/
static int __create_slab(struct gnix_mbox_alloc_handle *handle)
{
struct gnix_slab *slab;
gni_return_t status;
char error_buf[256];
char *error;
size_t total_size;
int ret;
int vmdh_index = -1;
int flags = GNI_MEM_READWRITE;
struct gnix_auth_key *info;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
slab = calloc(1, sizeof(*slab));
if (!slab) {
error = strerror_r(errno, error_buf, sizeof(error_buf));
GNIX_WARN(FI_LOG_EP_CTRL,
"Error allocating slab: %s\n",
error);
ret = -FI_ENOMEM;
goto err_slab_calloc;
}
total_size = handle->page_size * __page_count(handle);
GNIX_DEBUG(FI_LOG_EP_CTRL, "total_size requested for mmap: %zu.\n",
total_size);
slab->used = calloc(1, sizeof(*(slab->used)));
if (!slab->used) {
error = strerror_r(errno, error_buf, sizeof(error_buf));
GNIX_WARN(FI_LOG_EP_CTRL,
"Error allocating bitmap: %s\n",
error);
ret = -FI_ENOMEM;
goto err_bitmap_calloc;
}
slab->base = mmap(0, total_size, (PROT_READ | PROT_WRITE), MAP_SHARED,
handle->fd, handle->last_offset);
if (slab->base == MAP_FAILED) {
error = strerror_r(errno, error_buf, sizeof(error_buf));
GNIX_WARN(FI_LOG_EP_CTRL, "%s\n", error);
ret = -FI_ENOMEM;
goto err_mmap;
}
ret = _gnix_alloc_bitmap(slab->used, __mbox_count(handle), NULL);
if (ret) {
GNIX_WARN(FI_LOG_EP_CTRL, "Error allocating bitmap.\n");
goto err_alloc_bitmap;
}
COND_ACQUIRE(handle->nic_handle->requires_lock, &handle->nic_handle->lock);
if (handle->nic_handle->using_vmdh) {
info = _gnix_auth_key_lookup(GNIX_PROV_DEFAULT_AUTH_KEY,
GNIX_PROV_DEFAULT_AUTH_KEYLEN);
assert(info);
if (!handle->nic_handle->mdd_resources_set) {
/* check to see if the ptag registration limit was set
* yet or not -- becomes read-only after success */
_gnix_auth_key_enable(info);
status = GNI_SetMddResources(
handle->nic_handle->gni_nic_hndl,
(info->attr.prov_key_limit +
info->attr.user_key_limit));
assert(status == GNI_RC_SUCCESS);
handle->nic_handle->mdd_resources_set = 1;
}
vmdh_index = _gnix_get_next_reserved_key(info);
if (vmdh_index <= 0) {
GNIX_FATAL(FI_LOG_DOMAIN,
"failed to get reserved key for mbox "
"registration, rc=%d\n",
vmdh_index);
}
flags |= GNI_MEM_USE_VMDH;
}
status = GNI_MemRegister(handle->nic_handle->gni_nic_hndl,
(uint64_t) slab->base, total_size,
handle->cq_handle,
flags, vmdh_index,
&slab->memory_handle);
COND_RELEASE(handle->nic_handle->requires_lock, &handle->nic_handle->lock);
if (status != GNI_RC_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL, "GNI_MemRegister failed: %s\n",
gni_err_str[status]);
ret = gnixu_to_fi_errno(status);
goto err_memregister;
}
slab->allocator = handle;
gnix_slist_insert_tail(&slab->list_entry, &handle->slab_list);
handle->last_offset += total_size;
return ret;
err_memregister:
_gnix_free_bitmap(slab->used);
err_alloc_bitmap:
munmap(slab->base, total_size);
err_mmap:
free(slab->used);
err_bitmap_calloc:
free(slab);
err_slab_calloc:
return ret;
}
DIRECT_FN STATIC int gnix_accept(struct fid_ep *ep, const void *param,
size_t paramlen)
{
int ret;
struct gnix_vc *vc;
struct gnix_fid_ep *ep_priv;
struct gnix_pep_sock_conn *conn;
struct gnix_pep_sock_connresp resp;
struct fi_eq_cm_entry eq_entry, *eqe_ptr;
struct gnix_mbox *mbox = NULL;
struct gnix_av_addr_entry av_entry;
if (!ep || (paramlen && !param) || paramlen > GNIX_CM_DATA_MAX_SIZE)
return -FI_EINVAL;
ep_priv = container_of(ep, struct gnix_fid_ep, ep_fid.fid);
COND_ACQUIRE(ep_priv->requires_lock, &ep_priv->vc_lock);
/* Look up and unpack the connection request used to create this EP. */
conn = (struct gnix_pep_sock_conn *)ep_priv->info->handle;
if (!conn || conn->fid.fclass != FI_CLASS_CONNREQ) {
ret = -FI_EINVAL;
goto err_unlock;
}
/* Create new VC without CM data. */
av_entry.gnix_addr = ep_priv->dest_addr.gnix_addr;
av_entry.cm_nic_cdm_id = ep_priv->dest_addr.cm_nic_cdm_id;
ret = _gnix_vc_alloc(ep_priv, &av_entry, &vc);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL, "Failed to create VC: %d\n", ret);
goto err_unlock;
}
ep_priv->vc = vc;
ep_priv->vc->peer_caps = conn->req.peer_caps;
ep_priv->vc->peer_id = conn->req.vc_id;
ret = _gnix_mbox_alloc(vc->ep->nic->mbox_hndl, &mbox);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_DATA, "_gnix_mbox_alloc returned %s\n",
fi_strerror(-ret));
goto err_mbox_alloc;
}
vc->smsg_mbox = mbox;
/* Initialize the GNI connection. */
ret = _gnix_vc_smsg_init(vc, conn->req.vc_id,
&conn->req.vc_mbox_attr,
&conn->req.cq_irq_mdh);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"_gnix_vc_smsg_init returned %s\n",
fi_strerror(-ret));
goto err_smsg_init;
}
vc->conn_state = GNIX_VC_CONNECTED;
/* Send ACK with VC attrs to allow peer to initialize GNI connection. */
resp.cmd = GNIX_PEP_SOCK_RESP_ACCEPT;
resp.vc_id = vc->vc_id;
resp.vc_mbox_attr.msg_type = GNI_SMSG_TYPE_MBOX_AUTO_RETRANSMIT;
resp.vc_mbox_attr.msg_buffer = mbox->base;
resp.vc_mbox_attr.buff_size = vc->ep->nic->mem_per_mbox;
resp.vc_mbox_attr.mem_hndl = *mbox->memory_handle;
resp.vc_mbox_attr.mbox_offset = (uint64_t)mbox->offset;
resp.vc_mbox_attr.mbox_maxcredit =
ep_priv->domain->params.mbox_maxcredit;
resp.vc_mbox_attr.msg_maxsize =
ep_priv->domain->params.mbox_msg_maxsize;
resp.cq_irq_mdh = ep_priv->nic->irq_mem_hndl;
resp.peer_caps = ep_priv->caps;
resp.cm_data_len = paramlen;
if (paramlen) {
eqe_ptr = (struct fi_eq_cm_entry *)resp.eqe_buf;
memcpy(eqe_ptr->data, param, paramlen);
}
ret = write(conn->sock_fd, &resp, sizeof(resp));
if (ret != sizeof(resp)) {
GNIX_WARN(FI_LOG_EP_CTRL,
"Failed to send resp, errno: %d\n",
errno);
ret = -FI_EIO;
goto err_write;
}
/* Notify user that this side is connected. */
eq_entry.fid = &ep_priv->ep_fid.fid;
ret = fi_eq_write(&ep_priv->eq->eq_fid, FI_CONNECTED, &eq_entry,
sizeof(eq_entry), 0);
if (ret != sizeof(eq_entry)) {
GNIX_WARN(FI_LOG_EP_CTRL,
"fi_eq_write failed, err: %d\n", ret);
goto err_eq_write;
}
/* Free the connection request. */
free(conn);
ep_priv->conn_state = GNIX_EP_CONNECTED;
COND_RELEASE(ep_priv->requires_lock, &ep_priv->vc_lock);
GNIX_DEBUG(FI_LOG_EP_CTRL, "Sent conn accept: %p\n", ep_priv);
return FI_SUCCESS;
err_eq_write:
err_write:
err_smsg_init:
_gnix_mbox_free(ep_priv->vc->smsg_mbox);
ep_priv->vc->smsg_mbox = NULL;
err_mbox_alloc:
_gnix_vc_destroy(ep_priv->vc);
ep_priv->vc = NULL;
err_unlock:
COND_RELEASE(ep_priv->requires_lock, &ep_priv->vc_lock);
return ret;
}
DIRECT_FN STATIC int gnix_connect(struct fid_ep *ep, const void *addr,
const void *param, size_t paramlen)
{
int ret;
struct gnix_fid_ep *ep_priv;
struct sockaddr_in saddr;
struct gnix_pep_sock_connreq req;
struct fi_eq_cm_entry *eqe_ptr;
struct gnix_vc *vc;
struct gnix_mbox *mbox = NULL;
struct gnix_av_addr_entry av_entry;
if (!ep || !addr || (paramlen && !param) ||
paramlen > GNIX_CM_DATA_MAX_SIZE)
return -FI_EINVAL;
ep_priv = container_of(ep, struct gnix_fid_ep, ep_fid.fid);
COND_ACQUIRE(ep_priv->requires_lock, &ep_priv->vc_lock);
if (ep_priv->conn_state != GNIX_EP_UNCONNECTED) {
ret = -FI_EINVAL;
goto err_unlock;
}
ret = _gnix_pe_to_ip(addr, &saddr);
if (ret != FI_SUCCESS) {
GNIX_INFO(FI_LOG_EP_CTRL,
"Failed to translate gnix_ep_name to IP\n");
goto err_unlock;
}
/* Create new VC without CM data. */
av_entry.gnix_addr = ep_priv->dest_addr.gnix_addr;
av_entry.cm_nic_cdm_id = ep_priv->dest_addr.cm_nic_cdm_id;
ret = _gnix_vc_alloc(ep_priv, &av_entry, &vc);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"Failed to create VC:: %d\n",
ret);
goto err_unlock;
}
ep_priv->vc = vc;
ret = _gnix_mbox_alloc(vc->ep->nic->mbox_hndl, &mbox);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_DATA,
"_gnix_mbox_alloc returned %s\n",
fi_strerror(-ret));
goto err_mbox_alloc;
}
vc->smsg_mbox = mbox;
ep_priv->conn_fd = socket(AF_INET, SOCK_STREAM, 0);
if (ep_priv->conn_fd < 0) {
GNIX_WARN(FI_LOG_EP_CTRL,
"Failed to create connect socket, errno: %d\n",
errno);
ret = -FI_ENOSPC;
goto err_socket;
}
/* Currently blocks until connected. */
ret = connect(ep_priv->conn_fd, (struct sockaddr *)&saddr,
sizeof(saddr));
if (ret) {
GNIX_WARN(FI_LOG_EP_CTRL,
"Failed to connect, errno: %d\n",
errno);
ret = -FI_EIO;
goto err_connect;
}
req.info = *ep_priv->info;
/* Note addrs are swapped. */
memcpy(&req.dest_addr, (void *)&ep_priv->src_addr,
sizeof(req.dest_addr));
memcpy(&ep_priv->dest_addr, addr, sizeof(ep_priv->dest_addr));
memcpy(&req.src_addr, addr, sizeof(req.src_addr));
if (ep_priv->info->tx_attr)
req.tx_attr = *ep_priv->info->tx_attr;
if (ep_priv->info->rx_attr)
req.rx_attr = *ep_priv->info->rx_attr;
if (ep_priv->info->ep_attr)
req.ep_attr = *ep_priv->info->ep_attr;
if (ep_priv->info->domain_attr)
req.domain_attr = *ep_priv->info->domain_attr;
if (ep_priv->info->fabric_attr)
req.fabric_attr = *ep_priv->info->fabric_attr;
req.fabric_attr.fabric = NULL;
req.domain_attr.domain = NULL;
req.vc_id = vc->vc_id;
req.vc_mbox_attr.msg_type = GNI_SMSG_TYPE_MBOX_AUTO_RETRANSMIT;
req.vc_mbox_attr.msg_buffer = mbox->base;
req.vc_mbox_attr.buff_size = vc->ep->nic->mem_per_mbox;
req.vc_mbox_attr.mem_hndl = *mbox->memory_handle;
req.vc_mbox_attr.mbox_offset = (uint64_t)mbox->offset;
req.vc_mbox_attr.mbox_maxcredit =
ep_priv->domain->params.mbox_maxcredit;
req.vc_mbox_attr.msg_maxsize = ep_priv->domain->params.mbox_msg_maxsize;
req.cq_irq_mdh = ep_priv->nic->irq_mem_hndl;
req.peer_caps = ep_priv->caps;
req.cm_data_len = paramlen;
if (paramlen) {
eqe_ptr = (struct fi_eq_cm_entry *)req.eqe_buf;
memcpy(eqe_ptr->data, param, paramlen);
}
ret = write(ep_priv->conn_fd, &req, sizeof(req));
if (ret != sizeof(req)) {
GNIX_WARN(FI_LOG_EP_CTRL,
"Failed to send req, errno: %d\n",
errno);
ret = -FI_EIO;
goto err_write;
}
/* set fd to non-blocking now since we can't block within the eq
* progress system
*/
fi_fd_nonblock(ep_priv->conn_fd);
ep_priv->conn_state = GNIX_EP_CONNECTING;
COND_RELEASE(ep_priv->requires_lock, &ep_priv->vc_lock);
GNIX_DEBUG(FI_LOG_EP_CTRL, "Sent conn req: %p, %s\n",
ep_priv, inet_ntoa(saddr.sin_addr));
return FI_SUCCESS;
err_write:
err_connect:
close(ep_priv->conn_fd);
ep_priv->conn_fd = -1;
err_socket:
_gnix_mbox_free(ep_priv->vc->smsg_mbox);
ep_priv->vc->smsg_mbox = NULL;
err_mbox_alloc:
_gnix_vc_destroy(ep_priv->vc);
ep_priv->vc = NULL;
err_unlock:
COND_RELEASE(ep_priv->requires_lock, &ep_priv->vc_lock);
return ret;
}
/* Process a connection response on an FI_EP_MSG. */
static int __gnix_ep_connresp(struct gnix_fid_ep *ep,
struct gnix_pep_sock_connresp *resp)
{
int ret = FI_SUCCESS;
struct fi_eq_cm_entry *eq_entry;
int eqe_size;
switch (resp->cmd) {
case GNIX_PEP_SOCK_RESP_ACCEPT:
ep->vc->peer_caps = resp->peer_caps;
ep->vc->peer_id = resp->vc_id;
/* Initialize the GNI connection. */
ret = _gnix_vc_smsg_init(ep->vc, resp->vc_id,
&resp->vc_mbox_attr,
&resp->cq_irq_mdh);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"_gnix_vc_smsg_init returned %s\n",
fi_strerror(-ret));
return ret;
}
ep->vc->conn_state = GNIX_VC_CONNECTED;
ep->conn_state = GNIX_EP_CONNECTED;
/* Notify user that this side is connected. */
eq_entry = (struct fi_eq_cm_entry *)resp->eqe_buf;
eq_entry->fid = &ep->ep_fid.fid;
eqe_size = sizeof(*eq_entry) + resp->cm_data_len;
ret = fi_eq_write(&ep->eq->eq_fid, FI_CONNECTED, eq_entry,
eqe_size, 0);
if (ret != eqe_size) {
GNIX_WARN(FI_LOG_EP_CTRL,
"fi_eq_write failed, err: %d\n", ret);
return ret;
}
GNIX_DEBUG(FI_LOG_EP_CTRL, "Received conn accept: %p\n", ep);
break;
case GNIX_PEP_SOCK_RESP_REJECT:
/* Undo the connect and generate a failure EQE. */
close(ep->conn_fd);
ep->conn_fd = -1;
_gnix_mbox_free(ep->vc->smsg_mbox);
ep->vc->smsg_mbox = NULL;
_gnix_vc_destroy(ep->vc);
ep->vc = NULL;
ep->conn_state = GNIX_EP_UNCONNECTED;
/* Generate EQE. */
eq_entry = (struct fi_eq_cm_entry *)resp->eqe_buf;
eq_entry->fid = &ep->ep_fid.fid;
eq_entry = (struct fi_eq_cm_entry *)resp->eqe_buf;
ret = _gnix_eq_write_error(ep->eq, &ep->ep_fid.fid, NULL, 0,
FI_ECONNREFUSED, FI_ECONNREFUSED,
&eq_entry->data, resp->cm_data_len);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"fi_eq_write failed, err: %d\n", ret);
return ret;
}
GNIX_DEBUG(FI_LOG_EP_CTRL, "Conn rejected: %p\n", ep);
break;
default:
GNIX_INFO(FI_LOG_EP_CTRL, "Invalid response command: %d\n",
resp->cmd);
return -FI_EINVAL;
}
return FI_SUCCESS;
}
static int map_insert(struct gnix_fid_av *av_priv, const void *addr,
size_t count, fi_addr_t *fi_addr, uint64_t flags,
void *context)
{
int ret;
struct gnix_ep_name ep_name;
struct gnix_av_addr_entry *the_entry;
gnix_ht_key_t key;
size_t i;
struct gnix_av_block *blk = NULL;
int ret_cnt = count;
int *entry_err = context;
assert(av_priv->map_ht != NULL);
if (count == 0)
return 0;
blk = calloc(1, sizeof(struct gnix_av_block));
if (blk == NULL)
return -FI_ENOMEM;
blk->base = calloc(count, sizeof(struct gnix_av_addr_entry));
if (blk->base == NULL) {
free(blk);
return -FI_ENOMEM;
}
slist_insert_tail(&blk->slist, &av_priv->block_list);
for (i = 0; i < count; i++) {
_gnix_get_ep_name(addr, i, &ep_name, av_priv->domain);
/* check if this ep_name fits in the av context bits */
if (ep_name.name_type & GNIX_EPN_TYPE_SEP) {
if ((1 << av_priv->rx_ctx_bits) < ep_name.rx_ctx_cnt) {
if (flags & FI_SYNC_ERR) {
entry_err[i] = -FI_EINVAL;
fi_addr[i] = FI_ADDR_NOTAVAIL;
ret_cnt = -FI_EINVAL;
continue;
}
GNIX_DEBUG(FI_LOG_DEBUG, "ep_name doesn't fit "
"into the av context bits\n");
return -FI_EINVAL;
}
}
((struct gnix_address *)fi_addr)[i] = ep_name.gnix_addr;
the_entry = &blk->base[i];
memcpy(&the_entry->gnix_addr, &ep_name.gnix_addr,
sizeof(struct gnix_address));
the_entry->name_type = ep_name.name_type;
the_entry->cm_nic_cdm_id = ep_name.cm_nic_cdm_id;
the_entry->cookie = ep_name.cookie;
the_entry->rx_ctx_cnt = ep_name.rx_ctx_cnt;
memcpy(&key, &ep_name.gnix_addr, sizeof(gnix_ht_key_t));
ret = _gnix_ht_insert(av_priv->map_ht,
key,
the_entry);
if (flags & FI_SYNC_ERR) {
entry_err[i] = FI_SUCCESS;
}
/*
* we are okay with user trying to add more
* entries with same key.
*/
if ((ret != FI_SUCCESS) && (ret != -FI_ENOSPC)) {
GNIX_WARN(FI_LOG_AV,
"_gnix_ht_insert failed %d\n",
ret);
if (flags & FI_SYNC_ERR) {
entry_err[i] = ret;
fi_addr[i] = FI_ADDR_NOTAVAIL;
ret_cnt = ret;
continue;
}
return ret;
}
}
return ret_cnt;
}
