static int gnix_getinfo(uint32_t version, const char *node, const char *service,
uint64_t flags, struct fi_info *hints,
struct fi_info **info)
{
int ret = 0;
int ep_type_unspec = 1;
uint64_t mode = GNIX_FAB_MODES;
struct fi_info *gnix_info = NULL;
struct gnix_ep_name *dest_addr = NULL;
struct gnix_ep_name *src_addr = NULL;
struct gnix_ep_name *addr = NULL;
/*
* the code below for resolving a node/service to what
* will be a gnix_ep_name address is not fully implemented,
* but put a place holder in place
*/
if (node) {
addr = malloc(sizeof(*addr));
if (!addr) {
goto err;
}
/* resolve node/service to gnix_ep_name */
ret = gnix_resolve_name(node, service, flags, addr);
if (ret) {
goto err;
}
if (flags & FI_SOURCE) {
/* resolved address is the local address */
src_addr = addr;
if (hints && hints->dest_addr)
dest_addr = hints->dest_addr;
} else {
/* resolved address is a peer */
dest_addr = addr;
if (hints && hints->src_addr)
src_addr = hints->src_addr;
}
}
if (src_addr)
GNIX_INFO(FI_LOG_FABRIC, "src_pe: 0x%x src_port: 0x%lx\n",
src_addr->gnix_addr.device_addr,
src_addr->gnix_addr.cdm_id);
if (dest_addr)
GNIX_INFO(FI_LOG_FABRIC, "dest_pe: 0x%x dest_port: 0x%lx\n",
dest_addr->gnix_addr.device_addr,
dest_addr->gnix_addr.cdm_id);
/*
* fill in the gnix_info struct
*/
gnix_info = fi_allocinfo();
if (gnix_info == NULL) {
goto err;
}
/*
* Set the default values
*/
gnix_info->tx_attr->op_flags = 0;
gnix_info->rx_attr->op_flags = 0;
gnix_info->ep_attr->type = FI_EP_RDM;
gnix_info->ep_attr->protocol = FI_PROTO_GNI;
gnix_info->ep_attr->max_msg_size = GNIX_MAX_MSG_SIZE;
/* TODO: need to work on this */
gnix_info->ep_attr->mem_tag_format = 0x0;
gnix_info->ep_attr->tx_ctx_cnt = 1;
gnix_info->ep_attr->rx_ctx_cnt = 1;
gnix_info->domain_attr->threading = FI_THREAD_SAFE;
gnix_info->domain_attr->control_progress = FI_PROGRESS_AUTO;
gnix_info->domain_attr->data_progress = FI_PROGRESS_AUTO;
gnix_info->domain_attr->av_type = FI_AV_UNSPEC;
gnix_info->domain_attr->tx_ctx_cnt = gnix_max_nics_per_ptag;
/* only one aries per node */
gnix_info->domain_attr->name = strdup(gnix_dom_name);
gnix_info->domain_attr->cq_data_size = sizeof(uint64_t);
gnix_info->domain_attr->mr_mode = FI_MR_BASIC;
gnix_info->domain_attr->resource_mgmt = FI_RM_ENABLED;
gnix_info->next = NULL;
gnix_info->addr_format = FI_ADDR_GNI;
gnix_info->src_addrlen = sizeof(struct gnix_ep_name);
gnix_info->dest_addrlen = sizeof(struct gnix_ep_name);
gnix_info->src_addr = src_addr;
gnix_info->dest_addr = dest_addr;
/* prov_name gets filled in by fi_getinfo from the gnix_prov struct */
/* let's consider gni copyrighted :) */
gnix_info->tx_attr->msg_order = FI_ORDER_SAS;
gnix_info->tx_attr->comp_order = FI_ORDER_NONE;
gnix_info->tx_attr->size = GNIX_TX_SIZE_DEFAULT;
gnix_info->tx_attr->iov_limit = 1;
gnix_info->tx_attr->inject_size = GNIX_INJECT_SIZE;
gnix_info->tx_attr->rma_iov_limit = 1;
gnix_info->rx_attr->msg_order = FI_ORDER_SAS;
gnix_info->rx_attr->comp_order = FI_ORDER_NONE;
gnix_info->rx_attr->size = GNIX_RX_SIZE_DEFAULT;
gnix_info->rx_attr->iov_limit = 1;
if (hints) {
if (hints->ep_attr) {
/*
* support FI_EP_RDM, FI_EP_DGRAM endpoint types
*/
switch (hints->ep_attr->type) {
case FI_EP_UNSPEC:
break;
case FI_EP_RDM:
case FI_EP_DGRAM:
gnix_info->ep_attr->type = hints->ep_attr->type;
ep_type_unspec = 0;
break;
default:
goto err;
}
/*
* only support FI_PROTO_GNI protocol
*/
switch (hints->ep_attr->protocol) {
case FI_PROTO_UNSPEC:
case FI_PROTO_GNI:
break;
default:
goto err;
}
if (hints->ep_attr->tx_ctx_cnt > 1) {
goto err;
}
if (hints->ep_attr->rx_ctx_cnt > 1) {
goto err;
}
if (hints->ep_attr->max_msg_size > GNIX_MAX_MSG_SIZE) {
goto err;
}
}
/*
* check the mode field
*/
if (hints->mode) {
if ((hints->mode & GNIX_FAB_MODES) != GNIX_FAB_MODES) {
goto err;
}
mode = hints->mode & ~GNIX_FAB_MODES_CLEAR;
}
if (!hints->caps) {
/* Return all supported capabilities. */
gnix_info->caps = GNIX_EP_RDM_CAPS_FULL;
} else {
/* The provider must support all requested
* capabilities. */
if ((hints->caps & GNIX_EP_RDM_CAPS_FULL) !=
hints->caps) {
goto err;
}
/* The provider may silently enable secondary
* capabilities that do not introduce any overhead. */
gnix_info->caps = hints->caps | GNIX_EP_RDM_SEC_CAPS;
}
if (hints->tx_attr) {
if ((hints->tx_attr->op_flags & GNIX_EP_OP_FLAGS) !=
hints->tx_attr->op_flags) {
goto err;
}
if (hints->tx_attr->inject_size > GNIX_INJECT_SIZE) {
goto err;
}
gnix_info->tx_attr->op_flags =
hints->tx_attr->op_flags & GNIX_EP_OP_FLAGS;
}
if (hints->rx_attr) {
if ((hints->rx_attr->op_flags & GNIX_EP_OP_FLAGS) !=
hints->rx_attr->op_flags) {
goto err;
}
gnix_info->rx_attr->op_flags =
hints->rx_attr->op_flags & GNIX_EP_OP_FLAGS;
}
if (hints->fabric_attr && hints->fabric_attr->name &&
strncmp(hints->fabric_attr->name, gnix_fab_name,
strlen(gnix_fab_name))) {
goto err;
}
if (hints->domain_attr) {
if (hints->domain_attr->name &&
strncmp(hints->domain_attr->name, gnix_dom_name,
strlen(gnix_dom_name))) {
goto err;
}
if (hints->domain_attr->control_progress !=
FI_PROGRESS_UNSPEC)
gnix_info->domain_attr->control_progress =
hints->domain_attr->control_progress;
if (hints->domain_attr->data_progress !=
FI_PROGRESS_UNSPEC)
gnix_info->domain_attr->data_progress =
hints->domain_attr->data_progress;
switch (hints->domain_attr->mr_mode) {
case FI_MR_UNSPEC:
case FI_MR_BASIC:
gnix_info->domain_attr->mr_mode =
hints->domain_attr->mr_mode;
break;
case FI_MR_SCALABLE:
goto err;
}
switch (hints->domain_attr->threading) {
case FI_THREAD_COMPLETION:
gnix_info->domain_attr->threading =
hints->domain_attr->threading;
break;
default:
break;
}
ret = fi_check_domain_attr(&gnix_prov,
gnix_info->domain_attr,
hints->domain_attr,
FI_MATCH_EXACT);
if (ret)
goto err;
}
}
gnix_info->mode = mode;
gnix_info->fabric_attr->name = strdup(gnix_fab_name);
gnix_info->tx_attr->caps = gnix_info->caps;
gnix_info->tx_attr->mode = gnix_info->mode;
gnix_info->rx_attr->caps = gnix_info->caps;
gnix_info->rx_attr->mode = gnix_info->mode;
if (ep_type_unspec) {
struct fi_info *dg_info = fi_dupinfo(gnix_info);
if (!dg_info) {
GNIX_WARN(FI_LOG_FABRIC, "cannot copy info\n");
goto err;
}
dg_info->ep_attr->type = FI_EP_DGRAM;
gnix_info->next = dg_info;
}
*info = gnix_info;
return 0;
err:
if (gnix_info) {
if (gnix_info->tx_attr) free(gnix_info->tx_attr);
if (gnix_info->rx_attr) free(gnix_info->rx_attr);
if (gnix_info->ep_attr) free(gnix_info->ep_attr);
if (gnix_info->domain_attr) free(gnix_info->domain_attr);
if (gnix_info->fabric_attr) free(gnix_info->fabric_attr);
free(gnix_info);
}
/*
* for the getinfo method, we need to return -FI_ENODATA otherwise
* the fi_getinfo call will make an early exit without querying
* other providers which may be avaialble.
*/
return -FI_ENODATA;
}
static int gnix_getinfo(uint32_t version, const char *node, const char *service,
uint64_t flags, struct fi_info *hints,
struct fi_info **info)
{
int ret = 0;
uint64_t mode = GNIX_FAB_MODES;
enum fi_progress control_progress = FI_PROGRESS_AUTO;
enum fi_progress data_progress = FI_PROGRESS_AUTO;
struct fi_info *gnix_info = NULL;
struct gnix_ep_name *dest_addr = NULL;
struct gnix_ep_name *src_addr = NULL;
struct gnix_ep_name *addr = NULL;
/*
* the code below for resolving a node/service to what
* will be a gnix_ep_name address is not fully implemented,
* but put a place holder in place
*/
if (node) {
addr = malloc(sizeof(*addr));
if (!addr) {
ret = -FI_ENOMEM;
goto err;
}
/* resolve node/service to gnix_ep_name */
ret = gnix_resolve_name(node, service, flags, addr);
if (ret) {
goto err;
}
if (flags & FI_SOURCE) {
/* resolved address is the local address */
src_addr = addr;
if (hints && hints->dest_addr)
dest_addr = hints->dest_addr;
} else {
/* resolved address is a peer */
dest_addr = addr;
if (hints && hints->src_addr)
src_addr = hints->src_addr;
}
}
if (src_addr)
GNIX_INFO(FI_LOG_FABRIC, "src_pe: 0x%x src_port: 0x%lx\n",
src_addr->gnix_addr.device_addr,
src_addr->gnix_addr.cdm_id);
if (dest_addr)
GNIX_INFO(FI_LOG_FABRIC, "dest_pe: 0x%x dest_port: 0x%lx\n",
dest_addr->gnix_addr.device_addr,
dest_addr->gnix_addr.cdm_id);
if (hints) {
/*
* check for endpoint type, only support FI_EP_RDM for now
*/
if (hints->ep_attr) {
switch (hints->ep_attr->type) {
case FI_EP_UNSPEC:
case FI_EP_RDM:
break;
default:
ret = -FI_ENODATA;
goto err;
}
}
/*
* check the mode field
*/
if (hints->mode) {
if ((hints->mode & GNIX_FAB_MODES) != GNIX_FAB_MODES) {
ret = -FI_ENODATA;
goto err;
}
mode = hints->mode & ~GNIX_FAB_MODES_CLEAR;
}
if ((hints->caps & GNIX_EP_RDM_CAPS) != hints->caps) {
goto err;
}
if (hints->ep_attr) {
switch (hints->ep_attr->protocol) {
case FI_PROTO_UNSPEC:
case FI_PROTO_GNI:
break;
default:
ret = -FI_ENODATA;
goto err;
}
if (hints->ep_attr->tx_ctx_cnt > 1) {
ret = -FI_ENODATA;
goto err;
}
if (hints->ep_attr->rx_ctx_cnt > 1) {
ret = -FI_ENODATA;
goto err;
}
}
if (hints->tx_attr) {
if ((hints->tx_attr->op_flags & GNIX_EP_OP_FLAGS) !=
hints->tx_attr->op_flags) {
ret = -FI_ENODATA;
goto err;
}
if (hints->tx_attr->inject_size > GNIX_INJECT_SIZE) {
ret = -FI_ENODATA;
goto err;
}
}
if (hints->rx_attr &&
(hints->rx_attr->op_flags & GNIX_EP_OP_FLAGS) !=
hints->rx_attr->op_flags) {
ret = -FI_ENODATA;
goto err;
}
if (hints->fabric_attr && hints->fabric_attr->name &&
strncmp(hints->fabric_attr->name, gnix_fab_name,
strlen(gnix_fab_name))) {
ret = -FI_ENODATA;
goto err;
}
if (hints->domain_attr) {
if (hints->domain_attr->name &&
strncmp(hints->domain_attr->name, gnix_dom_name,
strlen(gnix_dom_name))) {
ret = -FI_ENODATA;
goto err;
}
if (hints->domain_attr->control_progress !=
FI_PROGRESS_UNSPEC)
control_progress =
hints->domain_attr->control_progress;
if (hints->domain_attr->data_progress !=
FI_PROGRESS_UNSPEC)
data_progress =
hints->domain_attr->control_progress;
switch (hints->domain_attr->mr_mode) {
case FI_MR_UNSPEC:
case FI_MR_BASIC:
break;
case FI_MR_SCALABLE:
ret = -FI_ENODATA;
goto err;
}
}
if (hints->ep_attr) {
if (hints->ep_attr->max_msg_size > GNIX_MAX_MSG_SIZE) {
ret = -FI_ENODATA;
goto err;
}
/*
* TODO: tag matching
* max_tag_value =
* fi_tag_bits(hints->ep_attr->mem_tag_format);
*/
}
}
/*
* fill in the gnix_info struct
*/
gnix_info = fi_allocinfo();
if (gnix_info == NULL) {
ret = -FI_ENOMEM;
goto err;
}
gnix_info->ep_attr->protocol = FI_PROTO_GNI;
gnix_info->ep_attr->max_msg_size = GNIX_MAX_MSG_SIZE;
/* TODO: need to work on this */
gnix_info->ep_attr->mem_tag_format = 0x0;
gnix_info->ep_attr->tx_ctx_cnt = 1;
gnix_info->ep_attr->rx_ctx_cnt = 1;
gnix_info->domain_attr->threading = FI_THREAD_SAFE;
gnix_info->domain_attr->control_progress = control_progress;
gnix_info->domain_attr->data_progress = data_progress;
gnix_info->domain_attr->av_type = FI_AV_UNSPEC;
gnix_info->domain_attr->tx_ctx_cnt = gnix_max_nics_per_ptag;
/* only one aries per node */
gnix_info->domain_attr->name = strdup(gnix_dom_name);
gnix_info->next = NULL;
gnix_info->ep_attr->type = FI_EP_RDM;
gnix_info->caps = hints ? hints->caps & GNIX_EP_RDM_CAPS :
GNIX_EP_RDM_CAPS;
gnix_info->mode = mode;
gnix_info->addr_format = FI_ADDR_GNI;
gnix_info->src_addrlen = sizeof(struct gnix_ep_name);
gnix_info->dest_addrlen = sizeof(struct gnix_ep_name);
gnix_info->src_addr = src_addr;
gnix_info->dest_addr = dest_addr;
gnix_info->fabric_attr->name = strdup(gnix_fab_name);
/* prov_name gets filled in by fi_getinfo from the gnix_prov struct */
/* let's consider gni copyrighted :) */
gnix_info->tx_attr->caps = gnix_info->caps;
gnix_info->tx_attr->mode = gnix_info->mode;
if (hints && hints->tx_attr && hints->tx_attr->op_flags) {
gnix_info->tx_attr->op_flags =
hints->tx_attr->op_flags & GNIX_EP_OP_FLAGS;
} else {
gnix_info->tx_attr->op_flags = 0;
}
gnix_info->tx_attr->msg_order = FI_ORDER_SAS;
gnix_info->tx_attr->comp_order = FI_ORDER_NONE;
gnix_info->tx_attr->size = GNIX_TX_SIZE_DEFAULT;
gnix_info->tx_attr->iov_limit = 1;
gnix_info->tx_attr->inject_size = GNIX_INJECT_SIZE;
gnix_info->tx_attr->rma_iov_limit = 1;
gnix_info->rx_attr->caps = gnix_info->caps;
gnix_info->rx_attr->mode = gnix_info->mode;
if (hints && hints->rx_attr && hints->rx_attr->op_flags) {
gnix_info->rx_attr->op_flags =
hints->rx_attr->op_flags & GNIX_EP_OP_FLAGS;
} else {
gnix_info->rx_attr->op_flags = 0;
}
gnix_info->rx_attr->msg_order = FI_ORDER_SAS;
gnix_info->rx_attr->comp_order = FI_ORDER_NONE;
gnix_info->rx_attr->size = GNIX_RX_SIZE_DEFAULT;
gnix_info->rx_attr->iov_limit = 1;
*info = gnix_info;
return 0;
err:
if (gnix_info) {
if (gnix_info->tx_attr) free(gnix_info->tx_attr);
if (gnix_info->rx_attr) free(gnix_info->rx_attr);
if (gnix_info->ep_attr) free(gnix_info->ep_attr);
if (gnix_info->domain_attr) free(gnix_info->domain_attr);
if (gnix_info->fabric_attr) free(gnix_info->fabric_attr);
free(gnix_info);
}
/*
* for the getinfo method, we need to return -FI_ENODATA otherwise
* the fi_getinfo call will make an early exit without querying
* other providers which may be avaialble.
*/
return -FI_ENODATA;
}
