static int psmx2_getinfo(uint32_t version, const char *node, const char *service, uint64_t flags, struct fi_info *hints, struct fi_info **info) { struct fi_info *psmx2_info; uint32_t cnt = 0; void *dest_addr = NULL; int ep_type = FI_EP_RDM; int av_type = FI_AV_UNSPEC; uint64_t mode = FI_CONTEXT; enum fi_mr_mode mr_mode = FI_MR_SCALABLE; enum fi_threading threading = FI_THREAD_COMPLETION; enum fi_progress control_progress = FI_PROGRESS_MANUAL; enum fi_progress data_progress = FI_PROGRESS_MANUAL; uint64_t caps = PSMX2_CAPS; uint64_t max_tag_value = -1ULL; int err = -FI_ENODATA; FI_INFO(&psmx2_prov, FI_LOG_CORE,"\n"); *info = NULL; if (psm2_ep_num_devunits(&cnt) || !cnt) { FI_INFO(&psmx2_prov, FI_LOG_CORE, "no PSM device is found.\n"); return -FI_ENODATA; } psmx2_init_env(); if (node && !(flags & FI_SOURCE)) { dest_addr = psmx2_resolve_name(node, 0); if (dest_addr) FI_INFO(&psmx2_prov, FI_LOG_CORE, "node '%s' resolved to <epid=0x%llx, vl=%d>\n", node, ((struct psmx2_ep_name *)dest_addr)->epid, ((struct psmx2_ep_name *)dest_addr)->vlane); else FI_INFO(&psmx2_prov, FI_LOG_CORE, "failed to resolve node '%s'.\n", node); } if (hints) { switch (hints->addr_format) { case FI_FORMAT_UNSPEC: case FI_ADDR_PSMX: break; default: FI_INFO(&psmx2_prov, FI_LOG_CORE, "hints->addr_format=%d, supported=%d,%d.\n", hints->addr_format, FI_FORMAT_UNSPEC, FI_ADDR_PSMX); goto err_out; } if (hints->ep_attr) { switch (hints->ep_attr->type) { case FI_EP_UNSPEC: case FI_EP_DGRAM: case FI_EP_RDM: break; default: FI_INFO(&psmx2_prov, FI_LOG_CORE, "hints->ep_attr->type=%d, supported=%d,%d,%d.\n", hints->ep_attr->type, FI_EP_UNSPEC, FI_EP_DGRAM, FI_EP_RDM); goto err_out; } switch (hints->ep_attr->protocol) { case FI_PROTO_UNSPEC: case FI_PROTO_PSMX: break; default: FI_INFO(&psmx2_prov, FI_LOG_CORE, "hints->protocol=%d, supported=%d %d\n", hints->ep_attr->protocol, FI_PROTO_UNSPEC, FI_PROTO_PSMX); goto err_out; } if (hints->ep_attr->tx_ctx_cnt > 1) { FI_INFO(&psmx2_prov, FI_LOG_CORE, "hints->ep_attr->tx_ctx_cnt=%d, supported=0,1\n", hints->ep_attr->tx_ctx_cnt); goto err_out; } if (hints->ep_attr->rx_ctx_cnt > 1) { FI_INFO(&psmx2_prov, FI_LOG_CORE, "hints->ep_attr->rx_ctx_cnt=%d, supported=0,1\n", hints->ep_attr->rx_ctx_cnt); goto err_out; } } if ((hints->caps & PSMX2_CAPS) != hints->caps) { FI_INFO(&psmx2_prov, FI_LOG_CORE, "hints->caps=0x%llx, supported=0x%llx\n", hints->caps, PSMX2_CAPS); goto err_out; } if (hints->tx_attr) { if ((hints->tx_attr->op_flags & PSMX2_OP_FLAGS) != hints->tx_attr->op_flags) { FI_INFO(&psmx2_prov, FI_LOG_CORE, "hints->tx->flags=0x%llx, " "supported=0x%llx\n", hints->tx_attr->op_flags, PSMX2_OP_FLAGS); goto err_out; } if (hints->tx_attr->inject_size > PSMX2_INJECT_SIZE) { FI_INFO(&psmx2_prov, FI_LOG_CORE, "hints->tx_attr->inject_size=%ld," "supported=%ld.\n", hints->tx_attr->inject_size, PSMX2_INJECT_SIZE); goto err_out; } } if (hints->rx_attr && (hints->rx_attr->op_flags & PSMX2_OP_FLAGS) != hints->rx_attr->op_flags) { FI_INFO(&psmx2_prov, FI_LOG_CORE, "hints->rx->flags=0x%llx, supported=0x%llx\n", hints->rx_attr->op_flags, PSMX2_OP_FLAGS); goto err_out; } if ((hints->caps & FI_TAGGED) || (hints->caps & FI_MSG)) { if ((hints->mode & FI_CONTEXT) != FI_CONTEXT) { FI_INFO(&psmx2_prov, FI_LOG_CORE, "hints->mode=0x%llx, required=0x%llx\n", hints->mode, FI_CONTEXT); goto err_out; } } else { mode = 0; } if (hints->fabric_attr && hints->fabric_attr->name && strcmp(hints->fabric_attr->name, PSMX2_FABRIC_NAME)) { FI_INFO(&psmx2_prov, FI_LOG_CORE, "hints->fabric_name=%s, supported=psm\n", hints->fabric_attr->name); goto err_out; } if (hints->domain_attr) { if (hints->domain_attr->name && strcmp(hints->domain_attr->name, PSMX2_DOMAIN_NAME)) { FI_INFO(&psmx2_prov, FI_LOG_CORE, "hints->domain_name=%s, supported=psm\n", hints->domain_attr->name); goto err_out; } switch (hints->domain_attr->av_type) { case FI_AV_UNSPEC: case FI_AV_MAP: case FI_AV_TABLE: av_type = hints->domain_attr->av_type; break; default: FI_INFO(&psmx2_prov, FI_LOG_CORE, "hints->domain_attr->av_type=%d, supported=%d %d %d\n", hints->domain_attr->av_type, FI_AV_UNSPEC, FI_AV_MAP, FI_AV_TABLE); goto err_out; } switch (hints->domain_attr->mr_mode) { case FI_MR_UNSPEC: break; case FI_MR_BASIC: case FI_MR_SCALABLE: mr_mode = hints->domain_attr->mr_mode; break; default: FI_INFO(&psmx2_prov, FI_LOG_CORE, "hints->domain_attr->mr_mode=%d, supported=%d %d %d\n", hints->domain_attr->mr_mode, FI_MR_UNSPEC, FI_MR_BASIC, FI_MR_SCALABLE); goto err_out; } switch (hints->domain_attr->threading) { case FI_THREAD_UNSPEC: break; case FI_THREAD_FID: case FI_THREAD_ENDPOINT: case FI_THREAD_COMPLETION: case FI_THREAD_DOMAIN: threading = hints->domain_attr->threading; break; default: FI_INFO(&psmx2_prov, FI_LOG_CORE, "hints->domain_attr->threading=%d, supported=%d %d %d %d %d\n", hints->domain_attr->threading, FI_THREAD_UNSPEC, FI_THREAD_FID, FI_THREAD_ENDPOINT, FI_THREAD_COMPLETION, FI_THREAD_DOMAIN); goto err_out; } switch (hints->domain_attr->control_progress) { case FI_PROGRESS_UNSPEC: break; case FI_PROGRESS_MANUAL: case FI_PROGRESS_AUTO: control_progress = hints->domain_attr->control_progress; break; default: FI_INFO(&psmx2_prov, FI_LOG_CORE, "hints->domain_attr->control_progress=%d, supported=%d %d %d\n", hints->domain_attr->control_progress, FI_PROGRESS_UNSPEC, FI_PROGRESS_MANUAL, FI_PROGRESS_AUTO); goto err_out; } switch (hints->domain_attr->data_progress) { case FI_PROGRESS_UNSPEC: break; case FI_PROGRESS_MANUAL: case FI_PROGRESS_AUTO: data_progress = hints->domain_attr->data_progress; break; default: FI_INFO(&psmx2_prov, FI_LOG_CORE, "hints->domain_attr->data_progress=%d, supported=%d %d %d\n", hints->domain_attr->data_progress, FI_PROGRESS_UNSPEC, FI_PROGRESS_MANUAL, FI_PROGRESS_AUTO); goto err_out; } } if (hints->ep_attr) { if (hints->ep_attr->max_msg_size > PSMX2_MAX_MSG_SIZE) { FI_INFO(&psmx2_prov, FI_LOG_CORE, "hints->ep_attr->max_msg_size=%ld," "supported=%ld.\n", hints->ep_attr->max_msg_size, PSMX2_MAX_MSG_SIZE); goto err_out; } max_tag_value = fi_tag_bits(hints->ep_attr->mem_tag_format); } if (hints->tx_attr) { if ((hints->tx_attr->msg_order & PSMX2_MSG_ORDER) != hints->tx_attr->msg_order) { FI_INFO(&psmx2_prov, FI_LOG_CORE, "hints->tx_attr->msg_order=%lx," "supported=%lx.\n", hints->tx_attr->msg_order, PSMX2_MSG_ORDER); goto err_out; } if ((hints->tx_attr->comp_order & PSMX2_COMP_ORDER) != hints->tx_attr->comp_order) { FI_INFO(&psmx2_prov, FI_LOG_CORE, "hints->tx_attr->msg_order=%lx," "supported=%lx.\n", hints->tx_attr->comp_order, PSMX2_COMP_ORDER); goto err_out; } if (hints->tx_attr->inject_size > PSMX2_INJECT_SIZE) { FI_INFO(&psmx2_prov, FI_LOG_CORE, "hints->tx_attr->inject_size=%ld," "supported=%d.\n", hints->tx_attr->inject_size, PSMX2_INJECT_SIZE); goto err_out; } if (hints->tx_attr->iov_limit > 1) { FI_INFO(&psmx2_prov, FI_LOG_CORE, "hints->tx_attr->iov_limit=%ld," "supported=1.\n", hints->tx_attr->iov_limit); goto err_out; } if (hints->tx_attr->rma_iov_limit > 1) { FI_INFO(&psmx2_prov, FI_LOG_CORE, "hints->tx_attr->rma_iov_limit=%ld," "supported=1.\n", hints->tx_attr->rma_iov_limit); goto err_out; } } if (hints->rx_attr) { if ((hints->rx_attr->msg_order & PSMX2_MSG_ORDER) != hints->rx_attr->msg_order) { FI_INFO(&psmx2_prov, FI_LOG_CORE, "hints->rx_attr->msg_order=%lx," "supported=%lx.\n", hints->rx_attr->msg_order, PSMX2_MSG_ORDER); goto err_out; } if ((hints->rx_attr->comp_order & PSMX2_COMP_ORDER) != hints->rx_attr->comp_order) { FI_INFO(&psmx2_prov, FI_LOG_CORE, "hints->rx_attr->msg_order=%lx," "supported=%lx.\n", hints->rx_attr->comp_order, PSMX2_COMP_ORDER); goto err_out; } if (hints->rx_attr->iov_limit > 1) { FI_INFO(&psmx2_prov, FI_LOG_CORE, "hints->rx_attr->iov_limit=%ld," "supported=1.\n", hints->rx_attr->iov_limit); goto err_out; } } if (hints->caps) caps = hints->caps; /* TODO: check other fields of hints */ } psmx2_info = fi_allocinfo(); if (!psmx2_info) { err = -FI_ENOMEM; goto err_out; } psmx2_info->ep_attr->type = ep_type; psmx2_info->ep_attr->protocol = FI_PROTO_PSMX; psmx2_info->ep_attr->protocol_version = PSM2_VERNO; psmx2_info->ep_attr->max_msg_size = PSMX2_MAX_MSG_SIZE; psmx2_info->ep_attr->mem_tag_format = fi_tag_format(max_tag_value); psmx2_info->ep_attr->tx_ctx_cnt = 1; psmx2_info->ep_attr->rx_ctx_cnt = 1; psmx2_info->domain_attr->threading = threading; psmx2_info->domain_attr->control_progress = control_progress; psmx2_info->domain_attr->data_progress = data_progress; psmx2_info->domain_attr->name = strdup(PSMX2_DOMAIN_NAME); psmx2_info->domain_attr->resource_mgmt = FI_RM_ENABLED; psmx2_info->domain_attr->av_type = av_type; psmx2_info->domain_attr->mr_mode = mr_mode; psmx2_info->domain_attr->mr_key_size = sizeof(uint64_t); psmx2_info->domain_attr->cq_data_size = 4; psmx2_info->domain_attr->cq_cnt = 65535; psmx2_info->domain_attr->ep_cnt = 65535; psmx2_info->domain_attr->tx_ctx_cnt = 1; psmx2_info->domain_attr->rx_ctx_cnt = 1; psmx2_info->domain_attr->max_ep_tx_ctx = 65535; psmx2_info->domain_attr->max_ep_rx_ctx = 1; psmx2_info->domain_attr->max_ep_stx_ctx = 65535; psmx2_info->domain_attr->max_ep_srx_ctx = 0; psmx2_info->next = NULL; psmx2_info->caps = caps; psmx2_info->mode = mode; psmx2_info->addr_format = FI_ADDR_PSMX; psmx2_info->src_addrlen = 0; psmx2_info->dest_addrlen = sizeof(struct psmx2_ep_name); psmx2_info->src_addr = NULL; psmx2_info->dest_addr = dest_addr; psmx2_info->fabric_attr->name = strdup(PSMX2_FABRIC_NAME); psmx2_info->fabric_attr->prov_name = NULL; psmx2_info->tx_attr->caps = psmx2_info->caps; psmx2_info->tx_attr->mode = psmx2_info->mode; psmx2_info->tx_attr->op_flags = (hints && hints->tx_attr && hints->tx_attr->op_flags) ? hints->tx_attr->op_flags : 0; psmx2_info->tx_attr->msg_order = PSMX2_MSG_ORDER; psmx2_info->tx_attr->comp_order = PSMX2_COMP_ORDER; psmx2_info->tx_attr->inject_size = PSMX2_INJECT_SIZE; psmx2_info->tx_attr->size = UINT64_MAX; psmx2_info->tx_attr->iov_limit = 1; psmx2_info->tx_attr->rma_iov_limit = 1; psmx2_info->rx_attr->caps = psmx2_info->caps; psmx2_info->rx_attr->mode = psmx2_info->mode; psmx2_info->rx_attr->op_flags = (hints && hints->rx_attr && hints->rx_attr->op_flags) ? hints->rx_attr->op_flags : 0; psmx2_info->rx_attr->msg_order = PSMX2_MSG_ORDER; psmx2_info->rx_attr->comp_order = PSMX2_COMP_ORDER; psmx2_info->rx_attr->total_buffered_recv = ~(0ULL); /* that's how PSM handles it internally! */ psmx2_info->rx_attr->size = UINT64_MAX; psmx2_info->rx_attr->iov_limit = 1; *info = psmx2_info; return 0; err_out: if (dest_addr) free(dest_addr); return err; }
static int psmx_getinfo(uint32_t version, const char *node, const char *service, uint64_t flags, struct fi_info *hints, struct fi_info **info) { struct fi_info *psmx_info; uint32_t cnt = 0; void *dest_addr = NULL; int ep_type = FI_EP_RDM; int av_type = FI_AV_UNSPEC; enum fi_mr_mode mr_mode = FI_MR_SCALABLE; int caps = 0; uint64_t max_tag_value = 0; int err = -FI_ENODATA; FI_INFO(&psmx_prov, FI_LOG_CORE,"\n"); *info = NULL; if (psm_ep_num_devunits(&cnt) || !cnt) { FI_INFO(&psmx_prov, FI_LOG_CORE, "no PSM device is found.\n"); return -FI_ENODATA; } if (node && !(flags & FI_SOURCE)) dest_addr = psmx_resolve_name(node, 0); if (hints) { switch (hints->addr_format) { case FI_FORMAT_UNSPEC: case FI_ADDR_PSMX: break; default: FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->addr_format=%d, supported=%d,%d.\n", hints->addr_format, FI_FORMAT_UNSPEC, FI_ADDR_PSMX); goto err_out; } if (hints->ep_attr) { switch (hints->ep_attr->type) { case FI_EP_UNSPEC: case FI_EP_DGRAM: case FI_EP_RDM: break; default: FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->ep_attr->type=%d, supported=%d,%d,%d.\n", hints->ep_attr->type, FI_EP_UNSPEC, FI_EP_DGRAM, FI_EP_RDM); goto err_out; } switch (hints->ep_attr->protocol) { case FI_PROTO_UNSPEC: case FI_PROTO_PSMX: break; default: FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->protocol=%d, supported=%d %d\n", hints->ep_attr->protocol, FI_PROTO_UNSPEC, FI_PROTO_PSMX); goto err_out; } if (hints->ep_attr->tx_ctx_cnt > 1) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->ep_attr->tx_ctx_cnt=%d, supported=0,1\n", hints->ep_attr->tx_ctx_cnt); goto err_out; } if (hints->ep_attr->rx_ctx_cnt > 1) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->ep_attr->rx_ctx_cnt=%d, supported=0,1\n", hints->ep_attr->rx_ctx_cnt); goto err_out; } } if ((hints->caps & PSMX_CAPS) != hints->caps && (hints->caps & PSMX_CAPS2) != hints->caps) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->caps=0x%llx, supported=0x%llx,0x%llx\n", hints->caps, PSMX_CAPS, PSMX_CAPS2); goto err_out; } if (hints->tx_attr) { if ((hints->tx_attr->op_flags & PSMX_OP_FLAGS) != hints->tx_attr->op_flags) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->tx->flags=0x%llx, " "supported=0x%llx\n", hints->tx_attr->op_flags, PSMX_OP_FLAGS); goto err_out; } if (hints->tx_attr->inject_size > PSMX_INJECT_SIZE) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->tx_attr->inject_size=%ld," "supported=%ld.\n", hints->tx_attr->inject_size, PSMX_INJECT_SIZE); goto err_out; } } if (hints->rx_attr && (hints->rx_attr->op_flags & PSMX_OP_FLAGS) != hints->rx_attr->op_flags) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->rx->flags=0x%llx, supported=0x%llx\n", hints->rx_attr->op_flags, PSMX_OP_FLAGS); goto err_out; } if ((hints->mode & PSMX_MODE) != PSMX_MODE) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->mode=0x%llx, required=0x%llx\n", hints->mode, PSMX_MODE); goto err_out; } if (hints->fabric_attr && hints->fabric_attr->name && strncmp(hints->fabric_attr->name, PSMX_FABRIC_NAME, PSMX_FABRIC_NAME_LEN)) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->fabric_name=%s, supported=psm\n", hints->fabric_attr->name); goto err_out; } if (hints->domain_attr) { if (hints->domain_attr->name && strncmp(hints->domain_attr->name, PSMX_DOMAIN_NAME, PSMX_DOMAIN_NAME_LEN)) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->domain_name=%s, supported=psm\n", hints->domain_attr->name); goto err_out; } switch (hints->domain_attr->av_type) { case FI_AV_UNSPEC: case FI_AV_MAP: case FI_AV_TABLE: av_type = hints->domain_attr->av_type; break; default: FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->domain_attr->av_type=%d, supported=%d %d %d\n", hints->domain_attr->av_type, FI_AV_UNSPEC, FI_AV_MAP, FI_AV_TABLE); goto err_out; } switch (hints->domain_attr->mr_mode) { case FI_MR_UNSPEC: break; case FI_MR_BASIC: case FI_MR_SCALABLE: mr_mode = hints->domain_attr->mr_mode; break; default: FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->domain_attr->mr_mode=%d, supported=%d %d %d\n", hints->domain_attr->mr_mode, FI_MR_UNSPEC, FI_MR_BASIC, FI_MR_SCALABLE); goto err_out; } } if (hints->ep_attr) { if (hints->ep_attr->max_msg_size > PSMX_MAX_MSG_SIZE) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->ep_attr->max_msg_size=%ld," "supported=%ld.\n", hints->ep_attr->max_msg_size, PSMX_MAX_MSG_SIZE); goto err_out; } max_tag_value = fi_tag_bits(hints->ep_attr->mem_tag_format); } caps = hints->caps; /* TODO: check other fields of hints */ } if (psmx_reserve_tag_bits(&caps, &max_tag_value) < 0) goto err_out; psmx_info = fi_allocinfo(); if (!psmx_info) { err = -FI_ENOMEM; goto err_out; } psmx_info->ep_attr->type = ep_type; psmx_info->ep_attr->protocol = FI_PROTO_PSMX; psmx_info->ep_attr->protocol_version = PSM_VERNO; psmx_info->ep_attr->max_msg_size = PSMX_MAX_MSG_SIZE; psmx_info->ep_attr->mem_tag_format = fi_tag_format(max_tag_value); psmx_info->ep_attr->tx_ctx_cnt = 1; psmx_info->ep_attr->rx_ctx_cnt = 1; psmx_info->domain_attr->threading = FI_THREAD_COMPLETION; psmx_info->domain_attr->control_progress = FI_PROGRESS_MANUAL; psmx_info->domain_attr->data_progress = FI_PROGRESS_MANUAL; psmx_info->domain_attr->name = strdup(PSMX_DOMAIN_NAME); psmx_info->domain_attr->resource_mgmt = FI_RM_ENABLED; psmx_info->domain_attr->av_type = av_type; psmx_info->domain_attr->mr_mode = mr_mode; psmx_info->domain_attr->mr_key_size = sizeof(uint64_t); psmx_info->domain_attr->cq_data_size = 4; psmx_info->domain_attr->cq_cnt = 65535; psmx_info->domain_attr->ep_cnt = 65535; psmx_info->domain_attr->tx_ctx_cnt = 1; psmx_info->domain_attr->rx_ctx_cnt = 1; psmx_info->domain_attr->max_ep_tx_ctx = 65535; psmx_info->domain_attr->max_ep_rx_ctx = 1; psmx_info->domain_attr->max_ep_stx_ctx = 65535; psmx_info->domain_attr->max_ep_srx_ctx = 0; psmx_info->next = NULL; psmx_info->caps = (hints && hints->caps) ? hints->caps : caps; psmx_info->mode = PSMX_MODE; psmx_info->addr_format = FI_ADDR_PSMX; psmx_info->src_addrlen = 0; psmx_info->dest_addrlen = sizeof(psm_epid_t); psmx_info->src_addr = NULL; psmx_info->dest_addr = dest_addr; psmx_info->fabric_attr->name = strdup(PSMX_FABRIC_NAME); psmx_info->fabric_attr->prov_name = strdup(PSMX_PROV_NAME); psmx_info->tx_attr->caps = psmx_info->caps; psmx_info->tx_attr->mode = psmx_info->mode; psmx_info->tx_attr->op_flags = (hints && hints->tx_attr && hints->tx_attr->op_flags) ? hints->tx_attr->op_flags : 0; psmx_info->tx_attr->msg_order = FI_ORDER_SAS; psmx_info->tx_attr->comp_order = FI_ORDER_NONE; psmx_info->tx_attr->inject_size = PSMX_INJECT_SIZE; psmx_info->tx_attr->size = UINT64_MAX; psmx_info->tx_attr->iov_limit = 1; psmx_info->rx_attr->caps = psmx_info->caps; psmx_info->rx_attr->mode = psmx_info->mode; psmx_info->rx_attr->op_flags = (hints && hints->rx_attr && hints->tx_attr->op_flags) ? hints->tx_attr->op_flags : 0; psmx_info->rx_attr->msg_order = FI_ORDER_SAS; psmx_info->rx_attr->comp_order = FI_ORDER_NONE; psmx_info->rx_attr->total_buffered_recv = ~(0ULL); /* that's how PSM handles it internally! */ psmx_info->rx_attr->size = UINT64_MAX; psmx_info->rx_attr->iov_limit = 1; *info = psmx_info; return 0; err_out: return err; }
static int psmx_getinfo(uint32_t version, const char *node, const char *service, uint64_t flags, struct fi_info *hints, struct fi_info **info) { struct fi_info *psmx_info; uint32_t cnt = 0; psm_epid_t *dest_addr = NULL; struct psmx_src_name *src_addr; int ep_type = FI_EP_RDM; int av_type = FI_AV_UNSPEC; uint64_t mode = FI_CONTEXT; enum fi_mr_mode mr_mode = FI_MR_SCALABLE; enum fi_threading threading = FI_THREAD_COMPLETION; enum fi_progress control_progress = FI_PROGRESS_MANUAL; enum fi_progress data_progress = FI_PROGRESS_MANUAL; int caps = 0; uint64_t max_tag_value = 0; int err = -FI_ENODATA; int svc0, svc = PSMX_ANY_SERVICE; FI_INFO(&psmx_prov, FI_LOG_CORE,"\n"); *info = NULL; if (psmx_init_lib()) return -FI_ENODATA; if (psm_ep_num_devunits(&cnt) || !cnt) { FI_INFO(&psmx_prov, FI_LOG_CORE, "no PSM device is found.\n"); return -FI_ENODATA; } psmx_init_env(); src_addr = calloc(1, sizeof(*src_addr)); if (!src_addr) { FI_INFO(&psmx_prov, FI_LOG_CORE, "failed to allocate src addr.\n"); return -FI_ENODATA; } src_addr->unit = PSMX_DEFAULT_UNIT; src_addr->port = PSMX_DEFAULT_PORT; src_addr->service = PSMX_ANY_SERVICE; if (flags & FI_SOURCE) { if (node) sscanf(node, "%*[^:]:%d:%d", &src_addr->unit, &src_addr->port); if (service) sscanf(service, "%d", &src_addr->service); FI_INFO(&psmx_prov, FI_LOG_CORE, "node '%s' service '%s' converted to <unit=%d, port=%d, service=%d>\n", node, service, src_addr->unit, src_addr->port, src_addr->service); } else if (node) { if (service) svc = atoi(service); svc0 = svc; dest_addr = psmx_ns_resolve_name(node, &svc); if (dest_addr) { FI_INFO(&psmx_prov, FI_LOG_CORE, "'%s:%u' resolved to <epid=0x%llx>:%u\n", node, svc0, *dest_addr, svc); } else { FI_INFO(&psmx_prov, FI_LOG_CORE, "failed to resolve '%s:%u'.\n", node, svc); err = -FI_ENODATA; goto err_out; } } if (hints) { switch (hints->addr_format) { case FI_FORMAT_UNSPEC: case FI_ADDR_PSMX: break; default: FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->addr_format=%d, supported=%d,%d.\n", hints->addr_format, FI_FORMAT_UNSPEC, FI_ADDR_PSMX); goto err_out; } if (hints->ep_attr) { switch (hints->ep_attr->type) { case FI_EP_UNSPEC: case FI_EP_DGRAM: case FI_EP_RDM: break; default: FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->ep_attr->type=%d, supported=%d,%d,%d.\n", hints->ep_attr->type, FI_EP_UNSPEC, FI_EP_DGRAM, FI_EP_RDM); goto err_out; } switch (hints->ep_attr->protocol) { case FI_PROTO_UNSPEC: case FI_PROTO_PSMX: break; default: FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->protocol=%d, supported=%d %d\n", hints->ep_attr->protocol, FI_PROTO_UNSPEC, FI_PROTO_PSMX); goto err_out; } if (hints->ep_attr->tx_ctx_cnt > 1) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->ep_attr->tx_ctx_cnt=%d, supported=0,1\n", hints->ep_attr->tx_ctx_cnt); goto err_out; } if (hints->ep_attr->rx_ctx_cnt > 1) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->ep_attr->rx_ctx_cnt=%d, supported=0,1\n", hints->ep_attr->rx_ctx_cnt); goto err_out; } } if ((hints->caps & PSMX_CAPS) != hints->caps && (hints->caps & PSMX_CAPS2) != hints->caps) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->caps=0x%llx, supported=0x%llx,0x%llx\n", hints->caps, PSMX_CAPS, PSMX_CAPS2); goto err_out; } if (hints->tx_attr) { if ((hints->tx_attr->op_flags & PSMX_OP_FLAGS) != hints->tx_attr->op_flags) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->tx->flags=0x%llx, " "supported=0x%llx\n", hints->tx_attr->op_flags, PSMX_OP_FLAGS); goto err_out; } if (hints->tx_attr->inject_size > PSMX_INJECT_SIZE) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->tx_attr->inject_size=%ld," "supported=%ld.\n", hints->tx_attr->inject_size, PSMX_INJECT_SIZE); goto err_out; } } if (hints->rx_attr && (hints->rx_attr->op_flags & PSMX_OP_FLAGS) != hints->rx_attr->op_flags) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->rx->flags=0x%llx, supported=0x%llx\n", hints->rx_attr->op_flags, PSMX_OP_FLAGS); goto err_out; } if ((hints->caps & FI_TAGGED) || ((hints->caps & FI_MSG) && !psmx_env.am_msg)) { if ((hints->mode & FI_CONTEXT) != FI_CONTEXT) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->mode=0x%llx, required=0x%llx\n", hints->mode, FI_CONTEXT); goto err_out; } } else { mode = 0; } if (hints->fabric_attr && hints->fabric_attr->name && strcmp(hints->fabric_attr->name, PSMX_FABRIC_NAME)) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->fabric_name=%s, supported=psm\n", hints->fabric_attr->name); goto err_out; } if (hints->domain_attr) { if (hints->domain_attr->name && strcmp(hints->domain_attr->name, PSMX_DOMAIN_NAME)) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->domain_name=%s, supported=psm\n", hints->domain_attr->name); goto err_out; } switch (hints->domain_attr->av_type) { case FI_AV_UNSPEC: case FI_AV_MAP: case FI_AV_TABLE: av_type = hints->domain_attr->av_type; break; default: FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->domain_attr->av_type=%d, supported=%d %d %d\n", hints->domain_attr->av_type, FI_AV_UNSPEC, FI_AV_MAP, FI_AV_TABLE); goto err_out; } switch (hints->domain_attr->mr_mode) { case FI_MR_UNSPEC: break; case FI_MR_BASIC: case FI_MR_SCALABLE: mr_mode = hints->domain_attr->mr_mode; break; default: FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->domain_attr->mr_mode=%d, supported=%d %d %d\n", hints->domain_attr->mr_mode, FI_MR_UNSPEC, FI_MR_BASIC, FI_MR_SCALABLE); goto err_out; } switch (hints->domain_attr->threading) { case FI_THREAD_UNSPEC: break; case FI_THREAD_FID: case FI_THREAD_ENDPOINT: case FI_THREAD_COMPLETION: case FI_THREAD_DOMAIN: threading = hints->domain_attr->threading; break; default: FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->domain_attr->threading=%d, supported=%d %d %d %d %d\n", hints->domain_attr->threading, FI_THREAD_UNSPEC, FI_THREAD_FID, FI_THREAD_ENDPOINT, FI_THREAD_COMPLETION, FI_THREAD_DOMAIN); goto err_out; } switch (hints->domain_attr->control_progress) { case FI_PROGRESS_UNSPEC: break; case FI_PROGRESS_MANUAL: case FI_PROGRESS_AUTO: control_progress = hints->domain_attr->control_progress; break; default: FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->domain_attr->control_progress=%d, supported=%d %d %d\n", hints->domain_attr->control_progress, FI_PROGRESS_UNSPEC, FI_PROGRESS_MANUAL, FI_PROGRESS_AUTO); goto err_out; } switch (hints->domain_attr->data_progress) { case FI_PROGRESS_UNSPEC: break; case FI_PROGRESS_MANUAL: case FI_PROGRESS_AUTO: data_progress = hints->domain_attr->data_progress; break; default: FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->domain_attr->data_progress=%d, supported=%d %d %d\n", hints->domain_attr->data_progress, FI_PROGRESS_UNSPEC, FI_PROGRESS_MANUAL, FI_PROGRESS_AUTO); goto err_out; } if (hints->domain_attr->caps & FI_SHARED_AV) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->domain_attr->caps=%lx, shared AV is unsupported\n", hints->domain_attr->caps); goto err_out; } } if (hints->ep_attr) { if (hints->ep_attr->max_msg_size > PSMX_MAX_MSG_SIZE) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->ep_attr->max_msg_size=%ld," "supported=%ld.\n", hints->ep_attr->max_msg_size, PSMX_MAX_MSG_SIZE); goto err_out; } max_tag_value = fi_tag_bits(hints->ep_attr->mem_tag_format); } if (hints->tx_attr) { if ((hints->tx_attr->msg_order & PSMX_MSG_ORDER) != hints->tx_attr->msg_order) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->tx_attr->msg_order=%lx," "supported=%lx.\n", hints->tx_attr->msg_order, PSMX_MSG_ORDER); goto err_out; } if ((hints->tx_attr->comp_order & PSMX_COMP_ORDER) != hints->tx_attr->comp_order) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->tx_attr->msg_order=%lx," "supported=%lx.\n", hints->tx_attr->comp_order, PSMX_COMP_ORDER); goto err_out; } if (hints->tx_attr->inject_size > PSMX_INJECT_SIZE) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->tx_attr->inject_size=%ld," "supported=%d.\n", hints->tx_attr->inject_size, PSMX_INJECT_SIZE); goto err_out; } if (hints->tx_attr->iov_limit > 1) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->tx_attr->iov_limit=%ld," "supported=1.\n", hints->tx_attr->iov_limit); goto err_out; } if (hints->tx_attr->rma_iov_limit > 1) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->tx_attr->rma_iov_limit=%ld," "supported=1.\n", hints->tx_attr->rma_iov_limit); goto err_out; } } if (hints->rx_attr) { if ((hints->rx_attr->msg_order & PSMX_MSG_ORDER) != hints->rx_attr->msg_order) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->rx_attr->msg_order=%lx," "supported=%lx.\n", hints->rx_attr->msg_order, PSMX_MSG_ORDER); goto err_out; } if ((hints->rx_attr->comp_order & PSMX_COMP_ORDER) != hints->rx_attr->comp_order) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->rx_attr->msg_order=%lx," "supported=%lx.\n", hints->rx_attr->comp_order, PSMX_COMP_ORDER); goto err_out; } if (hints->rx_attr->iov_limit > 1) { FI_INFO(&psmx_prov, FI_LOG_CORE, "hints->rx_attr->iov_limit=%ld," "supported=1.\n", hints->rx_attr->iov_limit); goto err_out; } } caps = hints->caps; /* TODO: check other fields of hints */ } if (psmx_reserve_tag_bits(&caps, &max_tag_value) < 0) goto err_out; psmx_info = fi_allocinfo(); if (!psmx_info) { err = -FI_ENOMEM; goto err_out; } psmx_info->ep_attr->type = ep_type; psmx_info->ep_attr->protocol = FI_PROTO_PSMX; psmx_info->ep_attr->protocol_version = PSM_VERNO; psmx_info->ep_attr->max_msg_size = PSMX_MAX_MSG_SIZE; psmx_info->ep_attr->mem_tag_format = fi_tag_format(max_tag_value); psmx_info->ep_attr->tx_ctx_cnt = 1; psmx_info->ep_attr->rx_ctx_cnt = 1; psmx_info->domain_attr->threading = threading; psmx_info->domain_attr->control_progress = control_progress; psmx_info->domain_attr->data_progress = data_progress; psmx_info->domain_attr->name = strdup(PSMX_DOMAIN_NAME); psmx_info->domain_attr->resource_mgmt = FI_RM_ENABLED; psmx_info->domain_attr->av_type = av_type; psmx_info->domain_attr->mr_mode = mr_mode; psmx_info->domain_attr->mr_key_size = sizeof(uint64_t); psmx_info->domain_attr->cq_data_size = 4; psmx_info->domain_attr->cq_cnt = 65535; psmx_info->domain_attr->ep_cnt = 65535; psmx_info->domain_attr->tx_ctx_cnt = 1; psmx_info->domain_attr->rx_ctx_cnt = 1; psmx_info->domain_attr->max_ep_tx_ctx = 1; psmx_info->domain_attr->max_ep_rx_ctx = 1; psmx_info->domain_attr->max_ep_stx_ctx = 65535; psmx_info->domain_attr->max_ep_srx_ctx = 0; psmx_info->domain_attr->cntr_cnt = 65535; psmx_info->domain_attr->mr_iov_limit = 65535; psmx_info->domain_attr->caps = PSMX_DOM_CAPS; psmx_info->domain_attr->mode = 0; psmx_info->next = NULL; psmx_info->caps = (hints && hints->caps) ? hints->caps : caps; psmx_info->mode = mode; psmx_info->addr_format = FI_ADDR_PSMX; psmx_info->src_addr = src_addr; psmx_info->src_addrlen = sizeof(*src_addr); psmx_info->dest_addr = dest_addr; psmx_info->dest_addrlen = sizeof(*dest_addr); psmx_info->fabric_attr->name = strdup(PSMX_FABRIC_NAME); psmx_info->fabric_attr->prov_name = NULL; psmx_info->fabric_attr->prov_version = PSMX_VERSION; psmx_info->tx_attr->caps = psmx_info->caps; psmx_info->tx_attr->mode = psmx_info->mode; psmx_info->tx_attr->op_flags = (hints && hints->tx_attr && hints->tx_attr->op_flags) ? hints->tx_attr->op_flags : 0; psmx_info->tx_attr->msg_order = PSMX_MSG_ORDER; psmx_info->tx_attr->comp_order = PSMX_COMP_ORDER; psmx_info->tx_attr->inject_size = PSMX_INJECT_SIZE; psmx_info->tx_attr->size = UINT64_MAX; psmx_info->tx_attr->iov_limit = 1; psmx_info->tx_attr->rma_iov_limit = 1; psmx_info->rx_attr->caps = psmx_info->caps; psmx_info->rx_attr->mode = psmx_info->mode; psmx_info->rx_attr->op_flags = (hints && hints->rx_attr && hints->rx_attr->op_flags) ? hints->rx_attr->op_flags : 0; psmx_info->rx_attr->msg_order = PSMX_MSG_ORDER; psmx_info->rx_attr->comp_order = PSMX_COMP_ORDER; psmx_info->rx_attr->total_buffered_recv = ~(0ULL); /* that's how PSM handles it internally! */ psmx_info->rx_attr->size = UINT64_MAX; psmx_info->rx_attr->iov_limit = 1; *info = psmx_info; return 0; err_out: free(dest_addr); free(src_addr); return err; }