static int set_mcast_group(struct pingpong_context *ctx,
struct perftest_parameters *user_param,
struct mcast_parameters *mcg_params)
{
struct ibv_port_attr port_attr;
if (ibv_query_gid(ctx->context,user_param->ib_port,user_param->gid_index,&mcg_params->port_gid)) {
return 1;
}
if (ibv_query_pkey(ctx->context,user_param->ib_port,DEF_PKEY_IDX,&mcg_params->pkey)) {
return 1;
}
if (ibv_query_port(ctx->context,user_param->ib_port,&port_attr)) {
return 1;
}
mcg_params->sm_lid = port_attr.sm_lid;
mcg_params->sm_sl = port_attr.sm_sl;
mcg_params->ib_port = user_param->ib_port;
if (!strcmp(link_layer_str(user_param->link_type),"IB")) {
/* Request for Mcast group create registery in SM. */
if (join_multicast_group(SUBN_ADM_METHOD_SET,mcg_params)) {
fprintf(stderr,"Couldn't Register the Mcast group on the SM\n");
return 1;
}
}
return 0;
}
int
ibv_query_pkey(struct ibv_context *context,
uint8_t port_num,
int index,
uint16_t *pkey)
{
JNOTE("******* WRAPPER for begin of ibv_query_pkey");
return ibv_query_pkey(context, port_num, index, pkey);
}
static int ucma_find_pkey(struct cma_device *cma_dev, uint8_t port_num,
uint16_t pkey, uint16_t *pkey_index)
{
int ret, i;
uint16_t chk_pkey;
for (i = 0, ret = 0; !ret; i++) {
ret = ibv_query_pkey(cma_dev->verbs, port_num, i, &chk_pkey);
if (!ret && pkey == chk_pkey) {
*pkey_index = (uint16_t) i;
return 0;
}
}
return ERR(EINVAL);
}
static ucs_status_t uct_ib_iface_init_pkey(uct_ib_iface_t *iface,
const uct_ib_iface_config_t *config)
{
uct_ib_device_t *dev = uct_ib_iface_device(iface);
uint16_t pkey_tbl_len = uct_ib_iface_port_attr(iface)->pkey_tbl_len;
uint16_t pkey_index, port_pkey, pkey;
if (config->pkey_value > UCT_IB_PKEY_PARTITION_MASK) {
ucs_error("Requested pkey 0x%x is invalid, should be in the range 0..0x%x",
config->pkey_value, UCT_IB_PKEY_PARTITION_MASK);
return UCS_ERR_INVALID_PARAM;
}
/* get the user's pkey value and find its index in the port's pkey table */
for (pkey_index = 0; pkey_index < pkey_tbl_len; ++pkey_index) {
/* get the pkey values from the port's pkeys table */
if (ibv_query_pkey(dev->ibv_context, iface->config.port_num, pkey_index,
&port_pkey))
{
ucs_error("ibv_query_pkey("UCT_IB_IFACE_FMT", index=%d) failed: %m",
UCT_IB_IFACE_ARG(iface), pkey_index);
}
pkey = ntohs(port_pkey);
if (!(pkey & UCT_IB_PKEY_MEMBERSHIP_MASK)) {
ucs_debug("skipping send-only pkey[%d]=0x%x", pkey_index, pkey);
continue;
}
/* take only the lower 15 bits for the comparison */
if ((pkey & UCT_IB_PKEY_PARTITION_MASK) == config->pkey_value) {
iface->pkey_index = pkey_index;
iface->pkey_value = pkey;
ucs_debug("using pkey[%d] 0x%x on "UCT_IB_IFACE_FMT, iface->pkey_index,
iface->pkey_value, UCT_IB_IFACE_ARG(iface));
return UCS_OK;
}
}
ucs_error("The requested pkey: 0x%x, cannot be used. "
"It wasn't found or the configured pkey doesn't have full membership.",
config->pkey_value);
return UCS_ERR_INVALID_PARAM;
}
static int set_mcast_group(struct pingpong_context *ctx,
struct perftest_parameters *user_param,
struct mcast_parameters *mcg_params) {
int i;
struct ibv_port_attr port_attr;
if (ibv_query_gid(ctx->context,user_param->ib_port,user_param->gid_index,&mcg_params->port_gid)) {
return 1;
}
if (ibv_query_pkey(ctx->context,user_param->ib_port,DEF_PKEY_IDX,&mcg_params->pkey)) {
return 1;
}
if (ibv_query_port(ctx->context,user_param->ib_port,&port_attr)) {
return 1;
}
mcg_params->sm_lid = port_attr.sm_lid;
mcg_params->sm_sl = port_attr.sm_sl;
mcg_params->ib_port = user_param->ib_port;
mcg_params->user_mgid = user_param->user_mgid;
set_multicast_gid(mcg_params,ctx->qp[0]->qp_num,(int)user_param->machine);
if (!strcmp(link_layer_str(user_param->link_type),"IB")) {
// Request for Mcast group create registery in SM.
if (join_multicast_group(SUBN_ADM_METHOD_SET,mcg_params)) {
fprintf(stderr," Failed to Join Mcast request\n");
return 1;
}
}
for (i=0; i < user_param->num_of_qps; i++) {
if (ibv_attach_mcast(ctx->qp[i],&mcg_params->mgid,mcg_params->mlid)) {
fprintf(stderr, "Couldn't attach QP to MultiCast group");
return 1;
}
}
mcg_params->mcast_state |= MCAST_IS_ATTACHED;
return 0;
}
int __ibv_query_pkey_1_0(struct ibv_context_1_0 *context, uint8_t port_num,
int index, uint16_t *pkey)
{
return ibv_query_pkey(context->real_context, port_num, index, pkey);
}
/*
* Opens the interface to the opp module.
* Must be called before any use of the path functions.
*
* device The verbs context for the HFI.
* Can be acquired via op_path_find_hfi.
*
* port_num The port to use for sending queries.
*
* This information is used for querying pkeys and
* calculating timeouts.
*
* Returns a pointer to the op_path context on success, or returns NULL
* and sets errno if the device could not be opened.
*/
void *
op_path_open(struct ibv_device *device, int p)
{
int i, err;
struct op_path_context *context;
if (!device) {
errno=ENXIO;
return NULL;
}
context = malloc(sizeof(struct op_path_context));
if (!context) {
errno=ENOMEM;
return NULL;
}
memset(context,0,sizeof(struct op_path_context));
context->ibv_context = ibv_open_device(device);
if (!context->ibv_context) {
errno=ENODEV;
goto open_device_failed;
}
context->port_num = p;
context->reader = op_ppath_allocate_reader();
if (!context->reader) {
errno=ENOMEM;
goto alloc_reader_failed;
}
err = op_ppath_create_reader(context->reader);
if (err) {
errno=err;
goto create_reader_failed;
}
if ((err=ibv_query_device(context->ibv_context,
&(context->device_attr)))) {
errno=EFAULT;
goto query_attr_failed;
}
if ((err=ibv_query_port(context->ibv_context,
context->port_num,
&(context->port_attr)))) {
errno=EFAULT;
goto query_attr_failed;
}
context->pkey_table = malloc(context->device_attr.max_pkeys* sizeof(int));
if (!context->pkey_table) {
errno= ENOMEM;
goto query_attr_failed;
}
memset(context->pkey_table,0,context->device_attr.max_pkeys* sizeof(int));
for (i = 0, err = 0; !err && i<context->device_attr.max_pkeys; i++) {
err = ibv_query_pkey(context->ibv_context, context->port_num, i,
&(context->pkey_table[i]));
if (err) {
errno=EFAULT;
goto query_pkey_failed;
}
}
return context;
query_pkey_failed:
free(context->pkey_table);
query_attr_failed:
op_ppath_close_reader(context->reader);
create_reader_failed:
free(context->reader);
alloc_reader_failed:
ibv_close_device(context->ibv_context);
open_device_failed:
free(context);
return NULL;
}
int __ibv_query_pkey_1_0(struct ibv_context_1_0 *context, uint8_t port_num,
int index, uint16_t *pkey)
{ fprintf(stderr, "%s:%s:%d \n", __func__, __FILE__, __LINE__);
return ibv_query_pkey(context->real_context, port_num, index, pkey);
}