static int send_set_up_connection(struct pingpong_context *ctx,
struct perftest_parameters *user_param,
struct pingpong_dest *my_dest,
struct mcast_parameters *mcg_params,
struct perftest_comm *comm)
{
int i;
if (set_up_connection(ctx,user_param,my_dest)) {
fprintf(stderr," Unable to set up my IB connection parameters\n");
return FAILURE;
}
if (user_param->use_mcg && (user_param->duplex || user_param->machine == SERVER)) {
mcg_params->user_mgid = user_param->user_mgid;
set_multicast_gid(mcg_params,ctx->qp[0]->qp_num,(int)user_param->machine);
if (set_mcast_group(ctx,user_param,mcg_params)) {
return 1;
}
for (i=0; i < user_param->num_of_qps; i++) {
if (ibv_attach_mcast(ctx->qp[i],&mcg_params->mgid,mcg_params->mlid)) {
fprintf(stderr, "Couldn't attach QP to MultiCast group");
return 1;
}
}
mcg_params->mcast_state |= MCAST_IS_ATTACHED;
my_dest->gid = mcg_params->mgid;
my_dest->lid = mcg_params->mlid;
my_dest->qpn = QPNUM_MCAST;
}
return 0;
}
static int ucma_process_join(struct cma_event *evt)
{
evt->mc->mgid = evt->event.param.ud.ah_attr.grh.dgid;
evt->mc->mlid = evt->event.param.ud.ah_attr.dlid;
if (!evt->id_priv->id.qp)
return 0;
return ibv_attach_mcast(evt->id_priv->id.qp, &evt->mc->mgid,
evt->mc->mlid);
}
static int set_mcast_group(struct pingpong_context *ctx,
struct perftest_parameters *user_param,
struct mcast_parameters *mcg_params) {
int i;
struct ibv_port_attr port_attr;
if (ibv_query_gid(ctx->context,user_param->ib_port,user_param->gid_index,&mcg_params->port_gid)) {
return 1;
}
if (ibv_query_pkey(ctx->context,user_param->ib_port,DEF_PKEY_IDX,&mcg_params->pkey)) {
return 1;
}
if (ibv_query_port(ctx->context,user_param->ib_port,&port_attr)) {
return 1;
}
mcg_params->sm_lid = port_attr.sm_lid;
mcg_params->sm_sl = port_attr.sm_sl;
mcg_params->ib_port = user_param->ib_port;
mcg_params->user_mgid = user_param->user_mgid;
set_multicast_gid(mcg_params,ctx->qp[0]->qp_num,(int)user_param->machine);
if (!strcmp(link_layer_str(user_param->link_type),"IB")) {
// Request for Mcast group create registery in SM.
if (join_multicast_group(SUBN_ADM_METHOD_SET,mcg_params)) {
fprintf(stderr," Failed to Join Mcast request\n");
return 1;
}
}
for (i=0; i < user_param->num_of_qps; i++) {
if (ibv_attach_mcast(ctx->qp[i],&mcg_params->mgid,mcg_params->mlid)) {
fprintf(stderr, "Couldn't attach QP to MultiCast group");
return 1;
}
}
mcg_params->mcast_state |= MCAST_IS_ATTACHED;
return 0;
}
static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
int tx_depth, int port,struct user_parameters *user_parm) {
struct pingpong_context *ctx;
struct ibv_device_attr device_attr;
ctx = malloc(sizeof *ctx);
if (!ctx)
return NULL;
ctx->size = size;
ctx->tx_depth = tx_depth;
/* in case of UD need space for the GRH */
if (user_parm->connection_type==UD) {
ctx->buf = memalign(page_size, ( size + 40 ) * 2);
if (!ctx->buf) {
fprintf(stderr, "Couldn't allocate work buf.\n");
return NULL;
}
memset(ctx->buf, 0, ( size + 40 ) * 2);
} else {
ctx->buf = memalign(page_size, size * 2);
if (!ctx->buf) {
fprintf(stderr, "Couldn't allocate work buf.\n");
return NULL;
}
memset(ctx->buf, 0, size * 2);
}
ctx->post_buf = (char*)ctx->buf + (size - 1);
ctx->poll_buf = (char*)ctx->buf + (2 * size - 1);
ctx->context = ibv_open_device(ib_dev);
if (!ctx->context) {
fprintf(stderr, "Couldn't get context for %s\n",
ibv_get_device_name(ib_dev));
return NULL;
}
if (user_parm->mtu == 0) {/*user did not ask for specific mtu */
if (ibv_query_device(ctx->context, &device_attr)) {
fprintf(stderr, "Failed to query device props");
return NULL;
}
if (device_attr.vendor_part_id == 23108 || user_parm->gid_index > -1) {
user_parm->mtu = 1024;
} else {
user_parm->mtu = 2048;
}
}
if (user_parm->use_event) {
ctx->channel = ibv_create_comp_channel(ctx->context);
if (!ctx->channel) {
fprintf(stderr, "Couldn't create completion channel\n");
return NULL;
}
} else
ctx->channel = NULL;
ctx->pd = ibv_alloc_pd(ctx->context);
if (!ctx->pd) {
fprintf(stderr, "Couldn't allocate PD\n");
return NULL;
}
if (user_parm->connection_type==UD) {
ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, (size + 40 ) * 2,
IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE);
if (!ctx->mr) {
fprintf(stderr, "Couldn't allocate MR\n");
return NULL;
}
} else {
ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size * 2,
IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE);
if (!ctx->mr) {
fprintf(stderr, "Couldn't allocate MR\n");
return NULL;
}
}
ctx->scq = ibv_create_cq(ctx->context, tx_depth, NULL, ctx->channel, 0);
if (!ctx->scq) {
fprintf(stderr, "Couldn't create CQ\n");
return NULL;
}
ctx->rcq = ibv_create_cq(ctx->context, tx_depth, NULL, ctx->channel, 0);
if (!ctx->rcq) {
fprintf(stderr, "Couldn't create Recieve CQ\n");
return NULL;
}
{
struct ibv_qp_init_attr attr;
memset(&attr, 0, sizeof(struct ibv_qp_init_attr));
attr.send_cq = ctx->scq;
attr.recv_cq = ctx->rcq;
attr.cap.max_send_wr = tx_depth;
/* Work around: driver doesnt support
* recv_wr = 0 */
attr.cap.max_recv_wr = tx_depth;
attr.cap.max_send_sge = 1;
attr.cap.max_recv_sge = 1;
attr.cap.max_inline_data = user_parm->inline_size;
switch (user_parm->connection_type) {
case RC :
attr.qp_type = IBV_QPT_RC;
break;
case UC :
attr.qp_type = IBV_QPT_UC;
break;
case UD :
attr.qp_type = IBV_QPT_UD;
break;
default:
fprintf(stderr, "Unknown connection type %d \n",user_parm->connection_type);
return NULL;
}
attr.sq_sig_all = 0;
ctx->qp = ibv_create_qp(ctx->pd, &attr);
if (!ctx->qp) {
fprintf(stderr, "Couldn't create QP\n");
return NULL;
}
}
{
struct ibv_qp_attr attr;
memset(&attr, 0, sizeof(struct ibv_qp_init_attr));
attr.qp_state = IBV_QPS_INIT;
attr.pkey_index = 0;
attr.port_num = port;
if (user_parm->connection_type==UD) {
attr.qkey = 0x11111111;
} else {
attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE;
}
if (user_parm->connection_type==UD) {
if (ibv_modify_qp(ctx->qp, &attr,
IBV_QP_STATE |
IBV_QP_PKEY_INDEX |
IBV_QP_PORT |
IBV_QP_QKEY)) {
fprintf(stderr, "Failed to modify UD QP to INIT\n");
return NULL;
}
if (user_parm->use_mcg) {
union ibv_gid gid;
uint8_t mcg_gid[16] = MCG_GID;
/* use the local QP number as part of the mcg */
mcg_gid[11] = (user_parm->servername) ? 0 : 1;
*(uint32_t *)(&mcg_gid[12]) = ctx->qp->qp_num;
memcpy(gid.raw, mcg_gid, 16);
if (ibv_attach_mcast(ctx->qp, &gid, MCG_LID)) {
fprintf(stderr, "Couldn't attach QP to mcg\n");
return NULL;
}
}
} else if (ibv_modify_qp(ctx->qp, &attr,
IBV_QP_STATE |
IBV_QP_PKEY_INDEX |
IBV_QP_PORT |
IBV_QP_ACCESS_FLAGS)) {
fprintf(stderr, "Failed to modify QP to INIT\n");
return NULL;
}
}
//send
ctx->wr.wr_id = PINGPONG_SEND_WRID;
ctx->wr.sg_list = &ctx->list;
ctx->wr.num_sge = 1;
ctx->wr.opcode = IBV_WR_SEND;
ctx->wr.next = NULL;
//recieve
ctx->rwr.wr_id = PINGPONG_RECV_WRID;
ctx->rwr.sg_list = &ctx->recv_list;
ctx->rwr.num_sge = 1;
ctx->rwr.next = NULL;
return ctx;
}
int __ibv_attach_mcast_1_0(struct ibv_qp_1_0 *qp, union ibv_gid *gid, uint16_t lid)
{
return ibv_attach_mcast(qp->real_qp, gid, lid);
}
int __ibv_attach_mcast_1_0(struct ibv_qp_1_0 *qp, union ibv_gid *gid, uint16_t lid)
{ fprintf(stderr, "%s:%s:%d \n", __func__, __FILE__, __LINE__);
return ibv_attach_mcast(qp->real_qp, gid, lid);
}
