static int send_local_dest(int sockfd, int index)
{
char msg[MSG_SIZE];
char gid[33];
uint32_t srq_num;
union ibv_gid local_gid;
if (ctx.gidx >= 0) {
if (ibv_query_gid(ctx.context, ctx.ib_port, ctx.gidx,
&local_gid)) {
fprintf(stderr, "can't read sgid of index %d\n",
ctx.gidx);
return -1;
}
} else {
memset(&local_gid, 0, sizeof(local_gid));
}
ctx.rem_dest[index].recv_psn = lrand48() & 0xffffff;
if (ibv_get_srq_num(ctx.srq, &srq_num)) {
fprintf(stderr, "Couldn't get SRQ num\n");
return -1;
}
inet_ntop(AF_INET6, &local_gid, gid, sizeof(gid));
printf(ADDR_FORMAT, "local", ctx.lid, ctx.recv_qp[index]->qp_num,
ctx.send_qp[index]->qp_num, ctx.rem_dest[index].recv_psn,
srq_num, gid);
gid_to_wire_gid(&local_gid, gid);
sprintf(msg, MSG_FORMAT, ctx.lid, ctx.recv_qp[index]->qp_num,
ctx.send_qp[index]->qp_num, ctx.rem_dest[index].recv_psn,
srq_num, gid);
if (write(sockfd, msg, MSG_SIZE) != MSG_SIZE) {
fprintf(stderr, "Couldn't send local address\n");
return -1;
}
return 0;
}
static int send_set_up_connection(struct pingpong_context *ctx,
struct perftest_parameters *user_param,
struct pingpong_dest *my_dest,
struct mcast_parameters *mcg_params,
struct perftest_comm *comm) {
int i;
srand48(getpid() * time(NULL));
union ibv_gid temp_gid;
if (user_param->gid_index != -1) {
if (ibv_query_gid(ctx->context,user_param->ib_port,user_param->gid_index,&temp_gid)) {
return -1;
}
}
for (i = 0; i < user_param->num_of_qps; i++)
{
if (user_param->use_mcg) {
if (set_mcast_group(ctx,user_param,mcg_params)) {
return 1;
}
my_dest[i].gid = mcg_params->mgid;
my_dest[i].lid = mcg_params->mlid;
my_dest[i].qpn = QPNUM_MCAST;
} else {
memcpy(my_dest[i].gid.raw,temp_gid.raw ,16);
my_dest[i].lid = ctx_get_local_lid(ctx->context,user_param->ib_port);
my_dest[i].qpn = ctx->qp[i]->qp_num;
}
my_dest[i].psn = lrand48() & 0xffffff;
// We do not fail test upon lid above RoCE.
if (user_param->gid_index < 0) {
if (!my_dest->lid) {
fprintf(stderr," Local lid 0x0 detected,without any use of gid. Is SM running?\n");
return -1;
}
}
#ifdef HAVE_XRCD
if (user_param->use_xrc || user_param->connection_type == DC) {
if (ibv_get_srq_num(ctx->srq,&(my_dest[i].srqn))) {
fprintf(stderr, "Couldn't get SRQ number\n");
return 1;
}
}
#endif
#ifdef HAVE_DC
if (user_param->connection_type == DC) {
if (ibv_get_srq_num(ctx->srq,&(my_dest[i].srqn))) {
fprintf(stderr, "Couldn't get SRQ number\n");
return 1;
}
}
#endif
}
return 0;
}
int set_up_connection(struct pingpong_context *ctx,
struct perftest_parameters *user_param,
struct pingpong_dest *my_dest)
{
int num_of_qps = user_param->num_of_qps;
int num_of_qps_per_port = user_param->num_of_qps / 2;
int i;
int is_ipv4;
union ibv_gid temp_gid;
union ibv_gid temp_gid2;
struct ibv_port_attr attr;
srand48(getpid() * time(NULL));
/*in xrc with bidirectional,
there are send qps and recv qps. the actual number of send/recv qps
is num_of_qps / 2.
*/
if ( (user_param->connection_type == DC || user_param->use_xrc) && (user_param->duplex || user_param->tst == LAT)) {
num_of_qps /= 2;
num_of_qps_per_port = num_of_qps / 2;
}
if (user_param->gid_index != -1) {
if (ibv_query_port(ctx->context,user_param->ib_port,&attr))
return 0;
if (user_param->use_gid_user) {
if (ibv_query_gid(ctx->context,user_param->ib_port,user_param->gid_index,&temp_gid)) {
return -1;
}
} else {
for (i=0 ; i < attr.gid_tbl_len; i++) {
if (ibv_query_gid(ctx->context,user_param->ib_port,i,&temp_gid)) {
return -1;
}
is_ipv4 = ipv6_addr_v4mapped((struct in6_addr *)temp_gid.raw);
if ((user_param->ipv6 && !is_ipv4) || (!user_param->ipv6 && is_ipv4)) {
user_param->gid_index = i;
break;
}
}
}
}
if (user_param->dualport==ON) {
if (user_param->gid_index2 != -1) {
if (ibv_query_port(ctx->context,user_param->ib_port2,&attr))
return 0;
if (user_param->use_gid_user) {
if (ibv_query_gid(ctx->context,user_param->ib_port2,user_param->gid_index,&temp_gid2))
return -1;
} else {
for (i=0 ; i < attr.gid_tbl_len; i++) {
if (ibv_query_gid(ctx->context,user_param->ib_port2,i,&temp_gid2)) {
return -1;
}
is_ipv4 = ipv6_addr_v4mapped((struct in6_addr *)temp_gid2.raw);
if ((user_param->ipv6 && !is_ipv4) || (!user_param->ipv6 && is_ipv4)) {
user_param->gid_index = i;
break;
}
}
}
}
}
for (i = 0; i < user_param->num_of_qps; i++) {
if (user_param->dualport == ON) {
/*first half of qps are for ib_port and second half are for ib_port2
in xrc with bidirectional, the first half of qps are xrc_send qps and
the second half are xrc_recv qps. the first half of the send/recv qps
are for ib_port1 and the second half are for ib_port2
*/
if (i % num_of_qps < num_of_qps_per_port) {
my_dest[i].lid = ctx_get_local_lid(ctx->context,user_param->ib_port);
my_dest[i].gid_index = user_param->gid_index;
} else {
my_dest[i].lid = ctx_get_local_lid(ctx->context,user_param->ib_port2);
my_dest[i].gid_index = user_param->gid_index2;
}
/*single-port case*/
} else {
my_dest[i].lid = ctx_get_local_lid(ctx->context,user_param->ib_port);
my_dest[i].gid_index = user_param->gid_index;
}
my_dest[i].qpn = ctx->qp[i]->qp_num;
my_dest[i].psn = lrand48() & 0xffffff;
my_dest[i].rkey = ctx->mr->rkey;
/* Each qp gives his receive buffer address.*/
my_dest[i].out_reads = user_param->out_reads;
my_dest[i].vaddr = (uintptr_t)ctx->buf + (user_param->num_of_qps + i)*BUFF_SIZE(ctx->size,ctx->cycle_buffer);
if (user_param->dualport==ON) {
if (i % num_of_qps < num_of_qps_per_port)
memcpy(my_dest[i].gid.raw,temp_gid.raw ,16);
else
memcpy(my_dest[i].gid.raw,temp_gid2.raw ,16);
} else {
memcpy(my_dest[i].gid.raw,temp_gid.raw ,16);
}
/*
We do not fail test upon lid above RoCE.
if ( (user_param->gid_index < 0) || ((user_param->gid_index2 < 0) && (user_param->dualport == ON)) ){
if (!my_dest[i].lid) {
fprintf(stderr," Local lid 0x0 detected. Is an SM running? \n");
return -1;
}
}
*/
}
#ifdef HAVE_XRCD
if (user_param->use_xrc) {
for (i=0; i < user_param->num_of_qps; i++) {
if (ibv_get_srq_num(ctx->srq,&(my_dest[i].srqn))) {
fprintf(stderr, "Couldn't get SRQ number\n");
return 1;
}
}
}
#endif
#ifdef HAVE_DC
if(user_param->machine == SERVER || user_param->duplex || user_param->tst == LAT) {
if (user_param->connection_type == DC) {
for (i=0; i < user_param->num_of_qps; i++) {
if (ibv_get_srq_num(ctx->srq, &(my_dest[i].srqn))) {
fprintf(stderr, "Couldn't get SRQ number\n");
return 1;
}
}
}
}
#endif
return 0;
}
