int RDMASock::sock_connect(struct sock_bind_t *sock_bind, struct sock_t *sock)
{
int rc;
if (sock_bind->is_daemon)
{
rc = sock_daemon_wait(sock_bind->port, sock_bind, sock);
rc |= bind_close(sock_bind); /// bug
}
else
{
rc = sock_client_connect(sock_bind->ip, sock_bind->port, sock);
}
return rc;
}
int RDMASock::sock_connect_multi(struct sock_bind_t *sock_bind, struct sock_t *sock)
{
int rc;
if (sock_bind->is_daemon)
{
printf("Running sock_daemon_wait.\n");
rc = sock_daemon_wait(sock_bind->port, sock_bind, sock);
}
else
{
printf("Running sock_client_connect, Server IP: %s\n", sock_bind->ip);
rc = sock_client_connect(sock_bind->ip, sock_bind->port, sock);
}
return rc;
}
static int resources_create(struct resources *res)
{
struct ibv_device **dev_list = NULL;
struct ibv_qp_init_attr qp_init_attr;
struct ibv_device *ib_dev = NULL;
size_t size;
int i;
int mr_flags = 0;
int cq_size = 0;
int num_devices;
/* if client side */
if (config.server_name) {
res->sock = sock_client_connect(config.server_name, config.tcp_port);
if (res->sock < 0) {
fprintf(stderr, "failed to establish TCP connection to server %s, port %d\n",
config.server_name, config.tcp_port);
return -1;
}
} else {
fprintf(stdout, "waiting on port %d for TCP connection\n", config.tcp_port);
res->sock = sock_daemon_connect(config.tcp_port);
if (res->sock < 0) {
fprintf(stderr, "failed to establish TCP connection with client on port %d\n",
config.tcp_port);
return -1;
}
}
fprintf(stdout, "TCP connection was established\n");
fprintf(stdout, "searching for IB devices in host\n");
/* get device names in the system */
dev_list = ibv_get_device_list(&num_devices);
if (!dev_list) {
fprintf(stderr, "failed to get IB devices list\n");
return 1;
}
/* if there isn't any IB device in host */
if (!num_devices) {
fprintf(stderr, "found %d device(s)\n", num_devices);
return 1;
}
fprintf(stdout, "found %d device(s)\n", num_devices);
/* search for the specific device we want to work with */
for (i = 0; i < num_devices; i ++) {
if (!config.dev_name) {
config.dev_name = strdup(ibv_get_device_name(dev_list[i]));
fprintf(stdout, "device not specified, using first one found: %s\n", config.dev_name);
}
if (!strcmp(ibv_get_device_name(dev_list[i]), config.dev_name)) {
ib_dev = dev_list[i];
break;
}
}
/* if the device wasn't found in host */
if (!ib_dev) {
fprintf(stderr, "IB device %s wasn't found\n", config.dev_name);
return 1;
}
/* get device handle */
res->ib_ctx = ibv_open_device(ib_dev);
if (!res->ib_ctx) {
fprintf(stderr, "failed to open device %s\n", config.dev_name);
return 1;
}
/* We are now done with device list, free it */
ibv_free_device_list(dev_list);
dev_list = NULL;
ib_dev = NULL;
/* query port properties */
if (ibv_query_port(res->ib_ctx, config.ib_port, &res->port_attr)) {
fprintf(stderr, "ibv_query_port on port %u failed\n", config.ib_port);
return 1;
}
/* allocate Protection Domain */
res->pd = ibv_alloc_pd(res->ib_ctx);
if (!res->pd) {
fprintf(stderr, "ibv_alloc_pd failed\n");
return 1;
}
/* each side will send only one WR, so Completion Queue with 1 entry is enough */
cq_size = 1;
res->cq = ibv_create_cq(res->ib_ctx, cq_size, NULL, NULL, 0);
if (!res->cq) {
fprintf(stderr, "failed to create CQ with %u entries\n", cq_size);
return 1;
}
/* allocate the memory buffer that will hold the data */
size = MSG_SIZE;
res->buf = malloc(size);
if (!res->buf) {
fprintf(stderr, "failed to malloc %Zu bytes to memory buffer\n", size);
return 1;
}
/* only in the daemon side put the message in the memory buffer */
if (!config.server_name) {
strcpy(res->buf, MSG);
fprintf(stdout, "going to send the message: '%s'\n", res->buf);
} else
memset(res->buf, 0, size);
/* register this memory buffer */
mr_flags = (config.server_name) ? IBV_ACCESS_LOCAL_WRITE : 0;
res->mr = ibv_reg_mr(res->pd, res->buf, size, mr_flags);
if (!res->mr) {
fprintf(stderr, "ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags);
return 1;
}
fprintf(stdout, "MR was registered with addr=%p, lkey=0x%x, rkey=0x%x, flags=0x%x\n",
res->buf, res->mr->lkey, res->mr->rkey, mr_flags);
/* create the Queue Pair */
memset(&qp_init_attr, 0, sizeof(qp_init_attr));
qp_init_attr.qp_type = IBV_QPT_RC;
qp_init_attr.sq_sig_all = 1;
qp_init_attr.send_cq = res->cq;
qp_init_attr.recv_cq = res->cq;
qp_init_attr.cap.max_send_wr = 1;
qp_init_attr.cap.max_recv_wr = 1;
qp_init_attr.cap.max_send_sge = 1;
qp_init_attr.cap.max_recv_sge = 1;
res->qp = ibv_create_qp(res->pd, &qp_init_attr);
if (!res->qp) {
fprintf(stderr, "failed to create QP\n");
return 1;
}
fprintf(stdout, "QP was created, QP number=0x%x\n", res->qp->qp_num);
return 0;
}
