int RDMASock::sock_connect(struct sock_bind_t *sock_bind, struct sock_t *sock) { int rc; if (sock_bind->is_daemon) { rc = sock_daemon_wait(sock_bind->port, sock_bind, sock); rc |= bind_close(sock_bind); /// bug } else { rc = sock_client_connect(sock_bind->ip, sock_bind->port, sock); } return rc; }
int RDMASock::sock_connect_multi(struct sock_bind_t *sock_bind, struct sock_t *sock) { int rc; if (sock_bind->is_daemon) { printf("Running sock_daemon_wait.\n"); rc = sock_daemon_wait(sock_bind->port, sock_bind, sock); } else { printf("Running sock_client_connect, Server IP: %s\n", sock_bind->ip); rc = sock_client_connect(sock_bind->ip, sock_bind->port, sock); } return rc; }
static int resources_create(struct resources *res) { struct ibv_device **dev_list = NULL; struct ibv_qp_init_attr qp_init_attr; struct ibv_device *ib_dev = NULL; size_t size; int i; int mr_flags = 0; int cq_size = 0; int num_devices; /* if client side */ if (config.server_name) { res->sock = sock_client_connect(config.server_name, config.tcp_port); if (res->sock < 0) { fprintf(stderr, "failed to establish TCP connection to server %s, port %d\n", config.server_name, config.tcp_port); return -1; } } else { fprintf(stdout, "waiting on port %d for TCP connection\n", config.tcp_port); res->sock = sock_daemon_connect(config.tcp_port); if (res->sock < 0) { fprintf(stderr, "failed to establish TCP connection with client on port %d\n", config.tcp_port); return -1; } } fprintf(stdout, "TCP connection was established\n"); fprintf(stdout, "searching for IB devices in host\n"); /* get device names in the system */ dev_list = ibv_get_device_list(&num_devices); if (!dev_list) { fprintf(stderr, "failed to get IB devices list\n"); return 1; } /* if there isn't any IB device in host */ if (!num_devices) { fprintf(stderr, "found %d device(s)\n", num_devices); return 1; } fprintf(stdout, "found %d device(s)\n", num_devices); /* search for the specific device we want to work with */ for (i = 0; i < num_devices; i ++) { if (!config.dev_name) { config.dev_name = strdup(ibv_get_device_name(dev_list[i])); fprintf(stdout, "device not specified, using first one found: %s\n", config.dev_name); } if (!strcmp(ibv_get_device_name(dev_list[i]), config.dev_name)) { ib_dev = dev_list[i]; break; } } /* if the device wasn't found in host */ if (!ib_dev) { fprintf(stderr, "IB device %s wasn't found\n", config.dev_name); return 1; } /* get device handle */ res->ib_ctx = ibv_open_device(ib_dev); if (!res->ib_ctx) { fprintf(stderr, "failed to open device %s\n", config.dev_name); return 1; } /* We are now done with device list, free it */ ibv_free_device_list(dev_list); dev_list = NULL; ib_dev = NULL; /* query port properties */ if (ibv_query_port(res->ib_ctx, config.ib_port, &res->port_attr)) { fprintf(stderr, "ibv_query_port on port %u failed\n", config.ib_port); return 1; } /* allocate Protection Domain */ res->pd = ibv_alloc_pd(res->ib_ctx); if (!res->pd) { fprintf(stderr, "ibv_alloc_pd failed\n"); return 1; } /* each side will send only one WR, so Completion Queue with 1 entry is enough */ cq_size = 1; res->cq = ibv_create_cq(res->ib_ctx, cq_size, NULL, NULL, 0); if (!res->cq) { fprintf(stderr, "failed to create CQ with %u entries\n", cq_size); return 1; } /* allocate the memory buffer that will hold the data */ size = MSG_SIZE; res->buf = malloc(size); if (!res->buf) { fprintf(stderr, "failed to malloc %Zu bytes to memory buffer\n", size); return 1; } /* only in the daemon side put the message in the memory buffer */ if (!config.server_name) { strcpy(res->buf, MSG); fprintf(stdout, "going to send the message: '%s'\n", res->buf); } else memset(res->buf, 0, size); /* register this memory buffer */ mr_flags = (config.server_name) ? IBV_ACCESS_LOCAL_WRITE : 0; res->mr = ibv_reg_mr(res->pd, res->buf, size, mr_flags); if (!res->mr) { fprintf(stderr, "ibv_reg_mr failed with mr_flags=0x%x\n", mr_flags); return 1; } fprintf(stdout, "MR was registered with addr=%p, lkey=0x%x, rkey=0x%x, flags=0x%x\n", res->buf, res->mr->lkey, res->mr->rkey, mr_flags); /* create the Queue Pair */ memset(&qp_init_attr, 0, sizeof(qp_init_attr)); qp_init_attr.qp_type = IBV_QPT_RC; qp_init_attr.sq_sig_all = 1; qp_init_attr.send_cq = res->cq; qp_init_attr.recv_cq = res->cq; qp_init_attr.cap.max_send_wr = 1; qp_init_attr.cap.max_recv_wr = 1; qp_init_attr.cap.max_send_sge = 1; qp_init_attr.cap.max_recv_sge = 1; res->qp = ibv_create_qp(res->pd, &qp_init_attr); if (!res->qp) { fprintf(stderr, "failed to create QP\n"); return 1; } fprintf(stdout, "QP was created, QP number=0x%x\n", res->qp->qp_num); return 0; }