/*
* Create SRQ and listen for XRC SRQN lookup request.
*/
static int xrc_create_srq_listen(struct sockaddr *addr, socklen_t addr_len)
{
struct rdma_addrinfo rai;
struct sockaddr_storage ss;
struct ibv_srq_init_attr attr;
int ret;
memset(&rai, 0, sizeof rai);
rai.ai_flags = RAI_PASSIVE;
rai.ai_family = addr->sa_family;
rai.ai_qp_type = IBV_QPT_UD; /* for now */
rai.ai_port_space = RDMA_PS_IB;
memcpy(&ss, addr, addr_len);
rai.ai_src_len = addr_len;
rai.ai_src_addr = (struct sockaddr *) &ss;
((struct sockaddr_in *) &ss)->sin_port = htons((short) atoi(port) + 1);
ret = rdma_create_ep(&srq_id, &rai, NULL, NULL);
if (ret) {
printf("rdma_create_ep srq ep %d\n", errno);
return ret;
}
if (!srq_id->verbs) {
printf("rdma_create_ep failed to bind to device.\n");
printf("XRC tests cannot use loopback addressing\n");
return -1;
}
memset(&attr, 0, sizeof attr);
attr.attr.max_wr = 1;
attr.attr.max_sge = 1;
attr.srq_type = IBV_SRQT_XRC;
attr.ext.xrc.xrcd = ibv_open_xrcd(srq_id->verbs, -1, 0);
if (!attr.ext.xrc.xrcd) {
printf("Unable to open xrcd\n");
return -1;
}
ret = rdma_create_srq(srq_id, NULL, &attr);
if (ret) {
printf("Unable to create srq %d\n", errno);
return ret;
}
ret = rdma_listen(srq_id, 0);
if (ret) {
printf("rdma_listen srq id %d\n", errno);
return ret;
}
return 0;
}
RDMAServerSocket::RDMAServerSocket(std::vector<std::string> hosts,
const std::string &port, uint32_t max_wr,
int cq_entries)
: ec(createEventChannel()), id(createCmId(hosts.back(), port, true)),
cc(id), cq(id, cc, cq_entries, 1, 0), running(true) {
assert(max_wr);
check_zero(rdma_migrate_id(id.get(), ec.get()));
ibv_srq_init_attr srq_attr = { nullptr, { max_wr, 1, 0 } };
check_zero(rdma_create_srq(id.get(), nullptr, &srq_attr));
log_info() << "Created id " << id.get() << " " << (void *)this;
hosts.pop_back();
for (const auto &host : hosts) {
ibv_qp_init_attr attr = {};
attr.cap.max_send_wr = 256;
attr.cap.max_recv_wr = 0;
attr.cap.max_send_sge = 1;
attr.cap.max_recv_sge = 0;
attr.recv_cq = cq;
attr.send_cq = cq;
attr.srq = id->srq;
attr.cap.max_inline_data = 72;
attr.sq_sig_all = 1;
auto client_id = createCmId(host, port, true, &attr, id->pd);
check_zero(rdma_migrate_id(client_id.get(), ec.get()));
ids.push_back(std::move(client_id));
log_info() << srq_attr;
}
cm_events();
if(id->verbs) {
ibv_device_attr attr;
check_zero(ibv_query_device(id->verbs, &attr));
log_info() << attr;
}
}