static int init_node(struct cmatest_node *node)
{
struct ibv_qp_init_attr init_qp_attr;
int cqe, ret;
int i;
struct ibv_cq **cqs[] = {&node->cq[SEND_CQ_INDEX],
&node->cq[RECV_CQ_INDEX]};
node->pd = ibv_alloc_pd(node->cma_id->verbs);
if (!node->pd) {
ret = -ENOMEM;
printf("cmatose: unable to allocate PD\n");
goto out;
}
cqe = message_count ? message_count : 1;
for (i = 0; i < sizeof(cqs)/sizeof(cqs[0]); i++) {
if (set_ts) {
struct ibv_exp_cq_init_attr cq_init_attr;
memset(&cq_init_attr, 0, sizeof(cq_init_attr));
cq_init_attr.flags = IBV_EXP_CQ_TIMESTAMP;
cq_init_attr.comp_mask = IBV_EXP_CQ_INIT_ATTR_FLAGS;
*cqs[i] = (struct ibv_cq *)ibv_exp_create_cq(
node->cma_id->verbs, cqe, node,
NULL, 0, &cq_init_attr);
} else {
*cqs[i] = ibv_create_cq(node->cma_id->verbs, cqe, node,
0, 0);
}
}
if (!node->cq[SEND_CQ_INDEX] || !node->cq[RECV_CQ_INDEX]) {
ret = -ENOMEM;
printf("cmatose: unable to create CQ\n");
goto out;
}
memset(&init_qp_attr, 0, sizeof init_qp_attr);
init_qp_attr.cap.max_send_wr = cqe;
init_qp_attr.cap.max_recv_wr = cqe;
init_qp_attr.cap.max_send_sge = 1;
init_qp_attr.cap.max_recv_sge = 1;
init_qp_attr.qp_context = node;
init_qp_attr.sq_sig_all = 1;
init_qp_attr.qp_type = IBV_QPT_RC;
init_qp_attr.send_cq = node->cq[SEND_CQ_INDEX];
init_qp_attr.recv_cq = node->cq[RECV_CQ_INDEX];
ret = rdma_create_qp(node->cma_id, node->pd, &init_qp_attr);
if (ret) {
perror("cmatose: unable to create QP");
goto out;
}
ret = create_message(node);
if (ret) {
printf("cmatose: failed to create messages: %d\n", ret);
goto out;
}
out:
return ret;
}
/*-----------------------------------------------------------------------------------*/
static void
low_level_init(struct netif *netif)
{
struct ibvif *ibvif;
int num_of_device, flags = IBV_ACCESS_LOCAL_WRITE;
struct ibv_qp_init_attr attr;
struct ibv_qp_attr qp_attr;
uint8_t port_num = 1;
int qp_flags;
struct ibv_device **ib_dev_list;
struct tcpip_thread *thread;
struct ibv_exp_cq_init_attr cq_attr;
ibvif = (struct ibvif *)netif->state;
/* Obtain MAC address from network interface. */
ibvif->ethaddr->addr[0] = 0x00;
ibvif->ethaddr->addr[1] = 0x02;
ibvif->ethaddr->addr[2] = 0xc9;
ibvif->ethaddr->addr[3] = 0xa4;
ibvif->ethaddr->addr[4] = 0x59;
ibvif->ethaddr->addr[5] = 0x41;
ibvif->buf_size = ALIGN_TO_PAGE_SIZE(PBUF_POOL_SIZE * TCP_MAX_PACKET_SIZE);
/* Do things needed for using Raw Packet Verbs */
ib_dev_list = ibv_get_device_list(&num_of_device);
if (num_of_device <= 0 || !ib_dev_list || !ib_dev_list[0]) {
perror("IBV no device found\n");
exit(1);
}
ibvif->context = ibv_open_device(ib_dev_list[1]);
if (!ibvif->context) {
perror("IBV can't open device\n");
exit(1);
}
ibv_free_device_list(ib_dev_list);
if (set_link_layer(ibvif->context, 1) == LINK_FAILURE) {
perror("IBV can't allocate PD\n");
exit(1);
}
ibvif->pd = ibv_alloc_pd(ibvif->context);
if (!ibvif->pd) {
perror("IBV can't allocate PD\n");
exit(1);
}
/*if (!ibv_buffer(ibvif)) {
LWIP_DEBUGF(NETIF_DEBUG, ("Buffer allocation failed\n"));
exit(1);
}*/
ibvif->recv_buf = netif->prot_thread->pbuf_rx_handle.buf;
ibvif->send_buf = netif->prot_thread->pbuf_tx_handle.buf;
ibvif->send_size = TCP_MAX_PACKET_SIZE;
ibvif->rx_depth = PBUF_POOL_SIZE;
ibvif->tx_depth = PBUF_POOL_SIZE;
ibvif->send_mr = ibv_reg_mr(ibvif->pd, ibvif->send_buf, ibvif->buf_size, flags);
if (!ibvif->send_mr) {
perror("IBV error reg send mr\n");
exit(1);
}
ibvif->recv_mr = ibv_reg_mr(ibvif->pd, ibvif->recv_buf, ibvif->buf_size, flags);
if (!ibvif->recv_mr) {
perror("IBV error reg recv mr\n");
exit(1);
}
ibvif->send_cq = ibv_create_cq(ibvif->context, ibvif->tx_depth, NULL, NULL, 0);
if (!ibvif->send_cq) {
perror("IBV can't create send cq\n");
exit(1);
}
cq_attr.flags = IBV_EXP_CQ_TIMESTAMP;
cq_attr.comp_mask = IBV_EXP_CQ_INIT_ATTR_FLAGS;
ibvif->recv_cq = ibv_exp_create_cq(ibvif->context, ibvif->rx_depth, NULL, NULL, 0, &cq_attr);
if (!ibvif->recv_cq) {
perror("IBV can't create recv cq\n");
exit(1);
}
memset(&attr, 0, sizeof(struct ibv_qp_init_attr));
attr.send_cq = ibvif->send_cq;
attr.recv_cq = ibvif->recv_cq;
attr.cap.max_send_wr = ibvif->tx_depth;
attr.cap.max_send_sge = 1;
attr.cap.max_recv_wr = ibvif->rx_depth;
attr.cap.max_recv_sge = 1;
attr.qp_type = IBV_QPT_RAW_PACKET;
ibvif->qp = ibv_create_qp(ibvif->pd, &attr);
if (!ibvif->qp) {
perror("IBV can't create QP\n");
exit(1);
}
qp_flags = IBV_QP_STATE | IBV_QP_PORT;
memset(&qp_attr, 0, sizeof(struct ibv_qp_attr));
qp_attr.qp_state = IBV_QPS_INIT;
qp_attr.pkey_index = 0;
qp_attr.port_num = port_num;
qp_attr.qp_access_flags = 0;
if (ibv_modify_qp(ibvif->qp, &qp_attr, qp_flags)) {
perror("IBV can't set qp to init\n");
exit(1);
}
ibv_attach_device(netif);
}
