uint32_t ib_ctx_time_converter::get_device_convertor_status(struct ibv_context* ctx) {
uint32_t dev_status = 0;
#ifdef DEFINED_IBV_EXP_CQ_TIMESTAMP
int rval;
// Checking if ibv_exp_query_device() is valid
struct ibv_exp_device_attr device_attr;
memset(&device_attr, 0, sizeof(device_attr));
device_attr.comp_mask = IBV_EXP_DEVICE_ATTR_WITH_HCA_CORE_CLOCK;
if ((rval = ibv_exp_query_device(ctx ,&device_attr)) || !device_attr.hca_core_clock) {
vlog_printf(VLOG_DEBUG, "ib_ctx_time_converter::get_device_convertor_status :Error in querying hca core clock "
"(ibv_exp_query_device() return value=%d ) (ibv context %p) (errno=%d %m)\n", rval, ctx, errno);
} else {
dev_status |= IBV_EXP_QUERY_DEVICE_SUPPORTED;
}
// Checking if ibv_exp_query_values() is valid
struct ibv_exp_values queried_values;
memset(&queried_values, 0, sizeof(queried_values));
queried_values.comp_mask = IBV_EXP_VALUES_HW_CLOCK;
if ((rval = ibv_exp_query_values(ctx,IBV_EXP_VALUES_HW_CLOCK, &queried_values)) || !queried_values.hwclock) {
vlog_printf(VLOG_DEBUG, "ib_ctx_time_converter::get_device_convertor_status :Error in querying hw clock, can't convert"
" hw time to system time (ibv_exp_query_values() return value=%d ) (ibv context %p) (errno=%d %m)\n", rval, ctx, errno);
} else {
dev_status |= IBV_EXP_QUERY_VALUES_SUPPORTED;
}
#else
NOT_IN_USE(ctx);
#endif
return dev_status;
}
bool ib_ctx_time_converter::sync_clocks(struct timespec* st, uint64_t* hw_clock){
struct timespec st1, st2, diff, st_min = TIMESPEC_INITIALIZER;
struct ibv_exp_values queried_values;
int64_t interval, best_interval = 0;
uint64_t hw_clock_min = 0;
memset(&queried_values, 0, sizeof(queried_values));
queried_values.comp_mask = IBV_EXP_VALUES_HW_CLOCK;
for (int i = 0 ; i < 10 ; i++) {
clock_gettime(CLOCK_REALTIME, &st1);
if (ibv_exp_query_values(m_p_ibv_context,IBV_EXP_VALUES_HW_CLOCK, &queried_values) || !queried_values.hwclock) {
return false;
}
clock_gettime(CLOCK_REALTIME, &st2);
interval = (st2.tv_sec - st1.tv_sec) * NSEC_PER_SEC + (st2.tv_nsec - st1.tv_nsec);
if (!best_interval || interval < best_interval) {
best_interval = interval;
hw_clock_min = queried_values.hwclock;
interval /= 2;
diff.tv_sec = interval / NSEC_PER_SEC;
diff.tv_nsec = interval - (diff.tv_sec * NSEC_PER_SEC);
ts_add(&st1, &diff, &st_min);
}
}
*st = st_min;
*hw_clock = hw_clock_min;
return true;
}
static int poll_cqs(enum CQ_INDEX index)
{
struct ibv_wc wc[8];
struct ibv_exp_wc exp_wc[8];
int done, i, ret, j;
struct ibv_wc *p_wc;
for (i = 0; i < connections; i++) {
if (!test.nodes[i].connected)
continue;
for (done = 0; done < message_count; done += ret) {
if (set_ts) {
ret = ibv_exp_poll_cq(test.nodes[i].cq[index], 8,
exp_wc, sizeof(exp_wc[0]));
p_wc = (struct ibv_wc *)exp_wc;
} else {
ret = ibv_poll_cq(test.nodes[i].cq[index], 8,
wc);
p_wc = (struct ibv_wc *)wc;
}
if (ret < 0) {
printf("cmatose: failed polling CQ: %d\n", ret);
return ret;
}
for (j = 0; j < ret; ++j) {
if (GET_WC(p_wc, j)->status != IBV_WC_SUCCESS) {
printf("wc[%d]: bad status %d\n",
j, GET_WC(p_wc, j)->status);
return -1;
}
}
if (set_ts && ret > 0) {
struct ibv_exp_values queried_values;
queried_values.comp_mask = IBV_EXP_VALUES_HW_CLOCK;
if (ibv_exp_query_values(test.nodes[i].cq[index]
->context,
IBV_EXP_VALUES_HW_CLOCK,
&queried_values) &&
(queried_values.comp_mask &
IBV_EXP_VALUES_HW_CLOCK)) {
printf("Error in querying hw clock\n");
queried_values.hwclock = 0;
}
for (j = 0; j < ret; j++)
printf("connection %d, batch %d, "
"packet %d/%d %s, "
"queried: %lu, timestamp: %lu\n",
i, done, j + 1, ret,
p_wc->opcode == IBV_WC_SEND ?
"tx" : "rx",
queried_values.hwclock,
exp_wc[j].timestamp);
}
}
}
return 0;
}
static int post_sends(struct cmatest_node *node)
{
struct ibv_send_wr send_wr, *bad_send_wr;
struct ibv_sge sge;
int i, ret = 0;
if (!node->connected || !message_count)
return 0;
send_wr.next = NULL;
send_wr.sg_list = &sge;
send_wr.num_sge = 1;
send_wr.opcode = IBV_WR_SEND;
send_wr.send_flags = 0;
send_wr.wr_id = (unsigned long)node;
sge.length = message_size;
sge.lkey = node->mr->lkey;
sge.addr = (uintptr_t) node->mem;
for (i = 0; i < message_count && !ret; i++) {
ret = ibv_post_send(node->cma_id->qp, &send_wr, &bad_send_wr);
if (ret) {
printf("failed to post sends: %d\n", ret);
} else if (set_ts) {
struct ibv_exp_values queried_values;
queried_values.comp_mask = IBV_EXP_VALUES_HW_CLOCK;
if (!ibv_exp_query_values(node->cma_id->verbs,
IBV_EXP_VALUES_HW_CLOCK,
&queried_values) &&
(queried_values.comp_mask &
IBV_EXP_VALUES_HW_CLOCK)) {
printf("Event: Post send was called in %lu\n",
queried_values.hwclock);
}
}
}
return ret;
}