int osmt_mtl_mad_post_recv_bufs(osmt_mtl_mad_res_t * res, void *buf_array,
u_int32_t num_o_bufs, u_int32_t size,
VAPI_wr_id_t start_id)
{
uint32_t i;
void *cur_buf;
VAPI_rr_desc_t rr;
VAPI_sg_lst_entry_t sg_entry;
VAPI_ret_t ret;
rr.opcode = VAPI_RECEIVE;
rr.comp_type = VAPI_SIGNALED; /* All with CQE (IB compliant) */
rr.sg_lst_len = 1; /* single buffers */
rr.sg_lst_p = &sg_entry;
sg_entry.lkey = res->l_key;
cur_buf = buf_array;
for (i = 0; i < num_o_bufs; i++) {
rr.id = start_id + i; /* WQE id used is the index to buffers ptr array */
ASSERT_VOIDP2UINTN(cur_buf);
sg_entry.addr = (VAPI_virt_addr_t) cur_buf;
sg_entry.len = size;
memset(cur_buf, 0x00, size); /* fill with 0 */
ret = VAPI_post_rr(res->hca_hndl, res->qp_hndl, &rr);
if (ret != VAPI_OK) {
MTL_ERROR1(__FUNCTION__
": failed posting RQ WQE (%s)\n",
VAPI_strerror_sym(ret));
return i;
}
MTL_DEBUG4(__FUNCTION__ ": posted buf at %p\n", cur_buf);
cur_buf += size;
}
return i; /* num of buffers posted */
}
VAPI_ret_t osmt_mtl_mad_send(osmt_mtl_mad_res_t * res, VAPI_wr_id_t id,
void *mad, VAPI_qp_num_t dest_qp, IB_sl_t sl,
u_int32_t dest_qkey, VAPI_ud_av_hndl_t avh)
{
VAPI_sr_desc_t sr;
VAPI_sg_lst_entry_t sg_entry;
VAPI_ret_t ret;
/* building SEND request */
sr.opcode = VAPI_SEND;
sr.remote_ah = avh;
sr.remote_qp = dest_qp;
sr.remote_qkey = dest_qkey;
sr.id = id;
sr.set_se = FALSE;
sr.fence = FALSE;
sr.comp_type = VAPI_SIGNALED;
sr.sg_lst_len = 1;
sr.sg_lst_p = &sg_entry;
ASSERT_VOIDP2UINTN(mad);
sg_entry.addr = (VAPI_virt_addr_t) (mad);
sg_entry.len = MAD_SIZE;
sg_entry.lkey = res->l_key;
ret = VAPI_post_sr(res->hca_hndl, res->qp_hndl, &sr);
if (ret != VAPI_OK) {
MTL_ERROR1(__FUNCTION__ ": failed VAPI_post_sr (%s)\n",
VAPI_strerror_sym(ret));
return ret;
}
return VAPI_OK;
}
static
int psib_poll(hca_info_t *hca_info, int blocking)
{
VAPI_ret_t rc;
do {
rc = psib_check_cq(hca_info);
} while (blocking && (rc != VAPI_CQ_EMPTY));
if (psib_debug &&
(rc != VAPI_CQ_EMPTY) &&
(rc != VAPI_OK)) {
printf("psib_poll: %s: %s\n", VAPI_strerror_sym(rc), VAPI_strerror(rc));
}
return (rc == VAPI_CQ_EMPTY);
}
static
void psib_err_rc(char *str, VAPI_ret_t rc)
{
const char *vapi_sym = VAPI_strerror_sym(rc);
const char *vapi_err = VAPI_strerror(rc);
int len = strlen(str) + strlen(vapi_sym) + strlen(vapi_err) + 20;
char *msg = malloc(len);
assert(msg);
strcpy(msg, str);
strcat(msg, " : ");
strcat(msg, vapi_sym);
strcat(msg, " - ");
strcat(msg, vapi_err);
psib_err(msg);
free(msg);
}
VAPI_ret_t osmt_mtl_create_av(osmt_mtl_mad_res_t * res, int16_t dlid,
VAPI_ud_av_hndl_t * avh_p)
{
VAPI_ud_av_t av;
VAPI_ret_t ret;
av.dlid = dlid;
av.port = res->port_num;
av.sl = 0; /* dest->sl; */
av.src_path_bits = 0; /* dest->ee_dlid.dst_path_bits; */
av.static_rate = 0;
/* GRH ? */
av.grh_flag = 0;
ret = VAPI_create_addr_hndl(res->hca_hndl, res->pd_hndl, &av, avh_p);
if (ret != VAPI_OK) {
MTL_ERROR1("%s: failed VAPI_create_addr_hndl (%s)\n", __func__,
VAPI_strerror_sym(ret));
return ret;
}
return VAPI_OK;
}
/*
* Wait for a trap on QPn
*
*/
ib_api_status_t
osmt_trap_wait(IN osmtest_t * const p_osmt, IN osmt_qp_ctx_t * p_qp_ctx)
{
ib_smp_t *p_smp = (ib_smp_t *) (p_qp_ctx->p_send_buf);
ib_sa_mad_t *p_sa_mad;
VAPI_ret_t vapi_ret;
VAPI_wc_desc_t wc_desc;
osm_log_t *p_log = &p_osmt->log;
ib_api_status_t status = IB_SUCCESS;
OSM_LOG_ENTER(p_log);
OSM_LOG(p_log, OSM_LOG_INFO,
"Waiting for Traps under QP:0x%X of SA LID:0x%X\n",
cl_ntoh16(p_osmt->local_port.sm_lid));
/* --------------------- RECV ------------------------- */
vapi_ret = osmt_mtl_mad_poll4cqe(p_qp_ctx->qp_bind_hndl.hca_hndl,
p_qp_ctx->qp_bind_hndl.rq_cq_hndl,
&wc_desc,
// 200,
p_osmt->opt.wait_time * 100,
10000, NULL);
if (vapi_ret != VAPI_SUCCESS) {
if (vapi_ret == VAPI_CQ_EMPTY) {
OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0130: "
"Timeout receiving mad (%s)\n",
VAPI_strerror_sym(vapi_ret));
status = IB_TIMEOUT;
} else {
OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0131: "
"Error receiving mad (%s)\n",
VAPI_strerror_sym(vapi_ret));
status = IB_ERROR;
}
goto Exit;
}
/* check to see if successful - by examination of the subscribe bit */
p_sa_mad = (ib_sa_mad_t *) (p_qp_ctx->p_recv_buf + GRH_LEN);
if (p_sa_mad->method == IB_MAD_METHOD_REPORT) {
if (p_sa_mad->attr_id == IB_MAD_ATTR_NOTICE) {
OSM_LOG(&p_osmt->log, OSM_LOG_INFO,
"Received the Report!\n");
status = IB_SUCCESS;
} else {
OSM_LOG(&p_osmt->log, OSM_LOG_ERROR, "ERR 1020"
"Did not receive a Report(Notice) but attr:%d\n",
cl_ntoh16(p_sa_mad->attr_id));
status = IB_ERROR;
}
} else {
OSM_LOG(&p_osmt->log, OSM_LOG_ERROR, "ERR 1020"
"Received an Unexpected Method:%d\n", p_smp->method);
status = IB_ERROR;
}
Exit:
OSM_LOG_EXIT(p_log);
return status;
}
/*
* Send a trap (Subn LID Route) Trap(Notice) through the regular
* connection QP connection (targeted at QP0)
*
* Wait for the trap repress
*/
ib_api_status_t
osmt_send_trap_wait_for_forward(IN osmtest_t * const p_osmt,
IN osmt_qp_ctx_t * p_qp_ctx)
{
ib_smp_t *p_smp = (ib_smp_t *) (p_qp_ctx->p_send_buf);
ib_mad_notice_attr_t *p_ntc = ib_smp_get_payload_ptr(p_smp);
ib_sa_mad_t *p_sa_mad;
IB_MGT_ret_t mgt_res;
VAPI_ret_t vapi_ret;
VAPI_wc_desc_t wc_desc;
VAPI_ud_av_hndl_t avh;
IB_ud_av_t av;
static VAPI_wr_id_t wrid = 2222;
osm_log_t *p_log = &p_osmt->log;
ib_api_status_t status = IB_SUCCESS;
OSM_LOG_ENTER(p_log);
OSM_LOG(p_log, OSM_LOG_INFO,
"Sending Traps to QP0 of SA LID:0x%X\n",
p_osmt->local_port.sm_lid);
/* init the MAD */
memset(p_smp, 0, sizeof(ib_smp_t));
ib_mad_init_new((ib_mad_t *) p_smp,
IB_MCLASS_SUBN_LID,
(uint8_t) 2,
IB_MAD_METHOD_TRAP, cl_hton64(wrid), (ib_net16_t) 0, 0);
wrid++;
p_smp->attr_id = IB_MAD_ATTR_NOTICE;
/* prepare the notice */
p_ntc->generic_type = 0x82; /* generic, type = 2 */
ib_notice_set_prod_type_ho(p_ntc, 1);
p_ntc->g_or_v.generic.trap_num = cl_hton16(0x26);
p_ntc->issuer_lid = cl_hton16(2);
/* --------------------- PREP ------------------------- */
if (osmt_mtl_mad_post_recv_bufs(&p_qp_ctx->qp_bind_hndl, p_qp_ctx->p_recv_buf, 1, /* we need to receive both trap repress and report */
GRH_LEN + MAD_BLOCK_SIZE, wrid) != 1) {
OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0127: "
"Error posting recv bufs\n");
status = IB_ERROR;
goto Exit;
}
OSM_LOG(p_log, OSM_LOG_DEBUG, "Posted recv bufs\n");
av.dlid = p_osmt->local_port.sm_lid;
av.grh_flag = FALSE;
/* EZ: returned in HACK: use constants */
av.static_rate = 0; /* p_mad_addr->static_rate; */
av.src_path_bits = 1; /* p_mad_addr->path_bits; */
av.sl = 0; /* p_mad_addr->addr_type.gsi.service_level; */
OSM_LOG(p_log, OSM_LOG_DEBUG,
"av.dlid 0x%X, av.static_rate %d, av.path_bits %d\n",
cl_ntoh16(av.dlid), av.static_rate, av.src_path_bits);
/* send it */
mgt_res = IB_MGT_send_mad(p_qp_ctx->ib_mgt_qp0_handle, p_smp, /* actual payload */
&av, /* address vector */
wrid, /* casting the mad wrapper pointer for err cb */
p_osmt->opt.transaction_timeout);
if (mgt_res != IB_MGT_OK) {
OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0128: "
"Error sending mad (%d)\n", mgt_res);
status = IB_ERROR;
goto Exit;
}
vapi_ret =
osmt_mtl_create_av(&p_qp_ctx->qp_bind_hndl,
p_osmt->local_port.sm_lid, &avh);
if (vapi_ret != VAPI_OK) {
OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0129: "
"Error Preparing AVH (%s)\n",
VAPI_strerror_sym(vapi_ret));
status = IB_ERROR;
goto Exit;
}
OSM_LOG(p_log, OSM_LOG_DEBUG, "Prepared AVH\n");
OSM_LOG(p_log, OSM_LOG_DEBUG, "Trap MAD Sent\n");
/* --------------------- RECV ------------------------- */
vapi_ret = osmt_mtl_mad_poll4cqe(p_qp_ctx->qp_bind_hndl.hca_hndl,
p_qp_ctx->qp_bind_hndl.rq_cq_hndl,
&wc_desc, 200, 10000, &avh);
if (vapi_ret != VAPI_SUCCESS) {
if (vapi_ret == VAPI_CQ_EMPTY) {
OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0130: "
"Timeout receiving mad (%s)\n",
VAPI_strerror_sym(vapi_ret));
status = IB_TIMEOUT;
} else {
OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0131: "
"Error receiving mad (%s)\n",
VAPI_strerror_sym(vapi_ret));
status = IB_ERROR;
}
goto Exit;
}
/* check to see if successful - by examination of the subscribe bit */
p_sa_mad = (ib_sa_mad_t *) (p_qp_ctx->p_recv_buf + GRH_LEN);
if (p_sa_mad->method == IB_MAD_METHOD_REPORT) {
if (p_sa_mad->attr_id == IB_MAD_ATTR_NOTICE) {
OSM_LOG(p_log, OSM_LOG_INFO, "Received the Report!\n");
status = IB_SUCCESS;
} else {
OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 1020"
"Did not receive a Report(Notice) but attr:%d\n",
cl_ntoh16(p_sa_mad->attr_id));
status = IB_ERROR;
}
} else {
OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 1020"
"Received an Unexpected Method:%d\n", p_smp->method);
status = IB_ERROR;
}
Exit:
OSM_LOG_EXIT(p_log);
return status;
}
/*
* Register/Unregister to receive the given InformInfo
*
* Uses the qp context to send the inform info mad.
* Wait for GetResp(InformInfoResp)
*
*/
ib_api_status_t
osmt_reg_unreg_inform_info(IN osmtest_t * p_osmt,
IN osmt_qp_ctx_t * p_qp_ctx,
IN ib_inform_info_t * p_inform_info,
IN uint8_t reg_flag)
{
ib_sa_mad_t *p_sa_mad = (ib_sa_mad_t *) (p_qp_ctx->p_send_buf);
ib_inform_info_t *p_ii = ib_sa_mad_get_payload_ptr(p_sa_mad); /* SA Payload */
VAPI_ret_t vapi_ret;
VAPI_wc_desc_t wc_desc;
VAPI_ud_av_hndl_t avh;
static VAPI_wr_id_t wrid = 16198;
osm_log_t *p_log = &p_osmt->log;
ib_api_status_t status = IB_SUCCESS;
OSM_LOG_ENTER(&p_osmt->log);
/* init the MAD */
ib_mad_init_new((ib_mad_t *) p_sa_mad,
IB_MCLASS_SUBN_ADM,
(uint8_t) 2,
IB_MAD_METHOD_SET, cl_hton64(wrid), (ib_net16_t) 0, 0);
wrid++;
p_sa_mad->attr_id = IB_MAD_ATTR_INFORM_INFO;
/* copy the reference inform info */
memcpy(p_ii, p_inform_info, sizeof(ib_inform_info_t));
if (reg_flag) {
OSM_LOG(&p_osmt->log, OSM_LOG_INFO,
"Subscribing InformInfo: Traps from lid:0x%X to 0x%X, trap num :0x%X\n",
p_ii->lid_range_begin, p_ii->lid_range_end,
p_ii->g_or_v.generic.trap_num);
} else {
OSM_LOG(&p_osmt->log, OSM_LOG_INFO,
"UnSubscribing InformInfo: Traps from lid:0x%X to 0x%X\n",
p_ii->lid_range_begin, p_ii->lid_range_end);
}
/* set the subscribe bit */
if (reg_flag) {
p_ii->subscribe = 1;
} else {
p_ii->subscribe = 0;
/*
* we need to set the QPN on the mad if we unsubscribe:
* o13-2.1.1 - QPN Field need to be set when unsubscribing.
*/
ib_inform_info_set_qpn(p_ii,
cl_hton32(p_qp_ctx->qp_bind_hndl.qp_id.
qp_num));
}
osm_dump_inform_info(&p_osmt->log, p_ii, OSM_LOG_DEBUG);
/* --------------------- PREP ------------------------- */
if (osmt_mtl_mad_post_recv_bufs(&p_qp_ctx->qp_bind_hndl, p_qp_ctx->p_recv_buf, 1, /* but we need only one mad at a time */
GRH_LEN + MAD_BLOCK_SIZE, wrid) != 1) {
OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0120: "
"Error posting recv bufs\n");
status = IB_ERROR;
goto Exit;
}
OSM_LOG(p_log, OSM_LOG_DEBUG, "Posted recv bufs\n");
vapi_ret =
osmt_mtl_create_av(&p_qp_ctx->qp_bind_hndl,
p_osmt->local_port.sm_lid, &avh);
if (vapi_ret != VAPI_OK) {
OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0121: "
"Error Preparing AVH (%s)\n",
VAPI_strerror_sym(vapi_ret));
status = IB_ERROR;
goto Exit;
}
OSM_LOG(p_log, OSM_LOG_DEBUG, "Prepared AVH\n");
if (osm_log_is_active(p_log, OSM_LOG_DEBUG)) {
osm_dump_sa_mad(p_log, (ib_sa_mad_t *) (p_qp_ctx->p_send_buf),
OSM_LOG_DEBUG);
#if 0
for (i = 56; i < 253; i++) {
if (i % 8 == 0) {
printf("\n %d : ", i);
}
printf("0x%02X ", p_qp_ctx->p_send_buf[i]);
}
#endif
printf("\n");
}
/* --------------------- SEND ------------------------- */
vapi_ret = osmt_mtl_mad_send(&p_qp_ctx->qp_bind_hndl, wrid, p_qp_ctx->p_send_buf, 1, /* SA is QP1 */
0, /* SL is 0 */
OSMT_MTL_REVERSE_QP1_WELL_KNOWN_Q_KEY,
avh);
if (vapi_ret != VAPI_OK) {
OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0122: "
"Error sending mad (%s)\n",
VAPI_strerror_sym(vapi_ret));
status = IB_ERROR;
goto Exit;
}
vapi_ret = osmt_mtl_mad_poll4cqe(p_qp_ctx->qp_bind_hndl.hca_hndl,
p_qp_ctx->qp_bind_hndl.sq_cq_hndl,
&wc_desc, 20, 10000, NULL);
if (vapi_ret != VAPI_OK) {
OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0123: "
"Error getting send completion (%s)\n",
VAPI_strerror_sym(vapi_ret));
status = IB_ERROR;
goto Exit;
}
if (wc_desc.status != VAPI_SUCCESS) {
OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0124: "
"Error on send completion (%s) (%d)\n",
VAPI_strerror_sym(wc_desc.status), wc_desc.status);
status = IB_ERROR;
goto Exit;
}
OSM_LOG(p_log, OSM_LOG_DEBUG, "Sent MAD\n");
/* --------------------- RECV ------------------------- */
vapi_ret = osmt_mtl_mad_poll4cqe(p_qp_ctx->qp_bind_hndl.hca_hndl,
p_qp_ctx->qp_bind_hndl.rq_cq_hndl,
&wc_desc, 20, 10000, &avh);
if (vapi_ret != VAPI_SUCCESS) {
if (vapi_ret == VAPI_CQ_EMPTY) {
status = IB_TIMEOUT;
} else {
OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0125: "
"Error receiving mad (%s)\n",
VAPI_strerror_sym(vapi_ret));
status = IB_ERROR;
}
goto Exit;
}
/* check to see if successful - by examination of the subscribe bit */
p_sa_mad = (ib_sa_mad_t *) (p_qp_ctx->p_recv_buf + GRH_LEN);
if (p_sa_mad->status != IB_SUCCESS) {
OSM_LOG(&p_osmt->log, OSM_LOG_ERROR, "Remote error = %s\n",
ib_get_mad_status_str((ib_mad_t *) p_sa_mad));
status = IB_REMOTE_ERROR;
goto Exit;
}
if (p_sa_mad->method != IB_MAD_METHOD_GET_RESP) {
OSM_LOG(&p_osmt->log, OSM_LOG_ERROR,
"Expected IB_MAD_METHOD_GET_RESP but got:(%X)\n",
p_sa_mad->method);
status = IB_REMOTE_ERROR;
goto Exit;
}
if (p_sa_mad->attr_id != IB_MAD_ATTR_INFORM_INFO) {
OSM_LOG(&p_osmt->log, OSM_LOG_ERROR,
"Expected IB_MAD_ATTR_INFORM_INFO but got:(%X)\n",
cl_ntoh16(p_sa_mad->attr_id));
status = IB_REMOTE_ERROR;
goto Exit;
}
p_ii = ib_sa_mad_get_payload_ptr(p_sa_mad);
if (!p_ii->subscribe) {
OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 0126: "
"Subscribe/Unsubscribe Failed\n");
status = IB_REMOTE_ERROR;
}
Exit:
OSM_LOG_EXIT(&p_osmt->log);
return status;
}
