ib_api_status_t
osm_resp_send(IN osm_sm_t * sm,
IN const osm_madw_t * const p_req_madw,
IN const ib_net16_t mad_status,
IN const uint8_t * const p_payload)
{
const ib_smp_t *p_req_smp;
ib_smp_t *p_smp;
osm_madw_t *p_madw;
ib_api_status_t status = IB_SUCCESS;
OSM_LOG_ENTER(sm->p_log);
CL_ASSERT(p_req_madw);
CL_ASSERT(p_payload);
/* do nothing if we are exiting ... */
if (osm_exit_flag)
goto Exit;
p_madw = osm_mad_pool_get(sm->p_mad_pool,
osm_madw_get_bind_handle(p_req_madw),
MAD_BLOCK_SIZE, NULL);
if (p_madw == NULL) {
OSM_LOG(sm->p_log, OSM_LOG_ERROR,
"ERR 1301: Unable to acquire MAD\n");
status = IB_INSUFFICIENT_RESOURCES;
goto Exit;
}
/*
Copy the request smp to the response smp, then just
update the necessary fields.
*/
p_smp = osm_madw_get_smp_ptr(p_madw);
p_req_smp = osm_madw_get_smp_ptr(p_req_madw);
osm_resp_make_resp_smp(sm, p_req_smp, mad_status, p_payload, p_smp);
p_madw->mad_addr.dest_lid =
p_req_madw->mad_addr.addr_type.smi.source_lid;
p_madw->mad_addr.addr_type.smi.source_lid =
p_req_madw->mad_addr.dest_lid;
p_madw->resp_expected = FALSE;
p_madw->fail_msg = CL_DISP_MSGID_NONE;
OSM_LOG(sm->p_log, OSM_LOG_DEBUG, "Responding to %s (0x%X)"
"\n\t\t\t\tattribute modifier 0x%X, TID 0x%" PRIx64 "\n",
ib_get_sm_attr_str(p_smp->attr_id), cl_ntoh16(p_smp->attr_id),
cl_ntoh32(p_smp->attr_mod), cl_ntoh64(p_smp->trans_id));
osm_vl15_post(sm->p_vl15, p_madw);
Exit:
OSM_LOG_EXIT(sm->p_log);
return (status);
}
static void vl15_send_mad(osm_vl15_t * p_vl, osm_madw_t * p_madw)
{
ib_api_status_t status;
boolean_t resp_expected = p_madw->resp_expected;
/*
Non-response-expected mads are not throttled on the wire
since we can have no confirmation that they arrived
at their destination.
*/
if (resp_expected)
/*
Note that other threads may not see the response MAD
arrive before send() even returns.
In that case, the wire count would temporarily go negative.
To avoid this confusion, preincrement the counts on the
assumption that send() will succeed.
*/
cl_atomic_inc(&p_vl->p_stats->qp0_mads_outstanding_on_wire);
else
cl_atomic_inc(&p_vl->p_stats->qp0_unicasts_sent);
cl_atomic_inc(&p_vl->p_stats->qp0_mads_sent);
status = osm_vendor_send(osm_madw_get_bind_handle(p_madw),
p_madw, p_madw->resp_expected);
if (status == IB_SUCCESS) {
OSM_LOG(p_vl->p_log, OSM_LOG_DEBUG,
"%u QP0 MADs on wire, %u outstanding, "
"%u unicasts sent, %u total sent\n",
p_vl->p_stats->qp0_mads_outstanding_on_wire,
p_vl->p_stats->qp0_mads_outstanding,
p_vl->p_stats->qp0_unicasts_sent,
p_vl->p_stats->qp0_mads_sent);
return;
}
OSM_LOG(p_vl->p_log, OSM_LOG_ERROR, "ERR 3E03: "
"MAD send failed (%s)\n", ib_get_err_str(status));
/*
The MAD was never successfully sent, so
fix up the pre-incremented count values.
*/
/* Decrement qp0_mads_sent that were incremented in the code above.
qp0_mads_outstanding will be decremented by send error callback
(called by osm_vendor_send() */
cl_atomic_dec(&p_vl->p_stats->qp0_mads_sent);
if (!resp_expected)
cl_atomic_dec(&p_vl->p_stats->qp0_unicasts_sent);
}
void osm_node_init_physp(IN osm_node_t * p_node, uint8_t port_num,
IN const osm_madw_t * p_madw)
{
ib_net64_t port_guid;
ib_smp_t *p_smp;
ib_node_info_t *p_ni;
p_smp = osm_madw_get_smp_ptr(p_madw);
p_ni = ib_smp_get_payload_ptr(p_smp);
port_guid = p_ni->port_guid;
CL_ASSERT(port_num < p_node->physp_tbl_size);
osm_physp_init(&p_node->physp_table[port_num],
port_guid, port_num, p_node,
osm_madw_get_bind_handle(p_madw),
p_smp->hop_count, p_smp->initial_path);
}
/* Send a MAD out on the GSI interface */
static ib_api_status_t
__osmv_send_sa_req(IN osmv_sa_bind_info_t * p_bind,
IN const osmv_sa_mad_data_t * const p_sa_mad_data,
IN const osmv_query_req_t * const p_query_req)
{
ib_api_status_t status;
ib_mad_t *p_mad_hdr;
ib_sa_mad_t *p_sa_mad;
osm_madw_t *p_madw;
osm_log_t *p_log = p_bind->p_log;
static atomic32_t trans_id;
boolean_t sync;
osmv_query_req_t *p_query_req_copy;
OSM_LOG_ENTER(p_log);
/*
since the sm_lid might change we obtain it every send
(actually it is cached in the bind object and refreshed
every 30sec by this proc)
*/
if (time(NULL) > p_bind->last_lids_update_sec + 30) {
status = update_umad_port(p_bind->p_vendor);
if (status != IB_SUCCESS) {
OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 5509: "
"Failed to obtain the SM lid\n");
goto Exit;
}
p_bind->last_lids_update_sec = time(NULL);
}
/* Get a MAD wrapper for the send */
p_madw = osm_mad_pool_get(p_bind->p_mad_pool,
p_bind->h_bind, MAD_BLOCK_SIZE, NULL);
if (p_madw == NULL) {
OSM_LOG(p_log, OSM_LOG_ERROR, "ERR 5510: "
"Unable to acquire MAD\n");
status = IB_INSUFFICIENT_RESOURCES;
goto Exit;
}
/* Initialize the Sent MAD: */
/* Initialize the MAD buffer for the send operation. */
p_mad_hdr = osm_madw_get_mad_ptr(p_madw);
p_sa_mad = osm_madw_get_sa_mad_ptr(p_madw);
/* Get a new transaction Id */
cl_atomic_inc(&trans_id);
/* Cleanup the MAD from any residue */
memset(p_sa_mad, 0, MAD_BLOCK_SIZE);
/* Initialize the standard MAD header. */
ib_mad_init_new(p_mad_hdr, /* mad pointer */
IB_MCLASS_SUBN_ADM, /* class */
(uint8_t) 2, /* version */
p_sa_mad_data->method, /* method */
cl_hton64((uint64_t) trans_id), /* tid */
p_sa_mad_data->attr_id, /* attr id */
p_sa_mad_data->attr_mod /* attr mod */);
/* Set the query information. */
p_sa_mad->sm_key = p_query_req->sm_key;
p_sa_mad->attr_offset = 0;
p_sa_mad->comp_mask = p_sa_mad_data->comp_mask;
#ifdef DUAL_SIDED_RMPP
if (p_sa_mad->method == IB_MAD_METHOD_GETMULTI)
p_sa_mad->rmpp_flags = IB_RMPP_FLAG_ACTIVE;
#endif
if (p_sa_mad->comp_mask) {
memcpy(p_sa_mad->data, p_sa_mad_data->p_attr,
ib_get_attr_size(p_sa_mad_data->attr_offset));
}
/*
Provide the address to send to
*/
p_madw->mad_addr.dest_lid =
cl_hton16(p_bind->p_vendor->umad_port.sm_lid);
p_madw->mad_addr.addr_type.smi.source_lid =
cl_hton16(p_bind->p_vendor->umad_port.base_lid);
p_madw->mad_addr.addr_type.gsi.remote_qp = CL_HTON32(1);
p_madw->resp_expected = TRUE;
p_madw->fail_msg = CL_DISP_MSGID_NONE;
/*
Provide MAD context such that the call back will know what to do.
We have to keep the entire request structure so we know the CB.
Since we can not rely on the client to keep it around until
the response - we duplicate it and will later dispose it (in CB).
To store on the MADW we cast it into what opensm has:
p_madw->context.ni_context.node_guid
*/
p_query_req_copy = malloc(sizeof(*p_query_req_copy));
*p_query_req_copy = *p_query_req;
p_madw->context.ni_context.node_guid =
(ib_net64_t) (long)p_query_req_copy;
/* we can support async as well as sync calls */
sync = ((p_query_req->flags & OSM_SA_FLAGS_SYNC) == OSM_SA_FLAGS_SYNC);
/* send the mad asynchronously */
status = osm_vendor_send(osm_madw_get_bind_handle(p_madw),
p_madw, p_madw->resp_expected);
/* if synchronous - wait on the event */
if (sync) {
OSM_LOG(p_log, OSM_LOG_DEBUG, "Waiting for async event\n");
cl_event_wait_on(&p_bind->sync_event, EVENT_NO_TIMEOUT, FALSE);
cl_event_reset(&p_bind->sync_event);
status = p_madw->status;
}
Exit:
OSM_LOG_EXIT(p_log);
return status;
}
/* Send a MAD out on the GSI interface */
ib_api_status_t
__osmv_send_sa_req(IN osmv_sa_bind_info_t * p_bind,
IN const osmv_sa_mad_data_t * const p_sa_mad_data,
IN const osmv_query_req_t * const p_query_req)
{
ib_api_status_t status;
ib_mad_t *p_mad_hdr;
ib_sa_mad_t *p_sa_mad;
osm_madw_t *p_madw;
osm_log_t *p_log = p_bind->p_log;
static atomic32_t trans_id;
boolean_t sync;
osmv_query_req_t *p_query_req_copy;
OSM_LOG_ENTER(p_log);
/*
since the sm_lid might change we obtain it every send
(actually it is cached in the bind object and refreshed
every 30sec by this proc )
*/
status =
__osmv_get_lid_and_sm_lid_by_port_guid(p_bind->p_vendor,
p_bind->port_guid,
&p_bind->
last_lids_update_sec,
&p_bind->lid,
&p_bind->sm_lid);
if (status != IB_SUCCESS) {
osm_log(p_log, OSM_LOG_ERROR,
"__osmv_send_sa_req: ERR 0509: "
"Fail to obtain the sm lid.\n");
goto Exit;
}
/* Get a MAD wrapper for the send */
p_madw = osm_mad_pool_get(p_bind->p_mad_pool,
p_bind->h_bind, MAD_BLOCK_SIZE, NULL);
if (p_madw == NULL) {
osm_log(p_log, OSM_LOG_ERROR,
"__osmv_send_sa_req: ERR 0510: "
"Unable to acquire MAD.\n");
status = IB_INSUFFICIENT_RESOURCES;
goto Exit;
}
/* Initialize the Sent MAD: */
/* Initialize the MAD buffer for the send operation. */
p_mad_hdr = osm_madw_get_mad_ptr(p_madw);
p_sa_mad = osm_madw_get_sa_mad_ptr(p_madw);
/* Get a new transaction Id */
cl_atomic_inc(&trans_id);
/* Cleanup the MAD from any residue */
memset(p_sa_mad, 0, MAD_BLOCK_SIZE);
/* Initialize the standard MAD header. */
ib_mad_init_new(p_mad_hdr, /* mad pointer */
IB_MCLASS_SUBN_ADM, /* class */
(uint8_t) 2, /* version */
p_sa_mad_data->method, /* method */
cl_hton64((uint64_t) trans_id), /* tid */
p_sa_mad_data->attr_id, /* attr id */
p_sa_mad_data->attr_mod /* attr mod */
);
/* Set the query information. */
p_sa_mad->sm_key = p_query_req->sm_key;
p_sa_mad->attr_offset = 0;
p_sa_mad->comp_mask = p_sa_mad_data->comp_mask;
if (p_sa_mad->comp_mask) {
memcpy(p_sa_mad->data, p_sa_mad_data->p_attr,
ib_get_attr_size(p_sa_mad_data->attr_offset));
}
/*
Provide the address to send to
*/
/* Patch to handle IBAL - host order , where it should take destination lid in network order */
#ifdef OSM_VENDOR_INTF_AL
p_madw->mad_addr.dest_lid = p_bind->sm_lid;
#else
p_madw->mad_addr.dest_lid = cl_hton16(p_bind->sm_lid);
#endif
p_madw->mad_addr.addr_type.smi.source_lid = cl_hton16(p_bind->lid);
p_madw->mad_addr.addr_type.gsi.remote_qp = CL_HTON32(1);
p_madw->mad_addr.addr_type.gsi.remote_qkey = IB_QP1_WELL_KNOWN_Q_KEY;
p_madw->mad_addr.addr_type.gsi.pkey_ix = 0;
p_madw->resp_expected = TRUE;
p_madw->fail_msg = CL_DISP_MSGID_NONE;
/*
Provide MAD context such that the call back will know what to do.
We have to keep the entire request structure so we know the CB.
Since we can not rely on the client to keep it arroud until
the response - we duplicate it and will later dispose it (in CB).
To store on the MADW we cast it into what opensm has:
p_madw->context.arb_context.context1
*/
p_query_req_copy = malloc(sizeof(*p_query_req_copy));
*p_query_req_copy = *p_query_req;
p_madw->context.arb_context.context1 = p_query_req_copy;
/* we can support async as well as sync calls */
sync = ((p_query_req->flags & OSM_SA_FLAGS_SYNC) == OSM_SA_FLAGS_SYNC);
/* send the mad asynchronously */
status = osm_vendor_send(osm_madw_get_bind_handle(p_madw),
p_madw, p_madw->resp_expected);
/* if synchronous - wait on the event */
if (sync) {
osm_log(p_log, OSM_LOG_DEBUG,
"__osmv_send_sa_req: " "Waiting for async event.\n");
cl_event_wait_on(&p_bind->sync_event, EVENT_NO_TIMEOUT, FALSE);
cl_event_reset(&p_bind->sync_event);
status = p_madw->status;
}
Exit:
OSM_LOG_EXIT(p_log);
return status;
}
void osm_pi_rcv_process(IN void *context, IN void *data)
{
osm_sm_t *sm = context;
osm_madw_t *p_madw = data;
ib_port_info_t *p_pi;
ib_smp_t *p_smp;
osm_port_t *p_port;
osm_physp_t *p_physp;
osm_dr_path_t *p_dr_path;
osm_node_t *p_node;
osm_pi_context_t *p_context;
ib_net64_t port_guid;
ib_net64_t node_guid;
uint8_t port_num;
OSM_LOG_ENTER(sm->p_log);
CL_ASSERT(sm);
CL_ASSERT(p_madw);
p_smp = osm_madw_get_smp_ptr(p_madw);
p_context = osm_madw_get_pi_context_ptr(p_madw);
p_pi = (ib_port_info_t *) ib_smp_get_payload_ptr(p_smp);
CL_ASSERT(p_smp->attr_id == IB_MAD_ATTR_PORT_INFO);
port_num = (uint8_t) cl_ntoh32(p_smp->attr_mod);
port_guid = p_context->port_guid;
node_guid = p_context->node_guid;
osm_dump_port_info(sm->p_log,
node_guid, port_guid, port_num, p_pi, OSM_LOG_DEBUG);
/* On receipt of client reregister, clear the reregister bit so
reregistering won't be sent again and again */
if (ib_port_info_get_client_rereg(p_pi)) {
OSM_LOG(sm->p_log, OSM_LOG_DEBUG,
"Client reregister received on response\n");
ib_port_info_set_client_rereg(p_pi, 0);
}
/*
we might get a response during a light sweep looking for a change in
the status of a remote port that did not respond in earlier sweeps.
So if the context of the Get was light_sweep - we do not need to
do anything with the response - just flag that we need a heavy sweep
*/
if (p_context->light_sweep == TRUE) {
OSM_LOG(sm->p_log, OSM_LOG_VERBOSE,
"Got light sweep response from remote port of parent node "
"GUID 0x%" PRIx64 " port 0x%016" PRIx64
", Commencing heavy sweep\n",
cl_ntoh64(node_guid), cl_ntoh64(port_guid));
sm->p_subn->force_heavy_sweep = TRUE;
sm->p_subn->ignore_existing_lfts = TRUE;
goto Exit;
}
CL_PLOCK_EXCL_ACQUIRE(sm->p_lock);
p_port = osm_get_port_by_guid(sm->p_subn, port_guid);
if (!p_port) {
CL_PLOCK_RELEASE(sm->p_lock);
OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 0F06: "
"No port object for port with GUID 0x%" PRIx64
"\n\t\t\t\tfor parent node GUID 0x%" PRIx64
", TID 0x%" PRIx64 "\n",
cl_ntoh64(port_guid),
cl_ntoh64(node_guid), cl_ntoh64(p_smp->trans_id));
goto Exit;
}
p_node = p_port->p_node;
CL_ASSERT(p_node);
/*
If we were setting the PortInfo, then receiving
this attribute was not part of sweeping the subnet.
In this case, just update the PortInfo attribute.
In an unfortunate blunder, the IB spec defines the
return method for Set() as a GetResp(). Thus, we can't
use the method (what would have been SetResp()) to determine
our course of action. So, we have to carry this extra
boolean around to determine if we were doing Get() or Set().
*/
if (p_context->set_method)
osm_pi_rcv_process_set(sm, p_node, port_num, p_madw);
else {
p_port->discovery_count++;
/*
This PortInfo arrived because we did a Get() method,
most likely due to a subnet sweep in progress.
*/
OSM_LOG(sm->p_log, OSM_LOG_VERBOSE,
"Discovered port num %u with GUID 0x%" PRIx64
" for parent node GUID 0x%" PRIx64
", TID 0x%" PRIx64 "\n",
port_num, cl_ntoh64(port_guid),
cl_ntoh64(node_guid), cl_ntoh64(p_smp->trans_id));
p_physp = osm_node_get_physp_ptr(p_node, port_num);
/*
Determine if we encountered a new Physical Port.
If so, initialize the new Physical Port then
continue processing as normal.
*/
if (!p_physp) {
OSM_LOG(sm->p_log, OSM_LOG_VERBOSE,
"Initializing port number %u\n", port_num);
p_physp = &p_node->physp_table[port_num];
osm_physp_init(p_physp,
port_guid,
port_num,
p_node,
osm_madw_get_bind_handle(p_madw),
p_smp->hop_count, p_smp->initial_path);
} else {
/*
Update the directed route path to this port
in case the old path is no longer usable.
*/
p_dr_path = osm_physp_get_dr_path_ptr(p_physp);
osm_dr_path_init(p_dr_path,
osm_madw_get_bind_handle(p_madw),
p_smp->hop_count, p_smp->initial_path);
}
/* if port just inited or reached INIT state (external reset)
request update for port related tables */
p_physp->need_update =
(ib_port_info_get_port_state(p_pi) == IB_LINK_INIT ||
p_physp->need_update > 1) ? 1 : 0;
switch (osm_node_get_type(p_node)) {
case IB_NODE_TYPE_CA:
case IB_NODE_TYPE_ROUTER:
__osm_pi_rcv_process_ca_or_router_port(sm,
p_node, p_physp,
p_pi);
break;
case IB_NODE_TYPE_SWITCH:
__osm_pi_rcv_process_switch_port(sm,
p_node, p_physp, p_pi);
break;
default:
OSM_LOG(sm->p_log, OSM_LOG_ERROR, "ERR 0F07: "
"Unknown node type %u with GUID 0x%" PRIx64
"\n", osm_node_get_type(p_node),
cl_ntoh64(node_guid));
break;
}
/*
Get the tables on the physp.
*/
if (p_physp->need_update || sm->p_subn->need_update)
__osm_pi_rcv_get_pkey_slvl_vla_tables(sm, p_node,
p_physp);
}
CL_PLOCK_RELEASE(sm->p_lock);
Exit:
/*
Release the lock before jumping here!!
*/
OSM_LOG_EXIT(sm->p_log);
}
