char *
check_switch(ib_portid_t *portid, int *nports, uint64_t *guid,
uint8_t *sw, char *nd)
{
uint8_t ni[IB_SMP_DATA_SIZE] = {0};
int type;
DEBUG("checking node type");
if (!smp_query(ni, portid, IB_ATTR_NODE_INFO, 0, 0)) {
xdump(stderr, "nodeinfo\n", ni, sizeof ni);
return "node info failed: valid addr?";
}
if (!smp_query(nd, portid, IB_ATTR_NODE_DESC, 0, 0))
return "node desc failed";
mad_decode_field(ni, IB_NODE_TYPE_F, &type);
if (type != IB_NODE_SWITCH)
return "not a switch";
DEBUG("Gathering information about switch");
mad_decode_field(ni, IB_NODE_NPORTS_F, nports);
mad_decode_field(ni, IB_NODE_GUID_F, guid);
if (!smp_query(sw, portid, IB_ATTR_SWITCH_INFO, 0, 0))
return "switch info failed: is a switch node?";
return 0;
}
static void aggregate_perfcounters_ext(void)
{
uint32_t val;
uint64_t val64;
mad_decode_field(pc, IB_PC_EXT_PORT_SELECT_F, &val);
perf_count_ext.portselect = val;
mad_decode_field(pc, IB_PC_EXT_COUNTER_SELECT_F, &val);
perf_count_ext.counterselect = val;
mad_decode_field(pc, IB_PC_EXT_XMT_BYTES_F, &val64);
aggregate_64bit(&perf_count_ext.portxmitdata, val64);
mad_decode_field(pc, IB_PC_EXT_RCV_BYTES_F, &val64);
aggregate_64bit(&perf_count_ext.portrcvdata, val64);
mad_decode_field(pc, IB_PC_EXT_XMT_PKTS_F, &val64);
aggregate_64bit(&perf_count_ext.portxmitpkts, val64);
mad_decode_field(pc, IB_PC_EXT_RCV_PKTS_F, &val64);
aggregate_64bit(&perf_count_ext.portrcvpkts, val64);
mad_decode_field(pc, IB_PC_EXT_XMT_UPKTS_F, &val64);
aggregate_64bit(&perf_count_ext.portunicastxmitpkts, val64);
mad_decode_field(pc, IB_PC_EXT_RCV_UPKTS_F, &val64);
aggregate_64bit(&perf_count_ext.portunicastrcvpkts, val64);
mad_decode_field(pc, IB_PC_EXT_XMT_MPKTS_F, &val64);
aggregate_64bit(&perf_count_ext.portmulticastxmitpkits, val64);
mad_decode_field(pc, IB_PC_EXT_RCV_MPKTS_F, &val64);
aggregate_64bit(&perf_count_ext.portmulticastrcvpkts, val64);
}
static char *node_desc(ib_portid_t * dest, char **argv, int argc)
{
int node_type, l;
uint64_t node_guid;
char nd[IB_SMP_DATA_SIZE] = { 0 };
uint8_t data[IB_SMP_DATA_SIZE] = { 0 };
char dots[128];
char *nodename = NULL;
if (!smp_query_via(data, dest, IB_ATTR_NODE_INFO, 0, 0, srcport))
return "node info query failed";
mad_decode_field(data, IB_NODE_TYPE_F, &node_type);
mad_decode_field(data, IB_NODE_GUID_F, &node_guid);
if (!smp_query_via(nd, dest, IB_ATTR_NODE_DESC, 0, 0, srcport))
return "node desc query failed";
nodename = remap_node_name(node_name_map, node_guid, nd);
l = strlen(nodename);
if (l < 32) {
memset(dots, '.', 32 - l);
dots[32 - l] = '\0';
} else {
dots[0] = '.';
dots[1] = '\0';
}
printf("Node Description:%s%s\n", dots, nodename);
free(nodename);
return 0;
}
void dump_portinfo(void *pi, int pisize, int tabs)
{
int field, i;
char val[64];
char buf[1024];
for (field = IB_PORT_FIRST_F; field < IB_PORT_LAST_F; field++) {
for (i=0;i<tabs;i++)
printf("\t");
if (field == IB_PORT_MKEY_F && show_keys == 0) {
snprint_field(buf, 1024, field, 32, NOT_DISPLAYED_STR);
} else {
mad_decode_field(pi, field, val);
if (!mad_dump_field(field, buf, 1024, val))
return;
}
printf("%s\n", buf);
}
for (field = IB_PORT_CAPMASK2_F;
field < IB_PORT_LINK_SPEED_EXT_LAST_F; field++) {
for (i=0;i<tabs;i++)
printf("\t");
mad_decode_field(pi, field, val);
if (!mad_dump_field(field, buf, 1024, val))
return;
printf("%s\n", buf);
}
}
static ibnd_node_t *create_node(smp_engine_t * engine, ib_portid_t * path,
uint8_t * node_info)
{
ibnd_fabric_t *fabric = ((ibnd_scan_t *) engine->user_data)->fabric;
ibnd_node_t *rc = calloc(1, sizeof(*rc));
if (!rc) {
IBND_ERROR("OOM: node creation failed\n");
return NULL;
}
/* decode just a couple of fields for quicker reference. */
mad_decode_field(node_info, IB_NODE_GUID_F, &rc->guid);
mad_decode_field(node_info, IB_NODE_TYPE_F, &rc->type);
mad_decode_field(node_info, IB_NODE_NPORTS_F, &rc->numports);
rc->ports = calloc(rc->numports + 1, sizeof(*rc->ports));
if (!rc->ports) {
free(rc);
IBND_ERROR("OOM: Failed to allocate the ports array\n");
return NULL;
}
rc->path_portid = *path;
memcpy(rc->info, node_info, sizeof(rc->info));
add_to_nodeguid_hash(rc, fabric->nodestbl);
/* add this to the all nodes list */
rc->next = fabric->nodes;
fabric->nodes = rc;
add_to_type_list(rc, fabric);
return rc;
}
int
ib_resolve_self_via(ib_portid_t *portid, int *portnum, ibmad_gid_t *gid,
const void *srcport)
{
ib_portid_t self = {0};
uint8_t portinfo[64];
uint8_t nodeinfo[64];
uint64_t guid, prefix;
if (!smp_query_via(nodeinfo, &self, IB_ATTR_NODE_INFO, 0, 0, srcport))
return -1;
if (!smp_query_via(portinfo, &self, IB_ATTR_PORT_INFO, 0, 0, srcport))
return -1;
mad_decode_field(portinfo, IB_PORT_LID_F, &portid->lid);
mad_decode_field(portinfo, IB_PORT_GID_PREFIX_F, &prefix);
mad_decode_field(nodeinfo, IB_NODE_PORT_GUID_F, &guid);
if (portnum)
mad_decode_field(nodeinfo, IB_NODE_LOCAL_PORT_F, portnum);
if (gid) {
mad_encode_field(*gid, IB_GID_PREFIX_F, &prefix);
mad_encode_field(*gid, IB_GID_GUID_F, &guid);
}
return 0;
}
static char *sl2vl_table(ib_portid_t * dest, char **argv, int argc)
{
uint8_t data[IB_SMP_DATA_SIZE] = { 0 };
int type, num_ports, portnum = 0;
int i;
char *ret;
if (argc > 0)
portnum = strtol(argv[0], 0, 0);
if (!smp_query_via(data, dest, IB_ATTR_NODE_INFO, 0, 0, srcport))
return "node info query failed";
mad_decode_field(data, IB_NODE_TYPE_F, &type);
mad_decode_field(data, IB_NODE_NPORTS_F, &num_ports);
if (portnum > num_ports)
return "invalid port number";
printf("# SL2VL table: %s\n", portid2str(dest));
printf("# SL: |");
for (i = 0; i < 16; i++)
printf("%2d|", i);
printf("\n");
if (type != IB_NODE_SWITCH)
return sl2vl_dump_table_entry(dest, 0, 0);
for (i = 0; i <= num_ports; i++) {
ret = sl2vl_dump_table_entry(dest, i, portnum);
if (ret)
return ret;
}
return 0;
}
/**
* read and reset IB counters (reset on demand)
*/
static int
read_ib_counter( )
{
uint32_t send_val;
uint32_t recv_val;
uint8_t pc[1024];
/* 32 bit counter FFFFFFFF */
uint32_t max_val = 4294967295;
/* if it is bigger than this -> reset */
uint32_t reset_limit = max_val * 0.7;
int mask = 0xFFFF;
if ( active_ib_port == NULL )
return 0;
/* reading cost ~70 mirco secs */
if ( !pma_query_via
( pc, &portid, ibportnum, ib_timeout, IB_GSI_PORT_COUNTERS,
srcport ) ) {
fprintf( stderr, "perfquery\n" );
exit( 1 );
}
mad_decode_field( pc, IB_PC_XMT_BYTES_F, &send_val );
mad_decode_field( pc, IB_PC_RCV_BYTES_F, &recv_val );
/* multiply the numbers read by 4 as the IB port counters are not
counting bytes. they always count 32dwords. see man page of
perfquery for details
internally a uint64_t ia used to sum up the values */
active_ib_port->sum_send_val +=
( send_val - active_ib_port->last_send_val ) * 4;
active_ib_port->sum_recv_val +=
( recv_val - active_ib_port->last_recv_val ) * 4;
active_ib_port->send_cntr->value = active_ib_port->sum_send_val;
active_ib_port->recv_cntr->value = active_ib_port->sum_recv_val;
if ( send_val > reset_limit || recv_val > reset_limit ) {
/* reset cost ~70 mirco secs */
if ( !performance_reset_via
( pc, &portid, ibportnum, mask, ib_timeout, IB_GSI_PORT_COUNTERS,
srcport ) ) {
fprintf( stderr, "perf reset\n" );
exit( 1 );
}
mad_decode_field( pc, IB_PC_XMT_BYTES_F,
&active_ib_port->last_send_val );
mad_decode_field( pc, IB_PC_RCV_BYTES_F,
&active_ib_port->last_recv_val );
} else {
active_ib_port->last_send_val = send_val;
active_ib_port->last_recv_val = recv_val;
}
return 0;
}
static char *pkey_table(ib_portid_t * dest, char **argv, int argc)
{
uint8_t data[IB_SMP_DATA_SIZE] = { 0 };
int i, j, k;
uint16_t *p;
unsigned mod;
int n, t, phy_ports;
int portnum = 0;
if (argc > 0)
portnum = strtol(argv[0], 0, 0);
/* Get the partition capacity */
if (!smp_query_via(data, dest, IB_ATTR_NODE_INFO, 0, 0, srcport))
return "node info query failed";
mad_decode_field(data, IB_NODE_TYPE_F, &t);
mad_decode_field(data, IB_NODE_NPORTS_F, &phy_ports);
if (portnum > phy_ports)
return "invalid port number";
if ((t == IB_NODE_SWITCH) && (portnum != 0)) {
if (!smp_query_via(data, dest, IB_ATTR_SWITCH_INFO, 0, 0,
srcport))
return "switch info failed";
mad_decode_field(data, IB_SW_PARTITION_ENFORCE_CAP_F, &n);
} else
mad_decode_field(data, IB_NODE_PARTITION_CAP_F, &n);
for (i = 0; i < (n + 31) / 32; i++) {
mod = i | (portnum << 16);
if (!smp_query_via(data, dest, IB_ATTR_PKEY_TBL, mod, 0,
srcport))
return "pkey table query failed";
if (i + 1 == (n + 31) / 32)
k = ((n + 7 - i * 32) / 8) * 8;
else
k = 32;
p = (uint16_t *) data;
for (j = 0; j < k; j += 8, p += 8) {
printf
("%4u: 0x%04x 0x%04x 0x%04x 0x%04x 0x%04x 0x%04x 0x%04x 0x%04x\n",
(i * 32) + j, ntohs(p[0]), ntohs(p[1]),
ntohs(p[2]), ntohs(p[3]), ntohs(p[4]), ntohs(p[5]),
ntohs(p[6]), ntohs(p[7]));
}
}
printf("%d pkeys capacity for this port\n", n);
return 0;
}
static char *vlarb_table(ib_portid_t * dest, char **argv, int argc)
{
uint8_t data[IB_SMP_DATA_SIZE] = { 0 };
int portnum = 0;
int type, enhsp0, lowcap, highcap;
char *ret = 0;
if (argc > 0)
portnum = strtol(argv[0], 0, 0);
/* port number of 0 could mean SP0 or port MAD arrives on */
if (portnum == 0) {
if (!smp_query_via(data, dest, IB_ATTR_NODE_INFO, 0, 0,
srcport))
return "node info query failed";
mad_decode_field(data, IB_NODE_TYPE_F, &type);
if (type == IB_NODE_SWITCH) {
memset(data, 0, sizeof(data));
if (!smp_query_via(data, dest, IB_ATTR_SWITCH_INFO, 0,
0, srcport))
return "switch info query failed";
mad_decode_field(data, IB_SW_ENHANCED_PORT0_F, &enhsp0);
if (!enhsp0) {
printf
("# No VLArbitration tables (BSP0): %s port %d\n",
portid2str(dest), 0);
return 0;
}
memset(data, 0, sizeof(data));
}
}
if (!smp_query_via(data, dest, IB_ATTR_PORT_INFO, portnum, 0, srcport))
return "port info query failed";
mad_decode_field(data, IB_PORT_VL_ARBITRATION_LOW_CAP_F, &lowcap);
mad_decode_field(data, IB_PORT_VL_ARBITRATION_HIGH_CAP_F, &highcap);
printf("# VLArbitration tables: %s port %d LowCap %d HighCap %d\n",
portid2str(dest), portnum, lowcap, highcap);
if (lowcap > 0)
ret = vlarb_dump_table(dest, portnum, "Low", 1, lowcap);
if (!ret && highcap > 0)
ret = vlarb_dump_table(dest, portnum, "High", 3, highcap);
return ret;
}
int ib_path_query_via(const struct ibmad_port *srcport, ibmad_gid_t srcgid,
ibmad_gid_t destgid, ib_portid_t * sm_id, void *buf)
{
ib_sa_call_t sa = { 0 };
uint8_t *p;
int dlid;
memset(&sa, 0, sizeof sa);
sa.method = IB_MAD_METHOD_GET;
sa.attrid = IB_SA_ATTR_PATHRECORD;
sa.mask = IB_PR_DEF_MASK;
sa.trid = mad_trid();
memset(buf, 0, IB_SA_PR_RECSZ);
mad_encode_field(buf, IB_SA_PR_DGID_F, destgid);
mad_encode_field(buf, IB_SA_PR_SGID_F, srcgid);
p = sa_rpc_call(srcport, buf, sm_id, &sa, 0);
if (!p) {
IBWARN("sa call path_query failed");
return -1;
}
mad_decode_field(p, IB_SA_PR_DLID_F, &dlid);
return dlid;
}
static int query_and_dump(char *buf, size_t size, ib_portid_t * portid,
ibnd_node_t * node, char *node_name, int portnum,
const char *attr_name, uint16_t attr_id,
int start_field, int end_field)
{
uint8_t pc[1024];
uint32_t val = 0;
int i, n;
memset(pc, 0, sizeof(pc));
if (!pma_query_via(pc, portid, portnum, ibd_timeout, attr_id,
ibmad_port)) {
IBWARN("%s query failed on %s, %s port %d", attr_name,
node_name, portid2str(portid), portnum);
summary.pma_query_failures++;
return 0;
}
for (n = 0, i = start_field; i < end_field; i++) {
mad_decode_field(pc, i, (void *)&val);
if (val)
n += snprintf(buf + n, size - n, " [%s == %u]",
mad_field_name(i), val);
}
return n;
}
static char *guid_info(ib_portid_t * dest, char **argv, int argc)
{
uint8_t data[IB_SMP_DATA_SIZE] = { 0 };
int i, j, k;
uint64_t *p;
unsigned mod;
int n;
/* Get the guid capacity */
if (!smp_query_via(data, dest, IB_ATTR_PORT_INFO, 0, 0, srcport))
return "port info failed";
mad_decode_field(data, IB_PORT_GUID_CAP_F, &n);
for (i = 0; i < (n + 7) / 8; i++) {
mod = i;
if (!smp_query_via(data, dest, IB_ATTR_GUID_INFO, mod, 0,
srcport))
return "guid info query failed";
if (i + 1 == (n + 7) / 8)
k = ((n + 1 - i * 8) / 2) * 2;
else
k = 8;
p = (uint64_t *) data;
for (j = 0; j < k; j += 2, p += 2) {
printf("%4u: 0x%016" PRIx64 " 0x%016" PRIx64 "\n",
(i * 8) + j, ntohll(p[0]), ntohll(p[1]));
}
}
printf("%d guids capacity for this port\n", n);
return 0;
}
int resolve_guid(char *ca_name, uint8_t ca_port, ib_portid_t *portid,
uint64_t *guid, ib_portid_t *sm_id,
const struct ibmad_port *srcport)
{
ib_portid_t sm_portid;
uint8_t buf[IB_SA_DATA_SIZE] = { 0 };
uint64_t prefix;
ibmad_gid_t selfgid;
if (!sm_id) {
sm_id = &sm_portid;
if (resolve_sm_portid(ca_name, ca_port, sm_id) < 0)
return -1;
}
if (resolve_self(ca_name, ca_port, NULL, NULL, &selfgid) < 0)
return -1;
memcpy(&prefix, portid->gid, sizeof(prefix));
if (!prefix)
mad_set_field64(portid->gid, 0, IB_GID_PREFIX_F,
IB_DEFAULT_SUBN_PREFIX);
if (guid)
mad_set_field64(portid->gid, 0, IB_GID_GUID_F, *guid);
if ((portid->lid =
ib_path_query_via(srcport, selfgid, portid->gid, sm_id, buf)) < 0)
return -1;
mad_decode_field(buf, IB_SA_PR_SL_F, &portid->sl);
return 0;
}
/**
* initialize one IB port so that we are able to read values from it
*/
static int
init_ib_port( ib_port * portdata )
{
int mgmt_classes[4] = { IB_SMI_CLASS, IB_SMI_DIRECT_CLASS, IB_SA_CLASS,
IB_PERFORMANCE_CLASS
};
char *ca = 0;
static uint8_t pc[1024];
int mask = 0xFFFF;
srcport = mad_rpc_open_port( ca, portdata->port_number, mgmt_classes, 4 );
if ( !srcport ) {
fprintf( stderr, "Failed to open '%s' port '%d'\n", ca,
portdata->port_number );
exit( 1 );
}
if ( ib_resolve_self_via( &portid, &ibportnum, 0, srcport ) < 0 ) {
fprintf( stderr, "can't resolve self port\n" );
exit( 1 );
}
/* PerfMgt ClassPortInfo is a required attribute */
/* might be redundant, could be left out for fast implementation */
if ( !pma_query_via
( pc, &portid, ibportnum, ib_timeout, CLASS_PORT_INFO, srcport ) ) {
fprintf( stderr, "classportinfo query\n" );
exit( 1 );
}
if ( !performance_reset_via
( pc, &portid, ibportnum, mask, ib_timeout, IB_GSI_PORT_COUNTERS,
srcport ) ) {
fprintf( stderr, "perf reset\n" );
exit( 1 );
}
/* read the initial values */
mad_decode_field( pc, IB_PC_XMT_BYTES_F, &portdata->last_send_val );
portdata->sum_send_val = 0;
mad_decode_field( pc, IB_PC_RCV_BYTES_F, &portdata->last_recv_val );
portdata->sum_recv_val = 0;
portdata->is_initialized = 1;
return 0;
}
char *
dump_unicast_tables(ib_portid_t *portid, int startlid, int endlid)
{
char lft[IB_SMP_DATA_SIZE];
char nd[IB_SMP_DATA_SIZE];
uint8_t sw[IB_SMP_DATA_SIZE];
char str[200], *s;
uint64_t nodeguid;
int block, i, e, nports, top;
int n = 0, startblock, endblock;
if ((s = check_switch(portid, &nports, &nodeguid, sw, nd)))
return s;
mad_decode_field(sw, IB_SW_LINEAR_FDB_TOP_F, &top);
if (!endlid || endlid > top)
endlid = top;
if (endlid > IB_MAX_UCAST_LID) {
IBWARN("ilegal lft top %d, truncate to %d", endlid, IB_MAX_UCAST_LID);
endlid = IB_MAX_UCAST_LID;
}
printf("Unicast lids [0x%x-0x%x] of switch %s guid 0x%016" PRIx64 " (%s):\n",
startlid, endlid, portid2str(portid), nodeguid, clean_nodedesc(nd));
DEBUG("Switch top is 0x%x\n", top);
printf(" Lid Out Destination\n");
printf(" Port Info \n");
startblock = startlid / IB_SMP_DATA_SIZE;
endblock = ALIGN(endlid, IB_SMP_DATA_SIZE) / IB_SMP_DATA_SIZE;
for (block = startblock; block <= endblock; block++) {
DEBUG("reading block %d", block);
if (!smp_query(lft, portid, IB_ATTR_LINEARFORWTBL, block, 0))
return "linear forwarding table get failed";
i = block * IB_SMP_DATA_SIZE;
e = i + IB_SMP_DATA_SIZE;
if (i < startlid)
i = startlid;
if (e > endlid + 1)
e = endlid + 1;
for (;i < e; i++) {
unsigned outport = lft[i % IB_SMP_DATA_SIZE];
unsigned valid = (outport <= nports);
if (!valid && !dump_all)
continue;
dump_lid(str, sizeof str, i, valid);
printf("0x%04x %03u %s\n", i, outport & 0xff, str);
n++;
}
}
printf("%d %slids dumped \n", n, dump_all ? "" : "valid ");
return 0;
}
int ib_resolve_smlid_via(ib_portid_t * sm_id, int timeout,
const struct ibmad_port *srcport)
{
ib_portid_t self = { 0 };
uint8_t portinfo[64];
int lid;
memset(sm_id, 0, sizeof(*sm_id));
if (!smp_query_via(portinfo, &self, IB_ATTR_PORT_INFO, 0, 0, srcport))
return -1;
mad_decode_field(portinfo, IB_PORT_SMLID_F, &lid);
mad_decode_field(portinfo, IB_PORT_SMSL_F, &sm_id->sl);
return ib_portid_set(sm_id, lid, 0, 0);
}
/*
* Returns 0 if non switch node is found, 1 if switch is found, -1 if error.
*/
int
get_node(Node *node, Port *port, ib_portid_t *portid)
{
char portinfo[64];
char switchinfo[64];
void *pi = portinfo, *ni = node->nodeinfo, *nd = node->nodedesc;
void *si = switchinfo;
if (!smp_query(ni, portid, IB_ATTR_NODE_INFO, 0, timeout))
return -1;
mad_decode_field(ni, IB_NODE_GUID_F, &node->nodeguid);
mad_decode_field(ni, IB_NODE_TYPE_F, &node->type);
mad_decode_field(ni, IB_NODE_NPORTS_F, &node->numports);
mad_decode_field(ni, IB_NODE_DEVID_F, &node->devid);
mad_decode_field(ni, IB_NODE_VENDORID_F, &node->vendid);
mad_decode_field(ni, IB_NODE_SYSTEM_GUID_F, &node->sysimgguid);
mad_decode_field(ni, IB_NODE_PORT_GUID_F, &node->portguid);
mad_decode_field(ni, IB_NODE_LOCAL_PORT_F, &node->localport);
port->portnum = node->localport;
port->portguid = node->portguid;
if (!smp_query(nd, portid, IB_ATTR_NODE_DESC, 0, timeout))
return -1;
if (!smp_query(pi, portid, IB_ATTR_PORT_INFO, 0, timeout))
return -1;
decode_port_info(pi, port);
if (node->type != SWITCH_NODE)
return 0;
node->smalid = port->lid;
node->smalmc = port->lmc;
/* after we have the sma information find out the real PortInfo for this port */
if (!smp_query(pi, portid, IB_ATTR_PORT_INFO, node->localport, timeout))
return -1;
decode_port_info(pi, port);
if (!smp_query(si, portid, IB_ATTR_SWITCH_INFO, 0, timeout))
node->smaenhsp0 = 0; /* assume base SP0 */
else
mad_decode_field(si, IB_SW_ENHANCED_PORT0_F, &node->smaenhsp0);
DEBUG("portid %s: got switch node %" PRIx64 " '%s'",
portid2str(portid), node->nodeguid, node->nodedesc);
return 1;
}
static int recv_switch_info(smp_engine_t * engine, ibnd_smp_t * smp,
uint8_t * mad, void *cb_data)
{
uint8_t *switch_info = mad + IB_SMP_DATA_OFFS;
ibnd_node_t *node = cb_data;
memcpy(node->switchinfo, switch_info, sizeof(node->switchinfo));
mad_decode_field(node->switchinfo, IB_SW_ENHANCED_PORT0_F,
&node->smaenhsp0);
return 0;
}
int ib_resolve_guid_via(ib_portid_t * portid, uint64_t * guid,
ib_portid_t * sm_id, int timeout,
const struct ibmad_port *srcport)
{
ib_portid_t sm_portid;
uint8_t buf[IB_SA_DATA_SIZE] = { 0 };
ib_portid_t self = { 0 };
uint64_t selfguid, prefix;
ibmad_gid_t selfgid;
uint8_t nodeinfo[64];
if (!sm_id) {
sm_id = &sm_portid;
if (ib_resolve_smlid_via(sm_id, timeout, srcport) < 0)
return -1;
}
if (!smp_query_via(nodeinfo, &self, IB_ATTR_NODE_INFO, 0, 0, srcport))
return -1;
mad_decode_field(nodeinfo, IB_NODE_PORT_GUID_F, &selfguid);
mad_set_field64(selfgid, 0, IB_GID_PREFIX_F, IB_DEFAULT_SUBN_PREFIX);
mad_set_field64(selfgid, 0, IB_GID_GUID_F, selfguid);
memcpy(&prefix, portid->gid, sizeof(prefix));
if (!prefix)
mad_set_field64(portid->gid, 0, IB_GID_PREFIX_F,
IB_DEFAULT_SUBN_PREFIX);
if (guid)
mad_set_field64(portid->gid, 0, IB_GID_GUID_F, *guid);
if ((portid->lid =
ib_path_query_via(srcport, selfgid, portid->gid, sm_id, buf)) < 0)
return -1;
mad_decode_field(buf, IB_SA_PR_SL_F, &portid->sl);
return 0;
}
static int
ib_resolve_addr(ib_portid_t *portid, int portnum, int show_lid, int show_gid)
{
char gid_str[INET6_ADDRSTRLEN];
uint8_t portinfo[64];
uint8_t nodeinfo[64];
uint64_t guid, prefix;
ibmad_gid_t gid;
int lmc;
if (!smp_query(nodeinfo, portid, IB_ATTR_NODE_INFO, 0, 0))
return -1;
if (!smp_query(portinfo, portid, IB_ATTR_PORT_INFO, portnum, 0))
return -1;
mad_decode_field(portinfo, IB_PORT_LID_F, &portid->lid);
mad_decode_field(portinfo, IB_PORT_GID_PREFIX_F, &prefix);
mad_decode_field(portinfo, IB_PORT_LMC_F, &lmc);
mad_decode_field(nodeinfo, IB_NODE_PORT_GUID_F, &guid);
mad_encode_field(gid, IB_GID_PREFIX_F, &prefix);
mad_encode_field(gid, IB_GID_GUID_F, &guid);
if (show_gid) {
printf("GID %s ", inet_ntop(AF_INET6, gid, gid_str,
sizeof gid_str));
}
if (show_lid > 0)
printf("LID start 0x%x end 0x%x", portid->lid, portid->lid + (1 << lmc) - 1);
else if (show_lid < 0)
printf("LID start %d end %d", portid->lid, portid->lid + (1 << lmc) - 1);
printf("\n");
return 0;
}
static int switch_lookup(Switch * sw, ib_portid_t * portid, int lid)
{
void *si = sw->switchinfo, *fdb = sw->fdb;
memset(si, 0, sizeof(sw->switchinfo));
if (!smp_query_via(si, portid, IB_ATTR_SWITCH_INFO, 0, timeout,
srcport))
return -1;
mad_decode_field(si, IB_SW_LINEAR_FDB_CAP_F, &sw->linearcap);
mad_decode_field(si, IB_SW_LINEAR_FDB_TOP_F, &sw->linearFDBtop);
if (lid >= sw->linearcap && lid > sw->linearFDBtop)
return -1;
memset(fdb, 0, sizeof(sw->fdb));
if (!smp_query_via(fdb, portid, IB_ATTR_LINEARFORWTBL, lid / 64,
timeout, srcport))
return -1;
DEBUG("portid %s: forward lid %d to port %d",
portid2str(portid), lid, sw->fdb[lid % 64]);
return sw->fdb[lid % 64];
}
void
decode_port_info(void *pi, Port *port)
{
mad_decode_field(pi, IB_PORT_LID_F, &port->lid);
mad_decode_field(pi, IB_PORT_LMC_F, &port->lmc);
mad_decode_field(pi, IB_PORT_STATE_F, &port->state);
mad_decode_field(pi, IB_PORT_PHYS_STATE_F, &port->physstate);
mad_decode_field(pi, IB_PORT_LINK_WIDTH_ACTIVE_F, &port->linkwidth);
mad_decode_field(pi, IB_PORT_LINK_SPEED_ACTIVE_F, &port->linkspeed);
}
static int
set_port_info(ib_portid_t *dest, uint8_t *data, int portnum, int port_op)
{
char buf[2048];
char val[64];
if (!smp_set(data, dest, IB_ATTR_PORT_INFO, portnum, 0))
return -1;
if (port_op != 4)
mad_dump_portstates(buf, sizeof buf, data, sizeof data);
else {
mad_decode_field(data, IB_PORT_LINK_SPEED_ENABLED_F, val);
mad_dump_field(IB_PORT_LINK_SPEED_ENABLED_F, buf, sizeof buf, val);
sprintf(buf+strlen(buf), "%s", "\n");
}
printf("\nAfter PortInfo set:\n");
printf("# Port info: %s port %d\n%s", portid2str(dest), portnum, buf);
return 0;
}
static int get_node(Node * node, Port * port, ib_portid_t * portid)
{
void *pi = port->portinfo, *ni = node->nodeinfo, *nd = node->nodedesc;
char *s, *e;
memset(ni, 0, sizeof(node->nodeinfo));
if (!smp_query_via(ni, portid, IB_ATTR_NODE_INFO, 0, timeout, srcport))
return -1;
memset(nd, 0, sizeof(node->nodedesc));
if (!smp_query_via(nd, portid, IB_ATTR_NODE_DESC, 0, timeout, srcport))
return -1;
for (s = nd, e = s + 64; s < e; s++) {
if (!*s)
break;
if (!isprint(*s))
*s = ' ';
}
memset(pi, 0, sizeof(port->portinfo));
if (!smp_query_via(pi, portid, IB_ATTR_PORT_INFO, 0, timeout, srcport))
return -1;
mad_decode_field(ni, IB_NODE_GUID_F, &node->nodeguid);
mad_decode_field(ni, IB_NODE_TYPE_F, &node->type);
mad_decode_field(ni, IB_NODE_NPORTS_F, &node->numports);
mad_decode_field(ni, IB_NODE_PORT_GUID_F, &port->portguid);
mad_decode_field(ni, IB_NODE_LOCAL_PORT_F, &port->portnum);
mad_decode_field(pi, IB_PORT_LID_F, &port->lid);
mad_decode_field(pi, IB_PORT_LMC_F, &port->lmc);
mad_decode_field(pi, IB_PORT_STATE_F, &port->state);
DEBUG("portid %s: got node %" PRIx64 " '%s'", portid2str(portid),
node->nodeguid, node->nodedesc);
return 0;
}
static void getLIDFromGUID(AllocFunction alloc, void *allocUd,
int fd, int agent, UInt16 smLid, UInt64 guid, UInt16 *lid)
{
UInt8 umad[256];
UInt8 *buf = NULL;
SInt64 len;
SInt32 status;
const SInt32 timeout = 1000;
fillMAD_Get_PathRecord(umad, sizeof(umad), smLid, guid, __LINE__ + guid);
libibumad_Send_MAD(fd, agent, umad, sizeof(umad), timeout, 0);
libibumad_Recv_MAD(alloc, allocUd, fd, &buf, &len, timeout);
status = mad_get_field(umad_get_mad(buf), 0, IB_MAD_STATUS_F);
if (UNLIKELY(0 != status)) {
FATAL("status is %d", status);
}
mad_decode_field((UInt8 *)umad_get_mad(buf) + IB_SA_DATA_OFFS, IB_SA_PR_DLID_F, lid);
buf = alloc(allocUd, buf, len, 0);
}
static int
get_port_info(ib_portid_t *dest, uint8_t *data, int portnum, int port_op)
{
char buf[2048];
char val[64];
if (!smp_query(data, dest, IB_ATTR_PORT_INFO, portnum, 0))
return -1;
if (port_op != 4) {
mad_dump_portstates(buf, sizeof buf, data, sizeof data);
mad_decode_field(data, IB_PORT_LINK_WIDTH_SUPPORTED_F, val);
mad_dump_field(IB_PORT_LINK_WIDTH_SUPPORTED_F, buf + strlen(buf), sizeof buf - strlen(buf), val);
sprintf(buf+strlen(buf), "%s", "\n");
mad_decode_field(data, IB_PORT_LINK_WIDTH_ENABLED_F, val);
mad_dump_field(IB_PORT_LINK_WIDTH_ENABLED_F, buf + strlen(buf), sizeof buf - strlen(buf), val);
sprintf(buf+strlen(buf), "%s", "\n");
mad_decode_field(data, IB_PORT_LINK_WIDTH_ACTIVE_F, val);
mad_dump_field(IB_PORT_LINK_WIDTH_ACTIVE_F, buf + strlen(buf), sizeof buf - strlen(buf), val);
sprintf(buf+strlen(buf), "%s", "\n");
mad_decode_field(data, IB_PORT_LINK_SPEED_SUPPORTED_F, val);
mad_dump_field(IB_PORT_LINK_SPEED_SUPPORTED_F, buf + strlen(buf), sizeof buf - strlen(buf), val);
sprintf(buf+strlen(buf), "%s", "\n");
mad_decode_field(data, IB_PORT_LINK_SPEED_ENABLED_F, val);
mad_dump_field(IB_PORT_LINK_SPEED_ENABLED_F, buf + strlen(buf), sizeof buf - strlen(buf), val);
sprintf(buf+strlen(buf), "%s", "\n");
mad_decode_field(data, IB_PORT_LINK_SPEED_ACTIVE_F, val);
mad_dump_field(IB_PORT_LINK_SPEED_ACTIVE_F, buf + strlen(buf), sizeof buf - strlen(buf), val);
sprintf(buf+strlen(buf), "%s", "\n");
} else {
mad_decode_field(data, IB_PORT_LINK_SPEED_ENABLED_F, val);
mad_dump_field(IB_PORT_LINK_SPEED_ENABLED_F, buf, sizeof buf, val);
sprintf(buf+strlen(buf), "%s", "\n");
}
printf("# Port info: %s port %d\n%s", portid2str(dest), portnum, buf);
return 0;
}
char *
dump_multicast_tables(ib_portid_t *portid, int startlid, int endlid)
{
char nd[IB_SMP_DATA_SIZE] = {0};
uint8_t sw[IB_SMP_DATA_SIZE] = {0};
char str[512];
char *s;
uint64_t nodeguid;
uint32_t mod;
int block, i, j, e, nports, cap, chunks;
int n = 0, startblock, lastblock;
if ((s = check_switch(portid, &nports, &nodeguid, sw, nd)))
return s;
mad_decode_field(sw, IB_SW_MCAST_FDB_CAP_F, &cap);
if (!endlid || endlid > IB_MIN_MCAST_LID + cap - 1)
endlid = IB_MIN_MCAST_LID + cap - 1;
if (!startlid)
startlid = IB_MIN_MCAST_LID;
if (startlid < IB_MIN_MCAST_LID) {
IBWARN("illegal start mlid %x, set to %x", startlid, IB_MIN_MCAST_LID);
startlid = IB_MIN_MCAST_LID;
}
if (endlid > IB_MAX_MCAST_LID) {
IBWARN("illegal end mlid %x, truncate to %x", endlid, IB_MAX_MCAST_LID);
endlid = IB_MAX_MCAST_LID;
}
printf("Multicast mlids [0x%x-0x%x] of switch %s guid 0x%016" PRIx64 " (%s):\n",
startlid, endlid, portid2str(portid), nodeguid, clean_nodedesc(nd));
if (brief)
printf(" MLid Port Mask\n");
else {
if (nports > 9) {
for (i = 0, s = str; i <= nports; i++) {
*s++ = (i%10) ? ' ' : '0' + i/10;
*s++ = ' ';
}
*s = 0;
printf(" %s\n", str);
}
for (i = 0, s = str; i <= nports; i++)
s += sprintf(s, "%d ", i%10);
printf(" Ports: %s\n", str);
printf(" MLid\n");
}
if (verbose)
printf("Switch muticast mlids capability is 0x%d\n", cap);
chunks = ALIGN(nports + 1, 16) / 16;
startblock = startlid / IB_MLIDS_IN_BLOCK;
lastblock = endlid / IB_MLIDS_IN_BLOCK;
for (block = startblock; block <= lastblock; block++) {
for (j = 0; j < chunks; j++) {
mod = (block - IB_MIN_MCAST_LID/IB_MLIDS_IN_BLOCK) | (j << 28);
DEBUG("reading block %x chunk %d mod %x", block, j, mod);
if (!smp_query(mft + j, portid, IB_ATTR_MULTICASTFORWTBL, mod, 0))
return "multicast forwarding table get failed";
}
i = block * IB_MLIDS_IN_BLOCK;
e = i + IB_MLIDS_IN_BLOCK;
if (i < startlid)
i = startlid;
if (e > endlid + 1)
e = endlid + 1;
for (; i < e; i++) {
if (dump_mlid(str, sizeof str, i, nports, mft) == 0)
continue;
printf("0x%04x %s\n", i, str);
n++;
}
}
printf("%d %smlids dumped \n", n, dump_all ? "" : "valid ");
return 0;
}
/*
* _read_ofed_values read the IB sensor and update last_update values and times
*/
static int _read_ofed_values(void)
{
static uint64_t last_update_xmtdata = 0;
static uint64_t last_update_rcvdata = 0;
static uint64_t last_update_xmtpkts = 0;
static uint64_t last_update_rcvpkts = 0;
static bool first = true;
int rc = SLURM_SUCCESS;
uint16_t cap_mask;
uint64_t send_val, recv_val, send_pkts, recv_pkts;
ofed_sens.last_update_time = ofed_sens.update_time;
ofed_sens.update_time = time(NULL);
if (first) {
char *ibd_ca = NULL;
int mgmt_classes[4] = {IB_SMI_CLASS, IB_SMI_DIRECT_CLASS,
IB_SA_CLASS, IB_PERFORMANCE_CLASS};
srcport = mad_rpc_open_port(ibd_ca, ofed_conf.port,
mgmt_classes, 4);
if (!srcport){
error("Failed to open '%s' port '%d'", ibd_ca,
ofed_conf.port);
debug("INFINIBAND: failed");
return SLURM_ERROR;
}
if (ib_resolve_self_via(&portid, &port, 0, srcport) < 0)
error("can't resolve self port %d", port);
memset(pc, 0, sizeof(pc));
if (!_slurm_pma_query_via(pc, &portid, port, ibd_timeout,
CLASS_PORT_INFO, srcport))
error("classportinfo query: %m");
memcpy(&cap_mask, pc + 2, sizeof(cap_mask));
if (!_slurm_pma_query_via(pc, &portid, port, ibd_timeout,
IB_GSI_PORT_COUNTERS_EXT, srcport)) {
error("ofed: %m");
return SLURM_ERROR;
}
mad_decode_field(pc, IB_PC_EXT_XMT_BYTES_F,
&last_update_xmtdata);
mad_decode_field(pc, IB_PC_EXT_RCV_BYTES_F,
&last_update_rcvdata);
mad_decode_field(pc, IB_PC_EXT_XMT_PKTS_F,
&last_update_xmtpkts);
mad_decode_field(pc, IB_PC_EXT_RCV_PKTS_F,
&last_update_rcvpkts);
if (debug_flags & DEBUG_FLAG_INFINIBAND)
info("%s ofed init", plugin_name);
first = 0;
return SLURM_SUCCESS;
}
memset(pc, 0, sizeof(pc));
memcpy(&cap_mask, pc + 2, sizeof(cap_mask));
if (!_slurm_pma_query_via(pc, &portid, port, ibd_timeout,
IB_GSI_PORT_COUNTERS_EXT, srcport)) {
error("ofed: %m");
return SLURM_ERROR;
}
mad_decode_field(pc, IB_PC_EXT_XMT_BYTES_F, &send_val);
mad_decode_field(pc, IB_PC_EXT_RCV_BYTES_F, &recv_val);
mad_decode_field(pc, IB_PC_EXT_XMT_PKTS_F, &send_pkts);
mad_decode_field(pc, IB_PC_EXT_RCV_PKTS_F, &recv_pkts);
ofed_sens.xmtdata = (send_val - last_update_xmtdata) * 4;
ofed_sens.total_xmtdata += ofed_sens.xmtdata;
ofed_sens.rcvdata = (recv_val - last_update_rcvdata) * 4;
ofed_sens.total_rcvdata += ofed_sens.rcvdata;
ofed_sens.xmtpkts = send_pkts - last_update_xmtpkts;
ofed_sens.total_xmtpkts += ofed_sens.xmtpkts;
ofed_sens.rcvpkts = recv_pkts - last_update_rcvpkts;
ofed_sens.total_rcvpkts += ofed_sens.rcvpkts;
last_update_xmtdata = send_val;
last_update_rcvdata = recv_val;
last_update_xmtpkts = send_pkts;
last_update_rcvpkts = recv_pkts;
return rc;
}
int
dump_lid(char *str, int strlen, int lid, int valid)
{
char nd[IB_SMP_DATA_SIZE] = {0};
uint8_t ni[IB_SMP_DATA_SIZE] = {0};
uint8_t pi[IB_SMP_DATA_SIZE] = {0};
ib_portid_t lidport = {0};
static int last_port_lid, base_port_lid;
char ntype[50], sguid[30], desc[64];
static uint64_t portguid;
int baselid, lmc, type;
if (brief) {
str[0] = 0;
return 0;
}
if (lid <= last_port_lid) {
if (!valid)
return snprintf(str, strlen, ": (path #%d - illegal port)",
lid - base_port_lid);
else if (!portguid)
return snprintf(str, strlen,
": (path #%d out of %d)",
lid - base_port_lid + 1,
last_port_lid - base_port_lid + 1);
else {
return snprintf(str, strlen,
": (path #%d out of %d: portguid %s)",
lid - base_port_lid + 1,
last_port_lid - base_port_lid + 1,
mad_dump_val(IB_NODE_PORT_GUID_F, sguid, sizeof sguid, &portguid));
}
}
if (!valid)
return snprintf(str, strlen, ": (illegal port)");
portguid = 0;
lidport.lid = lid;
if (!smp_query(nd, &lidport, IB_ATTR_NODE_DESC, 0, 100) ||
!smp_query(pi, &lidport, IB_ATTR_PORT_INFO, 0, 100) ||
!smp_query(ni, &lidport, IB_ATTR_NODE_INFO, 0, 100))
return snprintf(str, strlen, ": (unknown node and type)");
mad_decode_field(ni, IB_NODE_PORT_GUID_F, &portguid);
mad_decode_field(ni, IB_NODE_TYPE_F, &type);
mad_decode_field(pi, IB_PORT_LID_F, &baselid);
mad_decode_field(pi, IB_PORT_LMC_F, &lmc);
if (lmc > 0) {
base_port_lid = baselid;
last_port_lid = baselid + (1 << lmc) - 1;
}
return snprintf(str, strlen, ": (%s portguid %s: %s)",
mad_dump_val(IB_NODE_TYPE_F, ntype, sizeof ntype, &type),
mad_dump_val(IB_NODE_PORT_GUID_F, sguid, sizeof sguid, &portguid),
mad_dump_val(IB_NODE_DESC_F, desc, sizeof desc, clean_nodedesc(nd)));
}