static void get_sfb_slot(Node *node, Port *lineport)
{
ChassisRecord *ch = node->chrecord;
ch->chassisslot = SPINE_CS;
if (is_spine_9096(node)) {
ch->chassistype = ISR9096_CT;
ch->slotnum = spine4_slot_2_slb[lineport->portnum];
ch->anafanum = anafa_spine4_slot_2_slb[lineport->portnum];
} else if (is_spine_9288(node)) {
ch->chassistype = ISR9288_CT;
ch->slotnum = spine12_slot_2_slb[lineport->portnum];
ch->anafanum = anafa_spine12_slot_2_slb[lineport->portnum];
} else if (is_spine_2012(node)) {
ch->chassistype = ISR2012_CT;
ch->slotnum = spine12_slot_2_slb[lineport->portnum];
ch->anafanum = anafa_spine12_slot_2_slb[lineport->portnum];
} else if (is_spine_2004(node)) {
ch->chassistype = ISR2004_CT;
ch->slotnum = spine4_slot_2_slb[lineport->portnum];
ch->anafanum = anafa_spine4_slot_2_slb[lineport->portnum];
} else {
IBPANIC("Unexpected node found: guid 0x%016" PRIx64, node->nodeguid);
}
}
static int get_line_index(Node *node)
{
int retval = 3 * (node->chrecord->slotnum - 1) + node->chrecord->anafanum;
if (retval > LINES_MAX_NUM || retval < 1)
IBPANIC("Internal error");
return retval;
}
static char *ibsystat(ib_portid_t * portid, int attr)
{
ib_rpc_t rpc = { 0 };
int fd, agent, timeout, len;
void *data = (uint8_t *) umad_get_mad(buf) + IB_VENDOR_RANGE2_DATA_OFFS;
DEBUG("Sysstat ping..");
rpc.mgtclass = IB_VENDOR_OPENIB_SYSSTAT_CLASS;
rpc.method = IB_MAD_METHOD_GET;
rpc.attr.id = attr;
rpc.attr.mod = 0;
rpc.oui = oui;
rpc.timeout = 0;
rpc.datasz = IB_VENDOR_RANGE2_DATA_SIZE;
rpc.dataoffs = IB_VENDOR_RANGE2_DATA_OFFS;
portid->qp = 1;
if (!portid->qkey)
portid->qkey = IB_DEFAULT_QP1_QKEY;
if ((len = mad_build_pkt(buf, &rpc, portid, NULL, NULL)) < 0)
IBPANIC("cannot build packet.");
fd = mad_rpc_portid(srcport);
agent = mad_rpc_class_agent(srcport, rpc.mgtclass);
timeout = ibd_timeout ? ibd_timeout : MAD_DEF_TIMEOUT_MS;
if (umad_send(fd, agent, buf, len, timeout, 0) < 0)
IBPANIC("umad_send failed.");
len = sizeof(buf) - umad_size();
if (umad_recv(fd, buf, &len, timeout) < 0)
IBPANIC("umad_recv failed.");
if (umad_status(buf))
return strerror(umad_status(buf));
DEBUG("Got sysstat pong..");
if (attr != IB_PING_ATTR)
puts(data);
else
printf("sysstat ping succeeded\n");
return 0;
}
static void get_router_slot(Node *node, Port *spineport)
{
ChassisRecord *ch = node->chrecord;
int guessnum = 0;
if (!ch) {
if (!(node->chrecord = calloc(1, sizeof(ChassisRecord))))
IBPANIC("out of mem");
ch = node->chrecord;
}
ch->chassisslot = SRBD_CS;
if (is_spine_9096(spineport->node)) {
ch->chassistype = ISR9096_CT;
ch->slotnum = line_slot_2_sfb4[spineport->portnum];
ch->anafanum = ipr_slot_2_sfb4_port[spineport->portnum];
} else if (is_spine_9288(spineport->node)) {
ch->chassistype = ISR9288_CT;
ch->slotnum = line_slot_2_sfb12[spineport->portnum];
/* this is a smart guess based on nodeguids order on sFB-12 module */
guessnum = spineport->node->nodeguid % 4;
/* module 1 <--> remote anafa 3 */
/* module 2 <--> remote anafa 2 */
/* module 3 <--> remote anafa 1 */
ch->anafanum = (guessnum == 3 ? 1 : (guessnum == 1 ? 3 : 2));
} else if (is_spine_2012(spineport->node)) {
ch->chassistype = ISR2012_CT;
ch->slotnum = line_slot_2_sfb12[spineport->portnum];
/* this is a smart guess based on nodeguids order on sFB-12 module */
guessnum = spineport->node->nodeguid % 4;
// module 1 <--> remote anafa 3
// module 2 <--> remote anafa 2
// module 3 <--> remote anafa 1
ch->anafanum = (guessnum == 3? 1 : (guessnum == 1 ? 3 : 2));
} else if (is_spine_2004(spineport->node)) {
ch->chassistype = ISR2004_CT;
ch->slotnum = line_slot_2_sfb4[spineport->portnum];
ch->anafanum = ipr_slot_2_sfb4_port[spineport->portnum];
} else {
IBPANIC("Unexpected node found: guid 0x%016" PRIx64, spineport->node->nodeguid);
}
}
void
madrpc_init(char *dev_name, int dev_port, int *mgmt_classes, int num_classes)
{
if (umad_init() < 0)
IBPANIC("can't init UMAD library");
if ((mad_portid = umad_open_port(dev_name, dev_port)) < 0)
IBPANIC("can't open UMAD port (%s:%d)", dev_name, dev_port);
if (num_classes >= MAX_CLASS)
IBPANIC("too many classes %d requested", num_classes);
while (num_classes--) {
int rmpp_version = 0;
int mgmt = *mgmt_classes++;
if (mgmt == IB_SA_CLASS)
rmpp_version = 1;
if (mad_register_client(mgmt, rmpp_version) < 0)
IBPANIC("client_register for mgmt class %d failed", mgmt);
}
}
static int get_spine_index(Node *node)
{
int retval;
if (is_spine_9288(node) || is_spine_2012(node))
retval = 3 * (node->chrecord->slotnum - 1) + node->chrecord->anafanum;
else
retval = node->chrecord->slotnum;
if (retval > SPINES_MAX_NUM || retval < 1)
IBPANIC("Internal error");
return retval;
}
static void add_chassislist()
{
if (!(mylist.current = calloc(1, sizeof(ChassisList))))
IBPANIC("out of mem");
if (mylist.first == NULL) {
mylist.first = mylist.current;
mylist.last = mylist.current;
} else {
mylist.last->next = mylist.current;
mylist.current->next = NULL;
mylist.last = mylist.current;
}
}
/*
This function called for every Voltaire node in fabric
It could be optimized so, but time overhead is very small
and its only diag.util
*/
static void fill_chassis_record(Node *node)
{
Port *port;
Node *remnode = 0;
ChassisRecord *ch = 0;
if (node->chrecord) /* somehow this node has already been passed */
return;
if (!(node->chrecord = calloc(1, sizeof(ChassisRecord))))
IBPANIC("out of mem");
ch = node->chrecord;
/* node is router only in case of using unique lid */
/* (which is lid of chassis router port) */
/* in such case node->ports is actually a requested port... */
if (is_router(node) && is_spine(node->ports->remoteport->node))
get_router_slot(node, node->ports->remoteport);
else if (is_spine(node)) {
for (port = node->ports; port; port = port->next) {
if (!port->remoteport)
continue;
remnode = port->remoteport->node;
if (remnode->type != SWITCH_NODE) {
if (!remnode->chrecord)
get_router_slot(remnode, port);
continue;
}
if (!ch->chassistype)
/* we assume here that remoteport belongs to line */
get_sfb_slot(node, port->remoteport);
/* we could break here, but need to find if more routers connected */
}
} else if (is_line(node)) {
for (port = node->ports; port; port = port->next) {
if (port->portnum > 12)
continue;
if (!port->remoteport)
continue;
/* we assume here that remoteport belongs to spine */
get_slb_slot(ch, port->remoteport);
break;
}
}
return;
}
/**
* initialize the in/out connections
*
* @param basename base name for abstract namespace
*
* @return unix status
*/
static int sim_init_conn(char *basename)
{
union name_t name;
size_t size;
int fd, i;
DEBUG("initializing network connections (basename \"%s\")", basename);
// create ctl channel
fd = simctl = socket(remote_mode ? PF_INET : PF_LOCAL, SOCK_DGRAM, 0);
if (fd < 0)
IBPANIC("can't create socket for ctl");
if (maxfd < fd)
maxfd = fd;
size = make_name(&name, 0, listen_to_port, "%s:ctl", basename);
if (bind(fd, (struct sockaddr *)&name, size) < 0)
IBPANIC("can't bind socket %d to name %s",
fd, get_name(&name));
if (fcntl(fd, F_SETFL, O_NONBLOCK) < 0)
IBPANIC("can't set non blocking flags for ctl");
for (i = 0; i < IBSIM_MAX_CLIENTS; i++) {
fd = socket(remote_mode ? PF_INET : PF_LOCAL, SOCK_DGRAM, 0);
if (fd < 0)
IBPANIC("can't create socket for conn %d", i);
if (maxfd < fd)
maxfd = fd;
size = make_name(&name, 0, listen_to_port + i + 1,
"%s:out%d", basename, i);
if (bind(fd, (struct sockaddr *)&name, size) < 0)
IBPANIC("can't bind socket %d to name %s",
fd, get_name(&name));
if (fcntl(fd, F_SETFL, O_NONBLOCK) < 0)
IBPANIC("can't set non blocking flags for "
"client conn %d", i);
DEBUG("opening net connection fd %d %s", fd, get_name(&name));
clients[i].fd = fd;
}
return 0;
}
static void get_slb_slot(ChassisRecord *ch, Port *spineport)
{
ch->chassisslot = LINE_CS;
if (is_spine_9096(spineport->node)) {
ch->chassistype = ISR9096_CT;
ch->slotnum = line_slot_2_sfb4[spineport->portnum];
ch->anafanum = anafa_line_slot_2_sfb4[spineport->portnum];
} else if (is_spine_9288(spineport->node)) {
ch->chassistype = ISR9288_CT;
ch->slotnum = line_slot_2_sfb12[spineport->portnum];
ch->anafanum = anafa_line_slot_2_sfb12[spineport->portnum];
} else if (is_spine_2012(spineport->node)) {
ch->chassistype = ISR2012_CT;
ch->slotnum = line_slot_2_sfb12[spineport->portnum];
ch->anafanum = anafa_line_slot_2_sfb12[spineport->portnum];
} else if (is_spine_2004(spineport->node)) {
ch->chassistype = ISR2004_CT;
ch->slotnum = line_slot_2_sfb4[spineport->portnum];
ch->anafanum = anafa_line_slot_2_sfb4[spineport->portnum];
} else {
IBPANIC("Unexpected node found: guid 0x%016" PRIx64, spineport->node->nodeguid);
}
}
int
main(int argc, char *argv[])
{
char names[UMAD_MAX_DEVICES][UMAD_CA_NAME_LEN];
int dev_port = -1;
int list_only = 0, short_format = 0, list_ports = 0;
int n, i;
static char const str_opts[] = "dlspVhu";
static const struct option long_opts[] = {
{ "debug", 0, 0, 'd'},
{ "list_of_cas", 0, 0, 'l'},
{ "short", 0, 0, 's'},
{ "port_list", 0, 0, 'p'},
{ "Version", 0, 0, 'V'},
{ "help", 0, 0, 'h'},
{ "usage", 0, 0, 'u'},
{ }
};
argv0 = argv[0];
while (1) {
int ch = getopt_long(argc, argv, str_opts, long_opts, NULL);
if ( ch == -1 )
break;
switch(ch) {
case 'd':
debug++;
break;
case 'l':
list_only++;
break;
case 's':
short_format++;
break;
case 'p':
list_ports++;
break;
case 'V':
fprintf(stderr, "%s %s\n", argv0, get_build_version() );
exit(-1);
default:
usage();
break;
}
}
argc -= optind;
argv += optind;
if (argc > 1)
dev_port = strtol(argv[1], 0, 0);
if (umad_init() < 0)
IBPANIC("can't init UMAD library");
if ((n = umad_get_cas_names(names, UMAD_MAX_DEVICES)) < 0)
IBPANIC("can't list IB device names");
if (argc) {
for (i = 0; i < n; i++)
if (!strncmp(names[i], argv[0], sizeof names[i]))
break;
if (i >= n)
IBPANIC("'%s' IB device can't be found", argv[0]);
strncpy(names[i], argv[0], sizeof names[i]);
n = 1;
}
if (list_ports) {
if (ports_list(names, n) < 0)
IBPANIC("can't list ports");
return 0;
}
if (!list_only && argc) {
if (ca_stat(argv[0], dev_port, short_format) < 0)
IBPANIC("stat of IB device '%s' failed", argv[0]);
return 0;
}
for (i = 0; i < n; i++) {
if (list_only)
printf("%s\n", names[i]);
else
if (ca_stat(names[i], -1, short_format) < 0)
IBPANIC("stat of IB device '%s' failed", names[i]);
}
return 0;
}
int main(int argc, char **argv)
{
extern int alloc_core(void);
extern void free_core(void);
char *outfname = 0, *netfile;
FILE *infile, *outfile;
int status;
static char const str_opts[] = "rf:dpvIsN:S:P:L:M:l:Vhu";
static const struct option long_opts[] = {
{"remote", 0, 0, 'r'},
{"file", 1, 0, 'f'},
{"Nodes", 1, 0, 'N'},
{"Switches", 1, 0, 'S'},
{"Ports", 1, 0, 'P'},
{"Linearcap", 1, 0, 'L'},
{"Mcastcap", 1, 0, 'M'},
{"listen", 1, 0, 'l'},
{"Ignoredups", 0, 0, 'I'},
{"start", 0, 0, 's'},
{"debug", 0, 0, 'd'},
{"parsedebug", 0, 0, 'p'},
{"verbose", 0, 0, 'v'},
{"Version", 0, 0, 'V'},
{"help", 0, 0, 'h'},
{"usage", 0, 0, 'u'},
{}
};
while (1) {
int ch = getopt_long(argc, argv, str_opts, long_opts, NULL);
if (ch == -1)
break;
switch (ch) {
case 'r':
remote_mode = 1;
break;
case 'f':
outfname = optarg;
break;
case 'd':
ibdebug++;
break;
case 'p':
parsedebug++;
break;
case 'v':
simverb++;
break;
case 's':
netstarted = 1;
break;
case 'I':
ignoreduplicate = 1;
break;
case 'N':
maxnetnodes = strtoul(optarg, 0, 0);
break;
case 'S':
maxnetswitches = strtoul(optarg, 0, 0);
break;
case 'P':
maxnetports = strtoul(optarg, 0, 0);
break;
case 'L':
maxlinearcap = strtoul(optarg, 0, 0);
break;
case 'M':
maxmcastcap = strtoul(optarg, 0, 0);
break;
case 'l':
listen_to_port = strtoul(optarg, 0, 0);
break;
case 'V':
default:
usage(argv[0]);
}
}
maxnetaliases = maxnetports;
infile = stdin;
outfile = stdout;
if (outfname && (outfile = fopen(outfname, "w")) == 0)
IBPANIC("can't open out file %s for write", outfname);
if (optind >= argc)
usage(argv[0]);
netfile = argv[optind];
if (alloc_core() < 0)
IBPANIC("not enough memory for core structure");
DEBUG("initializing net \"%s\"", netfile);
status = sim_init_net(netfile, outfile);
if (status < 0)
IBPANIC("sim_init failed, status %d", status);
sim_init_console(outfile);
sim_run(fileno(infile));
free_core();
exit(0);
}
static int sim_ctl_new_client(Client * cl, struct sim_ctl * ctl, union name_t *from)
{
union name_t name;
size_t size;
Node *node;
struct sim_client_info *scl = (void *)ctl->data;
int id = scl->id;
int i;
DEBUG("connecting client pid %d", id);
// allocated free client
for (i = 0; i < IBSIM_MAX_CLIENTS; i++) {
cl = clients + i;
if (!cl->pid)
break;
}
if (i >= IBSIM_MAX_CLIENTS) {
IBWARN("can't open new connection for client pid %d", id);
ctl->type = SIM_CTL_ERROR;
return -1;
}
if (scl->nodeid[0]) {
if (!(node = find_node(scl->nodeid)) &&
!(node = find_node_by_desc(scl->nodeid))) {
IBWARN("client %d attempt to attach to unknown host"
" \"%s\"", i, scl->nodeid);
ctl->type = SIM_CTL_ERROR;
return -1;
}
cl->port = node_get_port(node, 0);
VERB("Attaching client %d at node \"%s\" port 0x%" PRIx64,
i, node->nodeid, cl->port->portguid);
} else {
VERB("Attaching client %d at default node \"%s\" port 0x%"
PRIx64, i, default_port->node->nodeid,
default_port->portguid);
cl->port = default_port;
}
if (scl->issm && sm_exists(cl->port->node)) {
IBWARN("client %d (pid %d) connection attempt failed:"
" SM already exists on \"%s\"",
i, id, cl->port->node->nodeid);
ctl->type = SIM_CTL_ERROR;
return -1;
}
size = make_name(&name, from->name_i.sin_addr.s_addr, id,
"%s:in%d", socket_basename, id);
if (connect(cl->fd, (struct sockaddr *)&name, size) < 0)
IBPANIC("can't connect to in socket %s - fd %d client pid %d",
get_name(&name), cl->fd, id);
cl->pid = id;
cl->id = i;
cl->qp = scl->qp;
cl->issm = scl->issm;
strncpy(scl->nodeid, cl->port->node->nodeid, sizeof(scl->nodeid) - 1);
scl->id = i;
DEBUG("client %d (%s) is connected - fd %d",
i, get_name(&name), cl->fd);
return 1;
}
/*
Main grouping function
Algorithm:
1. pass on every Voltaire node
2. catch spine chip for every Voltaire node
2.1 build/interpolate chassis around this chip
2.2 go to 1.
3. pass on non Voltaire nodes (SystemImageGUID based grouping)
4. now group non Voltaire nodes by SystemImageGUID
*/
ChassisList *group_nodes()
{
Node *node;
int dist;
int chassisnum = 0;
struct ChassisList *chassis;
mylist.first = NULL;
mylist.current = NULL;
mylist.last = NULL;
/* first pass on switches and build for every Voltaire node */
/* an appropriate chassis record (slotnum and position) */
/* according to internal connectivity */
/* not very efficient but clear code so... */
for (dist = 0; dist <= maxhops_discovered; dist++) {
for (node = nodesdist[dist]; node; node = node->dnext) {
if (node->vendid == VTR_VENDOR_ID)
fill_chassis_record(node);
}
}
/* separate every Voltaire chassis from each other and build linked list of them */
/* algorithm: catch spine and find all surrounding nodes */
for (dist = 0; dist <= maxhops_discovered; dist++) {
for (node = nodesdist[dist]; node; node = node->dnext) {
if (node->vendid != VTR_VENDOR_ID)
continue;
if (!node->chrecord || node->chrecord->chassisnum || !is_spine(node))
continue;
add_chassislist();
mylist.current->chassisnum = ++chassisnum;
build_chassis(node, mylist.current);
}
}
/* now make pass on nodes for chassis which are not Voltaire */
/* grouped by common SystemImageGUID */
for (dist = 0; dist <= maxhops_discovered; dist++) {
for (node = nodesdist[dist]; node; node = node->dnext) {
if (node->vendid == VTR_VENDOR_ID)
continue;
if (node->sysimgguid) {
chassis = find_chassisguid(node);
if (chassis)
chassis->nodecount++;
else {
/* Possible new chassis */
add_chassislist();
mylist.current->chassisguid = get_chassisguid(node);
mylist.current->nodecount = 1;
}
}
}
}
/* now, make another pass to see which nodes are part of chassis */
/* (defined as chassis->nodecount > 1) */
for (dist = 0; dist <= MAXHOPS; ) {
for (node = nodesdist[dist]; node; node = node->dnext) {
if (node->vendid == VTR_VENDOR_ID)
continue;
if (node->sysimgguid) {
chassis = find_chassisguid(node);
if (chassis && chassis->nodecount > 1) {
if (!chassis->chassisnum)
chassis->chassisnum = ++chassisnum;
if (!node->chrecord) {
if (!(node->chrecord = calloc(1, sizeof(ChassisRecord))))
IBPANIC("out of mem");
node->chrecord->chassisnum = chassis->chassisnum;
}
}
}
}
if (dist == maxhops_discovered)
dist = MAXHOPS; /* skip to CAs */
else
dist++;
}
return (mylist.first);
}
int main(int argc, char *argv[])
{
char names[UMAD_MAX_DEVICES][UMAD_CA_NAME_LEN];
int dev_port = -1;
int n, i;
const struct ibdiag_opt opts[] = {
{"list_of_cas", 'l', 0, NULL, "list all IB devices"},
{"short", 's', 0, NULL, "short output"},
{"port_list", 'p', 0, NULL, "show port list"},
{0}
};
char usage_args[] = "<ca_name> [portnum]";
const char *usage_examples[] = {
"-l # list all IB devices",
"mthca0 2 # stat port 2 of 'mthca0'",
NULL
};
ibdiag_process_opts(argc, argv, NULL, "CDeGKLPsty", opts, process_opt,
usage_args, usage_examples);
argc -= optind;
argv += optind;
if (argc > 1)
dev_port = strtol(argv[1], 0, 0);
if (umad_init() < 0)
IBPANIC("can't init UMAD library");
if ((n = umad_get_cas_names(names, UMAD_MAX_DEVICES)) < 0)
IBPANIC("can't list IB device names");
if (argc) {
for (i = 0; i < n; i++)
if (!strncmp(names[i], argv[0], sizeof names[i]))
break;
if (i >= n)
IBPANIC("'%s' IB device can't be found", argv[0]);
strncpy(names[0], argv[0], sizeof(names[0])-1);
names[0][sizeof(names[0])-1] = '\0';
n = 1;
}
if (list_ports) {
if (ports_list(names, n) < 0)
IBPANIC("can't list ports");
return 0;
}
for (i = 0; i < n; i++) {
if (list_only)
printf("%s\n", names[i]);
else if (ca_stat(names[i], dev_port, short_format) < 0)
IBPANIC("stat of IB device '%s' failed", names[i]);
}
return 0;
}
