static int rereg_port_gid(int port, int agent, ib_portid_t * dport,
uint8_t * umad, int len, ibmad_gid_t port_gid)
{
uint8_t data[IB_SA_DATA_SIZE];
uint64_t comp_mask;
comp_mask = build_mcm_rec(data, mgid_ipoib, port_gid);
build_mcm_rec_umad(umad, dport, IB_MAD_METHOD_DELETE, comp_mask, data);
if (umad_send(port, agent, umad, len, TMO, 0) < 0) {
err("umad_send leave failed: %s\n", strerror(errno));
return -1;
}
dbg("umad_send leave: tid = 0x%016" PRIx64 "\n",
mad_get_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F));
build_mcm_rec_umad(umad, dport, IB_MAD_METHOD_SET, comp_mask, data);
if (umad_send(port, agent, umad, len, TMO, 0) < 0) {
err("umad_send join failed: %s\n", strerror(errno));
return -1;
}
dbg("umad_send join: tid = 0x%016" PRIx64 "\n",
mad_get_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F));
return 0;
}
static void getAllMCMemberRecords(AllocFunction alloc, void *allocUd,
int fd, int agent, UInt16 smLid,
SInt32 *numMRs, struct MCMemberRecord **MRs)
{
UInt8 umad[256];
UInt8 *buf = NULL;
SInt64 len;
SInt32 status;
SInt64 i, n;
SInt32 offset;
struct _MCMemberRecord *p;
const SInt32 timeout = 1000;
fillMAD_Get_MCMemberRecord(umad, sizeof(umad), smLid, 0, __LINE__);
libibumad_Send_MAD(fd, agent, umad, sizeof(umad), timeout, 0);
libibumad_Recv_MAD(alloc, allocUd, fd, &buf, &len, timeout);
status = mad_get_field(umad_get_mad(buf), 0, IB_MAD_STATUS_F);
if (UNLIKELY(0 != status)) {
FATAL("status is %d", status);
}
/* RMPP packets.
*/
offset = mad_get_field(umad_get_mad(buf), 0, IB_SA_ATTROFFS_F);
if (UNLIKELY(0 == offset)) {
FATAL("SA attribute offset is zero.");
}
n = (len - IB_SA_DATA_OFFS)/(offset << 3); /* offset is in 8 byte units */
*numMRs = n;
*MRs = alloc(allocUd, NULL, 0, (*numMRs)*sizeof(struct MCMemberRecord));
for (i = 0 ; i < n; ++i) {
p = (struct _MCMemberRecord *)((UInt8 *)umad_get_mad(buf) + IB_SA_DATA_OFFS + i*(offset << 3));
memcpy((*MRs)[i].mgid, p->mgid, 16);
memcpy((*MRs)[i].portGid, p->portGid, 16);
(*MRs)[i].qkey = ntoh32(p->qkey);
(*MRs)[i].mlid = ntoh16(p->mlid);
(*MRs)[i].mtu = p->mtu;
(*MRs)[i].tclass = p->tclass;
(*MRs)[i].pkey = ntoh16(p->pkey);
(*MRs)[i].rate = p->rate;
(*MRs)[i].packetLife = p->packetLife;
/* FIXME
(*MRs)[i].serviceLevel = p->serviceLevel;
(*MRs)[i].flowLabel = ntoh32(p->flowLabel);
(*MRs)[i].hopLimit = p->hopLimit;
*/
(*MRs)[i].scope = (p->scope_joinState & 0xf0) >> 4;
(*MRs)[i].joinState = p->scope_joinState & 0x0f;
(*MRs)[i].proxyJoin = p->proxyJoin;
}
buf = alloc(allocUd, buf, len, 0);
}
static int
_do_madrpc(int port_id, void *sndbuf, void *rcvbuf, int agentid, int len,
int timeout)
{
uint32_t trid; /* only low 32 bits */
int retries;
int length, status;
if (!timeout)
timeout = def_madrpc_timeout;
if (ibdebug > 1) {
IBWARN(">>> sending: len %d pktsz %zu", len, umad_size() + len);
xdump(stderr, "send buf\n", sndbuf, umad_size() + len);
}
if (save_mad) {
memcpy(save_mad, umad_get_mad(sndbuf),
save_mad_len < len ? save_mad_len : len);
save_mad = 0;
}
trid = mad_get_field64(umad_get_mad(sndbuf), 0, IB_MAD_TRID_F);
for (retries = 0; retries < madrpc_retries; retries++) {
if (retries) {
ERRS("retry %d (timeout %d ms)", retries, timeout);
}
length = len;
if (umad_send(port_id, agentid, sndbuf, length, timeout, 0) < 0) {
IBWARN("send failed; %s", strerror(errno));
return -1;
}
/* Use same timeout on receive side just in case */
/* send packet is lost somewhere. */
do {
if (umad_recv(port_id, rcvbuf, &length, timeout) < 0) {
IBWARN("recv failed: %s", strerror(errno));
return -1;
}
if (ibdebug > 1) {
IBWARN("rcv buf:");
xdump(stderr, "rcv buf\n", umad_get_mad(rcvbuf), IB_MAD_SIZE);
}
} while ((uint32_t)mad_get_field64(umad_get_mad(rcvbuf), 0, IB_MAD_TRID_F) != trid);
status = umad_status(rcvbuf);
if (!status)
return length; /* done */
if (status == ENOMEM)
return length;
}
ERRS("timeout after %d retries, %d ms", retries, timeout * retries);
return -1;
}
static int rereg_recv_all(int port, int agent, ib_portid_t * dport,
struct guid_trid *list, unsigned cnt)
{
uint8_t *umad, *mad;
int len = umad_size() + 256;
uint64_t trid;
unsigned n, method, status;
unsigned i;
info("rereg_recv_all...\n");
umad = calloc(1, len);
if (!umad) {
err("cannot alloc mem for umad: %s\n", strerror(errno));
return -1;
}
n = 0;
while (rereg_recv(port, agent, dport, umad, len, TMO) > 0) {
dbg("rereg_recv_all: done %d\n", n);
n++;
mad = umad_get_mad(umad);
method = mad_get_field(mad, 0, IB_MAD_METHOD_F);
status = mad_get_field(mad, 0, IB_MAD_STATUS_F);
if (status)
dbg("MAD status %x, method %x\n", status, method);
if (status &&
(method & 0x7f) == (IB_MAD_METHOD_GET_RESPONSE & 0x7f)) {
trid = mad_get_field64(mad, 0, IB_MAD_TRID_F);
for (i = 0; i < cnt; i++)
if (trid == list[i].trid)
break;
if (i == cnt) {
err("cannot find trid 0x%016" PRIx64 "\n",
trid);
continue;
}
info("guid 0x%016" PRIx64
": method = %x status = %x. Resending\n",
ntohll(list[i].guid), method, status);
rereg_port_gid(port, agent, dport, umad, len,
list[i].gid);
list[i].trid =
mad_get_field64(umad_get_mad(umad), 0,
IB_MAD_TRID_F);
}
}
info("rereg_recv_all: got %u responses\n", n);
free(umad);
return 0;
}
static void fillMAD_Get_LFTRecord(UInt8 *umad, SInt64 len, UInt16 smLid, UInt16 destLid, UInt64 trId)
{
UInt64 mask;
struct _LinearForwardingTableRecord lft;
memset(umad, 0, len);
umad_set_addr(umad, smLid, MAD_QP1, MAD_DEFAULT_SL, IB_DEFAULT_QP1_QKEY);
/* Ignore GRH */
mad_set_field (umad_get_mad(umad), 0, IB_MAD_METHOD_F, IB_MAD_METHOD_GET_TABLE);
mad_set_field (umad_get_mad(umad), 0, IB_MAD_CLASSVER_F, MAD_HEADER_CLASS_VERSION_SA);
mad_set_field (umad_get_mad(umad), 0, IB_MAD_MGMTCLASS_F, IB_SA_CLASS);
mad_set_field (umad_get_mad(umad), 0, IB_MAD_BASEVER_F, 1);
mad_set_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F, trId);
mad_set_field (umad_get_mad(umad), 0, IB_MAD_ATTRID_F, IB_SA_ATTR_LFTRECORD);
if (destLid > 0) {
memset(&lft, 0, sizeof(lft));
lft.lid = hton16(destLid);
mask = 0x1; /* lid */
memcpy((UInt8 *)umad_get_mad(umad) + IB_SA_DATA_OFFS, &lft, sizeof(lft));
}
mad_set_field64(umad_get_mad(umad), 0, IB_SA_COMPMASK_F, mask);
}
static void getOneLinearForwardingTable(AllocFunction alloc, void *allocUd,
int fd, int agent, UInt16 smLid,
UInt16 lid, struct LinearForwardingTable *LFT)
{
UInt8 umad[256];
UInt8 *buf = NULL;
SInt64 len;
SInt32 status;
SInt64 i, n;
SInt32 j, offset;
struct _LinearForwardingTableRecord *p;
const SInt32 timeout = 1000;
/* Build the transaction id from the lid to ensure that we have unique transaction ids
* when looping over multiple lids.
*/
fillMAD_Get_LFTRecord(umad, sizeof(umad), smLid, lid, __LINE__ + lid);
libibumad_Send_MAD(fd, agent, umad, sizeof(umad), timeout, 0);
libibumad_Recv_MAD(alloc, allocUd, fd, &buf, &len, timeout);
status = mad_get_field(umad_get_mad(buf), 0, IB_MAD_STATUS_F);
if (UNLIKELY(0 != status)) {
FATAL("status is %d", status);
}
/* RMPP packets.
*/
offset = mad_get_field(umad_get_mad(buf), 0, IB_SA_ATTROFFS_F);
if (UNLIKELY(0 == offset)) {
FATAL("SA attribute offset is zero.");
}
n = (len - IB_SA_DATA_OFFS)/(offset << 3); /* offset is in 8 byte units */
LFT->len = 64*n;
LFT->lft = alloc(allocUd, NULL, 0, LFT->len*sizeof(UInt16));
for (i = 0 ; i < n; ++i) {
p = (struct _LinearForwardingTableRecord *)((UInt8 *)umad_get_mad(buf) + IB_SA_DATA_OFFS + i*(offset << 3));
LFT->lid = ntoh16(p->lid);
for (j = 0; j < 64; ++j) {
LFT->lft[64*ntoh16(p->block) + j] = p->lft[j];
}
}
buf = alloc(allocUd, buf, len, 0);
}
static void fillMAD_Delete_MCMemberRecord(UInt8 *umad, SInt64 len, UInt16 smLid, UInt64 guid, UInt64 trId, UInt8 *mgid, UInt8 *portGid)
{
UInt64 mask = 0;
struct _MCMemberRecord mr;
memset(umad, 0, len);
umad_set_addr(umad, smLid, MAD_QP1, MAD_DEFAULT_SL, IB_DEFAULT_QP1_QKEY);
/* Ignore GRH */
mad_set_field (umad_get_mad(umad), 0, IB_MAD_METHOD_F, IB_MAD_METHOD_DELETE);
mad_set_field (umad_get_mad(umad), 0, IB_MAD_CLASSVER_F, MAD_HEADER_CLASS_VERSION_SA);
mad_set_field (umad_get_mad(umad), 0, IB_MAD_MGMTCLASS_F, IB_SA_CLASS);
mad_set_field (umad_get_mad(umad), 0, IB_MAD_BASEVER_F, 1);
mad_set_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F, trId);
mad_set_field (umad_get_mad(umad), 0, IB_MAD_ATTRID_F, IB_SA_ATTR_MCRECORD);
memset(&mr, 0, sizeof(mr));
memcpy(mr.mgid, mgid, 16);
mask |= (1 << 0);
memcpy(mr.portGid, portGid, 16);
mask |= (1 << 1);
mr.scope_joinState = 0x1;
/* Scope remains unset */
mask |= (1 << 16);
mask |= (1 << 17);
memcpy((UInt8 *)umad_get_mad(umad) + IB_SA_DATA_OFFS, &mr, sizeof(mr));
mad_set_field64(umad_get_mad(umad), 0, IB_SA_COMPMASK_F, mask);
}
static void getAllNodeRecords(AllocFunction alloc, void *allocUd,
int fd, int agent, UInt16 smLid,
SInt32 *numNRs, struct NodeRecord **NRs)
{
UInt8 umad[256];
UInt8 *buf = NULL;
SInt64 len;
SInt32 status;
SInt64 i, n;
SInt32 offset;
struct _NodeRecord *p;
const SInt32 timeout = 1000;
fillMAD_Get_NodeRecord(umad, sizeof(umad), smLid, 0, __LINE__);
libibumad_Send_MAD(fd, agent, umad, sizeof(umad), timeout, 0);
libibumad_Recv_MAD(alloc, allocUd, fd, &buf, &len, timeout);
status = mad_get_field(umad_get_mad(buf), 0, IB_MAD_STATUS_F);
if (UNLIKELY(0 != status)) {
FATAL("status is %d", status);
}
/* RMPP packets.
*/
offset = mad_get_field(umad_get_mad(buf), 0, IB_SA_ATTROFFS_F);
if (UNLIKELY(0 == offset)) {
FATAL("SA attribute offset is zero.");
}
n = (len - IB_SA_DATA_OFFS)/(offset << 3); /* offset is in 8 byte units */
*numNRs = n;
*NRs = alloc(allocUd, NULL, 0, (*numNRs)*sizeof(struct NodeRecord));
for (i = 0 ; i < n; ++i) {
p = (struct _NodeRecord *)((UInt8 *)umad_get_mad(buf) + IB_SA_DATA_OFFS + i*(offset << 3));
(*NRs)[i].lid = ntoh16(p->lid);
(*NRs)[i].nodeType = p->nodeType;
(*NRs)[i].numPorts = p->numPorts;
(*NRs)[i].sysGuid = ntoh64(p->sysGuid);
(*NRs)[i].nodeGuid = ntoh64(p->nodeGuid);
(*NRs)[i].portGuid = ntoh64(p->portGuid);
memcpy((*NRs)[i].info, p->info, 64*sizeof(UInt8));
}
buf = alloc(allocUd, buf, len, 0);
}
static void getOneNodeRecord(AllocFunction alloc, void *allocUd,
int fd, int agent, UInt16 smLid,
UInt16 lid, struct NodeRecord *NR)
{
UInt8 umad[256];
UInt8 *buf = NULL;
SInt64 len;
SInt32 status;
SInt64 n;
SInt32 offset;
struct _NodeRecord *p;
const SInt32 timeout = 1000;
fillMAD_Get_NodeRecord(umad, sizeof(umad), smLid, lid, __LINE__ + lid);
libibumad_Send_MAD(fd, agent, umad, sizeof(umad), timeout, 0);
libibumad_Recv_MAD(alloc, allocUd, fd, &buf, &len, timeout);
status = mad_get_field(umad_get_mad(buf), 0, IB_MAD_STATUS_F);
if (UNLIKELY(0 != status)) {
FATAL("status is %d", status);
}
/* RMPP packets.
*/
offset = mad_get_field(umad_get_mad(buf), 0, IB_SA_ATTROFFS_F);
if (UNLIKELY(0 == offset)) {
FATAL("SA attribute offset is zero.");
}
n = (len - IB_SA_DATA_OFFS)/(offset << 3); /* offset is in 8 byte units */
if (UNLIKELY(1 != n)) {
FATAL("Expected one node record but found %d in answer", n);
}
p = (struct _NodeRecord *)((UInt8 *)umad_get_mad(buf) + IB_SA_DATA_OFFS + 0*(offset << 3));
NR->lid = ntoh16(p->lid);
NR->nodeType = p->nodeType;
NR->numPorts = p->numPorts;
NR->sysGuid = ntoh64(p->sysGuid);
NR->nodeGuid = ntoh64(p->nodeGuid);
NR->portGuid = ntoh64(p->portGuid);
memcpy(NR->info, p->info, 64*sizeof(UInt8));
buf = alloc(allocUd, buf, len, 0);
}
static int rereg_send_all(int port, int agent, ib_portid_t * dport,
struct guid_trid *list, unsigned cnt)
{
uint8_t *umad;
int len = umad_size() + 256;
unsigned i;
int ret;
info("rereg_send_all... cnt = %u\n", cnt);
umad = calloc(1, len);
if (!umad) {
err("cannot alloc mem for umad: %s\n", strerror(errno));
return -1;
}
for (i = 0; i < cnt; i++) {
ret =
rereg_port_gid(port, agent, dport, umad, len, list[i].gid);
if (ret < 0) {
err("rereg_send_all: rereg_port_gid 0x%016" PRIx64
" failed\n", ntohll(list[i].guid));
continue;
}
list[i].trid = mad_get_field64(umad_get_mad(umad), 0,
IB_MAD_TRID_F);
}
info("rereg_send_all: sent %u requests\n", cnt * 2);
free(umad);
return 0;
}
void leaveMCGroup(AllocFunction alloc, void *allocUd, char *CA, SInt16 port, UInt8 *mgid)
{
int fd, agent;
UInt16 smLid;
UInt8 umad[256];
SInt32 status;
SInt32 err;
UInt8 *buf = NULL;
SInt64 len;
UInt8 portGid[16];
setup(CA, port, &fd, &agent);
smLid = subnetManagerLocalIdentifier(CA, port);
err = portGlobalIdentifier(CA, port, 0, portGid);
if (UNLIKELY(0 != err)) {
FATAL("portGlobalIdentifier() failed.");
}
const SInt32 timeout = 1000;
fillMAD_Delete_MCMemberRecord(umad, sizeof(umad), smLid, 0, __LINE__, mgid, portGid);
libibumad_Send_MAD(fd, agent, umad, sizeof(umad), timeout, 0);
libibumad_Recv_MAD(alloc, allocUd, fd, &buf, &len, timeout);
status = mad_get_field(umad_get_mad(buf), 0, IB_MAD_STATUS_F);
if (UNLIKELY(0 != status)) {
FATAL("status is %d", status);
}
done(fd);
buf = alloc(allocUd, buf, len, 0);
}
static char *ibping_serv(void)
{
void *umad;
void *mad;
char *data;
DEBUG("starting to serve...");
while ((umad = mad_receive_via(0, -1, srcport))) {
if (umad_status(umad) == 0) {
mad = umad_get_mad(umad);
data = (char *)mad + IB_VENDOR_RANGE2_DATA_OFFS;
memcpy(data, host_and_domain, IB_VENDOR_RANGE2_DATA_SIZE);
DEBUG("Pong: %s", data);
if (mad_respond_via(umad, 0, 0, srcport) < 0)
DEBUG("respond failed");
}
mad_free(umad);
}
DEBUG("server out");
return 0;
}
static char *ibsystat_serv(void)
{
void *umad;
void *mad;
int attr, mod, size;
DEBUG("starting to serve...");
while ((umad = mad_receive_via(buf, -1, srcport))) {
if (umad_status(buf)) {
DEBUG("drop mad with status %x: %s", umad_status(buf),
strerror(umad_status(buf)));
continue;
}
mad = umad_get_mad(umad);
attr = mad_get_field(mad, 0, IB_MAD_ATTRID_F);
mod = mad_get_field(mad, 0, IB_MAD_ATTRMOD_F);
DEBUG("got packet: attr 0x%x mod 0x%x", attr, mod);
size =
mk_reply(attr, (uint8_t *) mad + IB_VENDOR_RANGE2_DATA_OFFS,
sizeof(buf) - umad_size() -
IB_VENDOR_RANGE2_DATA_OFFS);
if (server_respond(umad, IB_VENDOR_RANGE2_DATA_OFFS + size) < 0)
DEBUG("respond failed");
}
DEBUG("server out");
return 0;
}
void *
mad_rpc_rmpp(const void *port_id, ib_rpc_t *rpc, ib_portid_t *dport,
ib_rmpp_hdr_t *rmpp, void *data)
{
const struct ibmad_port *p = port_id;
int status, len;
uint8_t sndbuf[1024], rcvbuf[1024], *mad;
memset(sndbuf, 0, umad_size() + IB_MAD_SIZE);
DEBUG("rmpp %p data %p", rmpp, data);
if ((len = mad_build_pkt(sndbuf, rpc, dport, rmpp, data)) < 0)
return 0;
if ((len = _do_madrpc(p->port_id, sndbuf, rcvbuf,
p->class_agents[rpc->mgtclass],
len, rpc->timeout)) < 0) {
IBWARN("_do_madrpc failed; dport (%s)", portid2str(dport));
return 0;
}
mad = umad_get_mad(rcvbuf);
if ((status = mad_get_field(mad, 0, IB_MAD_STATUS_F)) != 0) {
ERRS("MAD completed with error status 0x%x; dport (%s)",
status, portid2str(dport));
return 0;
}
if (ibdebug) {
IBWARN("data offs %d sz %d", rpc->dataoffs, rpc->datasz);
xdump(stderr, "rmpp mad data\n", mad + rpc->dataoffs,
rpc->datasz);
}
if (rmpp) {
rmpp->flags = mad_get_field(mad, 0, IB_SA_RMPP_FLAGS_F);
if ((rmpp->flags & 0x3) &&
mad_get_field(mad, 0, IB_SA_RMPP_VERS_F) != 1) {
IBWARN("bad rmpp version");
return 0;
}
rmpp->type = mad_get_field(mad, 0, IB_SA_RMPP_TYPE_F);
rmpp->status = mad_get_field(mad, 0, IB_SA_RMPP_STATUS_F);
DEBUG("rmpp type %d status %d", rmpp->type, rmpp->status);
rmpp->d1.u = mad_get_field(mad, 0, IB_SA_RMPP_D1_F);
rmpp->d2.u = mad_get_field(mad, 0, IB_SA_RMPP_D2_F);
}
if (data)
memcpy(data, mad + rpc->dataoffs, rpc->datasz);
rpc->recsz = mad_get_field(mad, 0, IB_SA_ATTROFFS_F);
return data;
}
static void getLIDFromGUID(AllocFunction alloc, void *allocUd,
int fd, int agent, UInt16 smLid, UInt64 guid, UInt16 *lid)
{
UInt8 umad[256];
UInt8 *buf = NULL;
SInt64 len;
SInt32 status;
const SInt32 timeout = 1000;
fillMAD_Get_PathRecord(umad, sizeof(umad), smLid, guid, __LINE__ + guid);
libibumad_Send_MAD(fd, agent, umad, sizeof(umad), timeout, 0);
libibumad_Recv_MAD(alloc, allocUd, fd, &buf, &len, timeout);
status = mad_get_field(umad_get_mad(buf), 0, IB_MAD_STATUS_F);
if (UNLIKELY(0 != status)) {
FATAL("status is %d", status);
}
mad_decode_field((UInt8 *)umad_get_mad(buf) + IB_SA_DATA_OFFS, IB_SA_PR_DLID_F, lid);
buf = alloc(allocUd, buf, len, 0);
}
static void fillMAD_Set_MCMemberRecord(UInt8 *umad, SInt64 len, UInt16 smLid, UInt64 guid, UInt64 trId, UInt8 *mgid, UInt8 *portGid)
{
UInt64 mask = 0;
struct _MCMemberRecord mr;
memset(umad, 0, len);
umad_set_addr(umad, smLid, MAD_QP1, MAD_DEFAULT_SL, IB_DEFAULT_QP1_QKEY);
/* Ignore GRH */
mad_set_field (umad_get_mad(umad), 0, IB_MAD_METHOD_F, IB_MAD_METHOD_SET);
mad_set_field (umad_get_mad(umad), 0, IB_MAD_CLASSVER_F, MAD_HEADER_CLASS_VERSION_SA);
mad_set_field (umad_get_mad(umad), 0, IB_MAD_MGMTCLASS_F, IB_SA_CLASS);
mad_set_field (umad_get_mad(umad), 0, IB_MAD_BASEVER_F, 1);
mad_set_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F, trId);
mad_set_field (umad_get_mad(umad), 0, IB_MAD_ATTRID_F, IB_SA_ATTR_MCRECORD);
memset(&mr, 0, sizeof(mr));
memcpy(mr.mgid, mgid, 16);
mask |= (1 << 0);
memcpy(mr.portGid, portGid, 16);
mask |= (1 << 1);
mr.qkey = hton32(0x1);
mask |= (1 << 2);
mr.pkey = hton16(0xffff);
mask |= (1 << 7);
/* SL */
mask |= (1 << 12);
/* FlowLabel */
mask |= (1 << 13);
/* Traffic class */
mask |= (1 << 6);
mr.scope_joinState = (0x5 << 4) | 0x1;
mask |= (1 << 15);
mask |= (1 << 16);
memcpy((UInt8 *)umad_get_mad(umad) + IB_SA_DATA_OFFS, &mr, sizeof(mr));
mad_set_field64(umad_get_mad(umad), 0, IB_SA_COMPMASK_F, mask);
}
/* Returns 0 (invalid base version) on error */
static uint8_t get_base_version_from_ni(int fd, uint32_t aid, int pkey_index)
{
uint8_t rc;
void *umad_p = NULL;
struct umad_smp *send_mad;
size_t length;
umad_p = umad_alloc(1, sizeof(*send_mad) + umad_size());
if (!umad_p) {
OUTPUT_ERROR ("can't alloc umad for OPA check; send_size %ld\n", sizeof(*send_mad));
return 0;
}
memset(umad_p, 0, sizeof(*send_mad) + umad_size());
umad_set_grh(umad_p, 0);
send_mad = umad_get_mad(umad_p);
send_mad->base_version = UMAD_BASE_VERSION;
send_mad->mgmt_class = UMAD_CLASS_SUBN_DIRECTED_ROUTE;
send_mad->class_version = 0x01;
send_mad->method = UMAD_METHOD_GET;
send_mad->tid = htonl(0xDEADBEEF);
send_mad->attr_id = htons(UMAD_SM_ATTR_NODE_INFO);
send_mad->dr_slid = 0xffff;
send_mad->dr_dlid = 0xffff;
umad_set_pkey(umad_p, pkey_index);
umad_set_addr(umad_p, 0xffff, 0, 0, 0);
rc = 0;
if (umad_send(fd, aid, umad_p, sizeof(*send_mad), 100, 1) < 0)
goto free_mad;
length = sizeof(*send_mad);
if (umad_recv(fd, umad_p, (int *)&length, 100) < 0)
goto free_mad;
if (length < sizeof(*send_mad))
goto free_mad;
if (umad_status(umad_p) != 0)
goto free_mad;
rc = ((NODE_INFO *)(send_mad->data))->BaseVersion;
free_mad:
free(umad_p);
return rc;
}
static int server_respond(void *umad, int size)
{
ib_rpc_t rpc = { 0 };
ib_rmpp_hdr_t rmpp = { 0 };
ib_portid_t rport;
uint8_t *mad = umad_get_mad(umad);
ib_mad_addr_t *mad_addr;
if (!(mad_addr = umad_get_mad_addr(umad)))
return -1;
memset(&rport, 0, sizeof(rport));
rport.lid = ntohs(mad_addr->lid);
rport.qp = ntohl(mad_addr->qpn);
rport.qkey = ntohl(mad_addr->qkey);
rport.sl = mad_addr->sl;
if (!rport.qkey && rport.qp == 1)
rport.qkey = IB_DEFAULT_QP1_QKEY;
rpc.mgtclass = mad_get_field(mad, 0, IB_MAD_MGMTCLASS_F);
rpc.method = IB_MAD_METHOD_GET | IB_MAD_RESPONSE;
rpc.attr.id = mad_get_field(mad, 0, IB_MAD_ATTRID_F);
rpc.attr.mod = mad_get_field(mad, 0, IB_MAD_ATTRMOD_F);
rpc.oui = mad_get_field(mad, 0, IB_VEND2_OUI_F);
rpc.trid = mad_get_field64(mad, 0, IB_MAD_TRID_F);
if (size > IB_MAD_SIZE)
rmpp.flags = IB_RMPP_FLAG_ACTIVE;
DEBUG("responding %d bytes to %s, attr 0x%x mod 0x%x qkey %x",
size, portid2str(&rport), rpc.attr.id, rpc.attr.mod, rport.qkey);
if (mad_build_pkt(umad, &rpc, &rport, &rmpp, 0) < 0)
return -1;
if (ibdebug > 1)
xdump(stderr, "mad respond pkt\n", mad, IB_MAD_SIZE);
if (umad_send(mad_rpc_portid(srcport),
mad_rpc_class_agent(srcport, rpc.mgtclass), umad, size,
rpc.timeout, 0) < 0) {
DEBUG("send failed; %m");
return -1;
}
return 0;
}
static int rereg_query_all(int port, int agent, ib_portid_t * dport,
struct guid_trid *list, unsigned cnt)
{
uint8_t *umad, *mad;
int len = umad_size() + 256;
unsigned method, status;
unsigned i;
int ret;
info("rereg_query_all...\n");
umad = calloc(1, len);
if (!umad) {
err("cannot alloc mem for umad: %s\n", strerror(errno));
return -1;
}
for (i = 0; i < cnt; i++) {
ret = rereg_send(port, agent, dport, umad, len,
IB_MAD_METHOD_GET, list[i].gid);
if (ret < 0) {
err("query_all: rereg_send failed.\n");
continue;
}
ret = rereg_recv(port, agent, dport, umad, len, TMO);
if (ret < 0) {
err("query_all: rereg_recv failed.\n");
continue;
}
mad = umad_get_mad(umad);
method = mad_get_field(mad, 0, IB_MAD_METHOD_F);
status = mad_get_field(mad, 0, IB_MAD_STATUS_F);
if (status)
info("guid 0x%016" PRIx64 ": status %x, method %x\n",
ntohll(list[i].guid), status, method);
}
info("rereg_query_all: %u queried.\n", cnt);
free(umad);
return 0;
}
static char *ibsystat(ib_portid_t * portid, int attr)
{
ib_rpc_t rpc = { 0 };
int fd, agent, timeout, len;
void *data = (uint8_t *) umad_get_mad(buf) + IB_VENDOR_RANGE2_DATA_OFFS;
DEBUG("Sysstat ping..");
rpc.mgtclass = IB_VENDOR_OPENIB_SYSSTAT_CLASS;
rpc.method = IB_MAD_METHOD_GET;
rpc.attr.id = attr;
rpc.attr.mod = 0;
rpc.oui = oui;
rpc.timeout = 0;
rpc.datasz = IB_VENDOR_RANGE2_DATA_SIZE;
rpc.dataoffs = IB_VENDOR_RANGE2_DATA_OFFS;
portid->qp = 1;
if (!portid->qkey)
portid->qkey = IB_DEFAULT_QP1_QKEY;
if ((len = mad_build_pkt(buf, &rpc, portid, NULL, NULL)) < 0)
IBPANIC("cannot build packet.");
fd = mad_rpc_portid(srcport);
agent = mad_rpc_class_agent(srcport, rpc.mgtclass);
timeout = ibd_timeout ? ibd_timeout : MAD_DEF_TIMEOUT_MS;
if (umad_send(fd, agent, buf, len, timeout, 0) < 0)
IBPANIC("umad_send failed.");
len = sizeof(buf) - umad_size();
if (umad_recv(fd, buf, &len, timeout) < 0)
IBPANIC("umad_recv failed.");
if (umad_status(buf))
return strerror(umad_status(buf));
DEBUG("Got sysstat pong..");
if (attr != IB_PING_ATTR)
puts(data);
else
printf("sysstat ping succeeded\n");
return 0;
}
static void fillMAD_Get_MCMemberRecord(UInt8 *umad, SInt64 len, UInt16 smLid, UInt64 guid, UInt64 trId)
{
memset(umad, 0, len);
umad_set_addr(umad, smLid, MAD_QP1, MAD_DEFAULT_SL, IB_DEFAULT_QP1_QKEY);
/* Ignore GRH */
mad_set_field (umad_get_mad(umad), 0, IB_MAD_METHOD_F, IB_MAD_METHOD_GET_TABLE);
mad_set_field (umad_get_mad(umad), 0, IB_MAD_CLASSVER_F, MAD_HEADER_CLASS_VERSION_SA);
mad_set_field (umad_get_mad(umad), 0, IB_MAD_MGMTCLASS_F, IB_SA_CLASS);
mad_set_field (umad_get_mad(umad), 0, IB_MAD_BASEVER_F, 1);
mad_set_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F, trId);
mad_set_field (umad_get_mad(umad), 0, IB_MAD_ATTRID_F, IB_SA_ATTR_MCRECORD);
}
static int rereg_send(int port, int agent, ib_portid_t * dport,
uint8_t * umad, int len, int method, ibmad_gid_t port_gid)
{
uint8_t data[IB_SA_DATA_SIZE];
uint64_t comp_mask;
comp_mask = build_mcm_rec(data, mgid_ipoib, port_gid);
build_mcm_rec_umad(umad, dport, method, comp_mask, data);
if (umad_send(port, agent, umad, len, TMO, 0) < 0) {
err("umad_send %s failed: %s\n",
(method == IB_MAD_METHOD_GET) ? "query" : "non query",
strerror(errno));
return -1;
}
dbg("umad_send %d: tid = 0x%016" PRIx64 "\n", method,
mad_get_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F));
return 0;
}
static void fillMAD_Get_PathRecord(UInt8 *umad, SInt64 len, UInt16 smLid, UInt64 guid, UInt64 trId)
{
UInt8 dgid[16];
memset(umad, 0, len);
umad_set_addr(umad, smLid, MAD_QP1, MAD_DEFAULT_SL, IB_DEFAULT_QP1_QKEY);
/* Ignore GRH */
mad_set_field (umad_get_mad(umad), 0, IB_MAD_METHOD_F, IB_MAD_METHOD_GET);
mad_set_field (umad_get_mad(umad), 0, IB_MAD_CLASSVER_F, MAD_HEADER_CLASS_VERSION_SA);
mad_set_field (umad_get_mad(umad), 0, IB_MAD_MGMTCLASS_F, IB_SA_CLASS);
mad_set_field (umad_get_mad(umad), 0, IB_MAD_BASEVER_F, 1);
mad_set_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F, trId);
mad_set_field (umad_get_mad(umad), 0, IB_MAD_ATTRID_F, IB_SA_ATTR_PATHRECORD);
mad_set_field64(umad_get_mad(umad), 0, IB_SA_COMPMASK_F, 0x4);
mad_set_field64(dgid, 0, IB_GID_PREFIX_F, IB_DEFAULT_SUBN_PREFIX);
mad_set_field64(dgid, 0, IB_GID_GUID_F , guid);
/* 128 bit data
*/
mad_encode_field((UInt8 *)umad_get_mad(umad) + IB_SA_DATA_OFFS, IB_SA_PR_DGID_F, dgid);
}
void *
mad_rpc(const void *port_id, ib_rpc_t *rpc, ib_portid_t *dport, void *payload,
void *rcvdata)
{
const struct ibmad_port *p = port_id;
int status, len;
uint8_t sndbuf[1024], rcvbuf[1024], *mad;
len = 0;
memset(sndbuf, 0, umad_size() + IB_MAD_SIZE);
if ((len = mad_build_pkt(sndbuf, rpc, dport, 0, payload)) < 0)
return 0;
if ((len = _do_madrpc(p->port_id, sndbuf, rcvbuf,
p->class_agents[rpc->mgtclass],
len, rpc->timeout)) < 0) {
IBWARN("_do_madrpc failed; dport (%s)", portid2str(dport));
return 0;
}
mad = umad_get_mad(rcvbuf);
if ((status = mad_get_field(mad, 0, IB_DRSMP_STATUS_F)) != 0) {
ERRS("MAD completed with error status 0x%x; dport (%s)",
status, portid2str(dport));
return 0;
}
if (ibdebug) {
IBWARN("data offs %d sz %d", rpc->dataoffs, rpc->datasz);
xdump(stderr, "mad data\n", mad + rpc->dataoffs, rpc->datasz);
}
if (rcvdata)
memcpy(rcvdata, mad + rpc->dataoffs, rpc->datasz);
return rcvdata;
}
static int rereg_recv(int port, int agent, ib_portid_t * dport,
uint8_t * umad, int length, int tmo)
{
int ret, retry = 0;
int len = length;
while ((ret = umad_recv(port, umad, &len, tmo)) < 0 &&
errno == ETIMEDOUT) {
if (retry++ > 3)
return 0;
}
if (ret < 0) {
err("umad_recv %d failed: %s\n", ret, strerror(errno));
return -1;
}
dbg("umad_recv (retries %d), tid = 0x%016" PRIx64
": len = %d, status = %d\n", retry,
mad_get_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F), len,
umad_status(umad));
return 1;
}
/******************************************************************************
* join_multicast_group
******************************************************************************/
int join_multicast_group(subn_adm_method method,struct mcast_parameters *params) {
int portid = -1;
int agentid = -1;
void *umad_buff = NULL;
void *mad = NULL;
int length = MAD_SIZE;
int test_result = 0;
// mlid will be assigned to the new LID after the join
if (umad_init() < 0) {
fprintf(stderr, "failed to init the UMAD library\n");
goto cleanup;
}
/* use casting to loose the "const char0 *" */
portid = umad_open_port((char*)params->ib_devname,params->ib_port);
if (portid < 0) {
fprintf(stderr,"failed to open UMAD port %d\n",params->ib_port);
goto cleanup;
}
agentid = umad_register(portid,MANAGMENT_CLASS_SUBN_ADM, 2, 0, 0);
if (agentid < 0) {
fprintf(stderr,"failed to register UMAD agent for MADs\n");
goto cleanup;
}
umad_buff = umad_alloc(1, umad_size() + MAD_SIZE);
if (!umad_buff) {
fprintf(stderr, "failed to allocate MAD buffer\n");
goto cleanup;
}
mad = umad_get_mad(umad_buff);
prepare_mcast_mad(method,params,(struct sa_mad_packet_t *)mad);
if (umad_set_addr(umad_buff,params->sm_lid,1,params->sm_sl,QP1_WELL_KNOWN_Q_KEY) < 0) {
fprintf(stderr, "failed to set the destination address of the SMP\n");
goto cleanup;
}
if (umad_send(portid,agentid,umad_buff,MAD_SIZE,100,5) < 0) {
fprintf(stderr, "failed to send MAD\n");
goto cleanup;
}
if (umad_recv(portid,umad_buff,&length,5000) < 0) {
fprintf(stderr, "failed to receive MAD response\n");
goto cleanup;
}
if (check_mad_status((struct sa_mad_packet_t*)mad)) {
fprintf(stderr, "failed to get mlid from MAD\n");
goto cleanup;
}
// "Join multicast group" message was sent
if (method == SUBN_ADM_METHOD_SET) {
get_mlid_from_mad((struct sa_mad_packet_t*)mad,¶ms->mlid);
params->mcast_state |= MCAST_IS_JOINED;
// "Leave multicast group" message was sent
} else {
params->mcast_state &= ~MCAST_IS_JOINED;
}
cleanup:
if (umad_buff)
umad_free(umad_buff);
if (portid >= 0) {
if (agentid >= 0) {
if (umad_unregister(portid, agentid)) {
fprintf(stderr, "failed to deregister UMAD agent for MADs\n");
test_result = 1;
}
}
if (umad_close_port(portid)) {
fprintf(stderr, "failed to close UMAD portid\n");
test_result = 1;
}
}
return test_result;
}
static void getAllLinearForwardingTables(AllocFunction alloc, void *allocUd,
int fd, int agent, UInt16 smLid,
SInt32 *numLFTs, struct LinearForwardingTable **LFTs)
{
UInt8 umad[256];
UInt8 *buf = NULL;
SInt64 len;
SInt32 status;
SInt64 i, n;
SInt32 j, offset;
struct _LinearForwardingTableRecord *p;
struct LinearForwardingTable *current;
const SInt32 timeout = 1000;
fillMAD_Get_LFTRecord(umad, sizeof(umad), smLid, 0, __LINE__);
libibumad_Send_MAD(fd, agent, umad, sizeof(umad), timeout, 0);
libibumad_Recv_MAD(alloc, allocUd, fd, &buf, &len, timeout);
status = mad_get_field(umad_get_mad(buf), 0, IB_MAD_STATUS_F);
if (UNLIKELY(0 != status)) {
FATAL("status is %d", status);
}
/* RMPP packets.
*/
offset = mad_get_field(umad_get_mad(buf), 0, IB_SA_ATTROFFS_F);
if (UNLIKELY(0 == offset)) {
FATAL("SA attribute offset is zero.");
}
n = (len - IB_SA_DATA_OFFS)/(offset << 3); /* offset is in 8 byte units */
j = 0;
for (i = 0 ; i < n; ++i) {
p = (struct _LinearForwardingTableRecord *)((UInt8 *)umad_get_mad(buf) + IB_SA_DATA_OFFS + i*(offset << 3));
j += (0 == p->block);
}
*numLFTs = j;
*LFTs = alloc(allocUd, NULL, 0, (*numLFTs)*sizeof(struct LinearForwardingTable));
/* All LFTs have the same size.
*/
j = 64*n/(*numLFTs);
for (i = 0; i < (*numLFTs); ++i) {
(*LFTs)[i].len = j;
(*LFTs)[i].lft = alloc(allocUd, NULL, 0, j*sizeof(UInt16));
}
current = NULL;
for (i = 0 ; i < n; ++i) {
p = (struct _LinearForwardingTableRecord *)((UInt8 *)umad_get_mad(buf) + IB_SA_DATA_OFFS + i*(offset << 3));
if (0 == p->block) {
if (NULL == current) {
current = (*LFTs);
} else {
++current;
}
current->lid = ntoh16(p->lid);
}
for (j = 0; j < 64; ++j) {
current->lft[64*ntoh16(p->block) + j] = p->lft[j];
}
}
buf = alloc(allocUd, buf, len, 0);
}
/** ========================================================================= */
FSTATUS oib_recv_mad_no_alloc(struct oib_port *port, uint8_t *recv_mad, size_t *recv_size,
int timeout_ms, struct oib_mad_addr *addr)
{
size_t length = *recv_size;
ib_user_mad_t *umad = NULL;
int mad_agent;
uint32_t my_umad_status = 0;
FSTATUS status = FSUCCESS;
if (!port || !recv_mad || !*recv_size)
return FINVALID_PARAMETER;
umad = umad_alloc(1, length + umad_size());
if (!umad) {
OUTPUT_ERROR ("can't alloc umad length %ld\n", length);
status = FINSUFFICIENT_MEMORY;
goto done;
}
retry:
mad_agent = umad_recv(port->umad_fd, umad, (int *)&length, timeout_ms);
// There are 4 combinations:
// assorted errors: mad_agent < 0, length <= MAD_SIZE
// large RMPP response: mad_agent < 0, length > MAD_SIZE, umad_status==0
// got response: mad_agent >= 0, length <= MAD_SIZE, umad_status==0
// no response: mad_agent >= 0, length <= MAD_SIZE, umad_status == error
if (mad_agent < 0) {
if (length <= *recv_size) {
// no MAD returned. None available.
DBGPRINT ("recv error on umad (size %zu) (%s)\n", *recv_size,
strerror(errno));
if (errno == EINTR)
goto retry;
status = (errno == ETIMEDOUT) ? FNOT_DONE:FERROR;
goto done;
} else {
// this routine is not expecting large responses
OUTPUT_ERROR ("Rx Packet size %zu larger than mad-size %zu\n",
length, *recv_size);
status = FOVERRUN;
if (recv_mad)
memcpy(recv_mad, umad_get_mad(umad), *recv_size);
// Clean out Rx packet 'cause it will never go away..
umad_free(umad);
umad = umad_alloc(1, umad_size() + length);
if (!umad) {
OUTPUT_ERROR ("can't alloc umad for rx cleanup, length %ld\n", length);
status = FINSUFFICIENT_MEMORY;
goto done;
}
// just to be safe, we supply a timeout. However it
// should be unnecessary since we know we have a packet
retry2:
if (umad_recv(port->umad_fd, umad, (int *)&length, OIB_UTILS_DEF_TIMEOUT_MS) < 0) {
OUTPUT_ERROR ("recv error on cleanup, length %ld (%s)\n", length,
strerror(errno));
if (errno == EINTR)
goto retry2;
goto done;
}
if (dbg_file) {
umad_dump(umad);
oib_dump_mad(dbg_file, umad_get_mad(umad), length, "rcv mad discarded\n");
}
goto done;
}
}
if (mad_agent >= UMAD_CA_MAX_AGENTS) {
OUTPUT_ERROR ("invalid mad agent %d\n", mad_agent);
status = FERROR;
goto done;
}
my_umad_status = umad_status(umad);
DBGPRINT("UMAD Status: %s (%d)\n", strerror(my_umad_status), my_umad_status);
if (my_umad_status != 0) {
status = (my_umad_status == ETIMEDOUT) ? FTIMEOUT : FREJECT;
}
DBGPRINT("Received MAD: Agent %d, length=%ld\n", mad_agent, length);
if (dbg_file) {
umad_dump(umad);
oib_dump_mad(dbg_file, umad_get_mad(umad), length, "rcv mad\n");
}
// Copy the data
if (recv_mad && length > 0) {
*recv_size = length;
memcpy(recv_mad, umad_get_mad(umad), length);
}
if (addr != NULL) {
addr->lid = IB2STL_LID(ntoh16(umad->addr.lid));
addr->sl = umad->addr.sl;
addr->qkey = ntoh32(umad->addr.qkey);
addr->qpn = ntoh32(umad->addr.qpn);
addr->pkey = oib_find_pkey_from_idx(port, umad_get_pkey(umad));
}
done:
if (umad != NULL) {
umad_free(umad);
}
return status;
}
/** ========================================================================= */
FSTATUS oib_recv_mad_alloc(struct oib_port *port, uint8_t **recv_mad, size_t *recv_size,
int timeout_ms, struct oib_mad_addr *addr)
{
#define STL_MAD_SIZE 2048 // Note, hack should reference size value OFED standard header file
FSTATUS status = FSUCCESS;
ib_user_mad_t *umad = NULL;
int mad_agent;
uint32_t my_umad_status = 0;
size_t length;
if (!port || !recv_mad || !recv_size)
return FINVALID_PARAMETER;
length = STL_MAD_SIZE;
umad = umad_alloc(1, umad_size() + length);
if (!umad) {
OUTPUT_ERROR("can't alloc MAD sized umad\n");
status = FINSUFFICIENT_MEMORY;
goto done;
}
retry:
mad_agent = umad_recv(port->umad_fd, umad, (int *)&length, timeout_ms);
// There are 4 combinations:
// assorted errors: mad_agent < 0, length <= MAD_SIZE
// large RMPP response: mad_agent < 0, length > MAD_SIZE, umad_status==0
// got response: mad_agent >= 0, length <= MAD_SIZE, umad_status==0
// no response: mad_agent >= 0, length <= MAD_SIZE, umad_status == error
if (mad_agent < 0) {
if (length <= STL_MAD_SIZE) {
// no MAD returned. None available.
DBGPRINT ("recv error on MAD sized umad (%s) length=%ld\n",
strerror(errno), length);
if (errno == EINTR)
goto retry;
status = (errno == ETIMEDOUT)?FNOT_DONE:FERROR;
goto done;
} else {
/* Need a larger buffer for RMPP */
DBGPRINT ("Received 1st MAD length=%ld\n",length);
umad_free(umad);
umad = umad_alloc(1, umad_size() + length);
if (!umad) {
OUTPUT_ERROR ("can't alloc umad length %ld\n", length);
status = FINSUFFICIENT_MEMORY;
goto done;
}
// just to be safe, we supply a timeout. However it
// should be unnecessary since we know we have a packet
retry2:
if ((mad_agent = umad_recv(port->umad_fd, umad, (int *)&length, OIB_UTILS_DEF_TIMEOUT_MS)) < 0) {
OUTPUT_ERROR ("recv error on umad length %ld (%s)\n", length, strerror(errno));
if (errno == EINTR)
goto retry2;
status = FOVERRUN;
*recv_size = length;
goto done;
}
}
}
if (mad_agent >= UMAD_CA_MAX_AGENTS) {
OUTPUT_ERROR ("invalid mad agent %d - dropping\n", mad_agent);
status = FERROR;
goto done;
}
my_umad_status = umad_status(umad);
DBGPRINT("UMAD Status: %s (%d)\n", strerror(my_umad_status), my_umad_status);
if (my_umad_status != 0) {
status = (my_umad_status == ETIMEDOUT) ? FTIMEOUT : FREJECT;
}
DBGPRINT("Received MAD length=%ld, total umad size=%ld\n",length, length + umad_size());
if (dbg_file) {
struct umad_hdr * umad_hdr = (struct umad_hdr *)umad_get_mad(umad);
DBGPRINT(" Base_Version 0x%x Class 0x%x Method 0x%x attrId 0x%x attr_mod 0x%x status 0x%x\n",
umad_hdr->base_version, umad_hdr->mgmt_class, umad_hdr->method,
umad_hdr->attr_id, umad_hdr->attr_mod, umad_hdr->status);
umad_dump(umad);
oib_dump_mad(dbg_file, umad_get_mad(umad), length, "rcv mad\n");
}
// Allocate and copy to new buffer.
*recv_mad = calloc (1, length);
if (*recv_mad == NULL) {
OUTPUT_ERROR ("can't alloc return buffer length %ld\n", length);
status = FINSUFFICIENT_MEMORY;
goto done;
}
memcpy (*recv_mad, umad_get_mad(umad), length);
*recv_size = length;
if (addr != NULL) {
addr->lid = IB2STL_LID(ntoh16(umad->addr.lid));
addr->sl = umad->addr.sl;
addr->qkey = ntoh32(umad->addr.qkey);
addr->qpn = ntoh32(umad->addr.qpn);
addr->pkey = oib_find_pkey_from_idx(port, umad_get_pkey(umad));
}
done:
if (umad != NULL) {
umad_free(umad);
}
return status;
}
/** ========================================================================= */
FSTATUS oib_send_mad2(struct oib_port *port, uint8_t *send_mad, size_t send_size,
struct oib_mad_addr *addr, int timeout_ms, int retries)
{
FSTATUS status = FSUCCESS;
void *umad_p = NULL;
int response;
uint8_t mclass, class_ver;
int aid;
int correctedTimeout;
struct umad_hdr *mad_hdr = (struct umad_hdr *)send_mad;
uint16_t ib_lid;
int pkey_idx;
size_t padded_size;
if (!port || !send_mad || !send_size || !addr)
return FINVALID_PARAMETER;
ib_lid = addr->lid & 0xffff;
// Make sure we are registered for this class/version...
mclass = mad_hdr->mgmt_class;
class_ver = mad_hdr->class_version;
response = (mad_hdr->method & 0x80)
|| (mad_hdr->method == UMAD_METHOD_TRAP_REPRESS)
|| (mclass == UMAD_CLASS_BM &&
ntohl(mad_hdr->attr_mod) & BM_ATTRIB_MOD_RESPONSE);
aid = port->umad_agents[class_ver][mclass];
DBGPRINT (" Management Class 0x%x method 0x%x attrId 0x%x attrM 0x%x\n",mclass, mad_hdr->method,
ntohs(mad_hdr->attr_id), ntohl(mad_hdr->attr_mod));
DBGPRINT (" base_version 0x%x class_version 0x%x\n",mad_hdr->base_version, mad_hdr->class_version);
if (aid == OIB_INVALID_AGENTID) {
// automatically register for "send" only
int err = 0;
struct oib_class_args mgmt_class[2];
memset(mgmt_class, 0, sizeof(mgmt_class));
mgmt_class[0].base_version = mad_hdr->base_version;
mgmt_class[0].mgmt_class = mad_hdr->mgmt_class;
mgmt_class[0].class_version = mad_hdr->class_version;
mgmt_class[0].is_responding_client = 0;
mgmt_class[0].is_trap_client = 0;
mgmt_class[0].is_report_client = 0;
mgmt_class[0].kernel_rmpp = 1;
mgmt_class[0].use_methods = 0;
DBGPRINT ("auto registering class 0x%02x; version 0x%x for send only\n",
mclass, class_ver);
if ((err = oib_bind_classes(port, mgmt_class)) != 0) {
OUTPUT_ERROR ("Failed to auto register for class 0x%02x: %s\n",
mclass, strerror(err));
status = FERROR;
goto done;
}
aid = port->umad_agents[class_ver][mclass];
}
// Initialize the user mad.
// umad has limititation that outgoing packets must be > 36 bytes.
padded_size = ( MAX(send_size,36) + 7) & ~0x7;
DBGPRINT ("dlid %d qpn %d qkey %x sl %d\n", ib_lid, addr->qpn, addr->qkey, addr->sl);
umad_p = umad_alloc(1, padded_size + umad_size());
if (!umad_p) {
OUTPUT_ERROR ("can't alloc umad send_size %ld\n", padded_size);
status = FINSUFFICIENT_MEMORY;
goto done;
}
memset(umad_p, 0, padded_size + umad_size());
memcpy (umad_get_mad(umad_p), send_mad, send_size); /* Copy mad to umad */
umad_set_grh(umad_p, 0);
pkey_idx = oib_find_pkey(port, addr->pkey);
if (pkey_idx < 0) {
DBGPRINT("P_Key 0x%x not found in pkey table\n", addr->pkey);
if (addr->pkey == 0xffff) {
pkey_idx = oib_find_pkey(port, 0x7fff);
if (pkey_idx < 0) {
OUTPUT_ERROR("Failed to find 0x7fff pkey defaulting to index 1\n");
pkey_idx = 1;
} else {
DBGPRINT("... using 0x7fff found at index %d\n", pkey_idx);
}
} else {
// Previously, this code would try to find the limited management pkey
// if it could not find the requested pkey, and use that pkey instead.
// This would often "work" because all nodes should have the limited
// management pkey, but b/c it was a limited member, this would result
// in potential timeouts - especially where the full management pkey was
// required.
// Changed this code fail immediately without retrying a new pkey.
OUTPUT_ERROR("Failed to find requested pkey:0x%x, class 0x%x aid:0x%x \n",
addr->pkey, mclass, ntohs(mad_hdr->attr_id));
status = FPROTECTION;
goto done;
}
}
umad_set_pkey(umad_p, pkey_idx);
umad_set_addr(umad_p, ib_lid?ib_lid:0xffff, addr->qpn, addr->sl, addr->qkey);
if (dbg_file) {
DBGPRINT(">>> sending: len %ld pktsz %zu\n", send_size, umad_size() + padded_size);
umad_dump(umad_p);
oib_dump_mad(dbg_file, umad_get_mad(umad_p), send_size, "send mad\n");
}
correctedTimeout = (timeout_ms == OIB_SEND_TIMEOUT_DEFAULT)
? OIB_UTILS_DEF_TIMEOUT_MS : timeout_ms;
if (umad_send(port->umad_fd, aid, umad_p, padded_size, (response ? 0 : correctedTimeout), retries) < 0) {
OUTPUT_ERROR("send failed; %s, agent id %u MClass 0x%x method 0x%x attrId 0x%x attrM 0x%x\n",
strerror(errno), aid, mclass, mad_hdr->method,
ntohs(mad_hdr->attr_id), ntohl(mad_hdr->attr_mod));
status = FNOT_DONE;
goto done;
}
done:
// Free umad if allocated.
if (umad_p != NULL) {
umad_free(umad_p);
}
return status;
}
