static int _do_madrpc(int port_id, void *sndbuf, void *rcvbuf, int agentid, int len, int timeout) { uint32_t trid; /* only low 32 bits */ int retries; int length, status; if (!timeout) timeout = def_madrpc_timeout; if (ibdebug > 1) { IBWARN(">>> sending: len %d pktsz %zu", len, umad_size() + len); xdump(stderr, "send buf\n", sndbuf, umad_size() + len); } if (save_mad) { memcpy(save_mad, umad_get_mad(sndbuf), save_mad_len < len ? save_mad_len : len); save_mad = 0; } trid = mad_get_field64(umad_get_mad(sndbuf), 0, IB_MAD_TRID_F); for (retries = 0; retries < madrpc_retries; retries++) { if (retries) { ERRS("retry %d (timeout %d ms)", retries, timeout); } length = len; if (umad_send(port_id, agentid, sndbuf, length, timeout, 0) < 0) { IBWARN("send failed; %s", strerror(errno)); return -1; } /* Use same timeout on receive side just in case */ /* send packet is lost somewhere. */ do { if (umad_recv(port_id, rcvbuf, &length, timeout) < 0) { IBWARN("recv failed: %s", strerror(errno)); return -1; } if (ibdebug > 1) { IBWARN("rcv buf:"); xdump(stderr, "rcv buf\n", umad_get_mad(rcvbuf), IB_MAD_SIZE); } } while ((uint32_t)mad_get_field64(umad_get_mad(rcvbuf), 0, IB_MAD_TRID_F) != trid); status = umad_status(rcvbuf); if (!status) return length; /* done */ if (status == ENOMEM) return length; } ERRS("timeout after %d retries, %d ms", retries, timeout * retries); return -1; }
/* Returns 0 (invalid base version) on error */ static uint8_t get_base_version_from_ni(int fd, uint32_t aid, int pkey_index) { uint8_t rc; void *umad_p = NULL; struct umad_smp *send_mad; size_t length; umad_p = umad_alloc(1, sizeof(*send_mad) + umad_size()); if (!umad_p) { OUTPUT_ERROR ("can't alloc umad for OPA check; send_size %ld\n", sizeof(*send_mad)); return 0; } memset(umad_p, 0, sizeof(*send_mad) + umad_size()); umad_set_grh(umad_p, 0); send_mad = umad_get_mad(umad_p); send_mad->base_version = UMAD_BASE_VERSION; send_mad->mgmt_class = UMAD_CLASS_SUBN_DIRECTED_ROUTE; send_mad->class_version = 0x01; send_mad->method = UMAD_METHOD_GET; send_mad->tid = htonl(0xDEADBEEF); send_mad->attr_id = htons(UMAD_SM_ATTR_NODE_INFO); send_mad->dr_slid = 0xffff; send_mad->dr_dlid = 0xffff; umad_set_pkey(umad_p, pkey_index); umad_set_addr(umad_p, 0xffff, 0, 0, 0); rc = 0; if (umad_send(fd, aid, umad_p, sizeof(*send_mad), 100, 1) < 0) goto free_mad; length = sizeof(*send_mad); if (umad_recv(fd, umad_p, (int *)&length, 100) < 0) goto free_mad; if (length < sizeof(*send_mad)) goto free_mad; if (umad_status(umad_p) != 0) goto free_mad; rc = ((NODE_INFO *)(send_mad->data))->BaseVersion; free_mad: free(umad_p); return rc; }
static char *ibsystat(ib_portid_t * portid, int attr) { ib_rpc_t rpc = { 0 }; int fd, agent, timeout, len; void *data = (uint8_t *) umad_get_mad(buf) + IB_VENDOR_RANGE2_DATA_OFFS; DEBUG("Sysstat ping.."); rpc.mgtclass = IB_VENDOR_OPENIB_SYSSTAT_CLASS; rpc.method = IB_MAD_METHOD_GET; rpc.attr.id = attr; rpc.attr.mod = 0; rpc.oui = oui; rpc.timeout = 0; rpc.datasz = IB_VENDOR_RANGE2_DATA_SIZE; rpc.dataoffs = IB_VENDOR_RANGE2_DATA_OFFS; portid->qp = 1; if (!portid->qkey) portid->qkey = IB_DEFAULT_QP1_QKEY; if ((len = mad_build_pkt(buf, &rpc, portid, NULL, NULL)) < 0) IBPANIC("cannot build packet."); fd = mad_rpc_portid(srcport); agent = mad_rpc_class_agent(srcport, rpc.mgtclass); timeout = ibd_timeout ? ibd_timeout : MAD_DEF_TIMEOUT_MS; if (umad_send(fd, agent, buf, len, timeout, 0) < 0) IBPANIC("umad_send failed."); len = sizeof(buf) - umad_size(); if (umad_recv(fd, buf, &len, timeout) < 0) IBPANIC("umad_recv failed."); if (umad_status(buf)) return strerror(umad_status(buf)); DEBUG("Got sysstat pong.."); if (attr != IB_PING_ATTR) puts(data); else printf("sysstat ping succeeded\n"); return 0; }
static int rereg_recv(int port, int agent, ib_portid_t * dport, uint8_t * umad, int length, int tmo) { int ret, retry = 0; int len = length; while ((ret = umad_recv(port, umad, &len, tmo)) < 0 && errno == ETIMEDOUT) { if (retry++ > 3) return 0; } if (ret < 0) { err("umad_recv %d failed: %s\n", ret, strerror(errno)); return -1; } dbg("umad_recv (retries %d), tid = 0x%016" PRIx64 ": len = %d, status = %d\n", retry, mad_get_field64(umad_get_mad(umad), 0, IB_MAD_TRID_F), len, umad_status(umad)); return 1; }
/****************************************************************************** * join_multicast_group ******************************************************************************/ int join_multicast_group(subn_adm_method method,struct mcast_parameters *params) { int portid = -1; int agentid = -1; void *umad_buff = NULL; void *mad = NULL; int length = MAD_SIZE; int test_result = 0; // mlid will be assigned to the new LID after the join if (umad_init() < 0) { fprintf(stderr, "failed to init the UMAD library\n"); goto cleanup; } /* use casting to loose the "const char0 *" */ portid = umad_open_port((char*)params->ib_devname,params->ib_port); if (portid < 0) { fprintf(stderr,"failed to open UMAD port %d\n",params->ib_port); goto cleanup; } agentid = umad_register(portid,MANAGMENT_CLASS_SUBN_ADM, 2, 0, 0); if (agentid < 0) { fprintf(stderr,"failed to register UMAD agent for MADs\n"); goto cleanup; } umad_buff = umad_alloc(1, umad_size() + MAD_SIZE); if (!umad_buff) { fprintf(stderr, "failed to allocate MAD buffer\n"); goto cleanup; } mad = umad_get_mad(umad_buff); prepare_mcast_mad(method,params,(struct sa_mad_packet_t *)mad); if (umad_set_addr(umad_buff,params->sm_lid,1,params->sm_sl,QP1_WELL_KNOWN_Q_KEY) < 0) { fprintf(stderr, "failed to set the destination address of the SMP\n"); goto cleanup; } if (umad_send(portid,agentid,umad_buff,MAD_SIZE,100,5) < 0) { fprintf(stderr, "failed to send MAD\n"); goto cleanup; } if (umad_recv(portid,umad_buff,&length,5000) < 0) { fprintf(stderr, "failed to receive MAD response\n"); goto cleanup; } if (check_mad_status((struct sa_mad_packet_t*)mad)) { fprintf(stderr, "failed to get mlid from MAD\n"); goto cleanup; } // "Join multicast group" message was sent if (method == SUBN_ADM_METHOD_SET) { get_mlid_from_mad((struct sa_mad_packet_t*)mad,¶ms->mlid); params->mcast_state |= MCAST_IS_JOINED; // "Leave multicast group" message was sent } else { params->mcast_state &= ~MCAST_IS_JOINED; } cleanup: if (umad_buff) umad_free(umad_buff); if (portid >= 0) { if (agentid >= 0) { if (umad_unregister(portid, agentid)) { fprintf(stderr, "failed to deregister UMAD agent for MADs\n"); test_result = 1; } } if (umad_close_port(portid)) { fprintf(stderr, "failed to close UMAD portid\n"); test_result = 1; } } return test_result; }
/** ========================================================================= */ FSTATUS oib_recv_mad_no_alloc(struct oib_port *port, uint8_t *recv_mad, size_t *recv_size, int timeout_ms, struct oib_mad_addr *addr) { size_t length = *recv_size; ib_user_mad_t *umad = NULL; int mad_agent; uint32_t my_umad_status = 0; FSTATUS status = FSUCCESS; if (!port || !recv_mad || !*recv_size) return FINVALID_PARAMETER; umad = umad_alloc(1, length + umad_size()); if (!umad) { OUTPUT_ERROR ("can't alloc umad length %ld\n", length); status = FINSUFFICIENT_MEMORY; goto done; } retry: mad_agent = umad_recv(port->umad_fd, umad, (int *)&length, timeout_ms); // There are 4 combinations: // assorted errors: mad_agent < 0, length <= MAD_SIZE // large RMPP response: mad_agent < 0, length > MAD_SIZE, umad_status==0 // got response: mad_agent >= 0, length <= MAD_SIZE, umad_status==0 // no response: mad_agent >= 0, length <= MAD_SIZE, umad_status == error if (mad_agent < 0) { if (length <= *recv_size) { // no MAD returned. None available. DBGPRINT ("recv error on umad (size %zu) (%s)\n", *recv_size, strerror(errno)); if (errno == EINTR) goto retry; status = (errno == ETIMEDOUT) ? FNOT_DONE:FERROR; goto done; } else { // this routine is not expecting large responses OUTPUT_ERROR ("Rx Packet size %zu larger than mad-size %zu\n", length, *recv_size); status = FOVERRUN; if (recv_mad) memcpy(recv_mad, umad_get_mad(umad), *recv_size); // Clean out Rx packet 'cause it will never go away.. umad_free(umad); umad = umad_alloc(1, umad_size() + length); if (!umad) { OUTPUT_ERROR ("can't alloc umad for rx cleanup, length %ld\n", length); status = FINSUFFICIENT_MEMORY; goto done; } // just to be safe, we supply a timeout. However it // should be unnecessary since we know we have a packet retry2: if (umad_recv(port->umad_fd, umad, (int *)&length, OIB_UTILS_DEF_TIMEOUT_MS) < 0) { OUTPUT_ERROR ("recv error on cleanup, length %ld (%s)\n", length, strerror(errno)); if (errno == EINTR) goto retry2; goto done; } if (dbg_file) { umad_dump(umad); oib_dump_mad(dbg_file, umad_get_mad(umad), length, "rcv mad discarded\n"); } goto done; } } if (mad_agent >= UMAD_CA_MAX_AGENTS) { OUTPUT_ERROR ("invalid mad agent %d\n", mad_agent); status = FERROR; goto done; } my_umad_status = umad_status(umad); DBGPRINT("UMAD Status: %s (%d)\n", strerror(my_umad_status), my_umad_status); if (my_umad_status != 0) { status = (my_umad_status == ETIMEDOUT) ? FTIMEOUT : FREJECT; } DBGPRINT("Received MAD: Agent %d, length=%ld\n", mad_agent, length); if (dbg_file) { umad_dump(umad); oib_dump_mad(dbg_file, umad_get_mad(umad), length, "rcv mad\n"); } // Copy the data if (recv_mad && length > 0) { *recv_size = length; memcpy(recv_mad, umad_get_mad(umad), length); } if (addr != NULL) { addr->lid = IB2STL_LID(ntoh16(umad->addr.lid)); addr->sl = umad->addr.sl; addr->qkey = ntoh32(umad->addr.qkey); addr->qpn = ntoh32(umad->addr.qpn); addr->pkey = oib_find_pkey_from_idx(port, umad_get_pkey(umad)); } done: if (umad != NULL) { umad_free(umad); } return status; }
/** ========================================================================= */ FSTATUS oib_recv_mad_alloc(struct oib_port *port, uint8_t **recv_mad, size_t *recv_size, int timeout_ms, struct oib_mad_addr *addr) { #define STL_MAD_SIZE 2048 // Note, hack should reference size value OFED standard header file FSTATUS status = FSUCCESS; ib_user_mad_t *umad = NULL; int mad_agent; uint32_t my_umad_status = 0; size_t length; if (!port || !recv_mad || !recv_size) return FINVALID_PARAMETER; length = STL_MAD_SIZE; umad = umad_alloc(1, umad_size() + length); if (!umad) { OUTPUT_ERROR("can't alloc MAD sized umad\n"); status = FINSUFFICIENT_MEMORY; goto done; } retry: mad_agent = umad_recv(port->umad_fd, umad, (int *)&length, timeout_ms); // There are 4 combinations: // assorted errors: mad_agent < 0, length <= MAD_SIZE // large RMPP response: mad_agent < 0, length > MAD_SIZE, umad_status==0 // got response: mad_agent >= 0, length <= MAD_SIZE, umad_status==0 // no response: mad_agent >= 0, length <= MAD_SIZE, umad_status == error if (mad_agent < 0) { if (length <= STL_MAD_SIZE) { // no MAD returned. None available. DBGPRINT ("recv error on MAD sized umad (%s) length=%ld\n", strerror(errno), length); if (errno == EINTR) goto retry; status = (errno == ETIMEDOUT)?FNOT_DONE:FERROR; goto done; } else { /* Need a larger buffer for RMPP */ DBGPRINT ("Received 1st MAD length=%ld\n",length); umad_free(umad); umad = umad_alloc(1, umad_size() + length); if (!umad) { OUTPUT_ERROR ("can't alloc umad length %ld\n", length); status = FINSUFFICIENT_MEMORY; goto done; } // just to be safe, we supply a timeout. However it // should be unnecessary since we know we have a packet retry2: if ((mad_agent = umad_recv(port->umad_fd, umad, (int *)&length, OIB_UTILS_DEF_TIMEOUT_MS)) < 0) { OUTPUT_ERROR ("recv error on umad length %ld (%s)\n", length, strerror(errno)); if (errno == EINTR) goto retry2; status = FOVERRUN; *recv_size = length; goto done; } } } if (mad_agent >= UMAD_CA_MAX_AGENTS) { OUTPUT_ERROR ("invalid mad agent %d - dropping\n", mad_agent); status = FERROR; goto done; } my_umad_status = umad_status(umad); DBGPRINT("UMAD Status: %s (%d)\n", strerror(my_umad_status), my_umad_status); if (my_umad_status != 0) { status = (my_umad_status == ETIMEDOUT) ? FTIMEOUT : FREJECT; } DBGPRINT("Received MAD length=%ld, total umad size=%ld\n",length, length + umad_size()); if (dbg_file) { struct umad_hdr * umad_hdr = (struct umad_hdr *)umad_get_mad(umad); DBGPRINT(" Base_Version 0x%x Class 0x%x Method 0x%x attrId 0x%x attr_mod 0x%x status 0x%x\n", umad_hdr->base_version, umad_hdr->mgmt_class, umad_hdr->method, umad_hdr->attr_id, umad_hdr->attr_mod, umad_hdr->status); umad_dump(umad); oib_dump_mad(dbg_file, umad_get_mad(umad), length, "rcv mad\n"); } // Allocate and copy to new buffer. *recv_mad = calloc (1, length); if (*recv_mad == NULL) { OUTPUT_ERROR ("can't alloc return buffer length %ld\n", length); status = FINSUFFICIENT_MEMORY; goto done; } memcpy (*recv_mad, umad_get_mad(umad), length); *recv_size = length; if (addr != NULL) { addr->lid = IB2STL_LID(ntoh16(umad->addr.lid)); addr->sl = umad->addr.sl; addr->qkey = ntoh32(umad->addr.qkey); addr->qpn = ntoh32(umad->addr.qpn); addr->pkey = oib_find_pkey_from_idx(port, umad_get_pkey(umad)); } done: if (umad != NULL) { umad_free(umad); } return status; }