/* * BTL 2.0 version of module.btl_prepare_src. * * Note the "user" data the PML wishes to communicate and return a descriptor * that can be used for send or put. We create a frag (which is also a * descriptor by virtue of its base class) and populate it with enough * source information to complete a future send/put. * * We will create either a small send frag if < than an MTU, otherwise a large * send frag. The convertor will be saved for deferred packing if the user * buffer is noncontiguous. Otherwise it will be saved in one of the * descriptor's SGEs. * * NOTE that the *only* reason this routine is allowed to return a size smaller * than was requested is if the convertor cannot process the entire amount. */ mca_btl_base_descriptor_t* opal_btl_usnic_prepare_src( struct mca_btl_base_module_t* base_module, struct mca_btl_base_endpoint_t* endpoint, struct mca_mpool_base_registration_t* registration, struct opal_convertor_t* convertor, uint8_t order, size_t reserve, size_t* size, uint32_t flags) { OPAL_THREAD_LOCK(&btl_usnic_lock); opal_btl_usnic_module_t *module = (opal_btl_usnic_module_t*) base_module; opal_btl_usnic_send_frag_t *frag; uint32_t payload_len; #if MSGDEBUG2 size_t osize = *size; #endif /* Do we need to check the connectivity? If enabled, we'll check the connectivity at either first send to peer X or first ACK to peer X. */ opal_btl_usnic_check_connectivity(module, endpoint); /* * if total payload len fits in one MTU use small send, else large */ payload_len = *size + reserve; if (payload_len <= module->max_frag_payload) { frag = prepare_src_small(module, endpoint, convertor, order, reserve, size, flags); } else { frag = prepare_src_large(module, endpoint, convertor, order, reserve, size, flags); } #if MSGDEBUG2 opal_output(0, "prep_src: %s %s frag %p, size=%d+%u (was %u), conv=%p\n", module->linux_device_name, (reserve + *size) <= module->max_frag_payload?"small":"large", (void *)frag, (int)reserve, (unsigned)*size, (unsigned)osize, (void *)convertor); #if MSGDEBUG1 { unsigned i; mca_btl_base_descriptor_t *desc = &frag->sf_base.uf_base; for (i=0; i<desc->USNIC_SEND_LOCAL_COUNT; ++i) { opal_output(0, " %d: ptr:%p len:%d\n", i, (void *)desc->USNIC_SEND_LOCAL[i].seg_addr.pval, desc->USNIC_SEND_LOCAL[i].seg_len); } } #endif #endif OPAL_THREAD_UNLOCK(&btl_usnic_lock); return &frag->sf_base.uf_base; }
/* * Send an ACK */ void opal_btl_usnic_ack_send( opal_btl_usnic_module_t *module, opal_btl_usnic_endpoint_t *endpoint) { opal_btl_usnic_ack_segment_t *ack; /* Get an ACK frag. If we don't get one, just discard this ACK. */ ack = opal_btl_usnic_ack_segment_alloc(module); if (OPAL_UNLIKELY(NULL == ack)) { opal_output(0, "====================== No frag for sending the ACK -- skipped"); abort(); } /* send the seq of the lowest item in the window that we've received */ ack->ss_base.us_btl_header->ack_seq = endpoint->endpoint_next_contig_seq_to_recv - 1; ack->ss_base.us_sg_entry[0].length = sizeof(opal_btl_usnic_btl_header_t); #if MSGDEBUG1 { uint8_t mac[6]; char src_mac[32]; char dest_mac[32]; memset(src_mac, 0, sizeof(src_mac)); memset(dest_mac, 0, sizeof(dest_mac)); opal_btl_usnic_sprintf_mac(src_mac, module->if_mac); opal_btl_usnic_gid_to_mac(&endpoint->endpoint_remote_addr.gid, mac); opal_btl_usnic_sprintf_mac(dest_mac, mac); opal_output(0, "--> Sending ACK, sg_entry length %d, seq %" UDSEQ " to %s, qp %u", ack->ss_base.us_sg_entry[0].length, ack->ss_base.us_btl_header->ack_seq, dest_mac, endpoint->endpoint_remote_addr.qp_num[ack->ss_channel]); } #endif /* Do we need to check the connectivity? If enabled, we'll check the connectivity at either first send to peer X or first ACK to peer X. */ opal_btl_usnic_check_connectivity(module, endpoint); /* send the ACK */ opal_btl_usnic_post_segment(module, endpoint, ack); /* Stats */ ++module->stats.num_ack_sends; return; }
/* * Send an ACK */ void opal_btl_usnic_ack_send( opal_btl_usnic_module_t *module, opal_btl_usnic_endpoint_t *endpoint) { opal_btl_usnic_ack_segment_t *ack; /* Get an ACK frag. If we don't get one, just discard this ACK. */ ack = opal_btl_usnic_ack_segment_alloc(module); if (OPAL_UNLIKELY(NULL == ack)) { opal_output(0, "====================== No frag for sending the ACK -- skipped"); abort(); } /* send the seq of the lowest item in the window that we've received */ ack->ss_base.us_btl_header->ack_seq = endpoint->endpoint_next_contig_seq_to_recv - 1; ack->ss_len = sizeof(opal_btl_usnic_btl_header_t); #if MSGDEBUG1 { char remote_ip[IPV4STRADDRLEN]; struct opal_btl_usnic_modex_t *modex = &endpoint->endpoint_remote_modex; opal_btl_usnic_snprintf_ipv4_addr(remote_ip, sizeof(remote_ip), modex->ipv4_addr, modex->netmask); opal_output(0, "--> Sending ACK, length %d, seq %" UDSEQ " to %s, port %u", ack->ss_len, ack->ss_base.us_btl_header->ack_seq, remote_ip, modex->ports[ack->ss_channel]); } #endif /* Do we need to check the connectivity? If enabled, we'll check the connectivity at either first send to peer X or first ACK to peer X. */ opal_btl_usnic_check_connectivity(module, endpoint); /* send the ACK */ opal_btl_usnic_post_ack(module, endpoint, ack); /* Stats */ ++module->stats.num_ack_sends; return; }