/*
* BTL 2.0 version of module.btl_prepare_src.
*
* Note the "user" data the PML wishes to communicate and return a descriptor
* that can be used for send or put. We create a frag (which is also a
* descriptor by virtue of its base class) and populate it with enough
* source information to complete a future send/put.
*
* We will create either a small send frag if < than an MTU, otherwise a large
* send frag. The convertor will be saved for deferred packing if the user
* buffer is noncontiguous. Otherwise it will be saved in one of the
* descriptor's SGEs.
*
* NOTE that the *only* reason this routine is allowed to return a size smaller
* than was requested is if the convertor cannot process the entire amount.
*/
mca_btl_base_descriptor_t*
opal_btl_usnic_prepare_src(
struct mca_btl_base_module_t* base_module,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_mpool_base_registration_t* registration,
struct opal_convertor_t* convertor,
uint8_t order,
size_t reserve,
size_t* size,
uint32_t flags)
{
OPAL_THREAD_LOCK(&btl_usnic_lock);
opal_btl_usnic_module_t *module = (opal_btl_usnic_module_t*) base_module;
opal_btl_usnic_send_frag_t *frag;
uint32_t payload_len;
#if MSGDEBUG2
size_t osize = *size;
#endif
/* Do we need to check the connectivity? If enabled, we'll check
the connectivity at either first send to peer X or first ACK to
peer X. */
opal_btl_usnic_check_connectivity(module, endpoint);
/*
* if total payload len fits in one MTU use small send, else large
*/
payload_len = *size + reserve;
if (payload_len <= module->max_frag_payload) {
frag = prepare_src_small(module, endpoint, convertor,
order, reserve, size, flags);
} else {
frag = prepare_src_large(module, endpoint, convertor,
order, reserve, size, flags);
}
#if MSGDEBUG2
opal_output(0, "prep_src: %s %s frag %p, size=%d+%u (was %u), conv=%p\n",
module->linux_device_name,
(reserve + *size) <= module->max_frag_payload?"small":"large",
(void *)frag, (int)reserve, (unsigned)*size, (unsigned)osize,
(void *)convertor);
#if MSGDEBUG1
{
unsigned i;
mca_btl_base_descriptor_t *desc = &frag->sf_base.uf_base;
for (i=0; i<desc->USNIC_SEND_LOCAL_COUNT; ++i) {
opal_output(0, " %d: ptr:%p len:%d\n", i,
(void *)desc->USNIC_SEND_LOCAL[i].seg_addr.pval,
desc->USNIC_SEND_LOCAL[i].seg_len);
}
}
#endif
#endif
OPAL_THREAD_UNLOCK(&btl_usnic_lock);
return &frag->sf_base.uf_base;
}
/*
* Send an ACK
*/
void
opal_btl_usnic_ack_send(
opal_btl_usnic_module_t *module,
opal_btl_usnic_endpoint_t *endpoint)
{
opal_btl_usnic_ack_segment_t *ack;
/* Get an ACK frag. If we don't get one, just discard this ACK. */
ack = opal_btl_usnic_ack_segment_alloc(module);
if (OPAL_UNLIKELY(NULL == ack)) {
opal_output(0, "====================== No frag for sending the ACK -- skipped");
abort();
}
/* send the seq of the lowest item in the window that
we've received */
ack->ss_base.us_btl_header->ack_seq =
endpoint->endpoint_next_contig_seq_to_recv - 1;
ack->ss_base.us_sg_entry[0].length =
sizeof(opal_btl_usnic_btl_header_t);
#if MSGDEBUG1
{
uint8_t mac[6];
char src_mac[32];
char dest_mac[32];
memset(src_mac, 0, sizeof(src_mac));
memset(dest_mac, 0, sizeof(dest_mac));
opal_btl_usnic_sprintf_mac(src_mac, module->if_mac);
opal_btl_usnic_gid_to_mac(&endpoint->endpoint_remote_addr.gid, mac);
opal_btl_usnic_sprintf_mac(dest_mac, mac);
opal_output(0, "--> Sending ACK, sg_entry length %d, seq %" UDSEQ " to %s, qp %u",
ack->ss_base.us_sg_entry[0].length,
ack->ss_base.us_btl_header->ack_seq, dest_mac,
endpoint->endpoint_remote_addr.qp_num[ack->ss_channel]);
}
#endif
/* Do we need to check the connectivity? If enabled, we'll check
the connectivity at either first send to peer X or first ACK to
peer X. */
opal_btl_usnic_check_connectivity(module, endpoint);
/* send the ACK */
opal_btl_usnic_post_segment(module, endpoint, ack);
/* Stats */
++module->stats.num_ack_sends;
return;
}
/*
* Send an ACK
*/
void
opal_btl_usnic_ack_send(
opal_btl_usnic_module_t *module,
opal_btl_usnic_endpoint_t *endpoint)
{
opal_btl_usnic_ack_segment_t *ack;
/* Get an ACK frag. If we don't get one, just discard this ACK. */
ack = opal_btl_usnic_ack_segment_alloc(module);
if (OPAL_UNLIKELY(NULL == ack)) {
opal_output(0, "====================== No frag for sending the ACK -- skipped");
abort();
}
/* send the seq of the lowest item in the window that
we've received */
ack->ss_base.us_btl_header->ack_seq =
endpoint->endpoint_next_contig_seq_to_recv - 1;
ack->ss_len = sizeof(opal_btl_usnic_btl_header_t);
#if MSGDEBUG1
{
char remote_ip[IPV4STRADDRLEN];
struct opal_btl_usnic_modex_t *modex =
&endpoint->endpoint_remote_modex;
opal_btl_usnic_snprintf_ipv4_addr(remote_ip, sizeof(remote_ip),
modex->ipv4_addr,
modex->netmask);
opal_output(0, "--> Sending ACK, length %d, seq %" UDSEQ " to %s, port %u",
ack->ss_len,
ack->ss_base.us_btl_header->ack_seq,
remote_ip,
modex->ports[ack->ss_channel]);
}
#endif
/* Do we need to check the connectivity? If enabled, we'll check
the connectivity at either first send to peer X or first ACK to
peer X. */
opal_btl_usnic_check_connectivity(module, endpoint);
/* send the ACK */
opal_btl_usnic_post_ack(module, endpoint, ack);
/* Stats */
++module->stats.num_ack_sends;
return;
}
