/*
* Forcibly drain all pending output on an endpoint, without waiting for
* actual completion.
*/
void
ompi_btl_usnic_flush_endpoint(
ompi_btl_usnic_endpoint_t *endpoint)
{
ompi_btl_usnic_send_frag_t *frag;
/* First, free all pending fragments */
while (!opal_list_is_empty(&endpoint->endpoint_frag_send_queue)) {
frag = (ompi_btl_usnic_send_frag_t *)opal_list_remove_first(
&endpoint->endpoint_frag_send_queue);
/* _cond still needs to check ownership, but make sure the
* fragment is marked as done.
*/
frag->sf_ack_bytes_left = 0;
frag->sf_seg_post_cnt = 0;
ompi_btl_usnic_send_frag_return_cond(endpoint->endpoint_module, frag);
}
/* Now, ACK everything that is pending */
ompi_btl_usnic_handle_ack(endpoint, endpoint->endpoint_next_seq_to_send-1);
}
/*
* We have received an ACK for a given sequence number (either standalone
* or via piggy-back on a regular send)
*/
void
ompi_btl_usnic_handle_ack(
ompi_btl_usnic_endpoint_t *endpoint,
ompi_btl_usnic_seq_t ack_seq)
{
ompi_btl_usnic_seq_t is;
ompi_btl_usnic_send_segment_t *sseg;
ompi_btl_usnic_send_frag_t *frag;
ompi_btl_usnic_module_t *module;
uint32_t bytes_acked;
module = endpoint->endpoint_module;
/* ignore if this is an old ACK */
if (SEQ_LT(ack_seq, endpoint->endpoint_ack_seq_rcvd)) {
#if MSGDEBUG1
opal_output(0, "Got OLD DUP ACK seq %"UDSEQ" < %"UDSEQ"\n",
ack_seq, endpoint->endpoint_ack_seq_rcvd);
#endif
++module->stats.num_old_dup_acks;
return;
/* A duplicate ACK means next seg was lost */
} else if (ack_seq == endpoint->endpoint_ack_seq_rcvd) {
++module->stats.num_dup_acks;
ompi_btl_usnic_force_retrans(endpoint, ack_seq);
return;
}
/* Does this ACK have a new sequence number that we haven't
seen before? */
for (is = endpoint->endpoint_ack_seq_rcvd + 1; SEQ_LE(is, ack_seq); ++is) {
sseg = endpoint->endpoint_sent_segs[WINDOW_SIZE_MOD(is)];
#if MSGDEBUG1
opal_output(0, " Checking ACK/sent_segs window %p, index %lu, seq %lu, occupied=%p, seg_room=%d",
(void*) endpoint->endpoint_sent_segs,
WINDOW_SIZE_MOD(is), is, (void*)sseg, (sseg?sseg->ss_hotel_room:-2));
#endif
assert(sseg != NULL);
assert(sseg->ss_base.us_btl_header->pkt_seq == is);
#if MSGDEBUG1
if (sseg->ss_hotel_room == -1) {
opal_output(0, "=== ACKed frag in sent_frags array is not in hotel/enqueued, module %p, endpoint %p, seg %p, seq %" UDSEQ ", slot %lu",
(void*) module, (void*) endpoint,
(void*) sseg, is, WINDOW_SIZE_MOD(is));
}
#endif
/* Check the sending segment out from the hotel. NOTE: The
segment might not actually be in a hotel room if it has
already been evicted and queued for resend.
If it's not in the hotel, don't check it out! */
if (OPAL_LIKELY(sseg->ss_hotel_room != -1)) {
opal_hotel_checkout(&endpoint->endpoint_hotel, sseg->ss_hotel_room);
sseg->ss_hotel_room = -1;
/* hotel_room == -1 means queued for resend, remove it */
} else {
opal_list_remove_item((&module->pending_resend_segs),
&sseg->ss_base.us_list.super);
}
/* update the owning fragment */
bytes_acked = sseg->ss_base.us_btl_header->payload_len;
frag = sseg->ss_parent_frag;
#if MSGDEBUG1
opal_output(0, " ACKED seg %p frag %p ack_bytes=%"PRIu32" left=%zd dst_seg[0].seg_addr=%p des_flags=0x%x\n",
(void*)sseg, (void*)frag, bytes_acked,
frag->sf_ack_bytes_left-bytes_acked,
frag->sf_base.uf_dst_seg[0].seg_addr.pval,
frag->sf_base.uf_base.des_flags);
#endif
/* If all ACKs received, and this is a put or a regular send
* that needs a callback, perform the callback now
*
* NOTE on sf_ack_bytes_left - here we check for
* sf_ack_bytes_left == bytes_acked
* as opposed to adjusting sf_ack_bytes_left and checking for 0 because
* if we don't, the callback function may call usnic_free() and free
* the fragment out from under us which we do not want. If the
* fragment really needs to be freed, we'll take care of it in a few
* lines below.
*/
if (frag->sf_ack_bytes_left == bytes_acked &&
((frag->sf_base.uf_dst_seg[0].seg_addr.pval != NULL) ||
(frag->sf_base.uf_base.des_flags &
MCA_BTL_DES_SEND_ALWAYS_CALLBACK))) {
OMPI_BTL_USNIC_DO_SEND_FRAG_CB(module, frag, "send completion");
}
/* free this segment */
sseg->ss_ack_pending = false;
if (sseg->ss_send_posted == 0) {
ompi_btl_usnic_release_send_segment(module, frag, sseg);
}
/* when no bytes left to ACK, fragment send is truly done */
/* see note above on why this is done here as opposed to earlier */
frag->sf_ack_bytes_left -= bytes_acked;
/* OK to return this fragment? */
ompi_btl_usnic_send_frag_return_cond(module, frag);
/* indicate this segment has been ACKed */
endpoint->endpoint_sent_segs[WINDOW_SIZE_MOD(is)] = NULL;
}
/* update ACK received */
endpoint->endpoint_ack_seq_rcvd = ack_seq;
/* send window may have opened, possibly make endpoint ready-to-send */
ompi_btl_usnic_check_rts(endpoint);
}
