static ucs_status_t ucp_tag_eager_sync_contig_bcopy_single(uct_pending_req_t *self)
{
ucs_status_t status = ucp_do_am_bcopy_single(self, UCP_AM_ID_EAGER_SYNC_ONLY,
ucp_tag_pack_eager_sync_only_contig);
if (status == UCS_OK) {
ucp_request_t *req = ucs_container_of(self, ucp_request_t, send.uct);
ucp_tag_eager_sync_completion(req, UCP_REQUEST_FLAG_LOCAL_COMPLETED);
}
return status;
}
void
ucp_tag_eager_sync_zcopy_req_complete(ucp_request_t *req, ucs_status_t status)
{
if (req->send.state.dt.offset == req->send.length) {
ucp_request_send_buffer_dereg(req); /* TODO register+lane change */
ucp_tag_eager_sync_completion(req, UCP_REQUEST_FLAG_LOCAL_COMPLETED,
status);
} else if (status != UCS_OK) {
ucs_fatal("error handling is not supported with tag-sync protocol");
}
}
static ucs_status_t ucp_tag_eager_sync_bcopy_single(uct_pending_req_t *self)
{
ucs_status_t status = ucp_do_am_bcopy_single(self, UCP_AM_ID_EAGER_SYNC_ONLY,
ucp_tag_pack_eager_sync_only_dt);
if (status == UCS_OK) {
ucp_request_t *req = ucs_container_of(self, ucp_request_t, send.uct);
ucp_request_send_generic_dt_finish(req);
ucp_tag_eager_sync_completion(req, UCP_REQUEST_FLAG_LOCAL_COMPLETED,
UCS_OK);
} else if (status == UCP_STATUS_PENDING_SWITCH) {
status = UCS_OK;
}
return status;
}
static ucs_status_t ucp_tag_eager_sync_contig_bcopy_multi(uct_pending_req_t *self)
{
ucs_status_t status = ucp_do_am_bcopy_multi(self,
UCP_AM_ID_EAGER_SYNC_FIRST,
UCP_AM_ID_EAGER_MIDDLE,
UCP_AM_ID_EAGER_LAST,
sizeof(ucp_eager_sync_first_hdr_t),
sizeof(ucp_eager_hdr_t),
ucp_tag_pack_eager_sync_first_contig,
ucp_tag_pack_eager_middle_contig,
ucp_tag_pack_eager_last_contig);
if (status == UCS_OK) {
ucp_request_t *req = ucs_container_of(self, ucp_request_t, send.uct);
ucp_tag_eager_sync_completion(req, UCP_REQUEST_FLAG_LOCAL_COMPLETED);
}
return status;
}
static ucs_status_t ucp_tag_eager_sync_bcopy_multi(uct_pending_req_t *self)
{
ucs_status_t status = ucp_do_am_bcopy_multi(self,
UCP_AM_ID_EAGER_SYNC_FIRST,
UCP_AM_ID_EAGER_MIDDLE,
UCP_AM_ID_EAGER_MIDDLE,
sizeof(ucp_eager_middle_hdr_t),
ucp_tag_pack_eager_sync_first_dt,
ucp_tag_pack_eager_middle_dt,
ucp_tag_pack_eager_last_dt, 1);
if (status == UCS_OK) {
ucp_request_t *req = ucs_container_of(self, ucp_request_t, send.uct);
ucp_request_send_generic_dt_finish(req);
ucp_tag_eager_sync_completion(req, UCP_REQUEST_FLAG_LOCAL_COMPLETED,
UCS_OK);
} else if (status == UCP_STATUS_PENDING_SWITCH) {
status = UCS_OK;
}
return status;
}
static inline void ucp_tag_eager_sync_generic_complete(uct_pending_req_t *self)
{
ucp_request_t *req = ucs_container_of(self, ucp_request_t, send.uct);
ucp_request_generic_dt_finish(req);
ucp_tag_eager_sync_completion(req, UCP_REQUEST_FLAG_LOCAL_COMPLETED);
}
static inline void ucp_tag_eager_sync_contig_zcopy_req_complete(ucp_request_t *req)
{
ucp_request_send_buffer_dereg(req, UCP_EP_OP_AM);
ucp_tag_eager_sync_completion(req, UCP_REQUEST_FLAG_LOCAL_COMPLETED);
}
void ucp_tag_eager_sync_zcopy_req_complete(ucp_request_t *req)
{
ucp_request_send_buffer_dereg(req, req->send.lane); /* TODO register+lane change */
ucp_tag_eager_sync_completion(req, UCP_REQUEST_FLAG_LOCAL_COMPLETED);
}
