ucs_status_t ucp_ep_create_stub(ucp_worker_h worker, uint64_t dest_uuid,
const char *message, ucp_ep_h *ep_p)
{
ucs_status_t status;
ucp_ep_op_t optype;
ucp_ep_h ep = NULL;
status = ucp_ep_new(worker, dest_uuid, "??", message, &ep);
if (status != UCS_OK) {
goto err;
}
for (optype = 0; optype < UCP_EP_OP_LAST; ++optype) {
status = ucp_stub_ep_create(ep, optype, 0, NULL, &ep->uct_eps[optype]);
if (status != UCS_OK) {
goto err_destroy_uct_eps;
}
}
*ep_p = ep;
return UCS_OK;
err_destroy_uct_eps:
for (optype = 0; optype < UCP_EP_OP_LAST; ++optype) {
if (ep->uct_eps[optype] != NULL) {
uct_ep_destroy(ep->uct_eps[optype]);
}
}
ucp_ep_delete(ep);
err:
return status;
}
ucs_status_t ucp_ep_create_connected(ucp_worker_h worker, uint64_t dest_uuid,
const char *peer_name, unsigned address_count,
const ucp_address_entry_t *address_list,
const char *message, ucp_ep_h *ep_p)
{
ucs_status_t status;
ucp_ep_h ep = NULL;
status = ucp_ep_new(worker, dest_uuid, peer_name, message, &ep);
if (status != UCS_OK) {
goto err;
}
/* initialize transport endpoints */
status = ucp_ep_init_trasports(ep, address_count, address_list);
if (status != UCS_OK) {
goto err_delete;
}
*ep_p = ep;
return UCS_OK;
err_delete:
ucp_ep_delete(ep);
err:
return status;
}
ucs_status_t ucp_ep_create(ucp_worker_h worker, const ucp_address_t *address,
ucp_ep_h *ep_p)
{
char peer_name[UCP_WORKER_NAME_MAX];
uint8_t addr_indices[UCP_MAX_LANES];
ucp_address_entry_t *address_list;
unsigned address_count;
ucs_status_t status;
uint64_t dest_uuid;
ucp_ep_h ep;
UCS_ASYNC_BLOCK(&worker->async);
status = ucp_address_unpack(address, &dest_uuid, peer_name, sizeof(peer_name),
&address_count, &address_list);
if (status != UCS_OK) {
ucs_error("failed to unpack remote address: %s", ucs_status_string(status));
goto out;
}
ep = ucp_worker_ep_find(worker, dest_uuid);
if (ep != NULL) {
/* TODO handle a case where the existing endpoint is incomplete */
*ep_p = ep;
status = UCS_OK;
goto out_free_address;
}
/* allocate endpoint */
status = ucp_ep_new(worker, dest_uuid, peer_name, "from api call", &ep);
if (status != UCS_OK) {
goto out_free_address;
}
/* initialize transport endpoints */
status = ucp_wireup_init_lanes(ep, address_count, address_list, addr_indices);
if (status != UCS_OK) {
goto err_destroy_ep;
}
/* send initial wireup message */
if (!(ep->flags & UCP_EP_FLAG_LOCAL_CONNECTED)) {
status = ucp_wireup_send_request(ep);
if (status != UCS_OK) {
goto err_destroy_ep;
}
}
*ep_p = ep;
goto out_free_address;
err_destroy_ep:
ucp_ep_destroy(ep);
out_free_address:
ucs_free(address_list);
out:
UCS_ASYNC_UNBLOCK(&worker->async);
return status;
}
ucs_status_t ucp_ep_create_stub(ucp_worker_h worker, uint64_t dest_uuid,
const char *message, ucp_ep_h *ep_p)
{
ucs_status_t status;
ucp_ep_config_key_t key;
ucp_ep_h ep = NULL;
status = ucp_ep_new(worker, dest_uuid, "??", message, &ep);
if (status != UCS_OK) {
goto err;
}
/* all operations will use the first lane, which is a stub endpoint */
memset(&key, 0, sizeof(key));
key.rma_lane_map = 1;
key.amo_lane_map = 1;
key.reachable_md_map = 0; /* TODO */
key.am_lane = 0;
key.rndv_lane = 0;
key.wireup_msg_lane = 0;
key.lanes[0] = UCP_NULL_RESOURCE;
key.num_lanes = 1;
memset(key.amo_lanes, UCP_NULL_LANE, sizeof(key.amo_lanes));
ep->cfg_index = ucp_worker_get_ep_config(worker, &key);
ep->am_lane = 0;
status = ucp_stub_ep_create(ep, &ep->uct_eps[0]);
if (status != UCS_OK) {
goto err_destroy_uct_eps;
}
*ep_p = ep;
return UCS_OK;
err_destroy_uct_eps:
uct_ep_destroy(ep->uct_eps[0]);
ucp_ep_delete(ep);
err:
return status;
}
ucs_status_t ucp_ep_create(ucp_worker_h worker, ucp_address_t *address,
ucp_ep_h *ep_p)
{
uint64_t dest_uuid = ucp_address_uuid(address);
char peer_name[UCP_PEER_NAME_MAX];
ucs_status_t status;
ucp_ep_h ep;
UCS_ASYNC_BLOCK(&worker->async);
ep = ucp_worker_ep_find(worker, dest_uuid);
if (ep != NULL) {
ucs_debug("returning existing ep %p which is already connected to %"PRIx64,
ep, ep->dest_uuid);
goto out;
}
ucp_address_peer_name(address, peer_name);
status = ucp_ep_new(worker, dest_uuid, peer_name, " from api call", &ep);
if (status != UCS_OK) {
goto err;
}
status = ucp_wireup_start(ep, address);
if (status != UCS_OK) {
goto err_free;
}
out:
UCS_ASYNC_UNBLOCK(&worker->async);
*ep_p = ep;
return UCS_OK;
err_free:
ucs_free(ep);
err:
UCS_ASYNC_UNBLOCK(&worker->async);
return status;
}
static void ucp_wireup_process_request(ucp_worker_h worker, const ucp_wireup_msg_t *msg,
uint64_t uuid, const char *peer_name,
unsigned address_count,
const ucp_address_entry_t *address_list)
{
ucp_ep_h ep = ucp_worker_ep_find(worker, uuid);
ucp_rsc_index_t rsc_tli[UCP_MAX_LANES];
uint8_t addr_indices[UCP_MAX_LANES];
ucp_lane_index_t lane, remote_lane;
ucp_rsc_index_t rsc_index;
ucs_status_t status;
uint64_t tl_bitmap = 0;
ucs_trace("ep %p: got wireup request from %s", ep, peer_name);
/* Create a new endpoint if does not exist */
if (ep == NULL) {
status = ucp_ep_new(worker, uuid, peer_name, "remote-request", &ep);
if (status != UCS_OK) {
return;
}
}
/* Initialize lanes (possible destroy existing lanes) */
status = ucp_wireup_init_lanes(ep, address_count, address_list, addr_indices);
if (status != UCS_OK) {
return;
}
/* Connect p2p addresses to remote endpoint */
if (!(ep->flags & UCP_EP_FLAG_LOCAL_CONNECTED)) {
status = ucp_wireup_connect_local(ep, addr_indices, address_count,
address_list);
if (status != UCS_OK) {
return;
}
ep->flags |= UCP_EP_FLAG_LOCAL_CONNECTED;
/* Construct the list that tells the remote side with which address we
* have connected to each of its lanes.
*/
memset(rsc_tli, -1, sizeof(rsc_tli));
for (lane = 0; lane < ucp_ep_num_lanes(ep); ++lane) {
rsc_index = ucp_ep_get_rsc_index(ep, lane);
for (remote_lane = 0; remote_lane < UCP_MAX_LANES; ++remote_lane) {
/* If 'lane' has connected to 'remote_lane' ... */
if (addr_indices[lane] == msg->tli[remote_lane]) {
ucs_assert(ucp_worker_is_tl_p2p(worker, rsc_index));
rsc_tli[remote_lane] = rsc_index;
tl_bitmap |= UCS_BIT(rsc_index);
}
}
}
ucs_trace("ep %p: sending wireup reply", ep);
status = ucp_wireup_msg_send(ep, UCP_WIREUP_MSG_REPLY, tl_bitmap, rsc_tli);
if (status != UCS_OK) {
return;
}
}
}
