static UCS_CLASS_CLEANUP_FUNC(uct_mm_ep_t)
{
uct_mm_iface_t *iface = ucs_derived_of(self->super.super.iface, uct_mm_iface_t);
ucs_status_t status;
uct_mm_remote_seg_t *remote_seg;
struct sglib_hashed_uct_mm_remote_seg_t_iterator iter;
uct_mm_ep_signal_remote(self, UCT_MM_IFACE_SIGNAL_DISCONNECT);
uct_worker_progress_unregister(iface->super.worker, uct_mm_iface_progress,
iface);
for (remote_seg = sglib_hashed_uct_mm_remote_seg_t_it_init(&iter, self->remote_segments_hash);
remote_seg != NULL; remote_seg = sglib_hashed_uct_mm_remote_seg_t_it_next(&iter)) {
sglib_hashed_uct_mm_remote_seg_t_delete(self->remote_segments_hash, remote_seg);
/* detach the remote proceess's descriptors segment */
status = uct_mm_md_mapper_ops(iface->super.md)->detach(remote_seg);
if (status != UCS_OK) {
ucs_warn("Unable to detach shared memory segment of descriptors: %s",
ucs_status_string(status));
}
ucs_free(remote_seg);
}
/* detach the remote proceess's shared memory segment (remote recv FIFO) */
status = uct_mm_md_mapper_ops(iface->super.md)->detach(&self->mapped_desc);
if (status != UCS_OK) {
ucs_error("error detaching from remote FIFO");
}
uct_mm_ep_pending_purge(&self->super.super, NULL, NULL);
}
static UCS_CLASS_INIT_FUNC(uct_mm_ep_t, uct_iface_t *tl_iface,
const uct_device_addr_t *dev_addr,
const uct_iface_addr_t *iface_addr)
{
uct_mm_iface_t *iface = ucs_derived_of(tl_iface, uct_mm_iface_t);
const uct_mm_iface_addr_t *addr = (const void*)iface_addr;
ucs_status_t status;
size_t size_to_attach;
UCS_CLASS_CALL_SUPER_INIT(uct_base_ep_t, &iface->super);
/* Connect to the remote address (remote FIFO) */
/* Attach the address's memory */
size_to_attach = UCT_MM_GET_FIFO_SIZE(iface);
status =
uct_mm_md_mapper_ops(iface->super.md)->attach(addr->id,
size_to_attach,
(void *)addr->vaddr,
&self->mapped_desc.address,
&self->mapped_desc.cookie,
iface->path);
if (status != UCS_OK) {
ucs_error("failed to connect to remote peer with mm. remote mm_id: %zu",
addr->id);
return status;
}
self->mapped_desc.length = size_to_attach;
self->mapped_desc.mmid = addr->id;
uct_mm_set_fifo_ptrs(self->mapped_desc.address, &self->fifo_ctl, &self->fifo);
self->cached_tail = self->fifo_ctl->tail;
self->cached_signal_addrlen = self->fifo_ctl->signal_addrlen;
self->cached_signal_sockaddr = self->fifo_ctl->signal_sockaddr;
/* Send connect message to remote side so it will start polling */
status = uct_mm_ep_signal_remote(self, UCT_MM_IFACE_SIGNAL_CONNECT);
if (status != UCS_OK) {
uct_mm_md_mapper_ops(iface->super.md)->detach(&self->mapped_desc);
return status;
}
/* Initiate the hash which will keep the base_adresses of remote memory
* chunks that hold the descriptors for bcopy. */
sglib_hashed_uct_mm_remote_seg_t_init(self->remote_segments_hash);
ucs_arbiter_group_init(&self->arb_group);
/* Register for send side progress */
uct_worker_progress_register(iface->super.worker, uct_mm_iface_progress,
iface);
ucs_debug("mm: ep connected: %p, to remote_shmid: %zu", self, addr->id);
return UCS_OK;
}
static ucs_status_t uct_mm_iface_query(uct_iface_h tl_iface,
uct_iface_attr_t *iface_attr)
{
uct_mm_iface_t *iface = ucs_derived_of(tl_iface, uct_mm_iface_t);
memset(iface_attr, 0, sizeof(uct_iface_attr_t));
/* default values for all shared memory transports */
iface_attr->cap.put.max_short = UINT_MAX;
iface_attr->cap.put.max_bcopy = SIZE_MAX;
iface_attr->cap.put.min_zcopy = 0;
iface_attr->cap.put.max_zcopy = SIZE_MAX;
iface_attr->cap.put.opt_zcopy_align = UCS_SYS_CACHE_LINE_SIZE;
iface_attr->cap.put.align_mtu = iface_attr->cap.put.opt_zcopy_align;
iface_attr->cap.put.max_iov = 1;
iface_attr->cap.get.max_bcopy = SIZE_MAX;
iface_attr->cap.get.min_zcopy = 0;
iface_attr->cap.get.max_zcopy = SIZE_MAX;
iface_attr->cap.get.opt_zcopy_align = UCS_SYS_CACHE_LINE_SIZE;
iface_attr->cap.get.align_mtu = iface_attr->cap.get.opt_zcopy_align;
iface_attr->cap.get.max_iov = 1;
iface_attr->cap.am.max_short = iface->config.fifo_elem_size -
sizeof(uct_mm_fifo_element_t);
iface_attr->cap.am.max_bcopy = iface->config.seg_size;
iface_attr->cap.am.min_zcopy = 0;
iface_attr->cap.am.max_zcopy = 0;
iface_attr->cap.am.opt_zcopy_align = UCS_SYS_CACHE_LINE_SIZE;
iface_attr->cap.am.align_mtu = iface_attr->cap.am.opt_zcopy_align;
iface_attr->cap.am.max_iov = 1;
iface_attr->iface_addr_len = sizeof(uct_mm_iface_addr_t);
iface_attr->device_addr_len = UCT_SM_IFACE_DEVICE_ADDR_LEN;
iface_attr->ep_addr_len = 0;
iface_attr->cap.flags = UCT_IFACE_FLAG_PUT_SHORT |
UCT_IFACE_FLAG_PUT_BCOPY |
UCT_IFACE_FLAG_ATOMIC_ADD32 |
UCT_IFACE_FLAG_ATOMIC_ADD64 |
UCT_IFACE_FLAG_ATOMIC_FADD64 |
UCT_IFACE_FLAG_ATOMIC_FADD32 |
UCT_IFACE_FLAG_ATOMIC_SWAP64 |
UCT_IFACE_FLAG_ATOMIC_SWAP32 |
UCT_IFACE_FLAG_ATOMIC_CSWAP64 |
UCT_IFACE_FLAG_ATOMIC_CSWAP32 |
UCT_IFACE_FLAG_ATOMIC_CPU |
UCT_IFACE_FLAG_GET_BCOPY |
UCT_IFACE_FLAG_AM_SHORT |
UCT_IFACE_FLAG_AM_BCOPY |
UCT_IFACE_FLAG_PENDING |
UCT_IFACE_FLAG_AM_CB_SYNC |
UCT_IFACE_FLAG_CONNECT_TO_IFACE;
iface_attr->latency.overhead = 80e-9; /* 80 ns */
iface_attr->latency.growth = 0;
iface_attr->bandwidth = 6911 * 1024.0 * 1024.0;
iface_attr->overhead = 10e-9; /* 10 ns */
iface_attr->priority = uct_mm_md_mapper_ops(iface->super.md)->get_priority();
return UCS_OK;
}
static UCS_CLASS_CLEANUP_FUNC(uct_mm_ep_t)
{
uct_mm_iface_t *iface = ucs_derived_of(self->super.super.iface, uct_mm_iface_t);
ucs_status_t status;
uct_mm_remote_seg_t *remote_seg;
struct sglib_hashed_uct_mm_remote_seg_t_iterator iter;
/* don't send a disconnect message for now since it may prevent the receiver
* from progressing and reading incoming messages */
/* make sure the slow path function isn't invoked after the ep's cleanup */
if (self->cbq_elem_on) {
ucs_debug("Removing a remaining slow path progress function.");
uct_mm_ep_remove_slow_path_callback(iface, self);
}
uct_worker_progress_unregister(iface->super.worker, uct_mm_iface_progress,
iface);
for (remote_seg = sglib_hashed_uct_mm_remote_seg_t_it_init(&iter, self->remote_segments_hash);
remote_seg != NULL; remote_seg = sglib_hashed_uct_mm_remote_seg_t_it_next(&iter)) {
sglib_hashed_uct_mm_remote_seg_t_delete(self->remote_segments_hash, remote_seg);
/* detach the remote proceess's descriptors segment */
status = uct_mm_md_mapper_ops(iface->super.md)->detach(remote_seg);
if (status != UCS_OK) {
ucs_warn("Unable to detach shared memory segment of descriptors: %s",
ucs_status_string(status));
}
ucs_free(remote_seg);
}
/* detach the remote proceess's shared memory segment (remote recv FIFO) */
status = uct_mm_md_mapper_ops(iface->super.md)->detach(&self->mapped_desc);
if (status != UCS_OK) {
ucs_error("error detaching from remote FIFO");
}
uct_mm_ep_pending_purge(&self->super.super, NULL, NULL);
}
void *uct_mm_ep_attach_remote_seg(uct_mm_ep_t *ep, uct_mm_iface_t *iface, uct_mm_fifo_element_t *elem)
{
uct_mm_remote_seg_t *remote_seg, search;
ucs_status_t status;
/* take the mmid of the chunk that the desc belongs to, (the desc that the fifo_elem
* is 'assigned' to), and check if the ep has already attached to it.
*/
search.mmid = elem->desc_mmid;
remote_seg = sglib_hashed_uct_mm_remote_seg_t_find_member(ep->remote_segments_hash, &search);
if (remote_seg == NULL) {
/* not in the hash. attach to the memory the mmid refers to. the attach call
* will return the base address of the mmid's chunk -
* save this base address in a hash table (which maps mmid to base address). */
remote_seg = ucs_malloc(sizeof(*remote_seg), "mm_desc");
if (remote_seg == NULL) {
ucs_fatal("Failed to allocated memory for a remote segment identifier. %m");
}
status = uct_mm_md_mapper_ops(iface->super.md)->attach(elem->desc_mmid,
elem->desc_mpool_size,
elem->desc_chunk_base_addr,
&remote_seg->address,
&remote_seg->cookie,
iface->path);
if (status != UCS_OK) {
ucs_fatal("Failed to attach to remote mmid:%zu. %s ",
elem->desc_mmid, ucs_status_string(status));
}
remote_seg->mmid = elem->desc_mmid;
remote_seg->length = elem->desc_mpool_size;
/* put the base address into the ep's hash table */
sglib_hashed_uct_mm_remote_seg_t_add(ep->remote_segments_hash, remote_seg);
}
return remote_seg->address;
}
