Ejemplo n.º 1
0
static UCS_CLASS_INIT_FUNC(uct_self_iface_t, uct_md_h md, uct_worker_h worker,
                           const uct_iface_params_t *params,
                           const uct_iface_config_t *tl_config)
{
    ucs_status_t status;
    uct_self_iface_config_t *self_config = 0;

    ucs_trace_func("Creating a loop-back transport self=%p rxh=%lu",
                   self, params->rx_headroom);

    if (strcmp(params->dev_name, UCT_SELF_NAME) != 0) {
        ucs_error("No device was found: %s", params->dev_name);
        return UCS_ERR_NO_DEVICE;
    }

    UCS_CLASS_CALL_SUPER_INIT(uct_base_iface_t, &uct_self_iface_ops, md, worker,
                              tl_config UCS_STATS_ARG(params->stats_root)
                              UCS_STATS_ARG(UCT_SELF_NAME));

    self_config = ucs_derived_of(tl_config, uct_self_iface_config_t);

    self->id              = ucs_generate_uuid((uintptr_t)self);
    self->rx_headroom     = params->rx_headroom;
    self->data_length     = self_config->super.max_bcopy;
    self->release_desc.cb = uct_self_iface_release_desc;

    /* create a memory pool for data transferred */
    status = uct_iface_mpool_init(&self->super,
                                  &self->msg_desc_mp,
                                  sizeof(uct_recv_desc_t) + self->rx_headroom +
                                                            self->data_length,
                                  sizeof(uct_recv_desc_t) + self->rx_headroom,
                                  UCS_SYS_CACHE_LINE_SIZE,
                                  &self_config->mp,
                                  256,
                                  ucs_empty_function,
                                  "self_msg_desc");
    if (UCS_OK != status) {
        ucs_error("Failed to create a memory pool for the loop-back transport");
        goto err;
    }

    /* set the message descriptor for the loop-back */
    self->msg_cur_desc = ucs_mpool_get(&self->msg_desc_mp);
    VALGRIND_MAKE_MEM_DEFINED(self->msg_cur_desc, sizeof(*(self->msg_cur_desc)));
    if (NULL == self->msg_cur_desc) {
        ucs_error("Failed to get the first descriptor in loop-back MP storage");
        status = UCS_ERR_NO_RESOURCE;
        goto destroy_mpool;
    }

    ucs_debug("Created a loop-back iface. id=0x%lx, desc=%p, len=%u, tx_hdr=%lu",
              self->id, self->msg_cur_desc, self->data_length, self->rx_headroom);
    return UCS_OK;

destroy_mpool:
    ucs_mpool_cleanup(&self->msg_desc_mp, 1);
err:
    return status;
}
Ejemplo n.º 2
0
static UCS_CLASS_INIT_FUNC(uct_gdr_copy_iface_t, uct_md_h md, uct_worker_h worker,
                           const uct_iface_params_t *params,
                           const uct_iface_config_t *tl_config)
{
    UCS_CLASS_CALL_SUPER_INIT(uct_base_iface_t, &uct_gdr_copy_iface_ops, md, worker,
                              params, tl_config UCS_STATS_ARG(params->stats_root)
                              UCS_STATS_ARG(UCT_GDR_COPY_TL_NAME));

    if (strncmp(params->mode.device.dev_name,
                UCT_CUDA_DEV_NAME, strlen(UCT_CUDA_DEV_NAME)) != 0) {
        ucs_error("No device was found: %s", params->mode.device.dev_name);
        return UCS_ERR_NO_DEVICE;
    }

    self->id = ucs_generate_uuid((uintptr_t)self);

    return UCS_OK;
}
Ejemplo n.º 3
0
static UCS_CLASS_INIT_FUNC(uct_cm_iface_t, uct_pd_h pd, uct_worker_h worker,
                           const char *dev_name, size_t rx_headroom,
                           const uct_iface_config_t *tl_config)
{
    uct_cm_iface_config_t *config = ucs_derived_of(tl_config, uct_cm_iface_config_t);
    ucs_status_t status;
    int ret;

    ucs_trace_func("");

    UCS_CLASS_CALL_SUPER_INIT(uct_ib_iface_t, &uct_cm_iface_ops, pd, worker,
                              dev_name, rx_headroom, 0 /* rx_priv_len */,
                              0 /* rx_hdr_len */, 1 /* tx_cq_len */,
                              IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE, /* mss */
                              &config->super);

    if (worker->async == NULL) {
        ucs_error("cm must have async!=NULL");
        return UCS_ERR_INVALID_PARAM;
    }

    self->service_id          = (uint32_t)(ucs_generate_uuid((uintptr_t)self) &
                                             (~IB_CM_ASSIGN_SERVICE_ID_MASK));
    self->num_outstanding     = 0;

    self->config.timeout_ms   = (int)(config->timeout * 1e3 + 0.5);
    self->config.max_outstanding = config->max_outstanding;
    self->config.retry_count  = ucs_min(config->retry_count, UINT8_MAX);
    self->notify_q.head       = NULL;
    ucs_queue_head_init(&self->notify_q);

    self->outstanding = ucs_calloc(self->config.max_outstanding,
                                   sizeof(*self->outstanding),
                                   "cm_outstanding");
    if (self->outstanding == NULL) {
        status = UCS_ERR_NO_MEMORY;
        goto err;
    }

    self->cmdev = ib_cm_open_device(uct_ib_iface_device(&self->super)->ibv_context);
    if (self->cmdev == NULL) {
        ucs_error("ib_cm_open_device() failed: %m. Check if ib_ucm.ko module is loaded.");
        status = UCS_ERR_NO_DEVICE;
        goto err_free_outstanding;
    }

    status = ucs_sys_fcntl_modfl(self->cmdev->fd, O_NONBLOCK, 0);
    if (status != UCS_OK) {
        goto err_close_device;
    }

    ret = ib_cm_create_id(self->cmdev, &self->listen_id, self);
    if (ret) {
        ucs_error("ib_cm_create_id() failed: %m");
        status = UCS_ERR_NO_DEVICE;
        goto err_close_device;
    }

    ret = ib_cm_listen(self->listen_id, self->service_id, 0);
    if (ret) {
        ucs_error("ib_cm_listen() failed: %m");
        status = UCS_ERR_INVALID_ADDR;
        goto err_destroy_id;
    }

    if (config->async_mode == UCS_ASYNC_MODE_SIGNAL) {
        ucs_warn("ib_cm fd does not support SIGIO");
    }

    status = ucs_async_set_event_handler(config->async_mode, self->cmdev->fd,
                                         POLLIN, uct_cm_iface_event_handler, self,
                                         worker->async);
    if (status != UCS_OK) {
        ucs_error("failed to set event handler");
        goto err_destroy_id;
    }

    ucs_debug("listening for SIDR service_id 0x%x on fd %d", self->service_id,
              self->cmdev->fd);
    return UCS_OK;

err_destroy_id:
    ib_cm_destroy_id(self->listen_id);
err_close_device:
    ib_cm_close_device(self->cmdev);
err_free_outstanding:
    ucs_free(self->outstanding);
err:
    return status;
}