static int print_device_info(void) {
struct ibv_device ** ibv_devs;
int i = 0;
/*TODO: get num_devs automatically*/
int num_devs = 1;
/*NULL => get all devices*/
ibv_devs = ibv_get_device_list(NULL);
for (i = 0; i < num_devs; i++) {
struct ibv_context *ibv_contxt;
struct ibv_device_attr device_attr;
char *dev_name;
uint64_t dev_guid;
ibv_contxt = ibv_open_device (ibv_devs[i]);
dev_name = ibv_get_device_name(ibv_devs[i]);
dev_guid = ibv_get_device_guid(ibv_devs[i]);
printf("%s (%d):\n", dev_name, dev_guid);
ibv_query_device (ibv_contxt, &device_attr);
printf(" Record : %d\n", i);
printf(" max_mr_size : %llu\n", device_attr.max_mr_size);
printf(" max_mr : %llu\n", device_attr.max_mr);
ibv_close_device (ibv_contxt);
}
ibv_free_device_list(ibv_devs);
return 0;
}
void rdma_backend_fini(RdmaBackendDev *backend_dev)
{
mad_fini(backend_dev);
g_hash_table_destroy(ah_hash);
ibv_destroy_comp_channel(backend_dev->channel);
ibv_close_device(backend_dev->context);
}
/*
* Closes the interface to the opp module.
* No path functions will work after calling this function.
*/
void
op_path_close(void *uc)
{
struct op_path_context *context = (struct op_path_context *)uc;
op_ppath_close_reader(context->reader);
/*
* Close the HFI and release the memory being used.
*/
ibv_close_device(context->ibv_context);
/*
* Releases the memory used by the list - and decrements the usage
* counters on the HFIs. Note that this is important; all the HFIs
* in the device list are flagged as being used from the time
* the list is acquired until it is freed.
*/
if (dev_list) {
ibv_free_device_list(dev_list);
dev_list = NULL;
num_devices = 0;
}
free(context);
}
RDMAAdapter::~RDMAAdapter() {
StopInternalThread();
CHECK(!ibv_destroy_cq(cq_)) << "Failed to destroy CQ";
CHECK(!ibv_destroy_comp_channel(channel_)) << "Failed to destroy channel";
CHECK(!ibv_dealloc_pd(pd_)) << "Failed to deallocate PD";
CHECK(!ibv_close_device(context_)) << "Failed to release context";
}
static void ucma_cleanup(void)
{
if (cma_dev_cnt) {
while (cma_dev_cnt)
ibv_close_device(cma_dev_array[--cma_dev_cnt].verbs);
free(cma_dev_array);
cma_dev_cnt = 0;
}
}
int __ibv_close_device_1_0(struct ibv_context_1_0 *context)
{
int ret;
ret = ibv_close_device(context->real_context);
if (ret)
return ret;
free(context);
return 0;
}
int __ibv_close_device_1_0(struct ibv_context_1_0 *context)
{ fprintf(stderr, "%s:%s:%d \n", __func__, __FILE__, __LINE__);
int ret;
ret = ibv_close_device(context->real_context);
if (ret)
return ret;
free(context);
return 0;
}
int BClientContext::destroy_context () {
if (qp) TEST_NZ(ibv_destroy_qp (qp));
if (recv_memory_mr) TEST_NZ (ibv_dereg_mr (recv_memory_mr));
if (lock_result_mr) TEST_NZ (ibv_dereg_mr (lock_result_mr));
if (cq) TEST_NZ (ibv_destroy_cq (cq));
if (pd) TEST_NZ (ibv_dealloc_pd (pd));
if (ib_ctx) TEST_NZ (ibv_close_device (ib_ctx));
if (sockfd >= 0) TEST_NZ (close (sockfd));
return 0;
}
static
void psofed_scan_hca_ports(struct ibv_device *ib_dev)
{
struct ibv_context *ctx;
struct ibv_device_attr device_attr;
int rc;
unsigned port_cnt;
unsigned port;
const char *dev_name;
dev_name =ibv_get_device_name(ib_dev);
if (!dev_name) dev_name = "unknown";
ctx = ibv_open_device(ib_dev);
if (!ctx) goto err_open_dev;
rc = ibv_query_device(ctx, &device_attr);
if (!rc) {
port_cnt = device_attr.phys_port_cnt;
if (port_cnt > 128) port_cnt = 128;
} else {
// Query failed. Assume 2 ports.
port_cnt = 2;
}
for (port = 1; port <= port_cnt; port++) {
struct ibv_port_attr port_attr;
enum ibv_port_state port_state;
const char *marker;
rc = ibv_query_port(ctx, port, &port_attr);
port_state = !rc ? port_attr.state : 999 /* unknown */;
marker = "";
if (port_state == IBV_PORT_ACTIVE &&
(!psofed_hca || !strcmp(dev_name, psofed_hca)) &&
(!psofed_port || psofed_port == port)) {
// use this port for the communication:
if (!psofed_hca) psofed_hca = strdup(dev_name);
if (!psofed_port) psofed_port = port;
marker = "*";
}
psofed_dprint(3, "IB port <%s:%u>: %s%s",
dev_name, port, port_state_str(port_state), marker);
}
if (ctx) ibv_close_device(ctx);
err_open_dev:
return;
}
static int resources_destroy(struct resources *res)
{
int rc = 0;
if (res->qp) {
if (ibv_destroy_qp(res->qp)) {
fprintf(stderr, "failed to destroy QP\n");
rc = 1;
}
}
if (res->mr) {
if (ibv_dereg_mr(res->mr)) {
fprintf(stderr, "failed to deregister MR\n");
rc = 1;
}
}
if (res->buf)
free(res->buf);
if (res->cq) {
if (ibv_destroy_cq(res->cq)) {
fprintf(stderr, "failed to destroy CQ\n");
rc = 1;
}
}
if (res->pd) {
if (ibv_dealloc_pd(res->pd)) {
fprintf(stderr, "failed to deallocate PD\n");
rc = 1;
}
}
if (res->ib_ctx) {
if (ibv_close_device(res->ib_ctx)) {
fprintf(stderr, "failed to close device context\n");
rc = 1;
}
}
if (res->sock >= 0) {
if (close(res->sock)) {
fprintf(stderr, "failed to close socket\n");
rc = 1;
}
}
return rc;
}
static void device_item_destruct(opal_common_verbs_device_item_t *di)
{
if (NULL != di->device_name) {
free(di->device_name);
}
/* Only free the context if a) the device is open, and b) the
upper layer didn't tell us not to */
if (NULL != di->context && di->destructor_free_context) {
ibv_close_device(di->context);
}
/* Zero out all the fields */
device_item_construct(di);
}
static void
rdmasniff_cleanup(pcap_t *handle)
{
struct pcap_rdmasniff *priv = handle->priv;
ibv_dereg_mr(priv->mr);
ibv_destroy_flow(priv->flow);
ibv_destroy_qp(priv->qp);
ibv_destroy_cq(priv->cq);
ibv_dealloc_pd(priv->pd);
ibv_destroy_comp_channel(priv->channel);
ibv_close_device(priv->context);
free(priv->oneshot_buffer);
pcap_cleanup_live_common(handle);
}
int
resource_destroy(resource_t *res)
{
int rc = 0;
// Delete QP
if (res->qp && ibv_destroy_qp(res->qp)){
fprintf(stderr, "failed to destroy QP\n");
rc = 1;
}
// Deregister MR
if(res->mr_list && res->mr_size > 0){
int i;
for(i=0; i<res->mr_size; i++){
ibv_dereg_mr(res->mr_list[i]);
}
free(res->mr_list);
}
if(res->buf){
free(res->buf);
}
// Delete CQ
if (res->scq && ibv_destroy_cq(res->scq)){
fprintf(stderr, "failed to destroy SCQ\n");
rc = 1;
}
if (res->rcq && ibv_destroy_cq(res->rcq)){
fprintf(stderr, "failed to destroy RCQ\n");
rc = 1;
}
if(res->comp_ch && ibv_destroy_comp_channel(res->comp_ch)){
fprintf(stderr, "failed to destroy Complete CH\n");
rc = 1;
}
// Deallocate PD
if (res->pd && ibv_dealloc_pd(res->pd)){
fprintf(stderr, "failed to deallocate PD\n");
rc = 1;
}
if (res->ib_ctx && ibv_close_device(res->ib_ctx)){
fprintf(stderr, "failed to close device context\n");
rc = 1;
}
return rc;
}
static int destroy_ctx_resources(struct pingpong_context *ctx) {
int test_result = 0;
if (ibv_destroy_qp(ctx->qp)) {
fprintf(stderr, "failed to destroy QP\n");
test_result = 1;
}
if (ibv_destroy_cq(ctx->cq)) {
fprintf(stderr, "failed to destroy CQ\n");
test_result = 1;
}
if (ibv_dereg_mr(ctx->mr)) {
fprintf(stderr, "failed to deregister MR\n");
test_result = 1;
}
if (ctx->channel) {
if (ibv_destroy_comp_channel(ctx->channel)) {
fprintf(stderr, "failed to destroy channel \n");
test_result = 1;
}
}
if (ibv_dealloc_pd(ctx->pd)) {
fprintf(stderr, "failed to deallocate PD\n");
test_result = 1;
}
if (ibv_close_device(ctx->context)) {
fprintf(stderr, "failed to close device context\n");
test_result = 1;
}
free(ctx->buf);
free(ctx);
free(tstamp);
return test_result;
}
static int destroy_ctx_resources(struct pingpong_context *ctx,int num_qps) {
int i;
int test_result = 0;
for (i = 0; i < num_qps; i++) {
if (ibv_destroy_qp(ctx->qp[i])) {
fprintf(stderr, "failed to destroy QP\n");
test_result = 1;
}
}
if (ibv_destroy_cq(ctx->cq)) {
fprintf(stderr, "failed to destroy CQ\n");
test_result = 1;
}
if (ibv_dereg_mr(ctx->mr)) {
fprintf(stderr, "failed to deregister MR\n");
test_result = 1;
}
if (ibv_dealloc_pd(ctx->pd)) {
fprintf(stderr, "failed to deallocate PD\n");
test_result = 1;
}
if (ibv_close_device(ctx->context)) {
fprintf(stderr, "failed to close device context\n");
test_result = 1;
}
free(ctx->buf);
free(ctx->qp);
free(ctx->scnt);
free(ctx->ccnt);
free(ctx);
return test_result;
}
/*
* USNIC plugs into the verbs framework, but is not a usable device.
* Manually check for devices and fail gracefully if none are present.
* This avoids the lower libraries (libibverbs and librdmacm) from
* reporting error messages to stderr.
*/
static int fi_ibv_have_device(void)
{
struct ibv_device **devs;
struct ibv_context *verbs;
int i, ret = 0;
devs = ibv_get_device_list(NULL);
if (!devs)
return 0;
for (i = 0; devs[i]; i++) {
verbs = ibv_open_device(devs[i]);
if (verbs) {
ibv_close_device(verbs);
ret = 1;
break;
}
}
ibv_free_device_list(devs);
return ret;
}
/** ========================================================================= */
static int open_verbs_ctx(struct oib_port *port)
{
int i;
int num_devices;
struct ibv_device **dev_list;
dev_list = ibv_get_device_list(&num_devices);
for (i = 0; i < num_devices; ++i)
if (dev_list[i] != NULL
&& (strncmp(dev_list[i]->name, port->hfi_name, sizeof(dev_list[i]->name)) == 0) )
break;
if (i >= num_devices) {
ibv_free_device_list(dev_list);
OUTPUT_ERROR("failed to find verbs device\n");
return EIO;
}
port->verbs_ctx = ibv_open_device(dev_list[i]);
ibv_free_device_list(dev_list);
if (port->verbs_ctx == NULL)
{
OUTPUT_ERROR("failed to open verbs device\n");
return EIO;
}
if (sem_init(&port->lock,0,1) != 0)
{
ibv_close_device(port->verbs_ctx);
OUTPUT_ERROR("failed to init registry lock\n");
return EIO;
}
return 0;
}
/* ////////////////////////////////////////////////////////////////////////// */
static int
verbs_runtime_query(mca_base_module_t **module,
int *priority,
const char *hint)
{
int rc = OSHMEM_SUCCESS;
openib_device_t my_device;
openib_device_t *device = &my_device;
int num_devs = 0;
int i = 0;
*priority = 0;
*module = NULL;
memset(device, 0, sizeof(*device));
#ifdef HAVE_IBV_GET_DEVICE_LIST
device->ib_devs = ibv_get_device_list(&num_devs);
#else
#error unsupported ibv_get_device_list in infiniband/verbs.h
#endif
if (num_devs == 0 || !device->ib_devs) {
return OSHMEM_ERR_NOT_SUPPORTED;
}
/* Open device */
if (NULL != mca_sshmem_verbs_component.hca_name) {
for (i = 0; i < num_devs; i++) {
if (0 == strcmp(mca_sshmem_verbs_component.hca_name, ibv_get_device_name(device->ib_devs[i]))) {
device->ib_dev = device->ib_devs[i];
break;
}
}
} else {
device->ib_dev = device->ib_devs[0];
}
if (NULL == device->ib_dev) {
rc = OSHMEM_ERR_NOT_FOUND;
goto out;
}
if (NULL == (device->ib_dev_context = ibv_open_device(device->ib_dev))) {
rc = OSHMEM_ERR_RESOURCE_BUSY;
goto out;
}
/* Obtain device attributes */
if (ibv_query_device(device->ib_dev_context, &device->ib_dev_attr)) {
rc = OSHMEM_ERR_RESOURCE_BUSY;
goto out;
}
/* Allocate the protection domain for the device */
device->ib_pd = ibv_alloc_pd(device->ib_dev_context);
if (NULL == device->ib_pd) {
rc = OSHMEM_ERR_RESOURCE_BUSY;
goto out;
}
/* Allocate memory */
if (!rc) {
void *addr = NULL;
size_t size = getpagesize();
struct ibv_mr *ib_mr = NULL;
uint64_t access_flag = IBV_ACCESS_LOCAL_WRITE |
IBV_ACCESS_REMOTE_WRITE |
IBV_ACCESS_REMOTE_READ;
uint64_t exp_access_flag = 0;
OBJ_CONSTRUCT(&device->ib_mr_array, opal_value_array_t);
opal_value_array_init(&device->ib_mr_array, sizeof(struct ibv_mr *));
#if defined(MPAGE_ENABLE) && (MPAGE_ENABLE > 0)
exp_access_flag = IBV_EXP_ACCESS_ALLOCATE_MR |
IBV_EXP_ACCESS_SHARED_MR_USER_READ |
IBV_EXP_ACCESS_SHARED_MR_USER_WRITE;
#endif /* MPAGE_ENABLE */
struct ibv_exp_reg_mr_in in = {device->ib_pd, addr, size, access_flag|exp_access_flag, 0};
ib_mr = ibv_exp_reg_mr(&in);
if (NULL == ib_mr) {
rc = OSHMEM_ERR_OUT_OF_RESOURCE;
} else {
device->ib_mr_shared = ib_mr;
opal_value_array_append_item(&device->ib_mr_array, &ib_mr);
}
#if defined(MPAGE_ENABLE) && (MPAGE_ENABLE > 0)
if (!rc) {
struct ibv_exp_reg_shared_mr_in in_smr;
access_flag = IBV_ACCESS_LOCAL_WRITE |
IBV_ACCESS_REMOTE_WRITE |
IBV_ACCESS_REMOTE_READ|
IBV_EXP_ACCESS_NO_RDMA;
addr = (void *)mca_sshmem_base_start_address;
mca_sshmem_verbs_fill_shared_mr(&in_smr, device->ib_pd, device->ib_mr_shared->handle, addr, access_flag);
ib_mr = ibv_exp_reg_shared_mr(&in_smr);
if (NULL == ib_mr) {
mca_sshmem_verbs_component.has_shared_mr = 0;
} else {
opal_value_array_append_item(&device->ib_mr_array, &ib_mr);
mca_sshmem_verbs_component.has_shared_mr = 1;
}
}
#endif /* MPAGE_ENABLE */
}
/* all is well - rainbows and butterflies */
if (!rc) {
*priority = mca_sshmem_verbs_component.priority;
*module = (mca_base_module_t *)&mca_sshmem_verbs_module.super;
}
out:
if (device) {
if (opal_value_array_get_size(&device->ib_mr_array)) {
struct ibv_mr** array;
struct ibv_mr* ib_mr = NULL;
array = OPAL_VALUE_ARRAY_GET_BASE(&device->ib_mr_array, struct ibv_mr *);
while (opal_value_array_get_size(&device->ib_mr_array) > 0) {
ib_mr = array[0];
ibv_dereg_mr(ib_mr);
opal_value_array_remove_item(&device->ib_mr_array, 0);
}
if (device->ib_mr_shared) {
device->ib_mr_shared = NULL;
}
OBJ_DESTRUCT(&device->ib_mr_array);
}
if (device->ib_pd) {
ibv_dealloc_pd(device->ib_pd);
device->ib_pd = NULL;
}
if(device->ib_dev_context) {
ibv_close_device(device->ib_dev_context);
device->ib_dev_context = NULL;
}
if(device->ib_devs) {
ibv_free_device_list(device->ib_devs);
device->ib_devs = NULL;
}
}
return rc;
}
/** ========================================================================= */
static void close_verbs_ctx(struct oib_port *port)
{
sem_destroy(&port->lock);
ibv_close_device(port->verbs_ctx);
}
/*
* Opens the interface to the opp module.
* Must be called before any use of the path functions.
*
* device The verbs context for the HFI.
* Can be acquired via op_path_find_hfi.
*
* port_num The port to use for sending queries.
*
* This information is used for querying pkeys and
* calculating timeouts.
*
* Returns a pointer to the op_path context on success, or returns NULL
* and sets errno if the device could not be opened.
*/
void *
op_path_open(struct ibv_device *device, int p)
{
int i, err;
struct op_path_context *context;
if (!device) {
errno=ENXIO;
return NULL;
}
context = malloc(sizeof(struct op_path_context));
if (!context) {
errno=ENOMEM;
return NULL;
}
memset(context,0,sizeof(struct op_path_context));
context->ibv_context = ibv_open_device(device);
if (!context->ibv_context) {
errno=ENODEV;
goto open_device_failed;
}
context->port_num = p;
context->reader = op_ppath_allocate_reader();
if (!context->reader) {
errno=ENOMEM;
goto alloc_reader_failed;
}
err = op_ppath_create_reader(context->reader);
if (err) {
errno=err;
goto create_reader_failed;
}
if ((err=ibv_query_device(context->ibv_context,
&(context->device_attr)))) {
errno=EFAULT;
goto query_attr_failed;
}
if ((err=ibv_query_port(context->ibv_context,
context->port_num,
&(context->port_attr)))) {
errno=EFAULT;
goto query_attr_failed;
}
context->pkey_table = malloc(context->device_attr.max_pkeys* sizeof(int));
if (!context->pkey_table) {
errno= ENOMEM;
goto query_attr_failed;
}
memset(context->pkey_table,0,context->device_attr.max_pkeys* sizeof(int));
for (i = 0, err = 0; !err && i<context->device_attr.max_pkeys; i++) {
err = ibv_query_pkey(context->ibv_context, context->port_num, i,
&(context->pkey_table[i]));
if (err) {
errno=EFAULT;
goto query_pkey_failed;
}
}
return context;
query_pkey_failed:
free(context->pkey_table);
query_attr_failed:
op_ppath_close_reader(context->reader);
create_reader_failed:
free(context->reader);
alloc_reader_failed:
ibv_close_device(context->ibv_context);
open_device_failed:
free(context);
return NULL;
}
static int
mlx5_glue_close_device(struct ibv_context *context)
{
return ibv_close_device(context);
}
static ucs_status_t uct_ib_mlx5dv_md_open(struct ibv_device *ibv_device,
uct_ib_md_t **p_md)
{
uint32_t out[UCT_IB_MLX5DV_ST_SZ_DW(query_hca_cap_out)] = {};
uint32_t in[UCT_IB_MLX5DV_ST_SZ_DW(query_hca_cap_in)] = {};
struct mlx5dv_context_attr dv_attr = {};
ucs_status_t status = UCS_OK;
int atomic = 0, has_dc = 1;
struct ibv_context *ctx;
uct_ib_device_t *dev;
uct_ib_mlx5_md_t *md;
void *cap;
int ret;
#if HAVE_DECL_MLX5DV_IS_SUPPORTED
if (!mlx5dv_is_supported(ibv_device)) {
return UCS_ERR_UNSUPPORTED;
}
#endif
dv_attr.flags |= MLX5DV_CONTEXT_FLAGS_DEVX;
ctx = mlx5dv_open_device(ibv_device, &dv_attr);
if (ctx == NULL) {
ucs_debug("mlx5dv_open_device(%s) failed: %m", ibv_get_device_name(ibv_device));
status = UCS_ERR_UNSUPPORTED;
goto err;
}
md = ucs_calloc(1, sizeof(*md), "ib_mlx5_md");
if (md == NULL) {
status = UCS_ERR_NO_MEMORY;
goto err_free_context;
}
md->super.ops = &uct_ib_mlx5dv_md_ops;
dev = &md->super.dev;
dev->ibv_context = ctx;
IBV_EXP_DEVICE_ATTR_SET_COMP_MASK(&dev->dev_attr);
ret = ibv_query_device_ex(dev->ibv_context, NULL, &dev->dev_attr);
if (ret != 0) {
ucs_error("ibv_query_device() returned %d: %m", ret);
status = UCS_ERR_IO_ERROR;
goto err_free;
}
cap = UCT_IB_MLX5DV_ADDR_OF(query_hca_cap_out, out, capability);
UCT_IB_MLX5DV_SET(query_hca_cap_in, in, opcode, UCT_IB_MLX5_CMD_OP_QUERY_HCA_CAP);
UCT_IB_MLX5DV_SET(query_hca_cap_in, in, op_mod, UCT_IB_MLX5_HCA_CAP_OPMOD_GET_MAX |
(UCT_IB_MLX5_CAP_GENERAL << 1));
ret = mlx5dv_devx_general_cmd(ctx, in, sizeof(in), out, sizeof(out));
if (ret == 0) {
if (!UCT_IB_MLX5DV_GET(cmd_hca_cap, cap, dct)) {
has_dc = 0;
}
if (UCT_IB_MLX5DV_GET(cmd_hca_cap, cap, compact_address_vector)) {
dev->flags |= UCT_IB_DEVICE_FLAG_AV;
}
if (UCT_IB_MLX5DV_GET(cmd_hca_cap, cap, fixed_buffer_size)) {
md->flags |= UCT_IB_MLX5_MD_FLAG_KSM;
}
if (UCT_IB_MLX5DV_GET(cmd_hca_cap, cap, atomic)) {
atomic = 1;
}
} else if ((errno != EPERM) &&
(errno != EPROTONOSUPPORT) &&
(errno != EOPNOTSUPP)) {
ucs_error("MLX5_CMD_OP_QUERY_HCA_CAP failed: %m");
status = UCS_ERR_IO_ERROR;
goto err_free;
} else {
status = UCS_ERR_UNSUPPORTED;
goto err_free;
}
if (atomic) {
int ops = UCT_IB_MLX5_ATOMIC_OPS_CMP_SWAP |
UCT_IB_MLX5_ATOMIC_OPS_FETCH_ADD;
uint8_t arg_size;
int cap_ops, mode8b;
UCT_IB_MLX5DV_SET(query_hca_cap_in, in, op_mod, UCT_IB_MLX5_HCA_CAP_OPMOD_GET_MAX |
(UCT_IB_MLX5_CAP_ATOMIC << 1));
ret = mlx5dv_devx_general_cmd(ctx, in, sizeof(in), out, sizeof(out));
if (ret != 0) {
ucs_error("MLX5_CMD_OP_QUERY_HCA_CAP failed: %m");
return UCS_ERR_IO_ERROR;
}
arg_size = UCT_IB_MLX5DV_GET(atomic_caps, cap, atomic_size_qp);
cap_ops = UCT_IB_MLX5DV_GET(atomic_caps, cap, atomic_operations);
mode8b = UCT_IB_MLX5DV_GET(atomic_caps, cap, atomic_req_8B_endianness_mode);
if ((cap_ops & ops) == ops) {
dev->atomic_arg_sizes = sizeof(uint64_t);
if (!mode8b) {
dev->atomic_arg_sizes_be = sizeof(uint64_t);
}
}
ops |= UCT_IB_MLX5_ATOMIC_OPS_MASKED_CMP_SWAP |
UCT_IB_MLX5_ATOMIC_OPS_MASKED_FETCH_ADD;
if (has_dc) {
arg_size &= UCT_IB_MLX5DV_GET(query_hca_cap_out, out,
capability.atomic_caps.atomic_size_dc);
}
if ((cap_ops & ops) == ops) {
dev->ext_atomic_arg_sizes = arg_size;
if (mode8b) {
arg_size &= ~(sizeof(uint64_t));
}
dev->ext_atomic_arg_sizes_be = arg_size;
}
}
if (has_dc) {
status = uct_ib_mlx5_check_dc(dev);
}
dev->flags |= UCT_IB_DEVICE_FLAG_MLX5_PRM;
*p_md = &md->super;
return status;
err_free:
ucs_free(md);
err_free_context:
ibv_close_device(ctx);
err:
return status;
}
int main(int argc, char *argv[])
{
char *device_name = NULL;
struct ibv_async_event event;
struct ibv_context *ctx;
int ret = 0;
/* parse command line options */
while (1) {
int c;
c = getopt(argc, argv, "d:h");
if (c == -1)
break;
switch (c) {
case 'd':
device_name = strdup(optarg);
if (!device_name) {
fprintf(stderr, "Error, failed to allocate memory for the device name\n");
return -1;
}
break;
case 'h':
usage(argv[0]);
exit(1);
default:
fprintf(stderr, "Bad command line was used\n\n");
usage(argv[0]);
exit(1);
}
}
if (!device_name) {
fprintf(stderr, "Error, the device name is mandatory\n");
return -1;
}
ctx = get_device_context(device_name);
if (!ctx) {
fprintf(stderr, "Error, the context of the device name '%s' could not be opened\n", device_name);
free(device_name);
return -1;
}
printf("Listening on events for the device '%s'\n", device_name);
while (1) {
/* wait for the next async event */
ret = ibv_get_async_event(ctx, &event);
if (ret) {
fprintf(stderr, "Error, ibv_get_async_event() failed\n");
goto out;
}
/* print the event */
print_async_event(ctx, &event);
/* ack the event */
ibv_ack_async_event(&event);
}
ret = 0;
out:
if (ibv_close_device(ctx)) {
fprintf(stderr, "Error, failed to close the context of the device '%s'\n", device_name);
return -1;
}
printf("The context of the device name '%s' was closed\n", device_name);
free(device_name);
return ret;
}
int rdma_backend_init(RdmaBackendDev *backend_dev, PCIDevice *pdev,
RdmaDeviceResources *rdma_dev_res,
const char *backend_device_name, uint8_t port_num,
struct ibv_device_attr *dev_attr, CharBackend *mad_chr_be)
{
int i;
int ret = 0;
int num_ibv_devices;
struct ibv_device **dev_list;
memset(backend_dev, 0, sizeof(*backend_dev));
backend_dev->dev = pdev;
backend_dev->port_num = port_num;
backend_dev->rdma_dev_res = rdma_dev_res;
rdma_backend_register_comp_handler(dummy_comp_handler);
dev_list = ibv_get_device_list(&num_ibv_devices);
if (!dev_list) {
rdma_error_report("Failed to get IB devices list");
return -EIO;
}
if (num_ibv_devices == 0) {
rdma_error_report("No IB devices were found");
ret = -ENXIO;
goto out_free_dev_list;
}
if (backend_device_name) {
for (i = 0; dev_list[i]; ++i) {
if (!strcmp(ibv_get_device_name(dev_list[i]),
backend_device_name)) {
break;
}
}
backend_dev->ib_dev = dev_list[i];
if (!backend_dev->ib_dev) {
rdma_error_report("Failed to find IB device %s",
backend_device_name);
ret = -EIO;
goto out_free_dev_list;
}
} else {
backend_dev->ib_dev = *dev_list;
}
rdma_info_report("uverb device %s", backend_dev->ib_dev->dev_name);
backend_dev->context = ibv_open_device(backend_dev->ib_dev);
if (!backend_dev->context) {
rdma_error_report("Failed to open IB device %s",
ibv_get_device_name(backend_dev->ib_dev));
ret = -EIO;
goto out;
}
backend_dev->channel = ibv_create_comp_channel(backend_dev->context);
if (!backend_dev->channel) {
rdma_error_report("Failed to create IB communication channel");
ret = -EIO;
goto out_close_device;
}
ret = init_device_caps(backend_dev, dev_attr);
if (ret) {
rdma_error_report("Failed to initialize device capabilities");
ret = -EIO;
goto out_destroy_comm_channel;
}
ret = mad_init(backend_dev, mad_chr_be);
if (ret) {
rdma_error_report("Failed to initialize mad");
ret = -EIO;
goto out_destroy_comm_channel;
}
backend_dev->comp_thread.run = false;
backend_dev->comp_thread.is_running = false;
ah_cache_init();
goto out_free_dev_list;
out_destroy_comm_channel:
ibv_destroy_comp_channel(backend_dev->channel);
out_close_device:
ibv_close_device(backend_dev->context);
out_free_dev_list:
ibv_free_device_list(dev_list);
out:
return ret;
}
/* ////////////////////////////////////////////////////////////////////////// */
static int
segment_detach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
{
int rc = OSHMEM_SUCCESS;
openib_device_t *device = &memheap_device;
int i;
assert(ds_buf);
OPAL_OUTPUT_VERBOSE(
(70, oshmem_sshmem_base_framework.framework_output,
"%s: %s: detaching "
"(id: %d, addr: %p size: %lu, name: %s)\n",
mca_sshmem_verbs_component.super.base_version.mca_type_name,
mca_sshmem_verbs_component.super.base_version.mca_component_name,
ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
);
if (device) {
if (0 < (i = opal_value_array_get_size(&device->ib_mr_array))) {
struct ibv_mr** array;
struct ibv_mr* ib_mr = NULL;
array = OPAL_VALUE_ARRAY_GET_BASE(&device->ib_mr_array, struct ibv_mr *);
for (i--;i >= 0; i--) {
ib_mr = array[i];
if(ibv_dereg_mr(ib_mr)) {
OPAL_OUTPUT_VERBOSE(
(5, oshmem_sshmem_base_framework.framework_output,
"error ibv_dereg_mr(): %d: %s",
errno, strerror(errno))
);
rc = OSHMEM_ERROR;
}
opal_value_array_remove_item(&device->ib_mr_array, i);
}
if (!rc && device->ib_mr_shared) {
device->ib_mr_shared = NULL;
}
OBJ_DESTRUCT(&device->ib_mr_array);
}
if (!rc && device->ib_pd) {
if (ibv_dealloc_pd(device->ib_pd)) {
OPAL_OUTPUT_VERBOSE(
(5, oshmem_sshmem_base_framework.framework_output,
"error ibv_dealloc_pd(): %d: %s",
errno, strerror(errno))
);
rc = OSHMEM_ERROR;
} else {
device->ib_pd = NULL;
}
}
if(!rc && device->ib_dev_context) {
if(ibv_close_device(device->ib_dev_context)) {
OPAL_OUTPUT_VERBOSE(
(5, oshmem_sshmem_base_framework.framework_output,
"error ibv_close_device(): %d: %s",
errno, strerror(errno))
);
rc = OSHMEM_ERROR;
} else {
device->ib_dev_context = NULL;
}
}
if(!rc && device->ib_devs) {
ibv_free_device_list(device->ib_devs);
device->ib_devs = NULL;
}
}
/* reset the contents of the map_segment_t associated with this
* shared memory segment.
*/
shmem_ds_reset(ds_buf);
return rc;
}
struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size,
int rx_depth, int port, int use_event,
enum pp_wr_calc_op calc_op,
enum pp_wr_data_type calc_data_type,
char *calc_operands_str)
{
struct pingpong_context *ctx;
int rc;
ctx = malloc(sizeof *ctx);
if (!ctx)
return NULL;
memset(ctx, 0, sizeof *ctx);
ctx->size = size;
ctx->rx_depth = rx_depth;
ctx->calc_op.opcode = IBV_EXP_CALC_OP_NUMBER;
ctx->calc_op.data_type = IBV_EXP_CALC_DATA_TYPE_NUMBER;
ctx->calc_op.data_size = IBV_EXP_CALC_DATA_SIZE_NUMBER;
ctx->buf = memalign(page_size, size);
if (!ctx->buf) {
fprintf(stderr, "Couldn't allocate work buf.\n");
goto clean_ctx;
}
memset(ctx->buf, 0, size);
ctx->net_buf = memalign(page_size, size);
if (!ctx->net_buf) {
fprintf(stderr, "Couldn't allocate work buf.\n");
goto clean_buffer;
}
memset(ctx->net_buf, 0, size);
ctx->context = ibv_open_device(ib_dev);
if (!ctx->context) {
fprintf(stderr, "Couldn't get context for %s\n",
ibv_get_device_name(ib_dev));
goto clean_net_buf;
}
if (use_event) {
ctx->channel = ibv_create_comp_channel(ctx->context);
if (!ctx->channel) {
fprintf(stderr, "Couldn't create completion channel\n");
goto clean_device;
}
} else
ctx->channel = NULL;
ctx->pd = ibv_alloc_pd(ctx->context);
if (!ctx->pd) {
fprintf(stderr, "Couldn't allocate PD\n");
goto clean_comp_channel;
}
ctx->mr = ibv_reg_mr(ctx->pd, ctx->net_buf, size, IBV_ACCESS_LOCAL_WRITE);
if (!ctx->mr) {
fprintf(stderr, "Couldn't register MR\n");
goto clean_pd;
}
if (calc_op != PP_CALC_INVALID) {
int op_per_gather, num_op, max_num_op;
ctx->calc_op.opcode = IBV_EXP_CALC_OP_NUMBER;
ctx->calc_op.data_type = IBV_EXP_CALC_DATA_TYPE_NUMBER;
ctx->calc_op.data_size = IBV_EXP_CALC_DATA_SIZE_NUMBER;
num_op = pp_parse_calc_to_gather(calc_operands_str, calc_op, calc_data_type,
&ctx->calc_op, ctx->context, ctx->buf, ctx->net_buf);
if (num_op < 0) {
fprintf(stderr, "-E- failed parsing calc operators\n");
goto clean_mr;
}
rc = pp_query_calc_cap(ctx->context,
ctx->calc_op.opcode,
ctx->calc_op.data_type,
ctx->calc_op.data_size,
&op_per_gather, &max_num_op);
if (rc) {
fprintf(stderr, "-E- operation not supported on %s. valid ops are:\n",
ibv_get_device_name(ib_dev));
pp_print_dev_calc_ops(ctx->context);
goto clean_mr;
}
if (pp_prepare_sg_list(op_per_gather, num_op, ctx->mr->lkey, &ctx->calc_op, ctx->net_buf)) {
fprintf(stderr, "-failed to prepare the sg list\n");
goto clean_mr;
}
}
ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL,
ctx->channel, 0);
if (!ctx->cq) {
fprintf(stderr, "Couldn't create CQ\n");
goto clean_mr;
}
{
struct ibv_exp_qp_init_attr attr = {
.send_cq = ctx->cq,
.recv_cq = ctx->cq,
.cap = {
.max_send_wr = 16,
.max_recv_wr = rx_depth,
.max_send_sge = 16,
.max_recv_sge = 16
},
.qp_type = IBV_QPT_RC,
.pd = ctx->pd
};
attr.comp_mask |= IBV_EXP_QP_INIT_ATTR_CREATE_FLAGS | IBV_EXP_QP_INIT_ATTR_PD;
attr.exp_create_flags = IBV_EXP_QP_CREATE_CROSS_CHANNEL;
ctx->qp = ibv_exp_create_qp(ctx->context, &attr);
if (!ctx->qp) {
fprintf(stderr, "Couldn't create QP\n");
goto clean_cq;
}
}
{
struct ibv_qp_attr attr = {
.qp_state = IBV_QPS_INIT,
.pkey_index = 0,
.port_num = port,
.qp_access_flags = 0
};
if (ibv_modify_qp(ctx->qp, &attr,
IBV_QP_STATE |
IBV_QP_PKEY_INDEX |
IBV_QP_PORT |
IBV_QP_ACCESS_FLAGS)) {
fprintf(stderr, "Failed to modify QP to INIT\n");
goto clean_qp;
}
}
ctx->mcq = ibv_create_cq(ctx->context, rx_depth + 1, NULL,
ctx->channel, 0);
if (!ctx->mcq) {
fprintf(stderr, "Couldn't create CQ for MQP\n");
goto clean_qp;
}
{
struct ibv_exp_qp_init_attr mattr = {
.send_cq = ctx->mcq,
.recv_cq = ctx->mcq,
.cap = {
.max_send_wr = 1,
.max_recv_wr = rx_depth,
.max_send_sge = 16,
.max_recv_sge = 16
},
.qp_type = IBV_QPT_RC,
.pd = ctx->pd
};
mattr.comp_mask |= IBV_EXP_QP_INIT_ATTR_CREATE_FLAGS | IBV_EXP_QP_INIT_ATTR_PD;
mattr.exp_create_flags = IBV_EXP_QP_CREATE_CROSS_CHANNEL;
ctx->mqp = ibv_exp_create_qp(ctx->context, &mattr);
if (!ctx->qp) {
fprintf(stderr, "Couldn't create MQP\n");
goto clean_mcq;
}
}
{
struct ibv_qp_attr mattr = {
.qp_state = IBV_QPS_INIT,
.pkey_index = 0,
.port_num = port,
.qp_access_flags = 0
};
if (ibv_modify_qp(ctx->mqp, &mattr,
IBV_QP_STATE |
IBV_QP_PKEY_INDEX |
IBV_QP_PORT |
IBV_QP_ACCESS_FLAGS)) {
fprintf(stderr, "Failed to modify MQP to INIT\n");
goto clean_mqp;
}
}
return ctx;
clean_mqp:
ibv_destroy_qp(ctx->mqp);
clean_mcq:
ibv_destroy_cq(ctx->mcq);
clean_qp:
ibv_destroy_qp(ctx->qp);
clean_cq:
ibv_destroy_cq(ctx->cq);
clean_mr:
ibv_dereg_mr(ctx->mr);
clean_pd:
ibv_dealloc_pd(ctx->pd);
clean_comp_channel:
if (ctx->channel)
ibv_destroy_comp_channel(ctx->channel);
clean_device:
ibv_close_device(ctx->context);
clean_net_buf:
free(ctx->net_buf);
clean_buffer:
free(ctx->buf);
clean_ctx:
free(ctx);
return NULL;
}
int pp_close_ctx(struct pingpong_context *ctx)
{
if (ibv_destroy_qp(ctx->qp)) {
fprintf(stderr, "Couldn't destroy QP\n");
return 1;
}
if (ibv_destroy_qp(ctx->mqp)) {
fprintf(stderr, "Couldn't destroy MQP\n");
return 1;
}
if (ibv_destroy_cq(ctx->cq)) {
fprintf(stderr, "Couldn't destroy CQ\n");
return 1;
}
if (ibv_destroy_cq(ctx->mcq)) {
fprintf(stderr, "Couldn't destroy MCQ\n");
return 1;
}
if (ibv_dereg_mr(ctx->mr)) {
fprintf(stderr, "Couldn't deregister MR\n");
return 1;
}
if (ibv_dealloc_pd(ctx->pd)) {
fprintf(stderr, "Couldn't deallocate PD\n");
return 1;
}
if (ctx->channel) {
if (ibv_destroy_comp_channel(ctx->channel)) {
fprintf(stderr, "Couldn't destroy completion channel\n");
return 1;
}
}
if (ibv_close_device(ctx->context)) {
fprintf(stderr, "Couldn't release context\n");
return 1;
}
free(ctx->buf);
free(ctx->net_buf);
free(ctx);
return 0;
}
static int pp_post_recv(struct pingpong_context *ctx, int n)
{
int rc;
struct ibv_sge list = {
.addr = (uintptr_t) ctx->net_buf,
.length = ctx->size,
.lkey = ctx->mr->lkey
};
struct ibv_recv_wr wr = {
.wr_id = PP_RECV_WRID,
.sg_list = &list,
.num_sge = 1,
};
struct ibv_recv_wr *bad_wr;
int i;
for (i = 0; i < n; ++i) {
rc = ibv_post_recv(ctx->qp, &wr, &bad_wr);
if (rc)
return rc;
}
return i;
}
static int pp_post_send(struct pingpong_context *ctx)
{
int ret;
struct ibv_sge list = {
.addr = (uintptr_t) ctx->net_buf,
.length = ctx->size,
.lkey = ctx->mr->lkey
};
struct ibv_exp_send_wr wr = {
.wr_id = PP_SEND_WRID,
.sg_list = &list,
.num_sge = 1,
.exp_opcode = IBV_EXP_WR_SEND,
.exp_send_flags = IBV_EXP_SEND_SIGNALED,
};
struct ibv_exp_send_wr *bad_wr;
/* If this is a calc operation - set the required params in the wr */
if (ctx->calc_op.opcode != IBV_EXP_CALC_OP_NUMBER) {
wr.exp_opcode = IBV_EXP_WR_SEND;
wr.exp_send_flags |= IBV_EXP_SEND_WITH_CALC;
wr.sg_list = ctx->calc_op.gather_list;
wr.num_sge = ctx->calc_op.gather_list_size;
wr.op.calc.calc_op = ctx->calc_op.opcode;
wr.op.calc.data_type = ctx->calc_op.data_type;
wr.op.calc.data_size = ctx->calc_op.data_size;
}
ret = ibv_exp_post_send(ctx->qp, &wr, &bad_wr);
return ret;
}
int pp_post_ext_wqe(struct pingpong_context *ctx, enum ibv_exp_wr_opcode op)
{
int ret;
struct ibv_exp_send_wr wr = {
.wr_id = PP_CQE_WAIT,
.sg_list = NULL,
.num_sge = 0,
.exp_opcode = op,
.exp_send_flags = IBV_EXP_SEND_SIGNALED,
};
struct ibv_exp_send_wr *bad_wr;
switch (op) {
case IBV_EXP_WR_RECV_ENABLE:
case IBV_EXP_WR_SEND_ENABLE:
wr.task.wqe_enable.qp = ctx->qp;
wr.task.wqe_enable.wqe_count = 0;
wr.exp_send_flags |= IBV_EXP_SEND_WAIT_EN_LAST;
break;
case IBV_EXP_WR_CQE_WAIT:
wr.task.cqe_wait.cq = ctx->cq;
wr.task.cqe_wait.cq_count = 1;
wr.exp_send_flags |= IBV_EXP_SEND_WAIT_EN_LAST;
break;
default:
fprintf(stderr, "-E- unsupported m_wqe opcode %d\n", op);
return -1;
}
ret = ibv_exp_post_send(ctx->mqp, &wr, &bad_wr);
return ret;
}
int pp_poll_mcq(struct ibv_cq *cq, int num_cqe)
{
int ne;
int i;
struct ibv_wc wc[2];
if (num_cqe > 2) {
fprintf(stderr, "-E- max num cqe exceeded\n");
return -1;
}
do {
ne = ibv_poll_cq(cq, num_cqe, wc);
if (ne < 0) {
fprintf(stderr, "poll CQ failed %d\n", ne);
return 1;
}
} while (ne < 1);
for (i = 0; i < ne; ++i) {
if (wc[i].status != IBV_WC_SUCCESS) {
fprintf(stderr, "Failed %s status %s (%d)\n",
wr_id_str[(int)wc[i].wr_id],
ibv_wc_status_str(wc[i].status),
wc[i].status);
return 1;
}
if ((int) wc[i].wr_id != PP_CQE_WAIT) {
fprintf(stderr, "invalid wr_id %" PRIx64 "\n", wc[i].wr_id);
return -1;
}
}
return 0;
}
static int pp_calc_verify(struct pingpong_context *ctx,
enum pp_wr_data_type calc_data_type,
enum pp_wr_calc_op calc_opcode)
{
uint64_t *op1 = &(ctx->last_result);
uint64_t *op2 = (uint64_t *)ctx->buf + 2;
uint64_t *res = (uint64_t *)ctx->buf;
return !EXEC_VERIFY(calc_data_type, calc_opcode, 1, op1, op2, res);
}
static int pp_update_last_result(struct pingpong_context *ctx,
enum pp_wr_data_type calc_data_type,
enum pp_wr_calc_op calc_opcode)
{
/* EXEC_VERIFY derefence result parameter */
uint64_t *dummy;
uint64_t *op1 = (uint64_t *)ctx->buf;
uint64_t *op2 = (uint64_t *)ctx->buf + 2;
uint64_t res = (uint64_t)EXEC_VERIFY(calc_data_type, calc_opcode, 0, op1, op2, dummy);
ctx->last_result = res;
return 0;
}
static void usage(const char *argv0)
{
printf("Usage:\n");
printf(" %s start a server and wait for connection\n", argv0);
printf(" %s <host> connect to server at <host>\n", argv0);
printf("\n");
printf("Options:\n");
printf(" -p, --port=<port> listen on/connect to port <port> (default 18515)\n");
printf(" -d, --ib-dev=<dev> use IB device <dev> (default first device found)\n");
printf(" -i, --ib-port=<port> use port <port> of IB device (default 1)\n");
printf(" -s, --size=<size> size of message to exchange (default 4096 minimum 16)\n");
printf(" -m, --mtu=<size> path MTU (default 1024)\n");
printf(" -r, --rx-depth=<dep> number of receives to post at a time (default 500)\n");
printf(" -n, --iters=<iters> number of exchanges (default 1000)\n");
printf(" -l, --sl=<sl> service level value\n");
printf(" -e, --events sleep on CQ events (default poll)\n");
printf(" -c, --calc=<operation> calc operation\n");
printf(" -t, --op_type=<type> calc operands type\n");
printf(" -o, --operands=<o1,o2,...> comma separated list of operands\n");
printf(" -w, --wait_cq=cqn wait for entries on cq\n");
printf(" -v, --verbose print verbose information\n");
printf(" -V, --verify verify calc operations\n");
}
/**
* DPDK callback to register a PCI device.
*
* This function creates an Ethernet device for each port of a given
* PCI device.
*
* @param[in] pci_drv
* PCI driver structure (mlx5_driver).
* @param[in] pci_dev
* PCI device information.
*
* @return
* 0 on success, negative errno value on failure.
*/
static int
mlx5_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
{
struct ibv_device **list;
struct ibv_device *ibv_dev;
int err = 0;
struct ibv_context *attr_ctx = NULL;
struct ibv_device_attr device_attr;
unsigned int vf;
int idx;
int i;
(void)pci_drv;
assert(pci_drv == &mlx5_driver.pci_drv);
/* Get mlx5_dev[] index. */
idx = mlx5_dev_idx(&pci_dev->addr);
if (idx == -1) {
ERROR("this driver cannot support any more adapters");
return -ENOMEM;
}
DEBUG("using driver device index %d", idx);
/* Save PCI address. */
mlx5_dev[idx].pci_addr = pci_dev->addr;
list = ibv_get_device_list(&i);
if (list == NULL) {
assert(errno);
if (errno == ENOSYS) {
WARN("cannot list devices, is ib_uverbs loaded?");
return 0;
}
return -errno;
}
assert(i >= 0);
/*
* For each listed device, check related sysfs entry against
* the provided PCI ID.
*/
while (i != 0) {
struct rte_pci_addr pci_addr;
--i;
DEBUG("checking device \"%s\"", list[i]->name);
if (mlx5_ibv_device_to_pci_addr(list[i], &pci_addr))
continue;
if ((pci_dev->addr.domain != pci_addr.domain) ||
(pci_dev->addr.bus != pci_addr.bus) ||
(pci_dev->addr.devid != pci_addr.devid) ||
(pci_dev->addr.function != pci_addr.function))
continue;
vf = ((pci_dev->id.device_id ==
PCI_DEVICE_ID_MELLANOX_CONNECTX4VF) ||
(pci_dev->id.device_id ==
PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF));
INFO("PCI information matches, using device \"%s\" (VF: %s)",
list[i]->name, (vf ? "true" : "false"));
attr_ctx = ibv_open_device(list[i]);
err = errno;
break;
}
if (attr_ctx == NULL) {
ibv_free_device_list(list);
switch (err) {
case 0:
WARN("cannot access device, is mlx5_ib loaded?");
return 0;
case EINVAL:
WARN("cannot use device, are drivers up to date?");
return 0;
}
assert(err > 0);
return -err;
}
ibv_dev = list[i];
DEBUG("device opened");
if (ibv_query_device(attr_ctx, &device_attr))
goto error;
INFO("%u port(s) detected", device_attr.phys_port_cnt);
for (i = 0; i < device_attr.phys_port_cnt; i++) {
uint32_t port = i + 1; /* ports are indexed from one */
uint32_t test = (1 << i);
struct ibv_context *ctx = NULL;
struct ibv_port_attr port_attr;
struct ibv_pd *pd = NULL;
struct priv *priv = NULL;
struct rte_eth_dev *eth_dev;
#ifdef HAVE_EXP_QUERY_DEVICE
struct ibv_exp_device_attr exp_device_attr;
#endif /* HAVE_EXP_QUERY_DEVICE */
struct ether_addr mac;
#ifdef HAVE_EXP_QUERY_DEVICE
exp_device_attr.comp_mask =
IBV_EXP_DEVICE_ATTR_EXP_CAP_FLAGS |
IBV_EXP_DEVICE_ATTR_RX_HASH;
#endif /* HAVE_EXP_QUERY_DEVICE */
DEBUG("using port %u (%08" PRIx32 ")", port, test);
ctx = ibv_open_device(ibv_dev);
if (ctx == NULL)
goto port_error;
/* Check port status. */
err = ibv_query_port(ctx, port, &port_attr);
if (err) {
ERROR("port query failed: %s", strerror(err));
goto port_error;
}
if (port_attr.state != IBV_PORT_ACTIVE)
DEBUG("port %d is not active: \"%s\" (%d)",
port, ibv_port_state_str(port_attr.state),
port_attr.state);
/* Allocate protection domain. */
pd = ibv_alloc_pd(ctx);
if (pd == NULL) {
ERROR("PD allocation failure");
err = ENOMEM;
goto port_error;
}
mlx5_dev[idx].ports |= test;
/* from rte_ethdev.c */
priv = rte_zmalloc("ethdev private structure",
sizeof(*priv),
RTE_CACHE_LINE_SIZE);
if (priv == NULL) {
ERROR("priv allocation failure");
err = ENOMEM;
goto port_error;
}
priv->ctx = ctx;
priv->device_attr = device_attr;
priv->port = port;
priv->pd = pd;
priv->mtu = ETHER_MTU;
#ifdef HAVE_EXP_QUERY_DEVICE
if (ibv_exp_query_device(ctx, &exp_device_attr)) {
ERROR("ibv_exp_query_device() failed");
goto port_error;
}
priv->hw_csum =
((exp_device_attr.exp_device_cap_flags &
IBV_EXP_DEVICE_RX_CSUM_TCP_UDP_PKT) &&
(exp_device_attr.exp_device_cap_flags &
IBV_EXP_DEVICE_RX_CSUM_IP_PKT));
DEBUG("checksum offloading is %ssupported",
(priv->hw_csum ? "" : "not "));
priv->hw_csum_l2tun = !!(exp_device_attr.exp_device_cap_flags &
IBV_EXP_DEVICE_VXLAN_SUPPORT);
DEBUG("L2 tunnel checksum offloads are %ssupported",
(priv->hw_csum_l2tun ? "" : "not "));
priv->ind_table_max_size = exp_device_attr.rx_hash_caps.max_rwq_indirection_table_size;
DEBUG("maximum RX indirection table size is %u",
priv->ind_table_max_size);
#else /* HAVE_EXP_QUERY_DEVICE */
priv->ind_table_max_size = RSS_INDIRECTION_TABLE_SIZE;
#endif /* HAVE_EXP_QUERY_DEVICE */
priv->vf = vf;
/* Allocate and register default RSS hash keys. */
priv->rss_conf = rte_calloc(__func__, hash_rxq_init_n,
sizeof((*priv->rss_conf)[0]), 0);
if (priv->rss_conf == NULL) {
err = ENOMEM;
goto port_error;
}
err = rss_hash_rss_conf_new_key(priv,
rss_hash_default_key,
rss_hash_default_key_len,
ETH_RSS_PROTO_MASK);
if (err)
goto port_error;
/* Configure the first MAC address by default. */
if (priv_get_mac(priv, &mac.addr_bytes)) {
ERROR("cannot get MAC address, is mlx5_en loaded?"
" (errno: %s)", strerror(errno));
goto port_error;
}
INFO("port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x",
priv->port,
mac.addr_bytes[0], mac.addr_bytes[1],
mac.addr_bytes[2], mac.addr_bytes[3],
mac.addr_bytes[4], mac.addr_bytes[5]);
/* Register MAC and broadcast addresses. */
claim_zero(priv_mac_addr_add(priv, 0,
(const uint8_t (*)[ETHER_ADDR_LEN])
mac.addr_bytes));
claim_zero(priv_mac_addr_add(priv, (RTE_DIM(priv->mac) - 1),
&(const uint8_t [ETHER_ADDR_LEN])
{ "\xff\xff\xff\xff\xff\xff" }));
#ifndef NDEBUG
{
char ifname[IF_NAMESIZE];
if (priv_get_ifname(priv, &ifname) == 0)
DEBUG("port %u ifname is \"%s\"",
priv->port, ifname);
else
DEBUG("port %u ifname is unknown", priv->port);
}
#endif
/* Get actual MTU if possible. */
priv_get_mtu(priv, &priv->mtu);
DEBUG("port %u MTU is %u", priv->port, priv->mtu);
/* from rte_ethdev.c */
{
char name[RTE_ETH_NAME_MAX_LEN];
snprintf(name, sizeof(name), "%s port %u",
ibv_get_device_name(ibv_dev), port);
eth_dev = rte_eth_dev_allocate(name, RTE_ETH_DEV_PCI);
}
if (eth_dev == NULL) {
ERROR("can not allocate rte ethdev");
err = ENOMEM;
goto port_error;
}
eth_dev->data->dev_private = priv;
eth_dev->pci_dev = pci_dev;
eth_dev->driver = &mlx5_driver;
eth_dev->data->rx_mbuf_alloc_failed = 0;
eth_dev->data->mtu = ETHER_MTU;
priv->dev = eth_dev;
eth_dev->dev_ops = &mlx5_dev_ops;
eth_dev->data->mac_addrs = priv->mac;
TAILQ_INIT(ð_dev->link_intr_cbs);
/* Bring Ethernet device up. */
DEBUG("forcing Ethernet interface up");
priv_set_flags(priv, ~IFF_UP, IFF_UP);
continue;
port_error:
rte_free(priv->rss_conf);
rte_free(priv);
if (pd)
claim_zero(ibv_dealloc_pd(pd));
if (ctx)
claim_zero(ibv_close_device(ctx));
break;
}
static void mca_bcol_iboffload_device_destructor
(mca_bcol_iboffload_device_t *device)
{
int qp_index, num_qps = mca_bcol_iboffload_component.num_qps;
IBOFFLOAD_VERBOSE(10, ("Device %s will be destroyed.\n",
ibv_get_device_name(device->dev.ib_dev)));
if (NULL != device->frags_free) {
for (qp_index = 0; qp_index < num_qps; ++qp_index) {
mca_bcol_iboffload_dealloc_qps_resource_fn_t dealloc_resource =
mca_bcol_iboffload_component.qp_infos[qp_index].dealloc_resource;
if (NULL != dealloc_resource) {
dealloc_resource(qp_index, device);
}
}
free(device->frags_free);
}
if (NULL != device->mpool) {
IBOFFLOAD_VERBOSE(10, ("Mpool destroy - %p.\n", device->mpool));
if (OMPI_SUCCESS != mca_mpool_base_module_destroy(device->mpool)) {
IBOFFLOAD_ERROR(("Device %s, failed to destroy mpool",
ibv_get_device_name(device->dev.ib_dev)));
}
}
if (NULL != device->dummy_reg.mr) {
IBOFFLOAD_VERBOSE(10, ("Dummy memory MR unregister - %p.\n", device->dummy_reg.mr));
if (OMPI_SUCCESS !=
mca_bcol_iboffload_deregister_mr((void *) device, &device->dummy_reg.base)) {
IBOFFLOAD_ERROR(("Device %s: failed to unregister dummy memory MR.",
ibv_get_device_name(device->dev.ib_dev)));
}
}
if (NULL != device->ib_cq) {
if (ibv_destroy_cq(device->ib_cq)) {
IBOFFLOAD_ERROR(("Device %s, failed to destroy CQ, errno says %s",
ibv_get_device_name(device->dev.ib_dev), strerror(errno)));
}
}
if (NULL != device->ib_mq_cq) {
if (ibv_destroy_cq(device->ib_mq_cq)) {
IBOFFLOAD_ERROR(("Device %s, failed to destroy mq CQ, errno says %s",
ibv_get_device_name(device->dev.ib_dev), strerror(errno)));
}
}
/* Release IB PD if we have one */
if (NULL != device->ib_pd) {
if(ibv_dealloc_pd(device->ib_pd)) {
IBOFFLOAD_ERROR(("Device %s, failed to release PD, errno says %s",
ibv_get_device_name(device->dev.ib_dev), strerror(errno)));
}
}
/* close the device */
if (NULL != device->dev.ib_dev_context) {
if (ibv_close_device(device->dev.ib_dev_context)) {
IBOFFLOAD_ERROR(("Device %s "
", failed to close the device, errno says %s",
ibv_get_device_name(device->dev.ib_dev), strerror(errno)));
}
}
/* release memory */
if (NULL != device->ports) {
free(device->ports);
}
}
/**
* DPDK callback to close the device.
*
* Destroy all queues and objects, free memory.
*
* @param dev
* Pointer to Ethernet device structure.
*/
static void
mlx5_dev_close(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
void *tmp;
unsigned int i;
priv_lock(priv);
DEBUG("%p: closing device \"%s\"",
(void *)dev,
((priv->ctx != NULL) ? priv->ctx->device->name : ""));
/* In case mlx5_dev_stop() has not been called. */
priv_dev_interrupt_handler_uninstall(priv, dev);
priv_allmulticast_disable(priv);
priv_promiscuous_disable(priv);
priv_mac_addrs_disable(priv);
priv_destroy_hash_rxqs(priv);
/* Prevent crashes when queues are still in use. */
dev->rx_pkt_burst = removed_rx_burst;
dev->tx_pkt_burst = removed_tx_burst;
if (priv->rxqs != NULL) {
/* XXX race condition if mlx5_rx_burst() is still running. */
usleep(1000);
for (i = 0; (i != priv->rxqs_n); ++i) {
tmp = (*priv->rxqs)[i];
if (tmp == NULL)
continue;
(*priv->rxqs)[i] = NULL;
rxq_cleanup(tmp);
rte_free(tmp);
}
priv->rxqs_n = 0;
priv->rxqs = NULL;
}
if (priv->txqs != NULL) {
/* XXX race condition if mlx5_tx_burst() is still running. */
usleep(1000);
for (i = 0; (i != priv->txqs_n); ++i) {
tmp = (*priv->txqs)[i];
if (tmp == NULL)
continue;
(*priv->txqs)[i] = NULL;
txq_cleanup(tmp);
rte_free(tmp);
}
priv->txqs_n = 0;
priv->txqs = NULL;
}
if (priv->pd != NULL) {
assert(priv->ctx != NULL);
claim_zero(ibv_dealloc_pd(priv->pd));
claim_zero(ibv_close_device(priv->ctx));
} else
assert(priv->ctx == NULL);
if (priv->rss_conf != NULL) {
for (i = 0; (i != hash_rxq_init_n); ++i)
rte_free((*priv->rss_conf)[i]);
rte_free(priv->rss_conf);
}
priv_unlock(priv);
memset(priv, 0, sizeof(*priv));
}
int
resource_create(resource_t *res, int ib_port, int myrank)
{
struct ibv_device **dev_list = NULL;
struct ibv_qp_init_attr qp_init_attr;
struct ibv_device *ib_dev = NULL;
char *dev_name = NULL;
size_t size;
int i;
int mr_flags = 0;
int cq_size = 0;
int dev_numm;
int rc = 0;
/* Init structure */
memset(res, 0, sizeof(resource_t));
/* Get the device list */
dev_list = ibv_get_device_list(&dev_numm);
if(!dev_list) {
fprintf(stderr, "[%d] failed to get IB devices list\n", myrank);
return 1;
}
// if no device
if(!dev_numm) {
fprintf(stderr, "[%d] No IB device is found\n", myrank);
rc = 1;
goto err_exit;
}
DEBUG { printf("[%d] found %d IB device(s)\n", myrank, dev_numm); }
/* Open the requested device */
for(i = 0; i < dev_numm; i ++){
dev_name = strdup(ibv_get_device_name(dev_list[i]));
DEBUG { printf("[%d] IB device name: %s\n", myrank, dev_name); }
ib_dev = dev_list[i];
break;
}
if (!ib_dev){
fprintf(stderr, "[%d] IB device %s wasn't found\n", myrank, dev_name);
rc = 1;
goto err_exit;
}
res->ib_ctx = ibv_open_device(ib_dev);
DEBUG { printf("[%d] IB context = %lx\n", myrank, (uintptr_t)res->ib_ctx); }
if(!res->ib_ctx){
fprintf(stderr, "[%d] failed to open device %s\n", myrank, dev_name);
rc = 1;
goto err_exit;
}
// free device list
ibv_free_device_list(dev_list);
dev_list = NULL;
ib_dev = NULL;
// query prot properties
if(ibv_query_port(res->ib_ctx, ib_port, &res->port_attr)){
fprintf(stderr, "[%d] ibv_query_port on port %u failed\n", myrank, ib_port);
rc = 1;
goto err_exit;
}
/* Create a PD */
res->pd = ibv_alloc_pd(res->ib_ctx);
if (!res->pd){
fprintf(stderr, "[%d] ibv_alloc_pd failed\n", myrank);
rc = 1;
goto err_exit;
}
/* Create send/recv CQ
* inputs:
* device handle
* CQ capacity
* Output:
* CQ handle
*/
res->scq = ibv_create_cq(res->ib_ctx, MAX_CQ_CAPACITY, NULL, NULL, 0);
res->rcq = ibv_create_cq(res->ib_ctx, MAX_CQ_CAPACITY, NULL, NULL, 0);
if (!res->scq){
fprintf(stderr, "[%d] failed to create SCQ with %u entries\n", myrank, cq_size);
rc = 1;
goto err_exit;
}
if (!res->rcq){
fprintf(stderr, "[%d] failed to create SCQ with %u entries\n", myrank, cq_size);
rc = 1;
goto err_exit;
}
/* Allocate fix buffer */
size = MAX_FIX_BUF_SIZE;
res->buf_size = size;
res->buf = (char *)malloc(size * sizeof(char));
if (!res->buf ){
fprintf(stderr, "[%d] failed to malloc %Zu bytes to memory buffer\n", myrank, size);
rc = 1;
goto err_exit;
}
memset(res->buf, 0 , size);
/* Memory Region
* inputs:
* device handle
* PD
* Virtual Addr(addr of MR)
* Access Ctrl: LocalWrite, RemoteRead, RemoteWrite, RemoteAtomicOp, MemWindowBinding
* outputs:
* MR handle
* L_Key
* R_Key
*/
res->mr_list = malloc(sizeof(struct ibv_mr*) * MAX_MR_NUM);
res->mr_size = 1;
mr_flags = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ |
IBV_ACCESS_REMOTE_WRITE ;
res->mr_list[0] = ibv_reg_mr(res->pd, res->buf, size, mr_flags);
if (!res->mr_list[0]){
fprintf(stderr, "[%d] ibv_reg_mr failed with mr_flags=0x%x\n", myrank, mr_flags);
rc = 1;
goto err_exit;
}
DEBUG { printf("[%d] fixed MR was registered with addr=%p, lkey=0x%x, rkey=0x%x, flags=0x%x\n", myrank, res->buf, res->mr_list[0]->lkey, res->mr_list[0]->rkey, mr_flags); }
/* Create QP */
// inputs:
// PD
// CQs for SQ,RQ
// capacity of SQ,RQ
// Outputs:
// QP handle
memset(&qp_init_attr, 0, sizeof(qp_init_attr));
qp_init_attr.qp_type = IBV_QPT_RC;
qp_init_attr.sq_sig_all = 1;
qp_init_attr.send_cq = res->scq;
qp_init_attr.recv_cq = res->rcq;
// max SR/RR num in SQ/RQ
qp_init_attr.cap.max_send_wr = MAX_SQ_CAPACITY ;
qp_init_attr.cap.max_recv_wr = MAX_RQ_CAPACITY;
// max SGE num
qp_init_attr.cap.max_send_sge = MAX_SGE_CAPACITY;
qp_init_attr.cap.max_recv_sge = MAX_SGE_CAPACITY;
qp_init_attr.cap.max_inline_data = 256;
res->qp = ibv_create_qp(res->pd, &qp_init_attr);
if (!res->qp){
fprintf(stderr, "failed to create QP\n");
rc = 1;
goto err_exit;
}
DEBUG { printf("[%d] QP was created, QP number=0x%x\n", myrank, res->qp->qp_num); }
/* EXIT */
err_exit:
if(rc){
/* Error encountered, cleanup */
if(res->qp){
ibv_destroy_qp(res->qp);
res->qp = NULL;
}
if(res->mr_list && res->mr_size > 0){
int i;
for(i=0; i<res->mr_size; i++){
ibv_dereg_mr(res->mr_list[i]);
res->mr_list[i] = NULL;
}
free(res->mr_list);
}
if(res->buf){
free(res->buf);
res->buf = NULL;
}
if(res->scq){
ibv_destroy_cq(res->scq);
res->scq = NULL;
}
if(res->rcq){
ibv_destroy_cq(res->rcq);
res->rcq = NULL;
}
if(res->comp_ch){
ibv_destroy_comp_channel(res->comp_ch);
res->comp_ch = NULL;
}
if(res->pd){
ibv_dealloc_pd(res->pd);
res->pd = NULL;
}
if (res->ib_ctx) {
ibv_close_device(res->ib_ctx);
res->ib_ctx = NULL;
}
if (dev_list) {
ibv_free_device_list(dev_list);
dev_list = NULL;
}
}
return rc;
}
