static int mca_mpool_sm_register(void) { /* register SM component parameters */ (void) mca_base_var_group_component_register(&mca_mpool_sm_component.super.mpool_version, "Shared memory pool"); mca_mpool_sm_component.sm_allocator_name = "bucket"; (void) mca_base_component_var_register(&mca_mpool_sm_component.super.mpool_version, "allocator", "Name of allocator component " "to use with sm mpool", MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &mca_mpool_sm_component.sm_allocator_name); /* register as an unsigned long long to get up to 64 bits for the size */ ompi_mpool_sm_min_size = default_min; (void) mca_base_component_var_register(&mca_mpool_sm_component.super.mpool_version, "min_size", "Minimum size of the sm mpool shared memory file", MCA_BASE_VAR_TYPE_UNSIGNED_LONG_LONG, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_mpool_sm_min_size); ompi_mpool_sm_verbose = 0; (void) mca_base_component_var_register(&mca_mpool_sm_component.super.mpool_version, "verbose", "Enable verbose output for mpool sm component", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &ompi_mpool_sm_verbose); return OMPI_SUCCESS; }
static int mca_btl_self_component_register(void) { mca_base_var_group_component_register(&mca_btl_self_component.super.btl_version, "BTL for self communication"); /* register SELF component parameters */ mca_btl_self_component.free_list_num = 0; (void) mca_base_component_var_register(&mca_btl_self_component.super.btl_version, "free_list_num", "Number of fragments by default", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &mca_btl_self_component.free_list_num); mca_btl_self_component.free_list_max = -1; (void) mca_base_component_var_register(&mca_btl_self_component.super.btl_version, "free_list_max", "Maximum number of fragments", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &mca_btl_self_component.free_list_max); mca_btl_self_component.free_list_inc = 32; (void) mca_base_component_var_register(&mca_btl_self_component.super.btl_version, "free_list_inc", "Increment by this number of fragments", MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &mca_btl_self_component.free_list_inc); mca_btl_self.btl_exclusivity = MCA_BTL_EXCLUSIVITY_HIGH; mca_btl_self.btl_eager_limit = 128 * 1024; mca_btl_self.btl_rndv_eager_limit = 128 * 1024; mca_btl_self.btl_max_send_size = 256 * 1024; mca_btl_self.btl_rdma_pipeline_send_length = INT_MAX; mca_btl_self.btl_rdma_pipeline_frag_size = INT_MAX; mca_btl_self.btl_min_rdma_pipeline_size = 0; mca_btl_self.btl_flags = MCA_BTL_FLAGS_PUT | MCA_BTL_FLAGS_SEND_INPLACE; mca_btl_self.btl_seg_size = sizeof (mca_btl_base_segment_t); mca_btl_self.btl_bandwidth = 100; mca_btl_self.btl_latency = 0; mca_btl_base_param_register(&mca_btl_self_component.super.btl_version, &mca_btl_self); return OMPI_SUCCESS; }
static int btl_scif_component_register(void) { (void) mca_base_var_group_component_register(&mca_btl_scif_component.super.btl_version, "SCIF byte transport layer"); mca_btl_scif_component.scif_free_list_num = 8; (void) mca_base_component_var_register(&mca_btl_scif_component.super.btl_version, "free_list_num", "Initial fragment free list size", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_scif_component.scif_free_list_num); mca_btl_scif_component.scif_free_list_max = 16384; (void) mca_base_component_var_register(&mca_btl_scif_component.super.btl_version, "free_list_max", "Maximum fragment free list size", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_scif_component.scif_free_list_max); mca_btl_scif_component.scif_free_list_inc = 64; (void) mca_base_component_var_register(&mca_btl_scif_component.super.btl_version, "free_list_inc", "Fragment free list size increment", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_scif_component.scif_free_list_inc); mca_btl_scif_component.segment_size = 8 * 1024; (void) mca_base_component_var_register(&mca_btl_scif_component.super.btl_version, "segment_size", "Size of memory segment to " "allocate for each remote process (default: " "8k)", MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_scif_component.segment_size); mca_btl_scif_component.rma_use_cpu = false; (void) mca_base_component_var_register(&mca_btl_scif_component.super.btl_version, "rma_use_cpu", "Use CPU instead of DMA " "for RMA copies (default: false)", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_scif_component.rma_use_cpu); mca_btl_scif_component.rma_sync = true; (void) mca_base_component_var_register(&mca_btl_scif_component.super.btl_version, "rma_sync", "Use synchronous RMA instead of " "an RMA fence (default: true)", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_scif_component.rma_sync); #if defined(SCIF_TIMING) mca_btl_scif_component.aquire_buffer_time = 0.0; (void) mca_base_component_pvar_register(&mca_btl_scif_component.super.btl_version, "aquire_buffer_time", "Aggregate time spent " "aquiring send buffers", OPAL_INFO_LVL_9, MCA_BASE_PVAR_CLASS_AGGREGATE, MCA_BASE_VAR_TYPE_DOUBLE, NULL, MCA_BASE_VAR_BIND_NO_OBJECT, MCA_BASE_PVAR_FLAG_READONLY | MCA_BASE_PVAR_FLAG_CONTINUOUS, NULL, NULL, NULL, &mca_btl_scif_component.aquire_buffer_time); mca_btl_scif_component.send_time = 0.0; (void) mca_base_component_pvar_register(&mca_btl_scif_component.super.btl_version, "send_time", "Aggregate time spent writing to " "send buffers", OPAL_INFO_LVL_9, MCA_BASE_PVAR_CLASS_AGGREGATE, MCA_BASE_VAR_TYPE_DOUBLE, NULL, MCA_BASE_VAR_BIND_NO_OBJECT, MCA_BASE_PVAR_FLAG_READONLY | MCA_BASE_PVAR_FLAG_CONTINUOUS, NULL, NULL, NULL, &mca_btl_scif_component.send_time); mca_btl_scif_component.sendi_time = 0.0; (void) mca_base_component_pvar_register(&mca_btl_scif_component.super.btl_version, "sendi_time", "Aggregate time spent writing to " "send buffers in sendi", OPAL_INFO_LVL_9, MCA_BASE_PVAR_CLASS_AGGREGATE, MCA_BASE_VAR_TYPE_DOUBLE, NULL, MCA_BASE_VAR_BIND_NO_OBJECT, MCA_BASE_PVAR_FLAG_READONLY | MCA_BASE_PVAR_FLAG_CONTINUOUS, NULL, NULL, NULL, &mca_btl_scif_component.sendi_time); mca_btl_scif_component.get_time = 0.0; (void) mca_base_component_pvar_register(&mca_btl_scif_component.super.btl_version, "get_time", "Aggregate time spent in DMA read (scif_readfrom)", OPAL_INFO_LVL_9, MCA_BASE_PVAR_CLASS_AGGREGATE, MCA_BASE_VAR_TYPE_DOUBLE, NULL, MCA_BASE_VAR_BIND_NO_OBJECT, MCA_BASE_PVAR_FLAG_READONLY | MCA_BASE_PVAR_FLAG_CONTINUOUS, NULL, NULL, NULL, &mca_btl_scif_component.get_time); mca_btl_scif_component.get_count = 0; (void) mca_base_component_pvar_register(&mca_btl_scif_component.super.btl_version, "get_count", "Number of times btl_scif_get was called", OPAL_INFO_LVL_9, MCA_BASE_PVAR_CLASS_COUNTER, MCA_BASE_VAR_TYPE_UNSIGNED_LONG, NULL, MCA_BASE_VAR_BIND_NO_OBJECT, MCA_BASE_PVAR_FLAG_READONLY | MCA_BASE_PVAR_FLAG_CONTINUOUS, NULL, NULL, NULL, &mca_btl_scif_component.get_count); mca_btl_scif_component.put_time = 0.0; (void) mca_base_component_pvar_register(&mca_btl_scif_component.super.btl_version, "put_time", "Aggregate time spent in DMA write (scif_writeto)", OPAL_INFO_LVL_9, MCA_BASE_PVAR_CLASS_AGGREGATE, MCA_BASE_VAR_TYPE_DOUBLE, NULL, MCA_BASE_VAR_BIND_NO_OBJECT, MCA_BASE_PVAR_FLAG_READONLY | MCA_BASE_PVAR_FLAG_CONTINUOUS, NULL, NULL, NULL, &mca_btl_scif_component.put_time); mca_btl_scif_component.put_count = 0; (void) mca_base_component_pvar_register(&mca_btl_scif_component.super.btl_version, "put_count", "Number of times btl_scif_put was called", OPAL_INFO_LVL_9, MCA_BASE_PVAR_CLASS_COUNTER, MCA_BASE_VAR_TYPE_UNSIGNED_LONG, NULL, MCA_BASE_VAR_BIND_NO_OBJECT, MCA_BASE_PVAR_FLAG_READONLY | MCA_BASE_PVAR_FLAG_CONTINUOUS, NULL, NULL, NULL, &mca_btl_scif_component.put_count); #endif mca_btl_scif_module.super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_HIGH; mca_btl_scif_module.super.btl_eager_limit = 1 * 1024; mca_btl_scif_module.super.btl_rndv_eager_limit = 1 * 1024; mca_btl_scif_module.super.btl_rdma_pipeline_frag_size = 4 * 1024 * 1024; mca_btl_scif_module.super.btl_max_send_size = 1 * 1024; mca_btl_scif_module.super.btl_rdma_pipeline_send_length = 1 * 1024; /* threshold for put */ mca_btl_scif_module.super.btl_min_rdma_pipeline_size = 1 * 1024; mca_btl_scif_module.super.btl_flags = MCA_BTL_FLAGS_SEND | MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_SEND_INPLACE; mca_btl_scif_module.super.btl_registration_handle_size = sizeof (mca_btl_base_registration_handle_t); mca_btl_scif_module.super.btl_bandwidth = 50000; /* Mbs */ mca_btl_scif_module.super.btl_latency = 2; /* Microsecs */ /* Call the BTL based to register its MCA params */ mca_btl_base_param_register(&mca_btl_scif_component.super.btl_version, &mca_btl_scif_module.super); return OPAL_SUCCESS; }
static int btl_ugni_component_register(void) { mca_base_var_enum_t *new_enum; gni_nic_device_t device_type; int rc; (void) mca_base_var_group_component_register(&mca_btl_ugni_component.super.btl_version, "uGNI byte transport layer"); mca_btl_ugni_component.ugni_free_list_num = 8; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "free_list_num", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_free_list_num); mca_btl_ugni_component.ugni_free_list_max = 4096; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "free_list_max", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_free_list_max); mca_btl_ugni_component.ugni_free_list_inc = 64; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "free_list_inc", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_free_list_inc); mca_btl_ugni_component.ugni_eager_num = 16; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "eager_num", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_eager_num); mca_btl_ugni_component.ugni_eager_max = 128; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "eager_max", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_eager_max); mca_btl_ugni_component.ugni_eager_inc = 16; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "eager_inc", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_eager_inc); mca_btl_ugni_component.remote_cq_size = 40000; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "remote_cq_size", "Remote SMSG completion queue " "size (default 40000)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.remote_cq_size); mca_btl_ugni_component.local_cq_size = 8192; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "local_cq_size", "Local completion queue size " "(default 8192)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.local_cq_size); mca_btl_ugni_component.ugni_smsg_limit = 0; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "smsg_limit", "Maximum size message that " "will be sent using the SMSG/MSGQ protocol " "(0 - autoselect(default), 16k max)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_smsg_limit); mca_btl_ugni_component.smsg_max_credits = 32; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "smsg_max_credits", "Maximum number of " "outstanding SMSG/MSGQ message (default 32)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.smsg_max_credits); mca_btl_ugni_component.ugni_fma_limit = 1024; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "fma_limit", "Maximum size message that " "will be sent using the FMA (Fast Memory " "Access) protocol (default 1024, 64k max)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_fma_limit); mca_btl_ugni_component.rdma_max_retries = 16; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "rdma_max_retries", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.rdma_max_retries); mca_btl_ugni_component.smsg_max_retries = 16; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "smsg_max_retries", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.smsg_max_retries); mca_btl_ugni_component.max_mem_reg = 0; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "max_mem_reg", "Maximum number of " "memory registrations a process can " "hold (0 - autoselect, -1 - unlimited)" " (default 0)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.max_mem_reg); mca_btl_ugni_component.mbox_increment = 0; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "mbox_inc", "Number of SMSG mailboxes to " "allocate in each block (0 - autoselect(default))", MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.mbox_increment); mca_btl_ugni_component.smsg_page_size = 2 << 20; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "smsg_page_size", "Page size to use for SMSG " "mailbox allocation (default 2M)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.smsg_page_size); mca_btl_ugni_component.progress_thread_requested = 0; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "request_progress_thread", "Enable to request ugni btl progress thread - requires MPI_THREAD_MULTIPLE support", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.progress_thread_requested); /* performance variables */ mca_btl_ugni_progress_thread_wakeups = 0; (void) mca_base_component_pvar_register(&mca_btl_ugni_component.super.btl_version, "progress_thread_wakeups", "Number of times the progress thread " "has been woken", OPAL_INFO_LVL_9, MCA_BASE_PVAR_CLASS_COUNTER, MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, MCA_BASE_VAR_BIND_NO_OBJECT, MCA_BASE_PVAR_FLAG_READONLY | MCA_BASE_PVAR_FLAG_CONTINUOUS, NULL, NULL, NULL, &mca_btl_ugni_progress_thread_wakeups); /* btl/ugni can only support only a fixed set of mpools (these mpools have compatible resource * structures) */ rc = mca_base_var_enum_create ("btl_ugni_mpool", mpool_values, &new_enum); if (OPAL_SUCCESS != rc) { return rc; } mca_btl_ugni_component.mpool_type = MCA_BTL_UGNI_MPOOL_UDREG; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "mpool", "mpool to use", MCA_BASE_VAR_TYPE_INT, new_enum, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.mpool_type); OBJ_RELEASE(new_enum); /* ensure we loose send exclusivity to sm and vader if they are enabled */ mca_btl_ugni_module.super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_HIGH - 2; /* smsg threshold */ mca_btl_ugni_module.super.btl_eager_limit = 8 * 1024; mca_btl_ugni_module.super.btl_rndv_eager_limit = 8 * 1024; mca_btl_ugni_module.super.btl_rdma_pipeline_frag_size = 4 * 1024 * 1024; mca_btl_ugni_module.super.btl_max_send_size = 8 * 1024; mca_btl_ugni_module.super.btl_rdma_pipeline_send_length = 8 * 1024; mca_btl_ugni_module.super.btl_get_limit = 1 * 1024 * 1024; /* determine if there are get alignment restrictions */ GNI_GetDeviceType (&device_type); /* * see def. of ALIGNMENT_MASK to figure this one out */ /* both gemini and aries have a 4-byte alignment requirement on remote addresses */ mca_btl_ugni_module.super.btl_get_alignment = 4; /* threshold for put */ mca_btl_ugni_module.super.btl_min_rdma_pipeline_size = 8 * 1024; mca_btl_ugni_module.super.btl_flags = MCA_BTL_FLAGS_SEND | MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_ATOMIC_OPS | MCA_BTL_FLAGS_ATOMIC_FOPS; mca_btl_ugni_module.super.btl_atomic_flags = MCA_BTL_ATOMIC_SUPPORTS_ADD | MCA_BTL_ATOMIC_SUPPORTS_AND | MCA_BTL_ATOMIC_SUPPORTS_OR | MCA_BTL_ATOMIC_SUPPORTS_XOR | MCA_BTL_ATOMIC_SUPPORTS_CSWAP; mca_btl_ugni_module.super.btl_registration_handle_size = sizeof (mca_btl_base_registration_handle_t); mca_btl_ugni_module.super.btl_bandwidth = 40000; /* Mbs */ mca_btl_ugni_module.super.btl_latency = 2; /* Microsecs */ mca_btl_ugni_module.super.btl_get_local_registration_threshold = 0; mca_btl_ugni_module.super.btl_put_local_registration_threshold = mca_btl_ugni_component.ugni_fma_limit; /* Call the BTL based to register its MCA params */ mca_btl_base_param_register(&mca_btl_ugni_component.super.btl_version, &mca_btl_ugni_module.super); return OPAL_SUCCESS; }
static int mca_btl_vader_component_register (void) { (void) mca_base_var_group_component_register(&mca_btl_vader_component.super.btl_version, "XPMEM shared memory byte transport later"); /* register VADER component variables */ mca_btl_vader_component.vader_free_list_num = 8; (void) mca_base_component_var_register(&mca_btl_vader_component.super.btl_version, "free_list_num", "Initial number of fragments " "to allocate for shared memory communication.", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_vader_component.vader_free_list_num); mca_btl_vader_component.vader_free_list_max = 16384; (void) mca_base_component_var_register(&mca_btl_vader_component.super.btl_version, "free_list_max", "Maximum number of fragments " "to allocate for shared memory communication.", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_vader_component.vader_free_list_max); mca_btl_vader_component.vader_free_list_inc = 64; (void) mca_base_component_var_register(&mca_btl_vader_component.super.btl_version, "free_list_inc", "Number of fragments to create " "on each allocation.", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_vader_component.vader_free_list_inc); mca_btl_vader_component.memcpy_limit = 524288; (void) mca_base_component_var_register(&mca_btl_vader_component.super.btl_version, "memcpy_limit", "Message size to switch from using " "memove to memcpy. The relative speed of these two " "routines can vary by size.", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_vader_component.memcpy_limit); mca_btl_vader_component.log_attach_align = 21; (void) mca_base_component_var_register(&mca_btl_vader_component.super.btl_version, "log_align", "Log base 2 of the alignment to use for xpmem " "segments (default: 21, minimum: 12, maximum: 25)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_vader_component.log_attach_align); mca_btl_vader_component.segment_size = 1 << 24; (void) mca_base_component_var_register(&mca_btl_vader_component.super.btl_version, "segment_size", "Maximum size of all shared " "memory buffers (default: 16M)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_vader_component.segment_size); mca_btl_vader_component.max_inline_send = 256; (void) mca_base_component_var_register(&mca_btl_vader_component.super.btl_version, "max_inline_send", "Maximum size to transfer " "using copy-in copy-out semantics", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_vader_component.max_inline_send); mca_btl_vader.super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_HIGH; mca_btl_vader.super.btl_eager_limit = 32 * 1024; mca_btl_vader.super.btl_rndv_eager_limit = mca_btl_vader.super.btl_eager_limit; mca_btl_vader.super.btl_max_send_size = mca_btl_vader.super.btl_eager_limit; mca_btl_vader.super.btl_rdma_pipeline_send_length = mca_btl_vader.super.btl_eager_limit; mca_btl_vader.super.btl_rdma_pipeline_frag_size = mca_btl_vader.super.btl_eager_limit; mca_btl_vader.super.btl_min_rdma_pipeline_size = mca_btl_vader.super.btl_eager_limit; mca_btl_vader.super.btl_flags = MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_SEND_INPLACE; mca_btl_vader.super.btl_seg_size = sizeof (mca_btl_base_segment_t); mca_btl_vader.super.btl_bandwidth = 40000; /* Mbs */ mca_btl_vader.super.btl_latency = 1; /* Microsecs */ /* Call the BTL based to register its MCA params */ mca_btl_base_param_register(&mca_btl_vader_component.super.btl_version, &mca_btl_vader.super); return OMPI_SUCCESS; }
static int btl_ugni_component_register(void) { mca_base_var_enum_t *new_enum; gni_nic_device_t device_type; char *mpool_hints_tmp = NULL; int rc; (void) mca_base_var_group_component_register(&mca_btl_ugni_component.super.btl_version, "uGNI byte transport layer"); mca_btl_ugni_component.ugni_free_list_num = 8; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "free_list_num", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_free_list_num); mca_btl_ugni_component.ugni_free_list_max = 4096; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "free_list_max", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_free_list_max); mca_btl_ugni_component.ugni_free_list_inc = 64; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "free_list_inc", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_free_list_inc); mca_btl_ugni_component.ugni_eager_num = 16; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "eager_num", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_eager_num); mca_btl_ugni_component.ugni_eager_max = 128; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "eager_max", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_eager_max); mca_btl_ugni_component.ugni_eager_inc = 16; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "eager_inc", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_eager_inc); mca_btl_ugni_component.remote_cq_size = 40000; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "remote_cq_size", "Remote SMSG completion queue " "size (default 40000)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.remote_cq_size); mca_btl_ugni_component.local_cq_size = 8192; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "local_cq_size", "Local completion queue size " "(default 8192)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.local_cq_size); mca_btl_ugni_component.ugni_smsg_limit = 0; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "smsg_limit", "Maximum size message that " "will be sent using the SMSG/MSGQ protocol " "(0 - autoselect(default), 16k max)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_smsg_limit); mca_btl_ugni_component.smsg_max_credits = 32; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "smsg_max_credits", "Maximum number of " "outstanding SMSG/MSGQ message (default 32)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.smsg_max_credits); mca_btl_ugni_component.ugni_fma_limit = 1024; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "fma_limit", "Maximum size message that " "will be sent using the FMA (Fast Memory " "Access) protocol (default 1024, 64k max)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_fma_limit); mca_btl_ugni_component.rdma_max_retries = 16; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "rdma_max_retries", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.rdma_max_retries); mca_btl_ugni_component.smsg_max_retries = 16; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "smsg_max_retries", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.smsg_max_retries); mca_btl_ugni_component.max_mem_reg = 0; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "max_mem_reg", "Maximum number of " "memory registrations a process can " "hold (0 - autoselect, -1 - unlimited)" " (default 0)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.max_mem_reg); mca_btl_ugni_component.mbox_increment = 0; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "mbox_inc", "Number of SMSG mailboxes to " "allocate in each block (0 - autoselect(default))", MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.mbox_increment); /* communication domain flags */ rc = mca_base_var_enum_create_flag ("btl_ugni_cdm_flags", cdm_flags, (mca_base_var_enum_flag_t **) &new_enum); if (OPAL_SUCCESS != rc) { return rc; } mca_btl_ugni_component.cdm_flags = GNI_CDM_MODE_FORK_PARTCOPY | GNI_CDM_MODE_ERR_NO_KILL | GNI_CDM_MODE_FAST_DATAGRAM_POLL | GNI_CDM_MODE_MDD_SHARED | GNI_CDM_MODE_FMA_SHARED | GNI_CDM_MODE_FMA_SMALL_WINDOW; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "cdm_flags", "Flags to set when creating a communication domain " " (default: fork-fullcopy,cached-amo-enabled,err-no-kill,fast-datagram-poll," "fma-shared,fma-small-window)", MCA_BASE_VAR_TYPE_UNSIGNED_INT, new_enum, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.cdm_flags); OBJ_RELEASE(new_enum); mca_btl_ugni_component.virtual_device_count = 0; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "virtual_device_count", "Number of virtual devices to create. Higher numbers may " "result in better performance when using threads. (default: auto, max: 8)", MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.virtual_device_count); /* determine if there are get alignment restrictions */ GNI_GetDeviceType (&device_type); mca_btl_ugni_component.smsg_page_size = 2 << 20; if (GNI_DEVICE_GEMINI == device_type) { if (access ("/sys/class/gemini/ghal0/mrt", R_OK)) { int fd = open ("/sys/class/gemini/ghal0/mrt", O_RDONLY); char buffer[10]; if (0 <= fd) { memset (buffer, 0, sizeof (buffer)); read (fd, buffer, sizeof (buffer) - 1); close (fd); mca_btl_ugni_ugni_page_size = strtol (buffer, NULL, 10) * 1024; mca_btl_ugni_component.smsg_page_size = mca_btl_ugni_ugni_page_size; } } } (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "smsg_page_size", "Page size to use for SMSG mailbox allocation (default: detect)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.smsg_page_size); mca_btl_ugni_component.progress_thread_requested = 0; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "request_progress_thread", "Enable to request ugni btl progress thread - requires MPI_THREAD_MULTIPLE support", MCA_BASE_VAR_TYPE_BOOL, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.progress_thread_requested); /* performance variables */ mca_btl_ugni_progress_thread_wakeups = 0; (void) mca_base_component_pvar_register(&mca_btl_ugni_component.super.btl_version, "progress_thread_wakeups", "Number of times the progress thread " "has been woken", OPAL_INFO_LVL_9, MCA_BASE_PVAR_CLASS_COUNTER, MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, MCA_BASE_VAR_BIND_NO_OBJECT, MCA_BASE_PVAR_FLAG_READONLY | MCA_BASE_PVAR_FLAG_CONTINUOUS, NULL, NULL, NULL, &mca_btl_ugni_progress_thread_wakeups); /* register network statistics as performance variables */ for (int i = 0 ; i < GNI_NUM_STATS ; ++i) { char name[128], desc[128]; size_t str_len = strlen (gni_statistic_str[i]); assert (str_len < sizeof (name)); /* we can get an all-caps string for the variable from gni_statistic_str. need to make it lowercase * to match ompi standards */ for (size_t j = 0 ; j < str_len ; ++j) { name[j] = tolower (gni_statistic_str[i][j]); desc[j] = ('_' == name[j]) ? ' ' : name[j]; } name[str_len] = '\0'; desc[str_len] = '\0'; (void) mca_base_component_pvar_register (&mca_btl_ugni_component.super.btl_version, name, desc, OPAL_INFO_LVL_4, MCA_BASE_PVAR_CLASS_COUNTER, MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, MCA_BASE_VAR_BIND_NO_OBJECT, MCA_BASE_PVAR_FLAG_READONLY | MCA_BASE_PVAR_FLAG_CONTINUOUS, mca_btl_ugni_get_stat, NULL, mca_btl_ugni_notify_stat, (void *) (intptr_t) i); } /* btl/ugni can only support only a fixed set of rcache components (these rcache components have compatible resource * structures) */ rc = mca_base_var_enum_create ("btl_ugni_rcache", rcache_values, &new_enum); if (OPAL_SUCCESS != rc) { return rc; } /* NTH: there are known *serious* performance issues with udreg. if they are ever resolved it is the preferred rcache */ mca_btl_ugni_component.rcache_type = MCA_BTL_UGNI_RCACHE_GRDMA; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "rcache", "registration cache to use (default: grdma)", MCA_BASE_VAR_TYPE_INT, new_enum, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.rcache_type); OBJ_RELEASE(new_enum); if (mca_btl_ugni_ugni_page_size) { rc = asprintf (&mpool_hints_tmp, "page_size=%lu", mca_btl_ugni_ugni_page_size); if (rc < 0) { return OPAL_ERR_OUT_OF_RESOURCE; } mca_btl_ugni_component.mpool_hints = mpool_hints_tmp; } else { mca_btl_ugni_component.mpool_hints = "page_size=2M"; } (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "mpool_hints", "hints to use when selecting a memory pool (default: " "\"page_size=2M\")", MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.mpool_hints); free (mpool_hints_tmp); /* ensure we loose send exclusivity to sm and vader if they are enabled */ mca_btl_ugni_module.super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_HIGH - 2; /* smsg threshold */ mca_btl_ugni_module.super.btl_eager_limit = 8 * 1024; mca_btl_ugni_module.super.btl_rndv_eager_limit = 8 * 1024; mca_btl_ugni_module.super.btl_rdma_pipeline_frag_size = 4 * 1024 * 1024; mca_btl_ugni_module.super.btl_max_send_size = 8 * 1024; mca_btl_ugni_module.super.btl_rdma_pipeline_send_length = 8 * 1024; mca_btl_ugni_module.super.btl_get_limit = 1 * 1024 * 1024; /* * see def. of ALIGNMENT_MASK to figure this one out */ /* both gemini and aries have a 4-byte alignment requirement on remote addresses */ mca_btl_ugni_module.super.btl_get_alignment = 4; /* threshold for put */ mca_btl_ugni_module.super.btl_min_rdma_pipeline_size = 8 * 1024; mca_btl_ugni_module.super.btl_flags = MCA_BTL_FLAGS_SEND | MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_ATOMIC_OPS | MCA_BTL_FLAGS_ATOMIC_FOPS; mca_btl_ugni_module.super.btl_atomic_flags = MCA_BTL_ATOMIC_SUPPORTS_ADD | MCA_BTL_ATOMIC_SUPPORTS_AND | MCA_BTL_ATOMIC_SUPPORTS_OR | MCA_BTL_ATOMIC_SUPPORTS_XOR | MCA_BTL_ATOMIC_SUPPORTS_CSWAP; if (GNI_DEVICE_ARIES == device_type) { /* aries supports additional atomic operations */ mca_btl_ugni_module.super.btl_atomic_flags |= MCA_BTL_ATOMIC_SUPPORTS_MIN | MCA_BTL_ATOMIC_SUPPORTS_MAX | MCA_BTL_ATOMIC_SUPPORTS_LAND | MCA_BTL_ATOMIC_SUPPORTS_LOR | MCA_BTL_ATOMIC_SUPPORTS_LXOR | MCA_BTL_ATOMIC_SUPPORTS_32BIT | MCA_BTL_ATOMIC_SUPPORTS_FLOAT; } mca_btl_ugni_module.super.btl_registration_handle_size = sizeof (mca_btl_base_registration_handle_t); mca_btl_ugni_module.super.btl_bandwidth = 40000; /* Mbs */ mca_btl_ugni_module.super.btl_latency = 2; /* Microsecs */ mca_btl_ugni_module.super.btl_get_local_registration_threshold = 0; mca_btl_ugni_module.super.btl_put_local_registration_threshold = mca_btl_ugni_component.ugni_fma_limit; /* Call the BTL based to register its MCA params */ mca_btl_base_param_register(&mca_btl_ugni_component.super.btl_version, &mca_btl_ugni_module.super); return OPAL_SUCCESS; }
static int btl_ugni_component_register(void) { mca_base_var_enum_t *new_enum; int rc; (void) mca_base_var_group_component_register(&mca_btl_ugni_component.super.btl_version, "Gemini byte transport layer"); mca_btl_ugni_component.ugni_free_list_num = 8; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "free_list_num", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_free_list_num); mca_btl_ugni_component.ugni_free_list_max = 16384; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "free_list_max", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_free_list_max); mca_btl_ugni_component.ugni_free_list_inc = 64; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "free_list_inc", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_free_list_inc); mca_btl_ugni_component.ugni_eager_num = 16; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "eager_num", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_eager_num); mca_btl_ugni_component.ugni_eager_max = 128; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "eager_max", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_eager_max); mca_btl_ugni_component.ugni_eager_inc = 16; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "eager_inc", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_eager_inc); mca_btl_ugni_component.remote_cq_size = 40000; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "remote_cq_size", "Remote SMSG completion queue " "size (default 40000)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.remote_cq_size); mca_btl_ugni_component.local_cq_size = 8192; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "local_cq_size", "Local completion queue size " "(default 8192)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.local_cq_size); mca_btl_ugni_component.ugni_smsg_limit = 0; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "smsg_limit", "Maximum size message that " "will be sent using the SMSG/MSGQ protocol " "(0 - autoselect(default), 16k max)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_smsg_limit); mca_btl_ugni_component.smsg_max_credits = 32; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "smsg_max_credits", "Maximum number of " "outstanding SMSG/MSGQ message (default 32)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.smsg_max_credits); mca_btl_ugni_component.ugni_fma_limit = 1024; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "fma_limit", "Maximum size message that " "will be sent using the FMA (Fast Memory " "Access) protocol (default 1024, 64k max)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_fma_limit); mca_btl_ugni_component.ugni_get_limit = 1 * 1024 * 1024; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "get_limit", "Maximum size message that " "will be sent using a get protocol " "(default 1M)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_get_limit); mca_btl_ugni_component.rdma_max_retries = 16; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "rdma_max_retries", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.rdma_max_retries); mca_btl_ugni_component.smsg_max_retries = 16; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "smsg_max_retries", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.smsg_max_retries); mca_btl_ugni_component.max_mem_reg = 0; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "max_mem_reg", "Maximum number of " "memory registrations a process can " "hold (0 - autoselect, -1 - unlimited)" " (default 0)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.max_mem_reg); mca_btl_ugni_component.mbox_increment = 0; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "mbox_inc", "Number of SMSG mailboxes to " "allocate in each block (0 - autoselect(default))", MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.mbox_increment); mca_btl_ugni_component.smsg_page_size = 2 << 20; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "smsg_page_size", "Page size to use for SMSG " "mailbox allocation (default 2M)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.smsg_page_size); /* btl/ugni can only support only a fixed set of mpools (these mpools have compatible resource * structures) */ rc = mca_base_var_enum_create ("btl_ugni_mpool", mpool_values, &new_enum); if (OPAL_SUCCESS != rc) { return rc; } mca_btl_ugni_component.mpool_type = MCA_BTL_UGNI_MPOOL_UDREG; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "mpool", "mpool to use", MCA_BASE_VAR_TYPE_INT, new_enum, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.mpool_type); OBJ_RELEASE(new_enum); mca_btl_ugni_module.super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_HIGH; /* smsg threshold */ mca_btl_ugni_module.super.btl_eager_limit = 8 * 1024; mca_btl_ugni_module.super.btl_rndv_eager_limit = 8 * 1024; mca_btl_ugni_module.super.btl_rdma_pipeline_frag_size = 4 * 1024 * 1024; mca_btl_ugni_module.super.btl_max_send_size = 8 * 1024; mca_btl_ugni_module.super.btl_rdma_pipeline_send_length = 8 * 1024; /* threshold for put */ mca_btl_ugni_module.super.btl_min_rdma_pipeline_size = 8 * 1024; mca_btl_ugni_module.super.btl_flags = MCA_BTL_FLAGS_SEND | MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_SEND_INPLACE; mca_btl_ugni_module.super.btl_seg_size = sizeof (mca_btl_ugni_segment_t); mca_btl_ugni_module.super.btl_bandwidth = 40000; /* Mbs */ mca_btl_ugni_module.super.btl_latency = 2; /* Microsecs */ /* Call the BTL based to register its MCA params */ mca_btl_base_param_register(&mca_btl_ugni_component.super.btl_version, &mca_btl_ugni_module.super); return OMPI_SUCCESS; }
static int btl_ugni_component_register(void) { (void) mca_base_var_group_component_register(&mca_btl_ugni_component.super.btl_version, "Gemini byte transport layer"); mca_btl_ugni_component.ugni_free_list_num = 8; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "free_list_num", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_free_list_num); mca_btl_ugni_component.ugni_free_list_max = 16384; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "free_list_max", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_free_list_max); mca_btl_ugni_component.ugni_free_list_inc = 64; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "free_list_inc", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_free_list_inc); mca_btl_ugni_component.ugni_eager_num = 16; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "eager_num", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_eager_num); mca_btl_ugni_component.ugni_eager_max = 128; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "eager_max", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_eager_max); mca_btl_ugni_component.ugni_eager_inc = 16; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "eager_inc", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_eager_inc); mca_btl_ugni_component.remote_cq_size = 40000; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "remote_cq_size", "Remote SMSG completion queue " "size (default 40000)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.remote_cq_size); mca_btl_ugni_component.local_cq_size = 8192; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "local_cq_size", "Local completion queue size " "(default 8192)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.local_cq_size); mca_btl_ugni_component.ugni_smsg_limit = 0; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "smsg_limit", "Maximum size message that " "will be sent using the SMSG/MSGQ protocol " "(0 - autoselect(default), 16k max)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_smsg_limit); mca_btl_ugni_component.smsg_max_credits = 32; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "smsg_max_credits", "Maximum number of " "outstanding SMSG/MSGQ message (default 32)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.smsg_max_credits); mca_btl_ugni_component.ugni_fma_limit = 1024; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "fma_limit", "Maximum size message that " "will be sent using the FMA (Fast Memory " "Access) protocol (default 1024, 64k max)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_fma_limit); mca_btl_ugni_component.ugni_get_limit = 1 * 1024 * 1024; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "get_limit", "Maximum size message that " "will be sent using a get protocol " "(default 1M)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.ugni_get_limit); mca_btl_ugni_component.rdma_max_retries = 16; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "rdma_max_retries", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.rdma_max_retries); mca_btl_ugni_component.smsg_max_retries = 16; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "smsg_max_retries", NULL, MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_5, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.smsg_max_retries); mca_btl_ugni_component.max_mem_reg = 0; (void) mca_base_component_var_register(&mca_btl_ugni_component.super.btl_version, "max_mem_reg", "Maximum number of " "memory registrations a process can " "hold (0 - autoselect, -1 - unlimited)" " (default 0)", MCA_BASE_VAR_TYPE_INT, NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL, &mca_btl_ugni_component.max_mem_reg); mca_btl_ugni_module.super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_HIGH; /* smsg threshold */ mca_btl_ugni_module.super.btl_eager_limit = 8 * 1024; mca_btl_ugni_module.super.btl_rndv_eager_limit = 8 * 1024; mca_btl_ugni_module.super.btl_rdma_pipeline_frag_size = 4 * 1024 * 1024; mca_btl_ugni_module.super.btl_max_send_size = 8 * 1024; mca_btl_ugni_module.super.btl_rdma_pipeline_send_length = 8 * 1024; /* threshold for put */ mca_btl_ugni_module.super.btl_min_rdma_pipeline_size = 8 * 1024; mca_btl_ugni_module.super.btl_flags = MCA_BTL_FLAGS_SEND | MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_SEND_INPLACE; mca_btl_ugni_module.super.btl_seg_size = sizeof (mca_btl_ugni_segment_t); mca_btl_ugni_module.super.btl_bandwidth = 40000; /* Mbs */ mca_btl_ugni_module.super.btl_latency = 2; /* Microsecs */ /* Call the BTL based to register its MCA params */ mca_btl_base_param_register(&mca_btl_ugni_component.super.btl_version, &mca_btl_ugni_module.super); return OMPI_SUCCESS; }