ib_ctx_handler_collection::ib_ctx_handler_collection()
{
ibchc_logdbg("");
/* Read ib table from kernel and save it in local variable. */
update_tbl();
//Print table
print_val_tbl();
ibchc_logdbg("Done");
}
ib_ctx_handler_collection::~ib_ctx_handler_collection()
{
ibchc_logdbg("");
ib_context_map_t::iterator ib_ctx_iter;
while ((ib_ctx_iter = m_ib_ctx_map.begin()) != m_ib_ctx_map.end()) {
ib_ctx_handler* p_ib_ctx_handler = ib_ctx_iter->second;
delete p_ib_ctx_handler;
m_ib_ctx_map.erase(ib_ctx_iter);
}
ibchc_logdbg("Done");
}
size_t ib_ctx_handler_collection::mem_reg_on_all_devices(void* addr, size_t length,
ibv_mr** mr_array, size_t mr_array_sz,
uint64_t access)
{
ibchc_logfunc("");
size_t mr_pos = 0;
ib_context_map_t::iterator ib_ctx_iter;
for (ib_ctx_iter = m_ib_ctx_map.begin(); ib_ctx_iter != m_ib_ctx_map.end(), mr_pos<mr_array_sz; ib_ctx_iter++, mr_pos++) {
ib_ctx_handler* p_ib_ctx_handler = ib_ctx_iter->second;
mr_array[mr_pos] = p_ib_ctx_handler->mem_reg(addr, length, access);
BULLSEYE_EXCLUDE_BLOCK_START
if (mr_array[mr_pos] == NULL) {
ibchc_logwarn("Failure in mem_reg: addr=%p, length=%d, mr_pos=%d, mr_array[mr_pos]=%d, dev=%p, ibv_dev=%s",
addr, length, mr_pos, mr_array[mr_pos], p_ib_ctx_handler, p_ib_ctx_handler->get_ibv_device()->name);
return (size_t)-1;
}
BULLSEYE_EXCLUDE_BLOCK_END
errno = 0; //ibv_reg_mr() set errno=12 despite successful returning
#ifdef VMA_IBV_ACCESS_ALLOCATE_MR
if ((access & VMA_IBV_ACCESS_ALLOCATE_MR) != 0) { // contig pages mode
// When using 'IBV_ACCESS_ALLOCATE_MR', ibv_reg_mr will return a pointer that its 'addr' field will hold the address of the allocated memory.
// Second registration and above is done using 'IBV_ACCESS_LOCAL_WRITE' and the 'addr' we received from the first registration.
addr = mr_array[0]->addr;
access &= ~VMA_IBV_ACCESS_ALLOCATE_MR;
}
#endif
ibchc_logdbg("addr=%p, length=%d, pos=%d, mr[pos]->lkey=%u, dev1=%p, dev2=%p",
addr, length, mr_pos, mr_array[mr_pos]->lkey, mr_array[mr_pos]->context->device, p_ib_ctx_handler->get_ibv_device());
}
return mr_pos;
}
void ib_ctx_handler_collection::map_ib_devices() //return num_devices, can use rdma_get_devices()
{
struct ibv_context** pp_ibv_context_list = rdma_get_devices(&m_n_num_devices);
BULLSEYE_EXCLUDE_BLOCK_START
if (!pp_ibv_context_list) {
ibchc_logwarn("Failure in rdma_get_devices() (error=%d %m)", errno);
ibchc_logpanic("Please check OFED installation");
}
if (!m_n_num_devices) {
rdma_free_devices(pp_ibv_context_list);
ibchc_logpanic("No RDMA capable devices found!");
}
BULLSEYE_EXCLUDE_BLOCK_END
ibchc_logdbg("Mapping %d ibv devices", m_n_num_devices);
for (int i = 0; i < m_n_num_devices; i++) {
m_ib_ctx_map[pp_ibv_context_list[i]] = new ib_ctx_handler(pp_ibv_context_list[i]);
}
rdma_free_devices(pp_ibv_context_list);
}
void ib_ctx_handler_collection::update_tbl(const char *ifa_name)
{
struct ibv_device **dev_list = NULL;
ib_ctx_handler * p_ib_ctx_handler = NULL;
int num_devices = 0;
int i;
ibchc_logdbg("Checking for offload capable IB devices...");
dev_list = vma_ibv_get_device_list(&num_devices);
BULLSEYE_EXCLUDE_BLOCK_START
if (!dev_list) {
ibchc_logerr("Failure in vma_ibv_get_device_list() (error=%d %m)", errno);
ibchc_logerr("Please check rdma configuration");
throw_vma_exception("No IB capable devices found!");
}
if (!num_devices) {
vlog_levels_t _level = ifa_name ? VLOG_DEBUG : VLOG_ERROR; // Print an error only during initialization.
vlog_printf(_level, "VMA does not detect IB capable devices\n");
vlog_printf(_level, "No performance gain is expected in current configuration\n");
}
BULLSEYE_EXCLUDE_BLOCK_END
for (i = 0; i < num_devices; i++) {
struct ib_ctx_handler::ib_ctx_handler_desc desc = {dev_list[i]};
/* 2. Skip existing devices (compare by name) */
if (ifa_name && !check_device_name_ib_name(ifa_name, dev_list[i]->name)) {
continue;
}
if (ib_ctx_handler::is_mlx4(dev_list[i]->name)) {
// Note: mlx4 does not support this capability.
if(safe_mce_sys().enable_socketxtreme) {
ibchc_logdbg("Blocking offload: mlx4 interfaces in socketxtreme mode");
continue;
}
// Check if mlx4 steering creation is supported.
// Those setting are passed to the VM by the Hypervisor - NO NEED to specify the param on the VM.
if (safe_mce_sys().hypervisor == mce_sys_var::HYPER_NONE) {
check_flow_steering_log_num_mgm_entry_size();
}
}
/* 3. Add new ib devices */
p_ib_ctx_handler = new ib_ctx_handler(&desc);
if (!p_ib_ctx_handler) {
ibchc_logerr("failed allocating new ib_ctx_handler (errno=%d %m)", errno);
continue;
}
m_ib_ctx_map[p_ib_ctx_handler->get_ibv_device()] = p_ib_ctx_handler;
}
ibchc_logdbg("Check completed. Found %d offload capable IB devices", m_ib_ctx_map.size());
if (dev_list) {
ibv_free_device_list(dev_list);
}
}
