VAPI_ret_t osmt_mtl_mad_create_mr(osmt_mtl_mad_res_t * res) { VAPI_ret_t ret; VAPI_mrw_t mr_in, mr_out; res->buf_size = (MAD_SIZE + GRH_LEN) * (res->max_outs_sq + res->max_outs_rq + 1); /* Register single memory address region for all buffers */ res->buf_ptr = VMALLOC(res->buf_size); if (res->buf_ptr == ((VAPI_virt_addr_t) NULL)) { ret = VAPI_EAGAIN; VAPI_CHECK_RET; } /* Enable local and remote access to memory region */ mr_in.acl = VAPI_EN_LOCAL_WRITE | VAPI_EN_REMOTE_WRITE; mr_in.l_key = 0; mr_in.pd_hndl = res->pd_hndl; mr_in.r_key = 0; mr_in.size = res->buf_size; ASSERT_VOIDP2UINTN(res->buf_ptr); mr_in.start = (VAPI_virt_addr_t) (res->buf_ptr); mr_in.type = VAPI_MR; ret = VAPI_register_mr(res->hca_hndl, &mr_in, &(res->mr_hndl), &mr_out); VAPI_CHECK_RET; res->l_key = mr_out.l_key; return (ret); }
static int psib_vapi_alloc(IN hca_info_t *hca_info, IN int size, IN VAPI_mrw_acl_t perm, OUT mem_info_t *mem_info) /* IN VAPI_hca_hndl_t hca_hndl, IN VAPI_pd_hndl_t pd_hndl, IN VAPI_mrw_acl_t perm, IN int size, OUT VAPI_mr_hndl_t *mr_hndl, OUT void **vptr, OUT VAPI_lkey_t *lkey, OUT VAPI_rkey_t *rkey) */ { void *ptr; VAPI_mrw_t mrw; VAPI_ret_t rc; /* Region for buffers */ ptr = malloc(size); if (!ptr) goto err_malloc; memset(&mrw, 0, sizeof(mrw)); mrw.type = VAPI_MR; /* Memory region */ mrw.start = (VAPI_virt_addr_t)(MT_virt_addr_t)ptr; mrw.size = size; mrw.pd_hndl = hca_info->pd_hndl; mrw.acl = perm; //VAPI_EN_LOCAL_WRITE; rc = VAPI_register_mr(hca_info->hca_hndl, &mrw, &mem_info->mr_hndl, &mem_info->rep_mrw); if (rc != VAPI_SUCCESS) goto err_VAPI_register_mr; if (psib_debug > 1) { printf("lkey: 0x%08x rkey: 0x%08x ptr: %p start: 0x%016lx\n", mem_info->rep_mrw.l_key, mem_info->rep_mrw.r_key, ptr, (long)mem_info->rep_mrw.start); } mem_info->ptr = ptr; return 0; /* --- */ err_VAPI_register_mr: free(ptr); psib_err_rc("VAPI_register_mr() failed", rc); return -1; err_malloc: psib_err("malloc() failed!"); return -1; }
void MyMalloc(ArgStruct *p, int bufflen, int soffset, int roffset) { VAPI_ret_t ret; /* Allocate buffers */ p->r_buff = malloc(bufflen+MAX(soffset,roffset)); if(p->r_buff == NULL) { fprintf(stderr, "Error malloc'ing buffer\n"); exit(-1); } if(p->cache) { /* Infiniband spec says we can register same memory region * more than once, so just copy buffer address. We will register * the same buffer twice with Infiniband. */ p->s_buff = p->r_buff; } else { p->s_buff = malloc(bufflen+soffset); if(p->s_buff == NULL) { fprintf(stderr, "Error malloc'ing buffer\n"); exit(-1); } } /* Register buffers with Infiniband */ mr_in.acl = VAPI_EN_LOCAL_WRITE | VAPI_EN_REMOTE_WRITE; mr_in.l_key = 0; mr_in.pd_hndl = pd_hndl; mr_in.r_key = 0; mr_in.size = bufflen+MAX(soffset,roffset); mr_in.start = (VAPI_virt_addr_t)(MT_virt_addr_t)p->r_buff; mr_in.type = VAPI_MR; ret = VAPI_register_mr(hca_hndl, &mr_in, &r_mr_hndl, &r_mr_out); if(ret != VAPI_OK) { fprintf(stderr, "Error registering recv buffer: %s\n", VAPI_strerror(ret)); exit(-1); } else { LOGPRINTF("Registered Recv Buffer\n"); } mr_in.acl = VAPI_EN_LOCAL_WRITE; mr_in.l_key = 0; mr_in.pd_hndl = pd_hndl; mr_in.r_key = 0; mr_in.size = bufflen+soffset; mr_in.start = (VAPI_virt_addr_t)(MT_virt_addr_t)p->s_buff; mr_in.type = VAPI_MR; ret = VAPI_register_mr(hca_hndl, &mr_in, &s_mr_hndl, &s_mr_out); if(ret != VAPI_OK) { fprintf(stderr, "Error registering send buffer: %s\n", VAPI_strerror(ret)); exit(-1); } else { LOGPRINTF("Registered Send Buffer\n"); } }