VAPI_ret_t osmt_mtl_mad_create_mr(osmt_mtl_mad_res_t * res)
{
VAPI_ret_t ret;
VAPI_mrw_t mr_in, mr_out;
res->buf_size =
(MAD_SIZE + GRH_LEN) * (res->max_outs_sq + res->max_outs_rq + 1);
/* Register single memory address region for all buffers */
res->buf_ptr = VMALLOC(res->buf_size);
if (res->buf_ptr == ((VAPI_virt_addr_t) NULL)) {
ret = VAPI_EAGAIN;
VAPI_CHECK_RET;
}
/* Enable local and remote access to memory region */
mr_in.acl = VAPI_EN_LOCAL_WRITE | VAPI_EN_REMOTE_WRITE;
mr_in.l_key = 0;
mr_in.pd_hndl = res->pd_hndl;
mr_in.r_key = 0;
mr_in.size = res->buf_size;
ASSERT_VOIDP2UINTN(res->buf_ptr);
mr_in.start = (VAPI_virt_addr_t) (res->buf_ptr);
mr_in.type = VAPI_MR;
ret = VAPI_register_mr(res->hca_hndl, &mr_in, &(res->mr_hndl), &mr_out);
VAPI_CHECK_RET;
res->l_key = mr_out.l_key;
return (ret);
}
static
int psib_vapi_alloc(IN hca_info_t *hca_info, IN int size, IN VAPI_mrw_acl_t perm,
OUT mem_info_t *mem_info)
/*
IN VAPI_hca_hndl_t hca_hndl, IN VAPI_pd_hndl_t pd_hndl,
IN VAPI_mrw_acl_t perm, IN int size,
OUT VAPI_mr_hndl_t *mr_hndl, OUT void **vptr,
OUT VAPI_lkey_t *lkey, OUT VAPI_rkey_t *rkey)
*/
{
void *ptr;
VAPI_mrw_t mrw;
VAPI_ret_t rc;
/* Region for buffers */
ptr = malloc(size);
if (!ptr) goto err_malloc;
memset(&mrw, 0, sizeof(mrw));
mrw.type = VAPI_MR; /* Memory region */
mrw.start = (VAPI_virt_addr_t)(MT_virt_addr_t)ptr;
mrw.size = size;
mrw.pd_hndl = hca_info->pd_hndl;
mrw.acl = perm; //VAPI_EN_LOCAL_WRITE;
rc = VAPI_register_mr(hca_info->hca_hndl,
&mrw, &mem_info->mr_hndl, &mem_info->rep_mrw);
if (rc != VAPI_SUCCESS) goto err_VAPI_register_mr;
if (psib_debug > 1) {
printf("lkey: 0x%08x rkey: 0x%08x ptr: %p start: 0x%016lx\n",
mem_info->rep_mrw.l_key,
mem_info->rep_mrw.r_key,
ptr, (long)mem_info->rep_mrw.start);
}
mem_info->ptr = ptr;
return 0;
/* --- */
err_VAPI_register_mr:
free(ptr);
psib_err_rc("VAPI_register_mr() failed", rc);
return -1;
err_malloc:
psib_err("malloc() failed!");
return -1;
}
void MyMalloc(ArgStruct *p, int bufflen, int soffset, int roffset)
{
VAPI_ret_t ret;
/* Allocate buffers */
p->r_buff = malloc(bufflen+MAX(soffset,roffset));
if(p->r_buff == NULL) {
fprintf(stderr, "Error malloc'ing buffer\n");
exit(-1);
}
if(p->cache) {
/* Infiniband spec says we can register same memory region
* more than once, so just copy buffer address. We will register
* the same buffer twice with Infiniband.
*/
p->s_buff = p->r_buff;
} else {
p->s_buff = malloc(bufflen+soffset);
if(p->s_buff == NULL) {
fprintf(stderr, "Error malloc'ing buffer\n");
exit(-1);
}
}
/* Register buffers with Infiniband */
mr_in.acl = VAPI_EN_LOCAL_WRITE | VAPI_EN_REMOTE_WRITE;
mr_in.l_key = 0;
mr_in.pd_hndl = pd_hndl;
mr_in.r_key = 0;
mr_in.size = bufflen+MAX(soffset,roffset);
mr_in.start = (VAPI_virt_addr_t)(MT_virt_addr_t)p->r_buff;
mr_in.type = VAPI_MR;
ret = VAPI_register_mr(hca_hndl, &mr_in, &r_mr_hndl, &r_mr_out);
if(ret != VAPI_OK)
{
fprintf(stderr, "Error registering recv buffer: %s\n", VAPI_strerror(ret));
exit(-1);
}
else
{
LOGPRINTF("Registered Recv Buffer\n");
}
mr_in.acl = VAPI_EN_LOCAL_WRITE;
mr_in.l_key = 0;
mr_in.pd_hndl = pd_hndl;
mr_in.r_key = 0;
mr_in.size = bufflen+soffset;
mr_in.start = (VAPI_virt_addr_t)(MT_virt_addr_t)p->s_buff;
mr_in.type = VAPI_MR;
ret = VAPI_register_mr(hca_hndl, &mr_in, &s_mr_hndl, &s_mr_out);
if(ret != VAPI_OK) {
fprintf(stderr, "Error registering send buffer: %s\n", VAPI_strerror(ret));
exit(-1);
} else {
LOGPRINTF("Registered Send Buffer\n");
}
}
