void build_connection(struct rdma_cm_id *id) { struct connection *conn; struct ibv_qp_init_attr qp_attr; //init semaphores sem_init(&read_ops, 0, 0); sem_init(&done_ops, 0, 0); sem_init(&write_ops, 0, 1); build_context(id->verbs); build_qp_attr(&qp_attr); TEST_NZ(rdma_create_qp(id, s_ctx->pd, &qp_attr)); id->context = conn = (struct connection *)malloc(sizeof(struct connection)); conn->id = id; conn->qp = id->qp; conn->send_state = SS_INIT; conn->recv_state = RS_INIT; conn->connected = 0; register_memory(conn); post_receives(conn); }
void memory_registry::register_memory(const std::string& src_path, const std::string& dst_path, const checksum& builder_checksum) { register_memory(memory( src_path, make_file_state(src_path), make_file_state(dst_path), builder_checksum)); }
void build_connection(struct rdma_cm_id *id) { rdma_conn_t *conn; struct ibv_qp_init_attr qp_attr; build_context(id->verbs); build_qp_attr(&qp_attr); TEST_NZ(rdma_create_qp(id, s_ctx->pd, &qp_attr)); conn = malloc(sizeof(rdma_conn_t)); id->context = conn; rdma_conn = conn; conn->id = id; conn->qp = id->qp; conn->send_state = SS_INIT; conn->recv_state = RS_INIT; conn->connected = 0; register_memory(conn); post_receives(conn); }
int on_addr_resolved(struct rdma_cm_id *id) { struct ibv_qp_init_attr qp_attr; struct connection *conn; printf("address resolved.\n"); build_context(id->verbs); build_qp_attr(&qp_attr); TEST_NZ(rdma_create_qp(id, s_ctx->pd, &qp_attr)); id->context = conn = (struct connection *)malloc(sizeof(struct connection)); conn->id = id; conn->qp = id->qp; conn->num_completions = 0; register_memory(conn); post_receives(conn); TEST_NZ(rdma_resolve_route(id, TIMEOUT_IN_MS)); return 0; }
static int init_memory_block(struct memory_block **memory, struct mem_section *section, unsigned long state) { struct memory_block *mem; unsigned long start_pfn; int scn_nr; int ret = 0; mem = kzalloc(sizeof(*mem), GFP_KERNEL); if (!mem) return -ENOMEM; scn_nr = __section_nr(section); mem->start_section_nr = base_memory_block_id(scn_nr) * sections_per_block; mem->end_section_nr = mem->start_section_nr + sections_per_block - 1; mem->state = state; start_pfn = section_nr_to_pfn(mem->start_section_nr); mem->phys_device = arch_get_memory_phys_device(start_pfn); ret = register_memory(mem); *memory = mem; return ret; }
void re_register_vbufs() { vbuf_region *r = vbuf_region_head; lock_vbuf(); while (r) { r->mem_handle = register_memory(r->malloc_buf_start, r->count * viadev_vbuf_total_size); r = r->next; } unlock_vbuf(); }
int on_route_resolved(struct rdma_cm_id *id) { struct rdma_conn_param cm_params; struct timeval start, end, dt; printf("route resolved.\n"); build_params(&cm_params); TEST_NZ(rdma_connect(id, &cm_params)); gettimeofday(&start, NULL); register_memory((struct connection *)(id->context)); gettimeofday(&end, NULL); timersub(&end, &start, &dt); long usec = dt.tv_usec + 1000000 * dt.tv_sec; printf("[Register] takes %ld micro_secs.\n", usec); post_receive_for_msg(id->context); return 0; }
void build_connection(struct rdma_cm_id *id) { IbvConnection *conn; struct ibv_qp_init_attr qp_attr; id->context = conn = (IbvConnection *)malloc(sizeof(IbvConnection)); build_verbs(conn, id->verbs); build_qp_attr(conn, &qp_attr); TEST_NZ(rdma_create_qp(id, conn->pd, &qp_attr)); conn->id = id; conn->qp = id->qp; conn->connected = 0; register_memory(conn); post_receives(conn); }
static int init_memory_block(struct memory_block **memory, struct mem_section *section, unsigned long state) { struct memory_block *mem; unsigned long start_pfn; int scn_nr; int ret = 0; mem = kzalloc(sizeof(*mem), GFP_KERNEL); if (!mem) return -ENOMEM; scn_nr = __section_nr(section); mem->start_section_nr = base_memory_block_id(scn_nr) * sections_per_block; mem->end_section_nr = mem->start_section_nr + sections_per_block - 1; mem->state = state; mem->section_count++; mutex_init(&mem->state_mutex); start_pfn = section_nr_to_pfn(mem->start_section_nr); mem->phys_device = arch_get_memory_phys_device(start_pfn); ret = register_memory(mem); if (!ret) ret = mem_create_simple_file(mem, phys_index); if (!ret) ret = mem_create_simple_file(mem, end_phys_index); if (!ret) ret = mem_create_simple_file(mem, state); if (!ret) ret = mem_create_simple_file(mem, phys_device); if (!ret) ret = mem_create_simple_file(mem, removable); *memory = mem; return ret; }
int on_connect_request(struct rdma_cm_id *id) { struct ibv_qp_init_attr qp_attr; struct rdma_conn_param cm_params; struct connection *conn; printf("received connection request.\n"); build_context(id->verbs); build_qp_attr(&qp_attr); TEST_NZ(rdma_create_qp(id, s_ctx->pd, &qp_attr)); id->context = conn = (struct connection *)malloc(sizeof(struct connection)); conn->qp = id->qp; register_memory(conn); post_receives(conn); memset(&cm_params, 0, sizeof(cm_params)); TEST_NZ(rdma_accept(id, &cm_params)); return 0; }
void build_connection(struct rdma_cm_id *id){ struct ibv_qp_init_attr qp_attr; struct connection *conn; struct timeval start, end, dt; build_qp_attr(&qp_attr); TEST_NZ(rdma_create_qp(id, s_ctx->pd, &qp_attr)); id->context = conn = (struct connection *)malloc(sizeof(struct connection)); conn->id = id; conn->qp = id->qp; gettimeofday(&start, NULL); register_memory(conn); gettimeofday(&end, NULL); timersub(&end, &start, &dt); long usec = dt.tv_usec + 1000000 * dt.tv_sec; printf("[Register] takes %ld micro_secs.\n", usec); }
static long acdb_ioctl(struct file *f, unsigned int cmd, unsigned long arg) { int32_t result = 0; int32_t size; int32_t map_fd; uint32_t topology; uint32_t data[MAX_IOCTL_DATA]; struct msm_spk_prot_status prot_status; struct msm_spk_prot_status acdb_spk_status; pr_debug("%s\n", __func__); mutex_lock(&acdb_data.acdb_mutex); switch (cmd) { case AUDIO_REGISTER_PMEM: pr_debug("AUDIO_REGISTER_PMEM\n"); result = deregister_memory(); if (result < 0) pr_err("%s: deregister_memory failed returned %d!\n", __func__, result); if (copy_from_user(&map_fd, (void *)arg, sizeof(map_fd))) { pr_err("%s: fail to copy memory handle!\n", __func__); result = -EFAULT; } else { acdb_data.map_handle = map_fd; result = register_memory(); } goto done; case AUDIO_DEREGISTER_PMEM: pr_debug("AUDIO_DEREGISTER_PMEM\n"); result = deregister_memory(); goto done; case AUDIO_SET_VOICE_RX_TOPOLOGY: if (copy_from_user(&topology, (void *)arg, sizeof(topology))) { pr_err("%s: fail to copy topology!\n", __func__); result = -EFAULT; } store_voice_rx_topology(topology); goto done; case AUDIO_SET_VOICE_TX_TOPOLOGY: if (copy_from_user(&topology, (void *)arg, sizeof(topology))) { pr_err("%s: fail to copy topology!\n", __func__); result = -EFAULT; } store_voice_tx_topology(topology); goto done; case AUDIO_SET_ADM_RX_TOPOLOGY: if (copy_from_user(&topology, (void *)arg, sizeof(topology))) { pr_err("%s: fail to copy topology!\n", __func__); result = -EFAULT; } store_adm_rx_topology(topology); goto done; case AUDIO_SET_ADM_TX_TOPOLOGY: if (copy_from_user(&topology, (void *)arg, sizeof(topology))) { pr_err("%s: fail to copy topology!\n", __func__); result = -EFAULT; } store_adm_tx_topology(topology); goto done; case AUDIO_SET_ASM_TOPOLOGY: if (copy_from_user(&topology, (void *)arg, sizeof(topology))) { pr_err("%s: fail to copy topology!\n", __func__); result = -EFAULT; } store_asm_topology(topology); goto done; case AUDIO_SET_SPEAKER_PROT: if (copy_from_user(&acdb_data.spk_prot_cfg, (void *)arg, sizeof(acdb_data.spk_prot_cfg))) { pr_err("%s fail to copy spk_prot_cfg\n", __func__); result = -EFAULT; } goto done; case AUDIO_GET_SPEAKER_PROT: /*Indicates calibration was succesfull*/ if (acdb_data.spk_prot_cfg.mode == MSM_SPKR_PROT_CALIBRATED) { prot_status.r0 = acdb_data.spk_prot_cfg.r0; prot_status.status = 0; } else if (acdb_data.spk_prot_cfg.mode == MSM_SPKR_PROT_CALIBRATION_IN_PROGRESS) { /*Call AFE to query the status*/ acdb_spk_status.status = -EINVAL; acdb_spk_status.r0 = -1; get_spk_protection_status(&acdb_spk_status); prot_status.r0 = acdb_spk_status.r0; prot_status.status = acdb_spk_status.status; if (!acdb_spk_status.status) { acdb_data.spk_prot_cfg.mode = MSM_SPKR_PROT_CALIBRATED; acdb_data.spk_prot_cfg.r0 = prot_status.r0; } } else { /*Indicates calibration data is invalid*/ prot_status.status = -EINVAL; prot_status.r0 = -1; } if (copy_to_user((void *)arg, &prot_status, sizeof(prot_status))) { pr_err("%s: Failed to update prot_status\n", __func__); } goto done; case AUDIO_REGISTER_VOCPROC_VOL_TABLE: result = register_vocvol_table(); goto done; case AUDIO_DEREGISTER_VOCPROC_VOL_TABLE: result = deregister_vocvol_table(); goto done; case AUDIO_SET_HW_DELAY_RX: result = store_hw_delay(RX_CAL, (void *)arg); goto done; case AUDIO_SET_HW_DELAY_TX: result = store_hw_delay(TX_CAL, (void *)arg); goto done; case AUDIO_SET_META_INFO: result = store_meta_info((void *)arg); goto done; } if (copy_from_user(&size, (void *) arg, sizeof(size))) { result = -EFAULT; goto done; } if ((size <= 0) || (size > sizeof(data))) { pr_err("%s: Invalid size sent to driver: %d\n", __func__, size); result = -EFAULT; goto done; } switch (cmd) { case AUDIO_SET_VOCPROC_COL_CAL: result = store_voice_col_data(VOCPROC_CAL, size, (uint32_t *)arg); goto done; case AUDIO_SET_VOCSTRM_COL_CAL: result = store_voice_col_data(VOCSTRM_CAL, size, (uint32_t *)arg); goto done; case AUDIO_SET_VOCVOL_COL_CAL: result = store_voice_col_data(VOCVOL_CAL, size, (uint32_t *)arg); goto done; } if (copy_from_user(data, (void *)(arg + sizeof(size)), size)) { pr_err("%s: fail to copy table size %d\n", __func__, size); result = -EFAULT; goto done; } if (data == NULL) { pr_err("%s: NULL pointer sent to driver!\n", __func__); result = -EFAULT; goto done; } if (size > sizeof(struct cal_block)) pr_err("%s: More cal data for ioctl 0x%x then expected, size received: %d\n", __func__, cmd, size); switch (cmd) { case AUDIO_SET_AUDPROC_TX_CAL: result = store_audproc_cal(TX_CAL, (struct cal_block *)data); goto done; case AUDIO_SET_AUDPROC_RX_CAL: result = store_audproc_cal(RX_CAL, (struct cal_block *)data); goto done; case AUDIO_SET_AUDPROC_TX_STREAM_CAL: result = store_audstrm_cal(TX_CAL, (struct cal_block *)data); goto done; case AUDIO_SET_AUDPROC_RX_STREAM_CAL: result = store_audstrm_cal(RX_CAL, (struct cal_block *)data); goto done; case AUDIO_SET_AUDPROC_TX_VOL_CAL: result = store_audvol_cal(TX_CAL, (struct cal_block *)data); goto done; case AUDIO_SET_AUDPROC_RX_VOL_CAL: result = store_audvol_cal(RX_CAL, (struct cal_block *)data); goto done; case AUDIO_SET_AFE_TX_CAL: result = store_afe_cal(TX_CAL, (struct cal_block *)data); goto done; case AUDIO_SET_AFE_RX_CAL: result = store_afe_cal(RX_CAL, (struct cal_block *)data); goto done; case AUDIO_SET_VOCPROC_CAL: result = store_vocproc_cal((struct cal_block *)data); goto done; case AUDIO_SET_VOCPROC_STREAM_CAL: result = store_vocstrm_cal((struct cal_block *)data); goto done; case AUDIO_SET_VOCPROC_VOL_CAL: result = store_vocvol_cal((struct cal_block *)data); goto done; case AUDIO_SET_VOCPROC_DEV_CFG_CAL: result = store_vocproc_dev_cfg_cal((struct cal_block *)data); goto done; case AUDIO_SET_SIDETONE_CAL: store_sidetone_cal((struct sidetone_cal *)data); goto done; case AUDIO_SET_ANC_CAL: result = store_anc_cal((struct cal_block *)data); goto done; case AUDIO_SET_LSM_CAL: result = store_lsm_cal((struct cal_block *)data); goto done; case AUDIO_SET_ADM_CUSTOM_TOPOLOGY: result = store_adm_custom_topology((struct cal_block *)data); goto done; case AUDIO_SET_ASM_CUSTOM_TOPOLOGY: result = store_asm_custom_topology((struct cal_block *)data); goto done; case AUDIO_SET_AANC_CAL: result = store_aanc_cal((struct cal_block *)data); goto done; case AUDIO_LISTEN_SET_ULP_LSM_CAL: result = store_ulp_lsm_cal((struct cal_block *) data); goto done; case AUDIO_LISTEN_SET_ULP_AFE_CAL: result = store_ulp_afe_cal((struct cal_block *) data); goto done; default: pr_err("ACDB=> ACDB ioctl not found!\n"); result = -EFAULT; goto done; } done: mutex_unlock(&acdb_data.acdb_mutex); return result; }
int MV_Setup_Rndv_QPs() { int i; mvdev.rndv_pool_qps = (mv_qp_pool_entry *) malloc(sizeof(mv_qp_pool_entry) * mvparams.rndv_qps); mvdev.grh_buf = (char *) malloc(sizeof(char) * 40); mvdev.rndv_cq = (struct ibv_cq **) malloc(sizeof(struct ibv_cq *) * mvdev.num_hcas); mvdev.rndv_qp = (mv_qp *) malloc(sizeof(mv_qp) * mvdev.num_hcas); /* setup the pool of QPs */ for(i = 0; i < mvparams.rndv_qps; i++) { int hca = i % mvdev.num_hcas; mvdev.rndv_pool_qps[i].ud_cq = ibv_create_cq(mvdev.hca[hca].context, 8192 * 2, NULL, NULL, 0); if(!mvdev.rndv_pool_qps[i].ud_cq) { error_abort_all(IBV_RETURN_ERR, "Couldn't create RNDV CQ %d", i); return 0; } mvdev.rndv_si.recv_cq = mvdev.rndv_si.send_cq = mvdev.rndv_pool_qps[i].ud_cq; mvdev.rndv_si.sq_psn = mvparams.psn; mvdev.rndv_si.pd = mvdev.hca[hca].pd; mvdev.rndv_si.cap.max_send_wr = 1; mvdev.rndv_si.cap.max_recv_wr = 4096; mvdev.rndv_si.cap.max_send_sge = 1; mvdev.rndv_si.cap.max_recv_sge = 2; mvdev.rndv_si.cap.max_inline_data = 0; mvdev.rndv_pool_qps[i].ud_qp = MV_Setup_UD_QP(&mvdev.rndv_si); if(!mvdev.rndv_pool_qps[i].ud_qp) { error_abort_all(IBV_RETURN_ERR, "Couldn't create RNDV UD QP %d", i); } mvdev.rndv_pool_qps[i].associated_qpn = -1; mvdev.rndv_pool_qps[i].associated_rank = -1; mvdev.rndv_pool_qps[i].seqnum = 0; mvdev.rndv_pool_qps[i].ptr.next = NULL; mvdev.rndv_pool_qps[i].ptr.prev = NULL; mvdev.rndv_pool_qps[i].hca = &(mvdev.hca[hca]); } for(i = 0; i < mvparams.rndv_qps - 1; i++) { mvdev.rndv_pool_qps[i].ptr.next = &(mvdev.rndv_pool_qps[i + 1]); } mvdev.rndv_pool_qps_free_head = &(mvdev.rndv_pool_qps[0]); /* setup the cqs for completions of send ops */ for(i = 0; i < mvdev.num_hcas; i++) { mvdev.rndv_cq[i] = ibv_create_cq(mvdev.hca[i].context, 16384, NULL, NULL, 0); if (!mvdev.rndv_cq[i]) { error_abort_all(IBV_RETURN_ERR, "Couldn't create RNDV CQ"); return 0; } /* register the GRH buffer for each HCA */ mvdev.grh_mr[i] = register_memory(i, mvdev.grh_buf, 40); } /* setup the qps we send zcopy messages on */ for(i = 0; i < mvdev.num_hcas; i++) { mv_qp_setup_information si; si.send_cq = si.recv_cq = mvdev.rndv_cq[i]; si.sq_psn = mvparams.psn; si.pd = mvdev.hca[i].pd; si.cap.max_send_wr = 15000; si.cap.max_recv_wr = 1; si.cap.max_send_sge = 1; si.cap.max_recv_sge = 1; si.cap.max_inline_data = 0; mvdev.rndv_qp[i].qp = MV_Setup_UD_QP(&si); mvdev.rndv_qp[i].send_wqes_avail = 15000; mvdev.rndv_qp[i].send_wqes_total = 15000; mvdev.rndv_qp[i].ext_sendq_head = mvdev.rndv_qp[i].ext_sendq_tail = NULL; mvdev.rndv_qp[i].hca = &(mvdev.hca[i]); mvdev.rndv_qp[i].type = MVDEV_CH_UD_RQ; } return 0; }
static long acdb_ioctl(struct file *f, unsigned int cmd, unsigned long arg) { int32_t result = 0; int32_t size; int32_t map_fd; uint32_t topology; struct cal_block data[MAX_NETWORKS]; pr_debug("%s\n", __func__); switch (cmd) { case AUDIO_REGISTER_PMEM: pr_debug("AUDIO_REGISTER_PMEM\n"); if (atomic_read(&acdb_data.mem_len)) { deregister_memory(); pr_debug("Remove the existing memory\n"); } if (copy_from_user(&map_fd, (void *)arg, sizeof(map_fd))) { pr_err("%s: fail to copy memory handle!\n", __func__); result = -EFAULT; } else { atomic_set(&acdb_data.map_handle, map_fd); result = register_memory(); } goto done; case AUDIO_DEREGISTER_PMEM: pr_debug("AUDIO_DEREGISTER_PMEM\n"); deregister_memory(); goto done; case AUDIO_SET_VOICE_RX_TOPOLOGY: if (copy_from_user(&topology, (void *)arg, sizeof(topology))) { pr_err("%s: fail to copy topology!\n", __func__); result = -EFAULT; } store_voice_rx_topology(topology); goto done; case AUDIO_SET_VOICE_TX_TOPOLOGY: if (copy_from_user(&topology, (void *)arg, sizeof(topology))) { pr_err("%s: fail to copy topology!\n", __func__); result = -EFAULT; } store_voice_tx_topology(topology); goto done; case AUDIO_SET_ADM_RX_TOPOLOGY: if (copy_from_user(&topology, (void *)arg, sizeof(topology))) { pr_err("%s: fail to copy topology!\n", __func__); result = -EFAULT; } store_adm_rx_topology(topology); goto done; case AUDIO_SET_ADM_TX_TOPOLOGY: if (copy_from_user(&topology, (void *)arg, sizeof(topology))) { pr_err("%s: fail to copy topology!\n", __func__); result = -EFAULT; } store_adm_tx_topology(topology); goto done; case AUDIO_SET_ASM_TOPOLOGY: if (copy_from_user(&topology, (void *)arg, sizeof(topology))) { pr_err("%s: fail to copy topology!\n", __func__); result = -EFAULT; } store_asm_topology(topology); goto done; } if (copy_from_user(&size, (void *) arg, sizeof(size))) { result = -EFAULT; goto done; } if (size <= 0) { pr_err("%s: Invalid size sent to driver: %d\n", __func__, size); result = -EFAULT; goto done; } if (copy_from_user(data, (void *)(arg + sizeof(size)), size)) { pr_err("%s: fail to copy table size %d\n", __func__, size); result = -EFAULT; goto done; } if (data == NULL) { pr_err("%s: NULL pointer sent to driver!\n", __func__); result = -EFAULT; goto done; } switch (cmd) { case AUDIO_SET_AUDPROC_TX_CAL: if (size > sizeof(struct cal_block)) pr_err("%s: More Audproc Cal then expected, " "size received: %d\n", __func__, size); store_audproc_cal(TX_CAL, data); break; case AUDIO_SET_AUDPROC_RX_CAL: if (size > sizeof(struct cal_block)) pr_err("%s: More Audproc Cal then expected, " "size received: %d\n", __func__, size); store_audproc_cal(RX_CAL, data); break; case AUDIO_SET_AUDPROC_TX_STREAM_CAL: if (size > sizeof(struct cal_block)) pr_err("%s: More Audproc Cal then expected, " "size received: %d\n", __func__, size); store_audstrm_cal(TX_CAL, data); break; case AUDIO_SET_AUDPROC_RX_STREAM_CAL: if (size > sizeof(struct cal_block)) pr_err("%s: More Audproc Cal then expected, " "size received: %d\n", __func__, size); store_audstrm_cal(RX_CAL, data); break; case AUDIO_SET_AUDPROC_TX_VOL_CAL: if (size > sizeof(struct cal_block)) pr_err("%s: More Audproc Cal then expected, " "size received: %d\n", __func__, size); store_audvol_cal(TX_CAL, data); break; case AUDIO_SET_AUDPROC_RX_VOL_CAL: if (size > sizeof(struct cal_block)) pr_err("%s: More Audproc Cal then expected, " "size received: %d\n", __func__, size); store_audvol_cal(RX_CAL, data); break; case AUDIO_SET_AFE_TX_CAL: if (size > sizeof(struct cal_block)) pr_err("%s: More AFE Cal then expected, " "size received: %d\n", __func__, size); store_afe_cal(TX_CAL, data); break; case AUDIO_SET_AFE_RX_CAL: if (size > sizeof(struct cal_block)) pr_err("%s: More AFE Cal then expected, " "size received: %d\n", __func__, size); store_afe_cal(RX_CAL, data); break; case AUDIO_SET_VOCPROC_CAL: store_vocproc_cal(size / sizeof(struct cal_block), data); break; case AUDIO_SET_VOCPROC_STREAM_CAL: store_vocstrm_cal(size / sizeof(struct cal_block), data); break; case AUDIO_SET_VOCPROC_VOL_CAL: store_vocvol_cal(size / sizeof(struct cal_block), data); break; case AUDIO_SET_SIDETONE_CAL: if (size > sizeof(struct sidetone_cal)) pr_err("%s: More sidetone cal then expected, " "size received: %d\n", __func__, size); store_sidetone_cal((struct sidetone_cal *)data); break; case AUDIO_SET_ANC_CAL: store_anc_cal(data); break; default: pr_err("ACDB=> ACDB ioctl not found!\n"); } done: return result; }
int CMM_register_special_memory(CMM_handle *cmm_handle, void *data, CMM_free_func free_func) { return register_memory(cmm_handle, data, free_func); }
int CMM_register_memory(CMM_handle *cmm_handle, void *data) { return register_memory(cmm_handle, data, NULL); }
void vbuf_fast_rdma_alloc (viadev_connection_t * c, int dir) { vbuf * v; int vbuf_alignment = 64; int pagesize = getpagesize(); int i; struct ibv_mr *mem_handle; void *vbuf_ctrl_buf = NULL; void *vbuf_rdma_buf = NULL; lock_vbuf(); /* initialize revelant fields */ c->rdma_credit = 0; c->num_no_completion = 0; if (viadev_num_rdma_buffer) { /* allocate vbuf struct buffers */ if(posix_memalign((void **) &vbuf_ctrl_buf, vbuf_alignment, sizeof(struct vbuf) * viadev_num_rdma_buffer)) { error_abort_all(GEN_EXIT_ERR, "malloc: vbuf in vbuf_fast_rdma_alloc"); } memset(vbuf_ctrl_buf, 0, sizeof(struct vbuf) * viadev_num_rdma_buffer); /* allocate vbuf RDMA buffers */ if(posix_memalign((void **)&vbuf_rdma_buf, pagesize, viadev_vbuf_total_size * viadev_num_rdma_buffer)) { error_abort_all(GEN_EXIT_ERR, "malloc: vbuf DMA in vbuf_fast_rdma_alloc"); } memset(vbuf_rdma_buf, 0, viadev_vbuf_total_size * viadev_num_rdma_buffer); /* REGISTER RDMA SEND BUFFERS */ mem_handle = register_memory(vbuf_rdma_buf, viadev_vbuf_total_size * viadev_num_rdma_buffer); /* Connect the DMA buffer to the vbufs */ for (i = 0; i < viadev_num_rdma_buffer; i++) { v = ((vbuf *)vbuf_ctrl_buf) + i; v->head_flag = (VBUF_FLAG_TYPE *) ( (char *)(vbuf_rdma_buf) + (i * viadev_vbuf_total_size)); v->buffer = (unsigned char *) ((char *)(v->head_flag) + sizeof(VBUF_FLAG_TYPE)); } /* Some vbuf initialization */ for (i = 0; i < viadev_num_rdma_buffer; i++) { /* set global_rank */ ((vbuf *)vbuf_ctrl_buf + i)->grank = c->global_rank; if (dir==0) { ((vbuf *)vbuf_ctrl_buf + i)->desc.next = NULL; ((vbuf *)vbuf_ctrl_buf + i)->padding = FREE_FLAG; } else { ((vbuf *)vbuf_ctrl_buf + i)->padding = BUSY_FLAG; } ((vbuf *)vbuf_ctrl_buf + i)->ref_count = 0; ((vbuf *)vbuf_ctrl_buf + i)->len = 0; } if (dir==0) { c->RDMA_send_buf = vbuf_ctrl_buf; c->RDMA_send_buf_DMA = vbuf_rdma_buf; c->RDMA_send_buf_hndl = mem_handle; /* set pointers */ c->phead_RDMA_send = 0; c->ptail_RDMA_send = viadev_num_rdma_buffer - 1; } else { c->RDMA_recv_buf = vbuf_ctrl_buf; c->RDMA_recv_buf_DMA = vbuf_rdma_buf; c->RDMA_recv_buf_hndl = mem_handle; /* set pointers */ c->p_RDMA_recv = 0; c->p_RDMA_recv_tail = viadev_num_rdma_buffer - 1; /* Add the connection to the RDMA polling list */ viadev.RDMA_polling_group[viadev.RDMA_polling_group_size] = c; viadev.RDMA_polling_group_size++; } } unlock_vbuf(); }
static void allocate_vbuf_region(int nvbufs) { struct vbuf_region *reg; void *mem; void *vbuf_dma_buffer; int i; vbuf *cur; int alignment_vbuf = 64; int alignment_dma; alignment_dma = getpagesize(); if (free_vbuf_head != NULL) { error_abort_all(GEN_ASSERT_ERR, "free_vbuf_head = NULL"); } if (nvbufs <= 0) { error_abort_all(GEN_ASSERT_ERR, "Internal Error requested " "region size = %d", nvbufs); } /* are we limiting vbuf allocation? If so, make sure * we dont alloc more than allowed */ if (viadev_vbuf_max > 0) { nvbufs = MIN(nvbufs, viadev_vbuf_max - vbuf_n_allocated); if (nvbufs <= 0) { error_abort_all(GEN_EXIT_ERR, "VBUF alloc failure, limit exceeded"); } } reg = (struct vbuf_region *) malloc(sizeof(struct vbuf_region)); if (NULL == reg) { error_abort_all(GEN_EXIT_ERR, "Unable to malloc a new struct vbuf_region"); } if(posix_memalign((void **) &mem, alignment_vbuf, nvbufs * sizeof(vbuf))) { error_abort_all(GEN_EXIT_ERR, "unable to malloc vbuf struct"); } /* ALLOCATE THE DMA BUFFER */ if(posix_memalign((void **) &vbuf_dma_buffer, alignment_dma, nvbufs * viadev_vbuf_total_size)) { error_abort_all(GEN_EXIT_ERR, "unable to malloc vbufs DMA buffer"); } memset(mem, 0, nvbufs * sizeof(vbuf)); memset(vbuf_dma_buffer, 0, nvbufs * viadev_vbuf_total_size); vbuf_n_allocated += nvbufs; num_free_vbuf += nvbufs; reg->malloc_start = mem; reg->malloc_buf_start = vbuf_dma_buffer; reg->malloc_end = (void *) ((char *) mem + nvbufs * sizeof(vbuf)); reg->malloc_buf_end = (void *) ((char *) vbuf_dma_buffer + nvbufs * viadev_vbuf_total_size); reg->count = nvbufs; free_vbuf_head = (vbuf *) ((aint_t) mem); reg->vbuf_head = free_vbuf_head; D_PRINT("VBUF REGION ALLOCATION SZ %d TOT %d FREE %ld NF %ld NG %ld", nvbufs, vbuf_n_allocated, num_free_vbuf, num_vbuf_free, num_vbuf_get); reg->mem_handle = register_memory(vbuf_dma_buffer, nvbufs * viadev_vbuf_total_size); if (reg->mem_handle == NULL) { error_abort_all(GEN_EXIT_ERR, "unable to register vbuf DMA buffer"); } /* init the free list */ for (i = 0; i < nvbufs - 1; i++) { cur = free_vbuf_head + i; cur->desc.next = free_vbuf_head + i + 1; cur->region = reg; #ifdef ADAPTIVE_RDMA_FAST_PATH cur->head_flag = (VBUF_FLAG_TYPE *) ((char *)(vbuf_dma_buffer) + (i * viadev_vbuf_total_size)); cur->buffer = (unsigned char *) ((char *)(cur->head_flag) + sizeof(VBUF_FLAG_TYPE)); #else cur->buffer = (unsigned char *) ((char *)(vbuf_dma_buffer) + (i * viadev_vbuf_total_size)); #endif } /* last one needs to be set to NULL */ cur = free_vbuf_head + nvbufs - 1; cur->desc.next = NULL; cur->region = reg; #ifdef ADAPTIVE_RDMA_FAST_PATH cur->head_flag = (VBUF_FLAG_TYPE *) ((char *)vbuf_dma_buffer + ((nvbufs - 1) * viadev_vbuf_total_size)); cur->buffer = (unsigned char *) ((unsigned char *)(cur->head_flag) + sizeof(VBUF_FLAG_TYPE)); #else cur->buffer = (unsigned char *) ((char *)vbuf_dma_buffer + ((nvbufs - 1) * viadev_vbuf_total_size)); #endif /* thread region list */ reg->next = vbuf_region_head; vbuf_region_head = reg; }