/* * dapl_hca_alloc * * alloc and initialize an HCA struct * * Input: * name * port * * Output: * hca_ptr * * Returns: * none * */ DAPL_HCA *dapl_hca_alloc(char *name, char *port) { DAPL_HCA *hca_ptr; hca_ptr = dapl_os_alloc(sizeof(DAPL_HCA)); if (NULL == hca_ptr) { goto bail; } dapl_os_memzero(hca_ptr, sizeof(DAPL_HCA)); dapl_os_lock_init(&hca_ptr->lock); dapl_llist_init_head(&hca_ptr->ia_list_head); hca_ptr->name = dapl_os_strdup(name); if (NULL == hca_ptr->name) { goto bail; } hca_ptr->ib_hca_handle = IB_INVALID_HANDLE; hca_ptr->port_num = dapl_os_strtol(port, NULL, 0); return (hca_ptr); bail: if (NULL != hca_ptr) dapl_os_free(hca_ptr, sizeof(DAPL_HCA)); return NULL; }
/* * dapl_ib_setup_conn_listener * * Have the CM set up a connection listener. * * Input: * ibm_hca_handle HCA handle * qp_handle QP handle * * Output: * none * * Returns: * DAT_SUCCESS * DAT_INSUFFICIENT_RESOURCES * DAT_INVALID_PARAMETER * */ DAT_RETURN dapls_ib_setup_conn_listener(IN DAPL_IA *ia_ptr, IN DAT_UINT64 ServiceID, IN DAPL_SP *sp_ptr) { ib_hca_handle_t hca_hdl = ia_ptr->hca_ptr->ib_hca_handle; struct dapls_ib_cm_srvc_handle *srvc_hdl; dapl_service_register_t args; struct sockaddr *s; char addr_buf[64]; DAPL_EVD *evd_p = (DAPL_EVD *)sp_ptr->evd_handle; int retval; if (hca_hdl == NULL) { dapl_dbg_log(DAPL_DBG_TYPE_ERR, "setup_conn_listener: hca_handle == NULL\n"); return (DAT_INVALID_PARAMETER); } if (evd_p == NULL) { dapl_dbg_log(DAPL_DBG_TYPE_ERR, "setup_conn_listener: evd_p == NULL\n"); return (DAT_INVALID_PARAMETER); } srvc_hdl = (struct dapls_ib_cm_srvc_handle *) dapl_os_alloc(sizeof (*srvc_hdl)); if (srvc_hdl == NULL) { dapl_dbg_log(DAPL_DBG_TYPE_ERR, "setup_conn_listener: srvc_handle == NULL\n"); return (DAT_INSUFFICIENT_RESOURCES); } args.sr_sid = ServiceID; args.sr_evd_hkey = evd_p->ib_cq_handle->evd_hkey; args.sr_sp_cookie = (uintptr_t)sp_ptr; retval = ioctl(hca_hdl->ia_fd, DAPL_SERVICE_REGISTER, &args); if (retval != 0) { dapl_dbg_log(DAPL_DBG_TYPE_ERR, "setup_conn_listener: register failed %s\n", strerror(errno)); dapl_os_free(srvc_hdl, sizeof (*srvc_hdl)); return (dapls_convert_error(errno, retval)); } srvc_hdl->sv_sp_hkey = args.sr_sp_hkey; sp_ptr->cm_srvc_handle = srvc_hdl; sp_ptr->conn_qual = args.sr_retsid; s = (struct sockaddr *)&ia_ptr->hca_ptr->hca_address; dapl_dbg_log(DAPL_DBG_TYPE_CM, "setup_conn_listener: listening on ia_address %s, " "conn_qual %016llu\n\n", dapls_inet_ntop(s, addr_buf, 64), sp_ptr->conn_qual); return (DAT_SUCCESS); }
/* * dapls_os_thread_create * * Create a thread for dapl * * Input: * func function to invoke thread * f_arg argument to pass to function * * Output * thread_id handle for thread * * Returns: * DAT_SUCCESS */ DAT_RETURN dapl_os_thread_create(IN void (*func) (void *), IN void *data, OUT DAPL_OS_THREAD * thread_id) { pthread_attr_t thread_attr; struct thread_draft *thread_draft; int status; /* * Get default set of thread attributes */ status = pthread_attr_init(&thread_attr); if (status != 0) { return DAT_ERROR(DAT_INTERNAL_ERROR, 0); } /* Create dapl threads as detached from this process */ status = pthread_attr_setdetachstate(&thread_attr, PTHREAD_CREATE_DETACHED); if (status != 0) { return DAT_ERROR(DAT_INTERNAL_ERROR, 0); } thread_draft = dapl_os_alloc(sizeof(struct thread_draft)); thread_draft->func = func; thread_draft->data = data; /* Create the thread. Observe that we first invoke a local * routine to set up OS parameters, before invoking the user * specified routine. */ status = pthread_create(thread_id, &thread_attr, (void *(*)(void *))dapli_thread_init, (void *)thread_draft); /* clean up resources */ (void)pthread_attr_destroy(&thread_attr); if (status != 0) { return DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, 0); } return DAT_SUCCESS; }
DAPL_LMR *dapl_lmr_alloc(IN DAPL_IA * ia, IN DAT_MEM_TYPE mem_type, IN DAT_REGION_DESCRIPTION region_desc, IN DAT_VLEN length, IN DAT_PZ_HANDLE pz_handle, IN DAT_MEM_PRIV_FLAGS mem_priv) { DAPL_LMR *lmr; /* Allocate LMR */ lmr = (DAPL_LMR *) dapl_os_alloc(sizeof(DAPL_LMR)); if (NULL == lmr) { return (NULL); } /* zero the structure */ dapl_os_memzero(lmr, sizeof(DAPL_LMR)); /* * initialize the header */ lmr->header.provider = ia->header.provider; lmr->header.magic = DAPL_MAGIC_LMR; lmr->header.handle_type = DAT_HANDLE_TYPE_LMR; lmr->header.owner_ia = ia; lmr->header.user_context.as_64 = 0; lmr->header.user_context.as_ptr = NULL; dapl_llist_init_entry(&lmr->header.ia_list_entry); dapl_ia_link_lmr(ia, lmr); dapl_os_lock_init(&lmr->header.lock); /* * initialize the body */ lmr->param.ia_handle = (DAT_IA_HANDLE) ia; lmr->param.mem_type = mem_type; lmr->param.region_desc = region_desc; lmr->param.length = length; lmr->param.pz_handle = pz_handle; lmr->param.mem_priv = mem_priv; dapl_os_atomic_set(&lmr->lmr_ref_count, 0); return (lmr); }
/* * dapl_cno_alloc * * alloc and initialize an EVD struct * * Input: * ia * * Returns: * cno_ptr, or null on failure. */ DAPL_CNO * dapl_cno_alloc( IN DAPL_IA *ia_ptr, IN DAT_OS_WAIT_PROXY_AGENT wait_agent) { DAPL_CNO *cno_ptr; cno_ptr = (DAPL_CNO *) dapl_os_alloc(sizeof (DAPL_CNO)); if (!cno_ptr) { return (NULL); } /* zero the structure */ (void) dapl_os_memzero(cno_ptr, sizeof (DAPL_CNO)); /* * Initialize the header. */ cno_ptr->header.provider = ia_ptr->header.provider; cno_ptr->header.magic = DAPL_MAGIC_CNO; cno_ptr->header.handle_type = DAT_HANDLE_TYPE_CNO; cno_ptr->header.owner_ia = ia_ptr; cno_ptr->header.user_context.as_64 = 0; cno_ptr->header.user_context.as_ptr = NULL; dapl_llist_init_entry(&cno_ptr->header.ia_list_entry); dapl_llist_init_head(&cno_ptr->evd_list_head); dapl_os_lock_init(&cno_ptr->header.lock); /* * Initialize the body */ cno_ptr->cno_waiters = 0; cno_ptr->cno_ref_count = 0; cno_ptr->cno_state = DAPL_CNO_STATE_UNTRIGGERED; cno_ptr->cno_evd_triggered = NULL; cno_ptr->cno_wait_agent = wait_agent; (void) dapl_os_wait_object_init(&cno_ptr->cno_wait_object); return (cno_ptr); }
/* * dapl_sp_alloc * * alloc and initialize a PSP INFO struct * * Input: * IA INFO struct ptr * * Output: * sp_ptr * * Returns: * NULL * pointer to sp info struct * */ DAPL_SP * dapls_sp_alloc( IN DAPL_IA *ia_ptr, IN DAT_BOOLEAN is_psp) { DAPL_SP *sp_ptr; /* Allocate EP */ sp_ptr = (DAPL_SP *)dapl_os_alloc(sizeof (DAPL_SP)); if (sp_ptr == NULL) { return (NULL); } /* zero the structure */ (void) dapl_os_memzero(sp_ptr, sizeof (DAPL_SP)); /* * initialize the header */ sp_ptr->header.provider = ia_ptr->header.provider; if (is_psp) { sp_ptr->header.magic = DAPL_MAGIC_PSP; sp_ptr->header.handle_type = DAT_HANDLE_TYPE_PSP; } else { sp_ptr->header.magic = DAPL_MAGIC_RSP; sp_ptr->header.handle_type = DAT_HANDLE_TYPE_RSP; } sp_ptr->header.owner_ia = ia_ptr; sp_ptr->header.user_context.as_64 = 0; sp_ptr->header.user_context.as_ptr = NULL; dapl_llist_init_entry(&sp_ptr->header.ia_list_entry); dapl_os_lock_init(&sp_ptr->header.lock); /* * Initialize the Body (set to NULL above) */ dapl_llist_init_head(&sp_ptr->cr_list_head); return (sp_ptr); }
DAT_RETURN dapl_evd_resize( IN DAT_EVD_HANDLE evd_handle, IN DAT_COUNT req_evd_qlen) { int i; DAPL_EVD *evd_ptr; DAT_EVENT *event_ptr; DAT_EVENT *eventp; DAT_EVENT *event; DAT_EVENT *new_event; DAPL_RING_BUFFER free_event_queue; DAPL_RING_BUFFER pending_event_queue; DAT_RETURN dat_status; DAT_COUNT max_evd_qlen; DAT_COUNT evd_qlen; evd_ptr = (DAPL_EVD *)evd_handle; dat_status = DAT_SUCCESS; if (DAPL_BAD_HANDLE(evd_handle, DAPL_MAGIC_EVD)) { return (DAT_ERROR(DAT_INVALID_HANDLE, 0)); } if (req_evd_qlen < evd_ptr->qlen) { return (DAT_ERROR(DAT_INVALID_STATE, 0)); } if (req_evd_qlen == evd_ptr->qlen) { return (DAT_SUCCESS); } max_evd_qlen = evd_ptr->header.owner_ia->hca_ptr->ia_attr.max_evd_qlen; if (req_evd_qlen > max_evd_qlen) { return (DAT_ERROR(DAT_INVALID_STATE, 0)); } evd_qlen = DAPL_MIN_RESZ_QLEN; while (req_evd_qlen > evd_qlen) { evd_qlen <<= 1; if (evd_qlen > max_evd_qlen) evd_qlen = max_evd_qlen; } /* Allocate EVENTs */ event_ptr = (DAT_EVENT *) dapl_os_alloc(evd_qlen * sizeof (DAT_EVENT)); if (!event_ptr) { return (DAT_ERROR(DAT_INSUFFICIENT_RESOURCES, DAT_RESOURCE_MEMORY)); } /* allocate free event queue */ dat_status = dapls_rbuf_alloc(&free_event_queue, evd_qlen); if (dat_status != DAT_SUCCESS) { goto bail; } /* allocate pending event queue */ dat_status = dapls_rbuf_alloc(&pending_event_queue, evd_qlen); if (dat_status != DAT_SUCCESS) { goto bail; } /* need to resize the cq only for DTO/BIND evds */ if (0 != (evd_ptr->evd_flags & ~ (DAT_EVD_SOFTWARE_FLAG | DAT_EVD_CONNECTION_FLAG | DAT_EVD_CR_FLAG))) { dat_status = dapls_ib_cq_resize(evd_ptr, evd_qlen); if (dat_status != DAT_SUCCESS) goto bail; } /* add events to free event queue */ for (i = 0, eventp = event_ptr; i < evd_qlen; i++) { (void) dapls_rbuf_add(&free_event_queue, (void *)eventp); eventp++; } /* * copy pending events from evd to the new pending event queue */ while (event = (DAT_EVENT *) dapls_rbuf_remove(&evd_ptr->pending_event_queue)) { new_event = (DAT_EVENT *) dapls_rbuf_remove(&free_event_queue); dapl_os_assert(new_event); (void) dapl_os_memcpy(new_event, event, sizeof (DAT_EVENT)); dat_status = dapls_rbuf_add(&pending_event_queue, new_event); dapl_os_assert(dat_status == DAT_SUCCESS); dat_status = dapls_rbuf_add(&evd_ptr->free_event_queue, event); dapl_os_assert(dat_status == DAT_SUCCESS); } dapls_rbuf_destroy(&evd_ptr->free_event_queue); dapls_rbuf_destroy(&evd_ptr->pending_event_queue); if (evd_ptr->events) { dapl_os_free(evd_ptr->events, evd_ptr->qlen * sizeof (DAT_EVENT)); } evd_ptr->events = event_ptr; evd_ptr->free_event_queue = free_event_queue; evd_ptr->pending_event_queue = pending_event_queue; evd_ptr->qlen = evd_qlen; return (DAT_SUCCESS); bail: /* * If we are here means event_ptr was allocd but something else * failed */ dapl_os_free(event_ptr, evd_qlen * sizeof (DAT_EVENT)); dapls_rbuf_destroy(&free_event_queue); dapls_rbuf_destroy(&pending_event_queue); return (dat_status); }