static void ldes(lock_item_t *p) { uint32_t i; if(PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { segment_hdr_t *seg_hdr = (segment_hdr_t *)p->seg_desc->seg_info.seg_base_addr; if (p->lockfile) { unlink(p->lockfile); } for(i = 0; i < p->num_locks * 2; i++) { pthread_mutex_t *mutex = _GET_MUTEX_PTR(seg_hdr, i); if (0 != pthread_mutex_destroy(mutex)) { PMIX_ERROR_LOG(PMIX_ERROR); } } } if (p->lockfile) { free(p->lockfile); } if (p->seg_desc) { pmix_common_dstor_delete_sm_desc(p->seg_desc); } }
PMIX_EXPORT pmix_status_t PMIx_Allocation_request_nb(pmix_alloc_directive_t directive, pmix_info_t *info, size_t ninfo, pmix_info_cbfunc_t cbfunc, void *cbdata) { pmix_buffer_t *msg; pmix_cmd_t cmd = PMIX_ALLOC_CMD; pmix_status_t rc; pmix_query_caddy_t *cb; pmix_output_verbose(2, pmix_globals.debug_output, "pmix: allocate called"); PMIX_ACQUIRE_THREAD(&pmix_global_lock); if (pmix_globals.init_cntr <= 0) { PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_INIT; } /* if we are the server, then we just issue the request and * return the response */ if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { PMIX_RELEASE_THREAD(&pmix_global_lock); if (NULL == pmix_host_server.allocate) { /* nothing we can do */ return PMIX_ERR_NOT_SUPPORTED; } pmix_output_verbose(2, pmix_globals.debug_output, "pmix:allocate handed to RM"); rc = pmix_host_server.allocate(&pmix_globals.myid, directive, info, ninfo, cbfunc, cbdata); return rc; } /* if we are a client, then relay this request to the server */ /* if we aren't connected, don't attempt to send */ if (!pmix_globals.connected) { PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_UNREACH; } PMIX_RELEASE_THREAD(&pmix_global_lock); msg = PMIX_NEW(pmix_buffer_t); /* pack the cmd */ PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, msg, &cmd, 1, PMIX_COMMAND); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(msg); return rc; } /* pack the directive */ PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, msg, &directive, 1, PMIX_ALLOC_DIRECTIVE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(msg); return rc; } /* pack the info */ PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, msg, &ninfo, 1, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(msg); return rc; } if (0 < ninfo) { PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, msg, info, ninfo, PMIX_INFO); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(msg); return rc; } } /* create a callback object as we need to pass it to the * recv routine so we know which callback to use when * the return message is recvd */ cb = PMIX_NEW(pmix_query_caddy_t); cb->cbfunc = cbfunc; cb->cbdata = cbdata; /* push the message into our event base to send to the server */ PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver, msg, query_cbfunc, (void*)cb); if (PMIX_SUCCESS != rc) { PMIX_RELEASE(msg); PMIX_RELEASE(cb); } return rc; }
PMIX_EXPORT pmix_status_t PMIx_Query_info_nb(pmix_query_t queries[], size_t nqueries, pmix_info_cbfunc_t cbfunc, void *cbdata) { pmix_query_caddy_t *cd; pmix_cmd_t cmd = PMIX_QUERY_CMD; pmix_buffer_t *msg; pmix_status_t rc; PMIX_ACQUIRE_THREAD(&pmix_global_lock); pmix_output_verbose(2, pmix_globals.debug_output, "pmix:query non-blocking"); if (pmix_globals.init_cntr <= 0) { PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_INIT; } if (0 == nqueries || NULL == queries) { PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_BAD_PARAM; } /* if we are the server, then we just issue the query and * return the response */ if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { PMIX_RELEASE_THREAD(&pmix_global_lock); if (NULL == pmix_host_server.query) { /* nothing we can do */ return PMIX_ERR_NOT_SUPPORTED; } pmix_output_verbose(2, pmix_globals.debug_output, "pmix:query handed to RM"); pmix_host_server.query(&pmix_globals.myid, queries, nqueries, cbfunc, cbdata); return PMIX_SUCCESS; } /* if we aren't connected, don't attempt to send */ if (!pmix_globals.connected) { PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_UNREACH; } PMIX_RELEASE_THREAD(&pmix_global_lock); /* if we are a client, then relay this request to the server */ cd = PMIX_NEW(pmix_query_caddy_t); cd->cbfunc = cbfunc; cd->cbdata = cbdata; msg = PMIX_NEW(pmix_buffer_t); PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, msg, &cmd, 1, PMIX_COMMAND); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(msg); PMIX_RELEASE(cd); return rc; } PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, msg, &nqueries, 1, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(msg); PMIX_RELEASE(cd); return rc; } PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, msg, queries, nqueries, PMIX_QUERY); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(msg); PMIX_RELEASE(cd); return rc; } pmix_output_verbose(2, pmix_globals.debug_output, "pmix:query sending to server"); PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver, msg, query_cbfunc, (void*)cd); if (PMIX_SUCCESS != rc) { PMIX_RELEASE(cd); } return rc; }
/* * A file descriptor is available/ready for send. Check the state * of the socket and take the appropriate action. */ void pmix_usock_send_handler(int sd, short flags, void *cbdata) { pmix_peer_t *peer = (pmix_peer_t*)cbdata; pmix_ptl_send_t *msg = peer->send_msg; pmix_status_t rc; uint32_t nbytes; /* acquire the object */ PMIX_ACQUIRE_OBJECT(peer); pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "%s:%d usock:send_handler SENDING TO PEER %s:%d tag %u with %s msg", pmix_globals.myid.nspace, pmix_globals.myid.rank, peer->info->pname.nspace, peer->info->pname.rank, (NULL == msg) ? UINT_MAX : msg->hdr.tag, (NULL == msg) ? "NULL" : "NON-NULL"); if (NULL != msg) { if (!msg->hdr_sent) { if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { /* we have to convert the header back to host-byte order */ msg->hdr.pindex = ntohl(msg->hdr.pindex); msg->hdr.tag = ntohl(msg->hdr.tag); nbytes = msg->hdr.nbytes; msg->hdr.nbytes = ntohl(nbytes); } pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "usock:send_handler SENDING HEADER WITH MSG IDX %d TAG %d SIZE %lu", msg->hdr.pindex, msg->hdr.tag, (unsigned long)msg->hdr.nbytes); if (PMIX_SUCCESS == (rc = send_bytes(peer->sd, &msg->sdptr, &msg->sdbytes))) { /* header is completely sent */ pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "usock:send_handler HEADER SENT"); msg->hdr_sent = true; /* setup to send the data */ if (NULL == msg->data) { /* this was a zero-byte msg - nothing more to do */ PMIX_RELEASE(msg); peer->send_msg = NULL; goto next; } else { /* send the data as a single block */ msg->sdptr = msg->data->base_ptr; msg->sdbytes = msg->hdr.nbytes; } /* fall thru and let the send progress */ } else if (PMIX_ERR_RESOURCE_BUSY == rc || PMIX_ERR_WOULD_BLOCK == rc) { /* exit this event and let the event lib progress */ pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "usock:send_handler RES BUSY OR WOULD BLOCK"); if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { /* have to convert back again so we are correct when we re-enter */ msg->hdr.pindex = htonl(msg->hdr.pindex); msg->hdr.tag = htonl(msg->hdr.tag); nbytes = msg->hdr.nbytes; msg->hdr.nbytes = htonl(nbytes); } /* ensure we post the modified peer object before another thread * picks it back up */ PMIX_POST_OBJECT(peer); return; } else { // report the error event_del(&peer->send_event); peer->send_ev_active = false; PMIX_RELEASE(msg); peer->send_msg = NULL; pmix_ptl_base_lost_connection(peer, rc); /* ensure we post the modified peer object before another thread * picks it back up */ PMIX_POST_OBJECT(peer); return; } } if (msg->hdr_sent) { pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "usock:send_handler SENDING BODY OF MSG"); if (PMIX_SUCCESS == (rc = send_bytes(peer->sd, &msg->sdptr, &msg->sdbytes))) { // message is complete pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "usock:send_handler BODY SENT"); PMIX_RELEASE(msg); peer->send_msg = NULL; } else if (PMIX_ERR_RESOURCE_BUSY == rc || PMIX_ERR_WOULD_BLOCK == rc) { /* exit this event and let the event lib progress */ pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "usock:send_handler RES BUSY OR WOULD BLOCK"); /* ensure we post the modified peer object before another thread * picks it back up */ PMIX_POST_OBJECT(peer); return; } else { // report the error pmix_output(0, "pmix_usock_peer_send_handler: unable to send message ON SOCKET %d", peer->sd); pmix_event_del(&peer->send_event); peer->send_ev_active = false; PMIX_RELEASE(msg); peer->send_msg = NULL; pmix_ptl_base_lost_connection(peer, rc); /* ensure we post the modified peer object before another thread * picks it back up */ PMIX_POST_OBJECT(peer); return; } } next: /* if current message completed - progress any pending sends by * moving the next in the queue into the "on-deck" position. Note * that this doesn't mean we send the message right now - we will * wait for another send_event to fire before doing so. This gives * us a chance to service any pending recvs. */ peer->send_msg = (pmix_ptl_send_t*) pmix_list_remove_first(&peer->send_queue); } /* if nothing else to do unregister for send event notifications */ if (NULL == peer->send_msg && peer->send_ev_active) { pmix_event_del(&peer->send_event); peer->send_ev_active = false; } /* ensure we post the modified peer object before another thread * picks it back up */ PMIX_POST_OBJECT(peer); }
pmix_status_t pmix_gds_ds12_lock_init(pmix_common_dstor_lock_ctx_t *ctx, const char *base_path, const char * name, uint32_t local_size, uid_t uid, bool setuid) { size_t size = pmix_common_dstor_getpagesize(); pmix_status_t rc = PMIX_SUCCESS; pthread_rwlockattr_t attr; ds12_lock_pthread_ctx_t *lock_ctx = (ds12_lock_pthread_ctx_t*)ctx; if (*ctx != NULL) { return PMIX_SUCCESS; } lock_ctx = (ds12_lock_pthread_ctx_t*)malloc(sizeof(ds12_lock_pthread_ctx_t)); if (NULL == lock_ctx) { rc = PMIX_ERR_INIT; PMIX_ERROR_LOG(rc); goto error; } memset(lock_ctx, 0, sizeof(ds12_lock_pthread_ctx_t)); *ctx = (pmix_common_dstor_lock_ctx_t*)lock_ctx; lock_ctx->segment = (pmix_pshmem_seg_t *)malloc(sizeof(pmix_pshmem_seg_t)); if (NULL == lock_ctx->segment) { rc = PMIX_ERR_OUT_OF_RESOURCE; PMIX_ERROR_LOG(rc); goto error; } /* create a lock file to prevent clients from reading while server is writing * to the shared memory. This situation is quite often, especially in case of * direct modex when clients might ask for data simultaneously. */ if(0 > asprintf(&lock_ctx->lockfile, "%s/dstore_sm.lock", base_path)) { rc = PMIX_ERR_OUT_OF_RESOURCE; PMIX_ERROR_LOG(rc); goto error; } PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s _lockfile_name: %s", __FILE__, __LINE__, __func__, lock_ctx->lockfile)); if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { if (PMIX_SUCCESS != (rc = pmix_pshmem.segment_create(lock_ctx->segment, lock_ctx->lockfile, size))) { PMIX_ERROR_LOG(rc); goto error; } memset(lock_ctx->segment->seg_base_addr, 0, size); if (0 != setuid) { if (0 > chown(lock_ctx->lockfile, (uid_t) uid, (gid_t) -1)){ rc = PMIX_ERROR; PMIX_ERROR_LOG(rc); goto error; } /* set the mode as required */ if (0 > chmod(lock_ctx->lockfile, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP )) { rc = PMIX_ERROR; PMIX_ERROR_LOG(rc); goto error; } } lock_ctx->rwlock = (pthread_rwlock_t *)lock_ctx->segment->seg_base_addr; if (0 != pthread_rwlockattr_init(&attr)) { rc = PMIX_ERROR; PMIX_ERROR_LOG(rc); goto error; } if (0 != pthread_rwlockattr_setpshared(&attr, PTHREAD_PROCESS_SHARED)) { pthread_rwlockattr_destroy(&attr); rc = PMIX_ERR_INIT; PMIX_ERROR_LOG(rc); goto error; } #ifdef HAVE_PTHREAD_SETKIND if (0 != pthread_rwlockattr_setkind_np(&attr, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP)) { pthread_rwlockattr_destroy(&attr); PMIX_ERROR_LOG(PMIX_ERR_INIT); goto error; } #endif if (0 != pthread_rwlock_init(lock_ctx->rwlock, &attr)) { pthread_rwlockattr_destroy(&attr); PMIX_ERROR_LOG(PMIX_ERR_INIT); goto error; } if (0 != pthread_rwlockattr_destroy(&attr)) { PMIX_ERROR_LOG(PMIX_ERR_INIT); goto error; } } else { lock_ctx->segment->seg_size = size; snprintf(lock_ctx->segment->seg_name, PMIX_PATH_MAX, "%s", lock_ctx->lockfile); if (PMIX_SUCCESS != (rc = pmix_pshmem.segment_attach(lock_ctx->segment, PMIX_PSHMEM_RW))) { PMIX_ERROR_LOG(rc); goto error; } lock_ctx->rwlock = (pthread_rwlock_t *)lock_ctx->segment->seg_base_addr; } return PMIX_SUCCESS; error: if (NULL != lock_ctx) { if (lock_ctx->segment) { /* detach & unlink from current desc */ if (lock_ctx->segment->seg_cpid == getpid()) { pmix_pshmem.segment_unlink(lock_ctx->segment); } pmix_pshmem.segment_detach(lock_ctx->segment); lock_ctx->rwlock = NULL; } if (NULL != lock_ctx->lockfile) { free(lock_ctx->lockfile); } free(lock_ctx); *ctx = (pmix_common_dstor_lock_ctx_t*)NULL; } return rc; }
static pmix_status_t send_connect_ack(int sd) { char *msg; pmix_ptl_hdr_t hdr; size_t sdsize=0, csize=0; pmix_byte_object_t cred; char *sec, *bfrops, *gds; pmix_bfrop_buffer_type_t bftype; pmix_status_t rc; uint8_t flag; uid_t euid; gid_t egid; uint32_t u32; pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "pmix:tcp SEND CONNECT ACK"); /* if we are a server, then we shouldn't be here */ if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { return PMIX_ERR_NOT_SUPPORTED; } /* setup the header */ memset(&hdr, 0, sizeof(pmix_ptl_hdr_t)); hdr.pindex = -1; hdr.tag = UINT32_MAX; /* a security module was assigned to us during rte_init based * on a list of available security modules provided by our * local PMIx server, if known. Now use that module to * get a credential, if the security system provides one. Not * every psec module will do so, thus we must first check */ PMIX_BYTE_OBJECT_CONSTRUCT(&cred); PMIX_PSEC_CREATE_CRED(rc, pmix_globals.mypeer, NULL, 0, NULL, 0, &cred); if (PMIX_SUCCESS != rc) { pmix_output(0, "OUCH: %d", __LINE__); return rc; } /* allow space for a marker indicating client vs tool */ sdsize = 1; if (PMIX_PROC_IS_CLIENT(pmix_globals.mypeer)) { flag = 0; /* reserve space for our nspace and rank info */ sdsize += strlen(pmix_globals.myid.nspace) + 1 + sizeof(int); } else { flag = 1; /* add space for our uid/gid for ACL purposes */ sdsize += 2*sizeof(uint32_t); } /* add the name of our active sec module - we selected it * in pmix_client.c prior to entering here */ sec = pmix_globals.mypeer->nptr->compat.psec->name; /* add our active bfrops module name */ bfrops = pmix_globals.mypeer->nptr->compat.bfrops->version; /* and the type of buffer we are using */ bftype = pmix_globals.mypeer->nptr->compat.type; /* add our active gds module for working with the server */ gds = (char*)pmix_client_globals.myserver->nptr->compat.gds->name; /* set the number of bytes to be read beyond the header */ hdr.nbytes = sdsize + strlen(PMIX_VERSION) + 1 + strlen(sec) + 1 \ + strlen(bfrops) + 1 + sizeof(bftype) \ + strlen(gds) + 1 + sizeof(uint32_t) + cred.size; // must NULL terminate the strings! /* create a space for our message */ sdsize = (sizeof(hdr) + hdr.nbytes); if (NULL == (msg = (char*)malloc(sdsize))) { PMIX_BYTE_OBJECT_DESTRUCT(&cred); free(sec); return PMIX_ERR_OUT_OF_RESOURCE; } memset(msg, 0, sdsize); /* load the message */ csize=0; memcpy(msg, &hdr, sizeof(pmix_ptl_hdr_t)); csize += sizeof(pmix_ptl_hdr_t); /* provide our active psec module */ memcpy(msg+csize, sec, strlen(sec)); csize += strlen(sec)+1; /* load the length of the credential - we put this in uint32_t * format as that is a fixed size, and convert to network * byte order for heterogeneity */ u32 = htonl((uint32_t)cred.size); memcpy(msg+csize, &u32, sizeof(uint32_t)); csize += sizeof(uint32_t); /* load the credential */ if (0 < u32) { memcpy(msg+csize, cred.bytes, cred.size); csize += cred.size; } PMIX_BYTE_OBJECT_DESTRUCT(&cred); /* load our process type - this is a single byte, * so no worry about heterogeneity here */ memcpy(msg+csize, &flag, 1); csize += 1; if (PMIX_PROC_IS_CLIENT(pmix_globals.mypeer)) { /* if we are a client, provide our nspace/rank */ memcpy(msg+csize, pmix_globals.myid.nspace, strlen(pmix_globals.myid.nspace)); csize += strlen(pmix_globals.myid.nspace)+1; /* again, need to convert */ u32 = htonl((uint32_t)pmix_globals.myid.rank); memcpy(msg+csize, &u32, sizeof(uint32_t)); csize += sizeof(uint32_t); } else { /* if we are a tool, provide our uid/gid for ACL support - note * that we have to convert so we can handle heterogeneity */ euid = geteuid(); u32 = htonl(euid); memcpy(msg+csize, &u32, sizeof(uint32_t)); csize += sizeof(uint32_t); egid = getegid(); u32 = htonl(egid); memcpy(msg+csize, &u32, sizeof(uint32_t)); csize += sizeof(uint32_t); } /* provide our version */ memcpy(msg+csize, PMIX_VERSION, strlen(PMIX_VERSION)); csize += strlen(PMIX_VERSION)+1; /* provide our active bfrops module */ memcpy(msg+csize, bfrops, strlen(bfrops)); csize += strlen(bfrops)+1; /* provide the bfrops type */ memcpy(msg+csize, &bftype, sizeof(bftype)); csize += sizeof(bftype); /* provide the gds module */ memcpy(msg+csize, gds, strlen(gds)); csize += strlen(gds)+1; /* send the entire message across */ if (PMIX_SUCCESS != pmix_ptl_base_send_blocking(sd, msg, sdsize)) { free(msg); return PMIX_ERR_UNREACH; } free(msg); return PMIX_SUCCESS; }
PMIX_EXPORT pmix_status_t PMIx_Log_nb(const pmix_info_t data[], size_t ndata, const pmix_info_t directives[], size_t ndirs, pmix_op_cbfunc_t cbfunc, void *cbdata) { pmix_shift_caddy_t *cd; pmix_cmd_t cmd = PMIX_LOG_CMD; pmix_buffer_t *msg; pmix_status_t rc; size_t n; time_t timestamp = 0; pmix_proc_t *source = NULL; PMIX_ACQUIRE_THREAD(&pmix_global_lock); pmix_output_verbose(2, pmix_globals.debug_output, "pmix:log non-blocking"); if (pmix_globals.init_cntr <= 0) { PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_INIT; } if (0 == ndata || NULL == data) { PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_BAD_PARAM; } /* check the directives - if they requested a timestamp, then * get the time, also look for a source */ if (NULL != directives) { for (n=0; n < ndirs; n++) { if (0 == strncmp(directives[n].key, PMIX_LOG_GENERATE_TIMESTAMP, PMIX_MAX_KEYLEN)) { if (PMIX_INFO_TRUE(&directives[n])) { /* pickup the timestamp */ timestamp = time(NULL); } } else if (0 == strncmp(directives[n].key, PMIX_LOG_SOURCE, PMIX_MAX_KEYLEN)) { source = directives[n].value.data.proc; } } } /* if we are a client or tool, we never do this ourselves - we * always pass this request to our server for execution */ if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { /* if we aren't connected, don't attempt to send */ if (!pmix_globals.connected) { PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_UNREACH; } PMIX_RELEASE_THREAD(&pmix_global_lock); /* if we are not a server, then relay this request to the server */ cd = PMIX_NEW(pmix_shift_caddy_t); cd->cbfunc.opcbfn = cbfunc; cd->cbdata = cbdata; msg = PMIX_NEW(pmix_buffer_t); PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, msg, &cmd, 1, PMIX_COMMAND); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(msg); PMIX_RELEASE(cd); return rc; } /* provide the timestamp - zero will indicate * that it wasn't taken */ PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, msg, ×tamp, 1, PMIX_TIME); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(msg); PMIX_RELEASE(cd); return rc; } /* pack the number of data entries */ PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, msg, &ndata, 1, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(msg); PMIX_RELEASE(cd); return rc; } if (0 < ndata) { PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, msg, data, ndata, PMIX_INFO); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(msg); PMIX_RELEASE(cd); return rc; } } PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, msg, &ndirs, 1, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(msg); PMIX_RELEASE(cd); return rc; } if (0 < ndirs) { PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, msg, directives, ndirs, PMIX_INFO); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(msg); PMIX_RELEASE(cd); return rc; } } pmix_output_verbose(2, pmix_plog_base_framework.framework_output, "pmix:log sending to server"); PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver, msg, log_cbfunc, (void*)cd); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(cd); } return rc; } PMIX_RELEASE_THREAD(&pmix_global_lock); /* if no recorded source was found, then we must be it */ if (NULL == source) { source = &pmix_globals.myid; cd = PMIX_NEW(pmix_shift_caddy_t); cd->cbfunc.opcbfn = cbfunc; cd->cbdata = cbdata; cd->ndirs = ndirs + 1; PMIX_INFO_CREATE(cd->directives, cd->ndirs); for (n=0; n < ndirs; n++) { PMIX_INFO_XFER(&cd->directives[n], (pmix_info_t*)&directives[n]); } PMIX_INFO_LOAD(&cd->directives[ndirs], PMIX_LOG_SOURCE, &source, PMIX_PROC); /* call down to process the request - the various components * will thread shift as required */ rc = pmix_plog.log(source, data, ndata, cd->directives, cd->ndirs, localcbfunc, cd); if (PMIX_SUCCESS != rc) { PMIX_INFO_FREE(cd->directives, cd->ndirs); PMIX_RELEASE(cd); } } else if (0 == strncmp(source->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN) && source->rank == pmix_globals.myid.rank) { /* if I am the recorded source, then this is a re-submission of * something that got "upcalled" by a prior call. In this case, * we return a "not supported" error as clearly we couldn't * handle it, and neither could our host */ rc = PMIX_ERR_NOT_SUPPORTED; } else { /* call down to process the request - the various components * will thread shift as required */ rc = pmix_plog.log(source, data, ndata, directives, ndirs, cbfunc, cbdata); } return rc; }
pmix_status_t pmix_gds_ds21_lock_init(pmix_common_dstor_lock_ctx_t *ctx, const char *base_path, const char * name, uint32_t local_size, uid_t uid, bool setuid) { pthread_mutexattr_t attr; size_t size; uint32_t i; int page_size = pmix_common_dstor_getpagesize(); segment_hdr_t *seg_hdr; lock_item_t *lock_item = NULL; lock_ctx_t *lock_ctx = (lock_ctx_t*)*ctx; pmix_list_t *lock_tracker; pmix_status_t rc = PMIX_SUCCESS; if (NULL == *ctx) { lock_ctx = (lock_ctx_t*)malloc(sizeof(lock_ctx_t)); if (NULL == lock_ctx) { rc = PMIX_ERR_INIT; PMIX_ERROR_LOG(rc); goto error; } memset(lock_ctx, 0, sizeof(lock_ctx_t)); PMIX_CONSTRUCT(&lock_ctx->lock_traker, pmix_list_t); *ctx = lock_ctx; } lock_tracker = &lock_ctx->lock_traker; lock_item = PMIX_NEW(lock_item_t); if (NULL == lock_item) { rc = PMIX_ERR_INIT; PMIX_ERROR_LOG(rc); goto error; } pmix_list_append(lock_tracker, &lock_item->super); PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s local_size %d", __FILE__, __LINE__, __func__, local_size)); if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { size_t seg_align_size; size_t seg_hdr_size; if (0 != (seg_align_size = pmix_common_dstor_getcacheblocksize())) { seg_align_size = (sizeof(pthread_mutex_t) / seg_align_size + 1) * seg_align_size; } else { seg_align_size = sizeof(pthread_mutex_t); } seg_hdr_size = ((sizeof(segment_hdr_t) + sizeof(int32_t) * local_size) / seg_align_size + 1) * seg_align_size; size = ((seg_hdr_size + 2 * local_size * seg_align_size) /* array of mutexes */ / page_size + 1) * page_size; lock_item->seg_desc = pmix_common_dstor_create_new_lock_seg(base_path, size, name, 0, uid, setuid); if (NULL == lock_item->seg_desc) { rc = PMIX_ERR_OUT_OF_RESOURCE; PMIX_ERROR_LOG(rc); goto error; } if (0 != pthread_mutexattr_init(&attr)) { rc = PMIX_ERR_INIT; PMIX_ERROR_LOG(rc); goto error; } if (0 != pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED)) { pthread_mutexattr_destroy(&attr); rc = PMIX_ERR_INIT; PMIX_ERROR_LOG(rc); goto error; } segment_hdr_t *seg_hdr = (segment_hdr_t*)lock_item->seg_desc->seg_info.seg_base_addr; seg_hdr->num_locks = local_size; seg_hdr->seg_size = size; seg_hdr->align_size = seg_align_size; seg_hdr->mutex_offs = seg_hdr_size; lock_item->lockfile = strdup(lock_item->seg_desc->seg_info.seg_name); lock_item->num_locks = local_size; lock_item->mutex = _GET_MUTEX_ARR_PTR(seg_hdr); for(i = 0; i < local_size * 2; i++) { pthread_mutex_t *mutex = _GET_MUTEX_PTR(seg_hdr, i); if (0 != pthread_mutex_init(mutex, &attr)) { pthread_mutexattr_destroy(&attr); rc = PMIX_ERR_INIT; PMIX_ERROR_LOG(rc); goto error; } } if (0 != pthread_mutexattr_destroy(&attr)) { rc = PMIX_ERR_INIT; PMIX_ERROR_LOG(PMIX_ERR_INIT); goto error; } } else { int32_t *lock_idx_ptr; bool idx_found = false; size = pmix_common_dstor_getpagesize(); lock_item->seg_desc = pmix_common_dstor_attach_new_lock_seg(base_path, size, name, 0); if (NULL == lock_item->seg_desc) { rc = PMIX_ERR_NOT_FOUND; goto error; } seg_hdr = (segment_hdr_t*)lock_item->seg_desc->seg_info.seg_base_addr; if (seg_hdr->seg_size > size) { size = seg_hdr->seg_size; pmix_common_dstor_delete_sm_desc(lock_item->seg_desc); lock_item->seg_desc = pmix_common_dstor_attach_new_lock_seg(base_path, size, name, 0); if (NULL == lock_item->seg_desc) { rc = PMIX_ERR_NOT_FOUND; goto error; } } lock_item->num_locks = seg_hdr->num_locks; lock_idx_ptr = _GET_IDX_ARR_PTR(seg_hdr); lock_item->mutex = _GET_MUTEX_ARR_PTR(seg_hdr); for (i = 0; i < lock_item->num_locks; i++) { int32_t expected = 0; if (pmix_atomic_compare_exchange_strong_32(&lock_idx_ptr[i], &expected, 1)) { lock_item->lock_idx = i; lock_item->lockfile = strdup(lock_item->seg_desc->seg_info.seg_name); idx_found = true; break; } } if (false == idx_found) { rc = PMIX_ERR_NOT_FOUND; goto error; } } return rc; error: if (NULL != lock_item) { pmix_list_remove_item(lock_tracker, &lock_item->super); PMIX_RELEASE(lock_item); lock_item = NULL; } *ctx = NULL; return rc; }
static pmix_peer_t* find_peer(const pmix_proc_t *proc) { pmix_status_t rc; pmix_peer_t *peer; pmix_proc_t wildcard; pmix_value_t *value; int i; if (NULL == proc ) { return pmix_globals.mypeer; } /* if the target is someone in my nspace, then use my own peer */ if (0 == strncmp(proc->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN)) { return pmix_globals.mypeer; } if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { /* see if we know this proc */ for (i=0; i < pmix_server_globals.clients.size; i++) { if (NULL != (peer = (pmix_peer_t*)pmix_pointer_array_get_item(&pmix_server_globals.clients, i))) { continue; } if (0 == strncmp(proc->nspace, peer->nptr->nspace, PMIX_MAX_NSLEN)) { return peer; } } /* didn't find it, so try to get the library version of the target * from the host - the result will be cached, so we will only have * to retrieve it once */ (void)strncpy(wildcard.nspace, proc->nspace, PMIX_MAX_NSLEN); wildcard.rank = PMIX_RANK_WILDCARD; if (PMIX_SUCCESS != (rc = PMIx_Get(&wildcard, PMIX_BFROPS_MODULE, NULL, 0, &value))) { /* couldn't get it - nothing we can do */ return NULL; } /* setup a peer for this nspace */ peer = PMIX_NEW(pmix_peer_t); if (NULL == peer) { PMIX_RELEASE(value); return NULL; } peer->nptr = PMIX_NEW(pmix_nspace_t); if (NULL == peer->nptr) { PMIX_RELEASE(peer); PMIX_RELEASE(value); return NULL; } peer->nptr->nspace = strdup(proc->nspace); /* assign a module to it based on the returned version */ peer->nptr->compat.bfrops = pmix_bfrops_base_assign_module(value->data.string); PMIX_RELEASE(value); if (NULL == peer->nptr->compat.bfrops) { PMIX_RELEASE(peer); return NULL; } /* cache the peer object */ pmix_pointer_array_add(&pmix_server_globals.clients, peer); return peer; } // we are a client or tool /* If the target is for the server, then * pack it using that peer. */ if (0 == strncmp(proc->nspace, pmix_client_globals.myserver->info->pname.nspace, PMIX_MAX_NSLEN)) { return pmix_client_globals.myserver; } /* if the target is another member of my nspace, then * they must be using the same version */ if (0 == strncmp(proc->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN)) { return pmix_globals.mypeer; } /* try to get the library version of this peer - the result will be * cached, so we will only have to retrieve it once */ (void)strncpy(wildcard.nspace, proc->nspace, PMIX_MAX_NSLEN); wildcard.rank = PMIX_RANK_WILDCARD; if (PMIX_SUCCESS != (rc = PMIx_Get(&wildcard, PMIX_BFROPS_MODULE, NULL, 0, &value))) { /* couldn't get it - nothing we can do */ return NULL; } /* setup a peer for this nspace */ peer = PMIX_NEW(pmix_peer_t); if (NULL == peer) { PMIX_RELEASE(value); return NULL; } peer->nptr = PMIX_NEW(pmix_nspace_t); if (NULL == peer->nptr) { PMIX_RELEASE(peer); PMIX_RELEASE(value); return NULL; } peer->nptr->nspace = strdup(proc->nspace); /* assign a module to it based on the returned version */ peer->nptr->compat.bfrops = pmix_bfrops_base_assign_module(value->data.string); PMIX_RELEASE(value); if (NULL == peer->nptr->compat.bfrops) { PMIX_RELEASE(peer); return NULL; } /* need to cache the peer someplace so we can clean it * up later */ return peer; }