static void ldes(lock_item_t *p) {
uint32_t i;
if(PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) {
segment_hdr_t *seg_hdr = (segment_hdr_t *)p->seg_desc->seg_info.seg_base_addr;
if (p->lockfile) {
unlink(p->lockfile);
}
for(i = 0; i < p->num_locks * 2; i++) {
pthread_mutex_t *mutex = _GET_MUTEX_PTR(seg_hdr, i);
if (0 != pthread_mutex_destroy(mutex)) {
PMIX_ERROR_LOG(PMIX_ERROR);
}
}
}
if (p->lockfile) {
free(p->lockfile);
}
if (p->seg_desc) {
pmix_common_dstor_delete_sm_desc(p->seg_desc);
}
}
PMIX_EXPORT pmix_status_t PMIx_Allocation_request_nb(pmix_alloc_directive_t directive,
pmix_info_t *info, size_t ninfo,
pmix_info_cbfunc_t cbfunc, void *cbdata)
{
pmix_buffer_t *msg;
pmix_cmd_t cmd = PMIX_ALLOC_CMD;
pmix_status_t rc;
pmix_query_caddy_t *cb;
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix: allocate called");
PMIX_ACQUIRE_THREAD(&pmix_global_lock);
if (pmix_globals.init_cntr <= 0) {
PMIX_RELEASE_THREAD(&pmix_global_lock);
return PMIX_ERR_INIT;
}
/* if we are the server, then we just issue the request and
* return the response */
if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) &&
!PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) {
PMIX_RELEASE_THREAD(&pmix_global_lock);
if (NULL == pmix_host_server.allocate) {
/* nothing we can do */
return PMIX_ERR_NOT_SUPPORTED;
}
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix:allocate handed to RM");
rc = pmix_host_server.allocate(&pmix_globals.myid,
directive,
info, ninfo,
cbfunc, cbdata);
return rc;
}
/* if we are a client, then relay this request to the server */
/* if we aren't connected, don't attempt to send */
if (!pmix_globals.connected) {
PMIX_RELEASE_THREAD(&pmix_global_lock);
return PMIX_ERR_UNREACH;
}
PMIX_RELEASE_THREAD(&pmix_global_lock);
msg = PMIX_NEW(pmix_buffer_t);
/* pack the cmd */
PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
msg, &cmd, 1, PMIX_COMMAND);
if (PMIX_SUCCESS != rc) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
/* pack the directive */
PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
msg, &directive, 1, PMIX_ALLOC_DIRECTIVE);
if (PMIX_SUCCESS != rc) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
/* pack the info */
PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
msg, &ninfo, 1, PMIX_SIZE);
if (PMIX_SUCCESS != rc) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
if (0 < ninfo) {
PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
msg, info, ninfo, PMIX_INFO);
if (PMIX_SUCCESS != rc) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
return rc;
}
}
/* create a callback object as we need to pass it to the
* recv routine so we know which callback to use when
* the return message is recvd */
cb = PMIX_NEW(pmix_query_caddy_t);
cb->cbfunc = cbfunc;
cb->cbdata = cbdata;
/* push the message into our event base to send to the server */
PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver,
msg, query_cbfunc, (void*)cb);
if (PMIX_SUCCESS != rc) {
PMIX_RELEASE(msg);
PMIX_RELEASE(cb);
}
return rc;
}
PMIX_EXPORT pmix_status_t PMIx_Query_info_nb(pmix_query_t queries[], size_t nqueries,
pmix_info_cbfunc_t cbfunc, void *cbdata)
{
pmix_query_caddy_t *cd;
pmix_cmd_t cmd = PMIX_QUERY_CMD;
pmix_buffer_t *msg;
pmix_status_t rc;
PMIX_ACQUIRE_THREAD(&pmix_global_lock);
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix:query non-blocking");
if (pmix_globals.init_cntr <= 0) {
PMIX_RELEASE_THREAD(&pmix_global_lock);
return PMIX_ERR_INIT;
}
if (0 == nqueries || NULL == queries) {
PMIX_RELEASE_THREAD(&pmix_global_lock);
return PMIX_ERR_BAD_PARAM;
}
/* if we are the server, then we just issue the query and
* return the response */
if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) &&
!PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) {
PMIX_RELEASE_THREAD(&pmix_global_lock);
if (NULL == pmix_host_server.query) {
/* nothing we can do */
return PMIX_ERR_NOT_SUPPORTED;
}
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix:query handed to RM");
pmix_host_server.query(&pmix_globals.myid,
queries, nqueries,
cbfunc, cbdata);
return PMIX_SUCCESS;
}
/* if we aren't connected, don't attempt to send */
if (!pmix_globals.connected) {
PMIX_RELEASE_THREAD(&pmix_global_lock);
return PMIX_ERR_UNREACH;
}
PMIX_RELEASE_THREAD(&pmix_global_lock);
/* if we are a client, then relay this request to the server */
cd = PMIX_NEW(pmix_query_caddy_t);
cd->cbfunc = cbfunc;
cd->cbdata = cbdata;
msg = PMIX_NEW(pmix_buffer_t);
PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
msg, &cmd, 1, PMIX_COMMAND);
if (PMIX_SUCCESS != rc) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
PMIX_RELEASE(cd);
return rc;
}
PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
msg, &nqueries, 1, PMIX_SIZE);
if (PMIX_SUCCESS != rc) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
PMIX_RELEASE(cd);
return rc;
}
PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
msg, queries, nqueries, PMIX_QUERY);
if (PMIX_SUCCESS != rc) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
PMIX_RELEASE(cd);
return rc;
}
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix:query sending to server");
PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver,
msg, query_cbfunc, (void*)cd);
if (PMIX_SUCCESS != rc) {
PMIX_RELEASE(cd);
}
return rc;
}
/*
* A file descriptor is available/ready for send. Check the state
* of the socket and take the appropriate action.
*/
void pmix_usock_send_handler(int sd, short flags, void *cbdata)
{
pmix_peer_t *peer = (pmix_peer_t*)cbdata;
pmix_ptl_send_t *msg = peer->send_msg;
pmix_status_t rc;
uint32_t nbytes;
/* acquire the object */
PMIX_ACQUIRE_OBJECT(peer);
pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
"%s:%d usock:send_handler SENDING TO PEER %s:%d tag %u with %s msg",
pmix_globals.myid.nspace, pmix_globals.myid.rank,
peer->info->pname.nspace, peer->info->pname.rank,
(NULL == msg) ? UINT_MAX : msg->hdr.tag,
(NULL == msg) ? "NULL" : "NON-NULL");
if (NULL != msg) {
if (!msg->hdr_sent) {
if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) {
/* we have to convert the header back to host-byte order */
msg->hdr.pindex = ntohl(msg->hdr.pindex);
msg->hdr.tag = ntohl(msg->hdr.tag);
nbytes = msg->hdr.nbytes;
msg->hdr.nbytes = ntohl(nbytes);
}
pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
"usock:send_handler SENDING HEADER WITH MSG IDX %d TAG %d SIZE %lu",
msg->hdr.pindex, msg->hdr.tag, (unsigned long)msg->hdr.nbytes);
if (PMIX_SUCCESS == (rc = send_bytes(peer->sd, &msg->sdptr, &msg->sdbytes))) {
/* header is completely sent */
pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
"usock:send_handler HEADER SENT");
msg->hdr_sent = true;
/* setup to send the data */
if (NULL == msg->data) {
/* this was a zero-byte msg - nothing more to do */
PMIX_RELEASE(msg);
peer->send_msg = NULL;
goto next;
} else {
/* send the data as a single block */
msg->sdptr = msg->data->base_ptr;
msg->sdbytes = msg->hdr.nbytes;
}
/* fall thru and let the send progress */
} else if (PMIX_ERR_RESOURCE_BUSY == rc ||
PMIX_ERR_WOULD_BLOCK == rc) {
/* exit this event and let the event lib progress */
pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
"usock:send_handler RES BUSY OR WOULD BLOCK");
if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) {
/* have to convert back again so we are correct when we re-enter */
msg->hdr.pindex = htonl(msg->hdr.pindex);
msg->hdr.tag = htonl(msg->hdr.tag);
nbytes = msg->hdr.nbytes;
msg->hdr.nbytes = htonl(nbytes);
}
/* ensure we post the modified peer object before another thread
* picks it back up */
PMIX_POST_OBJECT(peer);
return;
} else {
// report the error
event_del(&peer->send_event);
peer->send_ev_active = false;
PMIX_RELEASE(msg);
peer->send_msg = NULL;
pmix_ptl_base_lost_connection(peer, rc);
/* ensure we post the modified peer object before another thread
* picks it back up */
PMIX_POST_OBJECT(peer);
return;
}
}
if (msg->hdr_sent) {
pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
"usock:send_handler SENDING BODY OF MSG");
if (PMIX_SUCCESS == (rc = send_bytes(peer->sd, &msg->sdptr, &msg->sdbytes))) {
// message is complete
pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
"usock:send_handler BODY SENT");
PMIX_RELEASE(msg);
peer->send_msg = NULL;
} else if (PMIX_ERR_RESOURCE_BUSY == rc ||
PMIX_ERR_WOULD_BLOCK == rc) {
/* exit this event and let the event lib progress */
pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
"usock:send_handler RES BUSY OR WOULD BLOCK");
/* ensure we post the modified peer object before another thread
* picks it back up */
PMIX_POST_OBJECT(peer);
return;
} else {
// report the error
pmix_output(0, "pmix_usock_peer_send_handler: unable to send message ON SOCKET %d",
peer->sd);
pmix_event_del(&peer->send_event);
peer->send_ev_active = false;
PMIX_RELEASE(msg);
peer->send_msg = NULL;
pmix_ptl_base_lost_connection(peer, rc);
/* ensure we post the modified peer object before another thread
* picks it back up */
PMIX_POST_OBJECT(peer);
return;
}
}
next:
/* if current message completed - progress any pending sends by
* moving the next in the queue into the "on-deck" position. Note
* that this doesn't mean we send the message right now - we will
* wait for another send_event to fire before doing so. This gives
* us a chance to service any pending recvs.
*/
peer->send_msg = (pmix_ptl_send_t*)
pmix_list_remove_first(&peer->send_queue);
}
/* if nothing else to do unregister for send event notifications */
if (NULL == peer->send_msg && peer->send_ev_active) {
pmix_event_del(&peer->send_event);
peer->send_ev_active = false;
}
/* ensure we post the modified peer object before another thread
* picks it back up */
PMIX_POST_OBJECT(peer);
}
pmix_status_t pmix_gds_ds12_lock_init(pmix_common_dstor_lock_ctx_t *ctx, const char *base_path,
const char * name, uint32_t local_size, uid_t uid, bool setuid)
{
size_t size = pmix_common_dstor_getpagesize();
pmix_status_t rc = PMIX_SUCCESS;
pthread_rwlockattr_t attr;
ds12_lock_pthread_ctx_t *lock_ctx = (ds12_lock_pthread_ctx_t*)ctx;
if (*ctx != NULL) {
return PMIX_SUCCESS;
}
lock_ctx = (ds12_lock_pthread_ctx_t*)malloc(sizeof(ds12_lock_pthread_ctx_t));
if (NULL == lock_ctx) {
rc = PMIX_ERR_INIT;
PMIX_ERROR_LOG(rc);
goto error;
}
memset(lock_ctx, 0, sizeof(ds12_lock_pthread_ctx_t));
*ctx = (pmix_common_dstor_lock_ctx_t*)lock_ctx;
lock_ctx->segment = (pmix_pshmem_seg_t *)malloc(sizeof(pmix_pshmem_seg_t));
if (NULL == lock_ctx->segment) {
rc = PMIX_ERR_OUT_OF_RESOURCE;
PMIX_ERROR_LOG(rc);
goto error;
}
/* create a lock file to prevent clients from reading while server is writing
* to the shared memory. This situation is quite often, especially in case of
* direct modex when clients might ask for data simultaneously. */
if(0 > asprintf(&lock_ctx->lockfile, "%s/dstore_sm.lock", base_path)) {
rc = PMIX_ERR_OUT_OF_RESOURCE;
PMIX_ERROR_LOG(rc);
goto error;
}
PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output,
"%s:%d:%s _lockfile_name: %s", __FILE__, __LINE__, __func__, lock_ctx->lockfile));
if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) {
if (PMIX_SUCCESS != (rc = pmix_pshmem.segment_create(lock_ctx->segment,
lock_ctx->lockfile, size))) {
PMIX_ERROR_LOG(rc);
goto error;
}
memset(lock_ctx->segment->seg_base_addr, 0, size);
if (0 != setuid) {
if (0 > chown(lock_ctx->lockfile, (uid_t) uid, (gid_t) -1)){
rc = PMIX_ERROR;
PMIX_ERROR_LOG(rc);
goto error;
}
/* set the mode as required */
if (0 > chmod(lock_ctx->lockfile, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP )) {
rc = PMIX_ERROR;
PMIX_ERROR_LOG(rc);
goto error;
}
}
lock_ctx->rwlock = (pthread_rwlock_t *)lock_ctx->segment->seg_base_addr;
if (0 != pthread_rwlockattr_init(&attr)) {
rc = PMIX_ERROR;
PMIX_ERROR_LOG(rc);
goto error;
}
if (0 != pthread_rwlockattr_setpshared(&attr, PTHREAD_PROCESS_SHARED)) {
pthread_rwlockattr_destroy(&attr);
rc = PMIX_ERR_INIT;
PMIX_ERROR_LOG(rc);
goto error;
}
#ifdef HAVE_PTHREAD_SETKIND
if (0 != pthread_rwlockattr_setkind_np(&attr,
PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP)) {
pthread_rwlockattr_destroy(&attr);
PMIX_ERROR_LOG(PMIX_ERR_INIT);
goto error;
}
#endif
if (0 != pthread_rwlock_init(lock_ctx->rwlock, &attr)) {
pthread_rwlockattr_destroy(&attr);
PMIX_ERROR_LOG(PMIX_ERR_INIT);
goto error;
}
if (0 != pthread_rwlockattr_destroy(&attr)) {
PMIX_ERROR_LOG(PMIX_ERR_INIT);
goto error;
}
}
else {
lock_ctx->segment->seg_size = size;
snprintf(lock_ctx->segment->seg_name, PMIX_PATH_MAX, "%s", lock_ctx->lockfile);
if (PMIX_SUCCESS != (rc = pmix_pshmem.segment_attach(lock_ctx->segment,
PMIX_PSHMEM_RW))) {
PMIX_ERROR_LOG(rc);
goto error;
}
lock_ctx->rwlock = (pthread_rwlock_t *)lock_ctx->segment->seg_base_addr;
}
return PMIX_SUCCESS;
error:
if (NULL != lock_ctx) {
if (lock_ctx->segment) {
/* detach & unlink from current desc */
if (lock_ctx->segment->seg_cpid == getpid()) {
pmix_pshmem.segment_unlink(lock_ctx->segment);
}
pmix_pshmem.segment_detach(lock_ctx->segment);
lock_ctx->rwlock = NULL;
}
if (NULL != lock_ctx->lockfile) {
free(lock_ctx->lockfile);
}
free(lock_ctx);
*ctx = (pmix_common_dstor_lock_ctx_t*)NULL;
}
return rc;
}
static pmix_status_t send_connect_ack(int sd)
{
char *msg;
pmix_ptl_hdr_t hdr;
size_t sdsize=0, csize=0;
pmix_byte_object_t cred;
char *sec, *bfrops, *gds;
pmix_bfrop_buffer_type_t bftype;
pmix_status_t rc;
uint8_t flag;
uid_t euid;
gid_t egid;
uint32_t u32;
pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
"pmix:tcp SEND CONNECT ACK");
/* if we are a server, then we shouldn't be here */
if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) {
return PMIX_ERR_NOT_SUPPORTED;
}
/* setup the header */
memset(&hdr, 0, sizeof(pmix_ptl_hdr_t));
hdr.pindex = -1;
hdr.tag = UINT32_MAX;
/* a security module was assigned to us during rte_init based
* on a list of available security modules provided by our
* local PMIx server, if known. Now use that module to
* get a credential, if the security system provides one. Not
* every psec module will do so, thus we must first check */
PMIX_BYTE_OBJECT_CONSTRUCT(&cred);
PMIX_PSEC_CREATE_CRED(rc, pmix_globals.mypeer,
NULL, 0, NULL, 0, &cred);
if (PMIX_SUCCESS != rc) {
pmix_output(0, "OUCH: %d", __LINE__);
return rc;
}
/* allow space for a marker indicating client vs tool */
sdsize = 1;
if (PMIX_PROC_IS_CLIENT(pmix_globals.mypeer)) {
flag = 0;
/* reserve space for our nspace and rank info */
sdsize += strlen(pmix_globals.myid.nspace) + 1 + sizeof(int);
} else {
flag = 1;
/* add space for our uid/gid for ACL purposes */
sdsize += 2*sizeof(uint32_t);
}
/* add the name of our active sec module - we selected it
* in pmix_client.c prior to entering here */
sec = pmix_globals.mypeer->nptr->compat.psec->name;
/* add our active bfrops module name */
bfrops = pmix_globals.mypeer->nptr->compat.bfrops->version;
/* and the type of buffer we are using */
bftype = pmix_globals.mypeer->nptr->compat.type;
/* add our active gds module for working with the server */
gds = (char*)pmix_client_globals.myserver->nptr->compat.gds->name;
/* set the number of bytes to be read beyond the header */
hdr.nbytes = sdsize + strlen(PMIX_VERSION) + 1 + strlen(sec) + 1 \
+ strlen(bfrops) + 1 + sizeof(bftype) \
+ strlen(gds) + 1 + sizeof(uint32_t) + cred.size; // must NULL terminate the strings!
/* create a space for our message */
sdsize = (sizeof(hdr) + hdr.nbytes);
if (NULL == (msg = (char*)malloc(sdsize))) {
PMIX_BYTE_OBJECT_DESTRUCT(&cred);
free(sec);
return PMIX_ERR_OUT_OF_RESOURCE;
}
memset(msg, 0, sdsize);
/* load the message */
csize=0;
memcpy(msg, &hdr, sizeof(pmix_ptl_hdr_t));
csize += sizeof(pmix_ptl_hdr_t);
/* provide our active psec module */
memcpy(msg+csize, sec, strlen(sec));
csize += strlen(sec)+1;
/* load the length of the credential - we put this in uint32_t
* format as that is a fixed size, and convert to network
* byte order for heterogeneity */
u32 = htonl((uint32_t)cred.size);
memcpy(msg+csize, &u32, sizeof(uint32_t));
csize += sizeof(uint32_t);
/* load the credential */
if (0 < u32) {
memcpy(msg+csize, cred.bytes, cred.size);
csize += cred.size;
}
PMIX_BYTE_OBJECT_DESTRUCT(&cred);
/* load our process type - this is a single byte,
* so no worry about heterogeneity here */
memcpy(msg+csize, &flag, 1);
csize += 1;
if (PMIX_PROC_IS_CLIENT(pmix_globals.mypeer)) {
/* if we are a client, provide our nspace/rank */
memcpy(msg+csize, pmix_globals.myid.nspace, strlen(pmix_globals.myid.nspace));
csize += strlen(pmix_globals.myid.nspace)+1;
/* again, need to convert */
u32 = htonl((uint32_t)pmix_globals.myid.rank);
memcpy(msg+csize, &u32, sizeof(uint32_t));
csize += sizeof(uint32_t);
} else {
/* if we are a tool, provide our uid/gid for ACL support - note
* that we have to convert so we can handle heterogeneity */
euid = geteuid();
u32 = htonl(euid);
memcpy(msg+csize, &u32, sizeof(uint32_t));
csize += sizeof(uint32_t);
egid = getegid();
u32 = htonl(egid);
memcpy(msg+csize, &u32, sizeof(uint32_t));
csize += sizeof(uint32_t);
}
/* provide our version */
memcpy(msg+csize, PMIX_VERSION, strlen(PMIX_VERSION));
csize += strlen(PMIX_VERSION)+1;
/* provide our active bfrops module */
memcpy(msg+csize, bfrops, strlen(bfrops));
csize += strlen(bfrops)+1;
/* provide the bfrops type */
memcpy(msg+csize, &bftype, sizeof(bftype));
csize += sizeof(bftype);
/* provide the gds module */
memcpy(msg+csize, gds, strlen(gds));
csize += strlen(gds)+1;
/* send the entire message across */
if (PMIX_SUCCESS != pmix_ptl_base_send_blocking(sd, msg, sdsize)) {
free(msg);
return PMIX_ERR_UNREACH;
}
free(msg);
return PMIX_SUCCESS;
}
PMIX_EXPORT pmix_status_t PMIx_Log_nb(const pmix_info_t data[], size_t ndata,
const pmix_info_t directives[], size_t ndirs,
pmix_op_cbfunc_t cbfunc, void *cbdata)
{
pmix_shift_caddy_t *cd;
pmix_cmd_t cmd = PMIX_LOG_CMD;
pmix_buffer_t *msg;
pmix_status_t rc;
size_t n;
time_t timestamp = 0;
pmix_proc_t *source = NULL;
PMIX_ACQUIRE_THREAD(&pmix_global_lock);
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix:log non-blocking");
if (pmix_globals.init_cntr <= 0) {
PMIX_RELEASE_THREAD(&pmix_global_lock);
return PMIX_ERR_INIT;
}
if (0 == ndata || NULL == data) {
PMIX_RELEASE_THREAD(&pmix_global_lock);
return PMIX_ERR_BAD_PARAM;
}
/* check the directives - if they requested a timestamp, then
* get the time, also look for a source */
if (NULL != directives) {
for (n=0; n < ndirs; n++) {
if (0 == strncmp(directives[n].key, PMIX_LOG_GENERATE_TIMESTAMP, PMIX_MAX_KEYLEN)) {
if (PMIX_INFO_TRUE(&directives[n])) {
/* pickup the timestamp */
timestamp = time(NULL);
}
} else if (0 == strncmp(directives[n].key, PMIX_LOG_SOURCE, PMIX_MAX_KEYLEN)) {
source = directives[n].value.data.proc;
}
}
}
/* if we are a client or tool, we never do this ourselves - we
* always pass this request to our server for execution */
if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) &&
!PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) {
/* if we aren't connected, don't attempt to send */
if (!pmix_globals.connected) {
PMIX_RELEASE_THREAD(&pmix_global_lock);
return PMIX_ERR_UNREACH;
}
PMIX_RELEASE_THREAD(&pmix_global_lock);
/* if we are not a server, then relay this request to the server */
cd = PMIX_NEW(pmix_shift_caddy_t);
cd->cbfunc.opcbfn = cbfunc;
cd->cbdata = cbdata;
msg = PMIX_NEW(pmix_buffer_t);
PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
msg, &cmd, 1, PMIX_COMMAND);
if (PMIX_SUCCESS != rc) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
PMIX_RELEASE(cd);
return rc;
}
/* provide the timestamp - zero will indicate
* that it wasn't taken */
PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
msg, ×tamp, 1, PMIX_TIME);
if (PMIX_SUCCESS != rc) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
PMIX_RELEASE(cd);
return rc;
}
/* pack the number of data entries */
PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
msg, &ndata, 1, PMIX_SIZE);
if (PMIX_SUCCESS != rc) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
PMIX_RELEASE(cd);
return rc;
}
if (0 < ndata) {
PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
msg, data, ndata, PMIX_INFO);
if (PMIX_SUCCESS != rc) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
PMIX_RELEASE(cd);
return rc;
}
}
PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
msg, &ndirs, 1, PMIX_SIZE);
if (PMIX_SUCCESS != rc) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
PMIX_RELEASE(cd);
return rc;
}
if (0 < ndirs) {
PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
msg, directives, ndirs, PMIX_INFO);
if (PMIX_SUCCESS != rc) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
PMIX_RELEASE(cd);
return rc;
}
}
pmix_output_verbose(2, pmix_plog_base_framework.framework_output,
"pmix:log sending to server");
PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver,
msg, log_cbfunc, (void*)cd);
if (PMIX_SUCCESS != rc) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(cd);
}
return rc;
}
PMIX_RELEASE_THREAD(&pmix_global_lock);
/* if no recorded source was found, then we must be it */
if (NULL == source) {
source = &pmix_globals.myid;
cd = PMIX_NEW(pmix_shift_caddy_t);
cd->cbfunc.opcbfn = cbfunc;
cd->cbdata = cbdata;
cd->ndirs = ndirs + 1;
PMIX_INFO_CREATE(cd->directives, cd->ndirs);
for (n=0; n < ndirs; n++) {
PMIX_INFO_XFER(&cd->directives[n], (pmix_info_t*)&directives[n]);
}
PMIX_INFO_LOAD(&cd->directives[ndirs], PMIX_LOG_SOURCE, &source, PMIX_PROC);
/* call down to process the request - the various components
* will thread shift as required */
rc = pmix_plog.log(source, data, ndata, cd->directives, cd->ndirs, localcbfunc, cd);
if (PMIX_SUCCESS != rc) {
PMIX_INFO_FREE(cd->directives, cd->ndirs);
PMIX_RELEASE(cd);
}
} else if (0 == strncmp(source->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN) &&
source->rank == pmix_globals.myid.rank) {
/* if I am the recorded source, then this is a re-submission of
* something that got "upcalled" by a prior call. In this case,
* we return a "not supported" error as clearly we couldn't
* handle it, and neither could our host */
rc = PMIX_ERR_NOT_SUPPORTED;
} else {
/* call down to process the request - the various components
* will thread shift as required */
rc = pmix_plog.log(source, data, ndata, directives, ndirs, cbfunc, cbdata);
}
return rc;
}
pmix_status_t pmix_gds_ds21_lock_init(pmix_common_dstor_lock_ctx_t *ctx, const char *base_path, const char * name,
uint32_t local_size, uid_t uid, bool setuid)
{
pthread_mutexattr_t attr;
size_t size;
uint32_t i;
int page_size = pmix_common_dstor_getpagesize();
segment_hdr_t *seg_hdr;
lock_item_t *lock_item = NULL;
lock_ctx_t *lock_ctx = (lock_ctx_t*)*ctx;
pmix_list_t *lock_tracker;
pmix_status_t rc = PMIX_SUCCESS;
if (NULL == *ctx) {
lock_ctx = (lock_ctx_t*)malloc(sizeof(lock_ctx_t));
if (NULL == lock_ctx) {
rc = PMIX_ERR_INIT;
PMIX_ERROR_LOG(rc);
goto error;
}
memset(lock_ctx, 0, sizeof(lock_ctx_t));
PMIX_CONSTRUCT(&lock_ctx->lock_traker, pmix_list_t);
*ctx = lock_ctx;
}
lock_tracker = &lock_ctx->lock_traker;
lock_item = PMIX_NEW(lock_item_t);
if (NULL == lock_item) {
rc = PMIX_ERR_INIT;
PMIX_ERROR_LOG(rc);
goto error;
}
pmix_list_append(lock_tracker, &lock_item->super);
PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output,
"%s:%d:%s local_size %d", __FILE__, __LINE__, __func__, local_size));
if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) {
size_t seg_align_size;
size_t seg_hdr_size;
if (0 != (seg_align_size = pmix_common_dstor_getcacheblocksize())) {
seg_align_size = (sizeof(pthread_mutex_t) / seg_align_size + 1)
* seg_align_size;
} else {
seg_align_size = sizeof(pthread_mutex_t);
}
seg_hdr_size = ((sizeof(segment_hdr_t)
+ sizeof(int32_t) * local_size)
/ seg_align_size + 1) * seg_align_size;
size = ((seg_hdr_size
+ 2 * local_size * seg_align_size) /* array of mutexes */
/ page_size + 1) * page_size;
lock_item->seg_desc = pmix_common_dstor_create_new_lock_seg(base_path,
size, name, 0, uid, setuid);
if (NULL == lock_item->seg_desc) {
rc = PMIX_ERR_OUT_OF_RESOURCE;
PMIX_ERROR_LOG(rc);
goto error;
}
if (0 != pthread_mutexattr_init(&attr)) {
rc = PMIX_ERR_INIT;
PMIX_ERROR_LOG(rc);
goto error;
}
if (0 != pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED)) {
pthread_mutexattr_destroy(&attr);
rc = PMIX_ERR_INIT;
PMIX_ERROR_LOG(rc);
goto error;
}
segment_hdr_t *seg_hdr = (segment_hdr_t*)lock_item->seg_desc->seg_info.seg_base_addr;
seg_hdr->num_locks = local_size;
seg_hdr->seg_size = size;
seg_hdr->align_size = seg_align_size;
seg_hdr->mutex_offs = seg_hdr_size;
lock_item->lockfile = strdup(lock_item->seg_desc->seg_info.seg_name);
lock_item->num_locks = local_size;
lock_item->mutex = _GET_MUTEX_ARR_PTR(seg_hdr);
for(i = 0; i < local_size * 2; i++) {
pthread_mutex_t *mutex = _GET_MUTEX_PTR(seg_hdr, i);
if (0 != pthread_mutex_init(mutex, &attr)) {
pthread_mutexattr_destroy(&attr);
rc = PMIX_ERR_INIT;
PMIX_ERROR_LOG(rc);
goto error;
}
}
if (0 != pthread_mutexattr_destroy(&attr)) {
rc = PMIX_ERR_INIT;
PMIX_ERROR_LOG(PMIX_ERR_INIT);
goto error;
}
}
else {
int32_t *lock_idx_ptr;
bool idx_found = false;
size = pmix_common_dstor_getpagesize();
lock_item->seg_desc = pmix_common_dstor_attach_new_lock_seg(base_path, size, name, 0);
if (NULL == lock_item->seg_desc) {
rc = PMIX_ERR_NOT_FOUND;
goto error;
}
seg_hdr = (segment_hdr_t*)lock_item->seg_desc->seg_info.seg_base_addr;
if (seg_hdr->seg_size > size) {
size = seg_hdr->seg_size;
pmix_common_dstor_delete_sm_desc(lock_item->seg_desc);
lock_item->seg_desc = pmix_common_dstor_attach_new_lock_seg(base_path, size, name, 0);
if (NULL == lock_item->seg_desc) {
rc = PMIX_ERR_NOT_FOUND;
goto error;
}
}
lock_item->num_locks = seg_hdr->num_locks;
lock_idx_ptr = _GET_IDX_ARR_PTR(seg_hdr);
lock_item->mutex = _GET_MUTEX_ARR_PTR(seg_hdr);
for (i = 0; i < lock_item->num_locks; i++) {
int32_t expected = 0;
if (pmix_atomic_compare_exchange_strong_32(&lock_idx_ptr[i], &expected, 1)) {
lock_item->lock_idx = i;
lock_item->lockfile = strdup(lock_item->seg_desc->seg_info.seg_name);
idx_found = true;
break;
}
}
if (false == idx_found) {
rc = PMIX_ERR_NOT_FOUND;
goto error;
}
}
return rc;
error:
if (NULL != lock_item) {
pmix_list_remove_item(lock_tracker, &lock_item->super);
PMIX_RELEASE(lock_item);
lock_item = NULL;
}
*ctx = NULL;
return rc;
}
static pmix_peer_t* find_peer(const pmix_proc_t *proc)
{
pmix_status_t rc;
pmix_peer_t *peer;
pmix_proc_t wildcard;
pmix_value_t *value;
int i;
if (NULL == proc ) {
return pmix_globals.mypeer;
}
/* if the target is someone in my nspace, then use my own peer */
if (0 == strncmp(proc->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN)) {
return pmix_globals.mypeer;
}
if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) {
/* see if we know this proc */
for (i=0; i < pmix_server_globals.clients.size; i++) {
if (NULL != (peer = (pmix_peer_t*)pmix_pointer_array_get_item(&pmix_server_globals.clients, i))) {
continue;
}
if (0 == strncmp(proc->nspace, peer->nptr->nspace, PMIX_MAX_NSLEN)) {
return peer;
}
}
/* didn't find it, so try to get the library version of the target
* from the host - the result will be cached, so we will only have
* to retrieve it once */
(void)strncpy(wildcard.nspace, proc->nspace, PMIX_MAX_NSLEN);
wildcard.rank = PMIX_RANK_WILDCARD;
if (PMIX_SUCCESS != (rc = PMIx_Get(&wildcard, PMIX_BFROPS_MODULE, NULL, 0, &value))) {
/* couldn't get it - nothing we can do */
return NULL;
}
/* setup a peer for this nspace */
peer = PMIX_NEW(pmix_peer_t);
if (NULL == peer) {
PMIX_RELEASE(value);
return NULL;
}
peer->nptr = PMIX_NEW(pmix_nspace_t);
if (NULL == peer->nptr) {
PMIX_RELEASE(peer);
PMIX_RELEASE(value);
return NULL;
}
peer->nptr->nspace = strdup(proc->nspace);
/* assign a module to it based on the returned version */
peer->nptr->compat.bfrops = pmix_bfrops_base_assign_module(value->data.string);
PMIX_RELEASE(value);
if (NULL == peer->nptr->compat.bfrops) {
PMIX_RELEASE(peer);
return NULL;
}
/* cache the peer object */
pmix_pointer_array_add(&pmix_server_globals.clients, peer);
return peer;
}
// we are a client or tool
/* If the target is for the server, then
* pack it using that peer. */
if (0 == strncmp(proc->nspace, pmix_client_globals.myserver->info->pname.nspace, PMIX_MAX_NSLEN)) {
return pmix_client_globals.myserver;
}
/* if the target is another member of my nspace, then
* they must be using the same version */
if (0 == strncmp(proc->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN)) {
return pmix_globals.mypeer;
}
/* try to get the library version of this peer - the result will be
* cached, so we will only have to retrieve it once */
(void)strncpy(wildcard.nspace, proc->nspace, PMIX_MAX_NSLEN);
wildcard.rank = PMIX_RANK_WILDCARD;
if (PMIX_SUCCESS != (rc = PMIx_Get(&wildcard, PMIX_BFROPS_MODULE, NULL, 0, &value))) {
/* couldn't get it - nothing we can do */
return NULL;
}
/* setup a peer for this nspace */
peer = PMIX_NEW(pmix_peer_t);
if (NULL == peer) {
PMIX_RELEASE(value);
return NULL;
}
peer->nptr = PMIX_NEW(pmix_nspace_t);
if (NULL == peer->nptr) {
PMIX_RELEASE(peer);
PMIX_RELEASE(value);
return NULL;
}
peer->nptr->nspace = strdup(proc->nspace);
/* assign a module to it based on the returned version */
peer->nptr->compat.bfrops = pmix_bfrops_base_assign_module(value->data.string);
PMIX_RELEASE(value);
if (NULL == peer->nptr->compat.bfrops) {
PMIX_RELEASE(peer);
return NULL;
}
/* need to cache the peer someplace so we can clean it
* up later */
return peer;
}
