/*\ server actions triggered by client request to ATTACH
\*/
void armci_server_ipc(request_header_t* msginfo, void* descr,
void* buffer, int buflen)
{
double *ptr;
long *idlist = (long*)descr;
long size = *(long*)buffer;
int rlen = *(int*)(sizeof(long)+(char*)buffer);
extern int **_armci_int_mutexes;
ARMCI_PR_DBG("enter",0);
if(size<0) armci_die("armci_server_ipc: size<0",(int)size);
if(DEBUG_MEM)printf("\n%d:got idlist+1 %p, size %d, idlist[0] %d, idlist[1] %d",armci_me,idlist+1,size,idlist[0],idlist[1]);
ptr=(double*)Attach_Shared_Region(idlist+1,size,idlist[0]);
if(!ptr)armci_die("armci_server_ipc: failed to attach",0);
/* provide data server with access to the memory lock data structures */
if(allocate_memlock){
allocate_memlock = 0;
server_alloc_memlock(ptr);
}
if(_armci_int_mutexes==NULL){
printf("unresolved portals external\n");
abort();
# ifdef PORTALS_UNRESOLVED
extern int _armci_server_mutex_ready;
extern void *_armci_server_mutex_ptr;
if(_armci_server_mutex_ready){
_armci_int_mutexes=(int **)_armci_server_mutex_ptr;
}
# endif
}
if(size>0)armci_set_mem_offset(ptr);
if(msginfo->datalen != sizeof(long)+sizeof(int))
armci_die("armci_server_ipc: bad msginfo->datalen ",msginfo->datalen);
if(rlen==sizeof(ptr)){
msginfo->datalen = rlen;
armci_send_data(msginfo, &ptr);
}
else armci_die("armci_server_ipc: bad rlen",rlen);
ARMCI_PR_DBG("exit",0);
}
/*\ Collective Memory Allocation on shared memory systems
\*/
void armci_shmem_malloc(void *ptr_arr[], armci_size_t bytes)
{
void *myptr=NULL, *ptr=NULL;
long idlist[SHMIDLEN];
long size=0, offset=0;
long *size_arr;
void **ptr_ref_arr;
int i,cn, len;
int nproc = armci_clus_info[armci_clus_me].nslave;
ARMCI_PR_DBG("enter",0);
bzero((char*)ptr_arr,armci_nproc*sizeof(void*));
/* allocate work arrays */
size_arr = (long*)calloc(armci_nproc,sizeof(long));
if(!size_arr)armci_die("armci_malloc:calloc failed",armci_nproc);
/* allocate arrays for cluster address translations */
# if defined(DATA_SERVER)
len = armci_nclus;
# else
len = nproc;
# endif
ptr_ref_arr = calloc(len,sizeof(void*)); /* must be zero */
if(!ptr_ref_arr)armci_die("armci_malloc:calloc 2 failed",len);
/* combine all memory requests into size_arr */
size_arr[armci_me] = bytes;
armci_msg_lgop(size_arr, armci_nproc, "+");
/* determine aggregate request size on the cluster node */
for(i=0, size=0; i< nproc; i++) size += size_arr[i+armci_master];
/* master process creates shmem region and then others attach to it */
if(armci_me == armci_master ){
/* can malloc if there is no data server process and has 1 process/node*/
# ifndef RMA_NEEDS_SHMEM
if(nproc == 1)
myptr = kr_malloc(size, &ctx_localmem);
else
# endif
myptr = Create_Shared_Region(idlist+1,size,idlist);
if(!myptr && size>0 )armci_die("armci_malloc: could not create", (int)(size>>10));
/* place its address at begining of attached region for others to see */
if(size)armci_master_exp_attached_ptr(myptr);
if(DEBUG_){
printf("%d:armci_malloc addr mptr=%p size=%ld\n",armci_me,myptr,size);
fflush(stdout);
}
}
/* broadcast shmem id to other processes on the same cluster node */
armci_msg_clus_brdcst(idlist, SHMIDLEN*sizeof(long));
if(armci_me != armci_master){
myptr=(double*)Attach_Shared_Region(idlist+1,size,idlist[0]);
if(!myptr)armci_die("armci_malloc: could not attach", (int)(size>>10));
/* now every process in a SMP node needs to find out its offset
* w.r.t. master - this offset is necessary to use memlock table
*/
if(size) armci_set_mem_offset(myptr);
if(DEBUG_){
printf("%d:armci_malloc attached addr mptr=%p ref=%p size=%ld\n",
armci_me,myptr, *(void**)myptr,size); fflush(stdout);
}
}
void test()
{
double *a, start=1., end=-1.;
int len=100;
long size = len*sizeof(double);
long idlist[SHMIDLEN];
int numlock=10, i;
lockset_t lockid;
/* shared memory test */
if(me==0){
printf("Test shared memory\n");
a=(double*)Create_Shared_Region(idlist+1,size,idlist);
assert(a);
a[0]= start;
a[len-1]=end;
}
MPI_Bcast(idlist,SHMIDLEN,MPI_LONG,0,MPI_COMM_WORLD);
if(me){
a=(double*)Attach_Shared_Region(idlist+1,size,idlist[0]);
assert(a);
}
if(me==nproc-1){
printf("%d: start=%f end=%f\n",me,a[0],a[len-1]);
if(a[0]== start && a[len-1]== end) printf("Works!\n");
}
/*printf("%d: a=%x\n",me,a); */
MPI_Barrier(MPI_COMM_WORLD);
/* allocate locks */
if(me == 0){
a[0]=0.;
CreateInitLocks(numlock, &lockid);
printf("\nMutual exclusion test\n");
}
MPI_Bcast(&lockid,sizeof(lockid),MPI_BYTE,0,MPI_COMM_WORLD);
if(me)InitLocks(numlock, lockid);
/* mutual exclusion test:
* everybody increments shared variable 1000 times
*/
# define TIMES 1000
MPI_Barrier(MPI_COMM_WORLD);
for(i=0;i<TIMES; i++){
NATIVE_LOCK(0);
a[0]++;
NATIVE_UNLOCK(0);
}
MPI_Barrier(MPI_COMM_WORLD);
if(me==nproc-1){
printf("value of shared variable =%f should be %f\n",
a[0],1.0*nproc*TIMES);
if(a[0]==1.0*nproc*TIMES ) printf("Works!\n\n");
}
/* cleanup of IPC resources */
if(me==0){
DeleteLocks(lockid);
Delete_All_Regions();
}
MPI_Barrier(MPI_COMM_WORLD);
}
