void armci_region_exchange(void *start, long size)
{
int found=0, i;
armci_region_t *reg=0;
#ifdef REGIONS_REQUIRE_MEMHDL
ARMCI_MEMHDL_T *hdlarr;
hdlarr = calloc(armci_nclus,sizeof(ARMCI_MEMHDL_T));
#endif
if(!allow_pin)return;
if(armci_nclus<=1)return;
found=armci_region_clus_found(armci_clus_me, start,size);
if(found>-1){
if(!exch_rem[found]){
reg = (clus_regions+armci_clus_me)->list+found;
exch_rem[found]=1;
}
}else{
found= armci_region_loc_found(start,size);
if(found>-1){
if(!exch_loc[found]){
reg = (&loc_regions_arr)->list+found;
exch_loc[found]=1;
}
}
}
bzero(exch_list,2*armci_nclus*sizeof(void *));
if( reg && (armci_me == armci_master)){
exch_list[2*armci_clus_me] = reg->start;
exch_list[2*armci_clus_me+1] = reg->end;
#ifdef REGIONS_REQUIRE_MEMHDL
armci_copy(®->memhdl,&hdlarr[armci_clus_me],sizeof(ARMCI_MEMHDL_T));
#endif
}
/* exchange info on new regions with other nodes */
armci_exchange_address(exch_list,2*armci_nclus);
#ifdef REGIONS_REQUIRE_MEMHDL
i = armci_nclus*sizeof(ARMCI_MEMHDL_T)/sizeof(int);
armci_msg_gop_scope(SCOPE_ALL,hdlarr,i,"+",ARMCI_INT);
#endif
for(i=0; i<armci_nclus; i++){
armci_reglist_t *r=clus_regions+i;
if(i==armci_clus_me) continue;
if(exch_list[2*i]){
#if 0
printf("%d recording clus=%d mem %p-%p n=%d\n",armci_me,i,exch_list[2*i],
exch_list[2*i+1],r->n);
fflush(stdout);
#endif
#ifdef REGIONS_REQUIRE_MEMHDL
armci_copy(&hdlarr[i],&(r->list+r->n)->memhdl,sizeof(ARMCI_MEMHDL_T));
#endif
armci_region_record(exch_list[2*i],exch_list[2*i+1], r);
}
}
}
void armci_shmalloc_exchange_offsets(context_t *ctx_local)
{
void **ptr_arr;
void *ptr;
armci_size_t bytes = 128;
int i;
ptr_arr = (void**)malloc(armci_nproc*sizeof(void*));
offset_arr = (long*) malloc(armci_nproc*sizeof(long));
if(!ptr_arr || !offset_arr) armci_die("armci_shmalloc_get_offsets: malloc failed", 0);
/* get memory with same size on all procs */
ptr = kr_malloc(bytes, ctx_local);
if(!ptr) armci_die("armci_shmalloc_get_offsets: kr_malloc failed",bytes);
bzero((char*)ptr_arr,armci_nproc*sizeof(void*));
ptr_arr[armci_me] = ptr;
/* now combine individual addresses into a single array */
armci_exchange_address(ptr_arr, armci_nproc);
/* identify offets */
for (i=0; i<armci_nproc; i++)
{
offset_arr[i] = (long) ((char*)ptr - (char*)ptr_arr[i]);
}
/* release memory */
kr_free(ptr, ctx_local);
}
void armci_shmalloc_exchange_address(void **ptr_arr)
{
int i;
/* now combine individual addresses into a single array */
armci_exchange_address(ptr_arr, armci_nproc);
/* since shmalloc may not give symmetric addresses (especially on Linux),
* adjust addresses based on offset calculated during initialization */
for (i=0; i<armci_nproc; i++)
{
ptr_arr[i] = (char*)ptr_arr[i] + offset_arr[i];
}
}
int PARMCI_Malloc(void *ptr_arr[], armci_size_t bytes)
{
int mynslave = armci_clus_info[armci_clus_me].nslave;
void *servptr,*mynodeptrs[mynslave];
int id,nodeids[mynslave],mynodeid=armci_me-armci_master;
ARMCI_PR_DBG("enter",0);
#ifdef DEBUG_MEM
fprintf(stderr,"%d bytes in armci_malloc %d\n",armci_me, (int)bytes);
fflush(stderr);
armci_msg_barrier();
#endif
if(bytes>0){
if(mynslave>1){
#ifdef DEBUG_MEM
printf("\n%d:%s:mynslave is %d",armci_me,__FUNCTION__,mynslave);fflush(stdout);
#endif
bzero((void *)nodeids,sizeof(int)*mynslave);
id =nodeids[mynodeid]= armci_shmget(bytes,"PARMCI_Malloc");
armci_msg_gop_scope(SCOPE_NODE,nodeids,mynslave,"+",ARMCI_INT);
for(int i=0;i<mynslave;i++){
if((long)((mynodeptrs[i] = shmat(nodeids[i],mynodeptrs[i],0))) == -1L){
char command[64];
CLEANUP_CMD(command);
if(system(command) == -1)
printf("clean shared memory (id=%d): see man ipcrm\n",nodeids[i]);
armci_die("allocate: failed to attach to shared id=",nodeids[i]);
}
}
/* mark my id for rem so OS cleans when all attached processes vanish*/
if(shmctl( id, IPC_RMID, (struct shmid_ds *)NULL))
fprintf(stderr,"failed to remove shm id=%d\n",id);
#ifdef DEBUG_MEM
printf("%d:attach:id=%d paddr=%p size=%ld\n",armci_me,id,mynodeptrs[mynodeid],bytes);
fflush(stdout);
#endif
if(armci_nclus>1){
servptr = armci_server_ptr(id);
}
else servptr = mynodeptrs[mynodeid];
}
else{
#ifdef DEBUG_MEM
printf("\n%d:%s:mynslave is %d, doing malloc",armci_me,__FUNCTION__,mynslave);fflush(stdout);
#endif
mynodeptrs[mynodeid] = servptr = malloc(bytes);
}
}
else{
mynodeptrs[mynodeid] = servptr = NULL;
}
bzero((char*)ptr_arr,armci_nproc*sizeof(void*));
/*ptr_arr[armci_me] = servptr;*/
ptr_arr[armci_me] = mynodeptrs[mynodeid];
armci_exchange_address(ptr_arr,armci_nproc);
if(mynslave>1)for(int i=0;i<mynslave;i++){
ptr_arr[armci_master+i] = mynodeptrs[i];
}
if(armci_nclus>1){
armci_portals_memsetup((long)servptr-(long)ptr_arr[armci_me]);
}
ARMCI_PR_DBG("exit",0);
return(0);
}