void armci_region_exchange(void *start, long size)
{
int found=0, i;
armci_region_t *reg=0;
#ifdef REGIONS_REQUIRE_MEMHDL
ARMCI_MEMHDL_T *hdlarr;
hdlarr = calloc(armci_nclus,sizeof(ARMCI_MEMHDL_T));
#endif
if(!allow_pin)return;
if(armci_nclus<=1)return;
found=armci_region_clus_found(armci_clus_me, start,size);
if(found>-1){
if(!exch_rem[found]){
reg = (clus_regions+armci_clus_me)->list+found;
exch_rem[found]=1;
}
}else{
found= armci_region_loc_found(start,size);
if(found>-1){
if(!exch_loc[found]){
reg = (&loc_regions_arr)->list+found;
exch_loc[found]=1;
}
}
}
bzero(exch_list,2*armci_nclus*sizeof(void *));
if( reg && (armci_me == armci_master)){
exch_list[2*armci_clus_me] = reg->start;
exch_list[2*armci_clus_me+1] = reg->end;
#ifdef REGIONS_REQUIRE_MEMHDL
armci_copy(®->memhdl,&hdlarr[armci_clus_me],sizeof(ARMCI_MEMHDL_T));
#endif
}
/* exchange info on new regions with other nodes */
armci_exchange_address(exch_list,2*armci_nclus);
#ifdef REGIONS_REQUIRE_MEMHDL
i = armci_nclus*sizeof(ARMCI_MEMHDL_T)/sizeof(int);
armci_msg_gop_scope(SCOPE_ALL,hdlarr,i,"+",ARMCI_INT);
#endif
for(i=0; i<armci_nclus; i++){
armci_reglist_t *r=clus_regions+i;
if(i==armci_clus_me) continue;
if(exch_list[2*i]){
#if 0
printf("%d recording clus=%d mem %p-%p n=%d\n",armci_me,i,exch_list[2*i],
exch_list[2*i+1],r->n);
fflush(stdout);
#endif
#ifdef REGIONS_REQUIRE_MEMHDL
armci_copy(&hdlarr[i],&(r->list+r->n)->memhdl,sizeof(ARMCI_MEMHDL_T));
#endif
armci_region_record(exch_list[2*i],exch_list[2*i+1], r);
}
}
}
void armci_msg_dgop(double *x, int n, char *op) {
armci_msg_gop_scope(SCOPE_ALL, x, n, op, ARMCI_DOUBLE);
}
void armci_msg_fgop(float *x, int n, char *op) {
armci_msg_gop_scope(SCOPE_ALL, x, n, op, ARMCI_FLOAT);
}
void armci_msg_llgop(long long *x, int n, char *op) {
armci_msg_gop_scope(SCOPE_ALL, x, n, op, ARMCI_LONG_LONG);
}
void armci_msg_igop(int *x, int n, char *op) {
armci_msg_gop_scope(SCOPE_ALL, x, n, op, ARMCI_INT);
}
int PARMCI_Malloc(void *ptr_arr[], armci_size_t bytes)
{
int mynslave = armci_clus_info[armci_clus_me].nslave;
void *servptr,*mynodeptrs[mynslave];
int id,nodeids[mynslave],mynodeid=armci_me-armci_master;
ARMCI_PR_DBG("enter",0);
#ifdef DEBUG_MEM
fprintf(stderr,"%d bytes in armci_malloc %d\n",armci_me, (int)bytes);
fflush(stderr);
armci_msg_barrier();
#endif
if(bytes>0){
if(mynslave>1){
#ifdef DEBUG_MEM
printf("\n%d:%s:mynslave is %d",armci_me,__FUNCTION__,mynslave);fflush(stdout);
#endif
bzero((void *)nodeids,sizeof(int)*mynslave);
id =nodeids[mynodeid]= armci_shmget(bytes,"PARMCI_Malloc");
armci_msg_gop_scope(SCOPE_NODE,nodeids,mynslave,"+",ARMCI_INT);
for(int i=0;i<mynslave;i++){
if((long)((mynodeptrs[i] = shmat(nodeids[i],mynodeptrs[i],0))) == -1L){
char command[64];
CLEANUP_CMD(command);
if(system(command) == -1)
printf("clean shared memory (id=%d): see man ipcrm\n",nodeids[i]);
armci_die("allocate: failed to attach to shared id=",nodeids[i]);
}
}
/* mark my id for rem so OS cleans when all attached processes vanish*/
if(shmctl( id, IPC_RMID, (struct shmid_ds *)NULL))
fprintf(stderr,"failed to remove shm id=%d\n",id);
#ifdef DEBUG_MEM
printf("%d:attach:id=%d paddr=%p size=%ld\n",armci_me,id,mynodeptrs[mynodeid],bytes);
fflush(stdout);
#endif
if(armci_nclus>1){
servptr = armci_server_ptr(id);
}
else servptr = mynodeptrs[mynodeid];
}
else{
#ifdef DEBUG_MEM
printf("\n%d:%s:mynslave is %d, doing malloc",armci_me,__FUNCTION__,mynslave);fflush(stdout);
#endif
mynodeptrs[mynodeid] = servptr = malloc(bytes);
}
}
else{
mynodeptrs[mynodeid] = servptr = NULL;
}
bzero((char*)ptr_arr,armci_nproc*sizeof(void*));
/*ptr_arr[armci_me] = servptr;*/
ptr_arr[armci_me] = mynodeptrs[mynodeid];
armci_exchange_address(ptr_arr,armci_nproc);
if(mynslave>1)for(int i=0;i<mynslave;i++){
ptr_arr[armci_master+i] = mynodeptrs[i];
}
if(armci_nclus>1){
armci_portals_memsetup((long)servptr-(long)ptr_arr[armci_me]);
}
ARMCI_PR_DBG("exit",0);
return(0);
}
/*\
* for server thread to know which region is what in cases where there is one
* process per node
\*/
void armci_global_region_exchange(void *start, long size)
{
#ifdef REGIONS_REQUIRE_MEMHDL
ARMCI_MEMHDL_T *hdlarr;
hdlarr = calloc(armci_nclus,sizeof(ARMCI_MEMHDL_T));
#endif
if(!allow_pin)return;
if(armci_nclus<=1)return;
armci_region_exchange(start,size);
#ifdef REGIONS_REQUIRE_MEMHDL
{
int foundclus=0, foundserv=0, i,loc=0;
armci_reglist_t *reglist=NULL,*clreglist=NULL;
armci_region_t *reg=NULL;
foundclus=armci_region_clus_found(armci_clus_me, start,size);
foundserv=armci_region_serv_found(armci_clus_me, start,size);
if(foundclus==-1){
foundclus = armci_region_loc_found(start,size);
loc=1;
}
if(foundclus!=-1 && foundserv==-1){
reglist = (serv_regions+armci_clus_me);
if(loc)
clreglist = &(loc_regions_arr);
else
clreglist = (clus_regions+armci_clus_me);
#if defined(DATA_SERVER) || defined(PORTALS)
# if defined(PORTALS)
((reglist->list+reglist->n)->memhdl).regid=(reglist->n);
# endif
armci_serv_register_req((clreglist->list+foundclus)->start,((char *)(clreglist->list+foundclus)->end-(char *)((clreglist->list+foundclus)->start)),&((reglist->list+reglist->n)->memhdl));
#endif
(void)armci_region_record((clreglist->list+foundclus)->start,(clreglist->list+foundclus)->end,reglist);
#ifdef LAPI_RDMA
armci_copy(&(clreglist->list+foundclus)->memhdl, &(reglist->list+foundclus)->memhdl, sizeof(ARMCI_MEMHDL_T));
#endif
#if DEBUG
printf("\n%d:serv recording st=%p end=%p sz=%d from %d n=%d sz=%d\n",armci_me,(clreglist->list+foundclus)->start,(clreglist->list+foundclus)->end,(clreglist->list+foundclus)->end-(clreglist->list+foundclus)->start,armci_clus_me,reglist->n,sizeof(ARMCI_MEMHDL_T));fflush(stdout);
#endif
foundserv=armci_region_serv_found(armci_clus_me, start,size);
reg = (serv_regions+armci_clus_me)->list+foundserv;
}
if(reg)
armci_copy(®->memhdl,&hdlarr[armci_clus_me],sizeof(ARMCI_MEMHDL_T));
i = armci_nclus*sizeof(ARMCI_MEMHDL_T)/sizeof(int);
armci_msg_gop_scope(SCOPE_ALL,hdlarr,i,"+",ARMCI_INT);
for(i=0; i<armci_nclus; i++){
armci_reglist_t *r=serv_regions+i;
armci_reglist_t *rc=clus_regions+i;
if(i==armci_clus_me) continue;
if((rc->list+r->n)->start){
#if 0
printf("\n%d:serv recording %p from %d n=%d \n",armci_me,(rc->list+r->n)->start,i,r->n);fflush(stdout);
#endif
armci_copy(&hdlarr[i],&(r->list+r->n)->memhdl,sizeof(ARMCI_MEMHDL_T));
armci_region_record((rc->list+r->n)->start,(rc->list+r->n)->end,r);
}
}
}
#endif
}
