void armci_region_exchange(void *start, long size) { int found=0, i; armci_region_t *reg=0; #ifdef REGIONS_REQUIRE_MEMHDL ARMCI_MEMHDL_T *hdlarr; hdlarr = calloc(armci_nclus,sizeof(ARMCI_MEMHDL_T)); #endif if(!allow_pin)return; if(armci_nclus<=1)return; found=armci_region_clus_found(armci_clus_me, start,size); if(found>-1){ if(!exch_rem[found]){ reg = (clus_regions+armci_clus_me)->list+found; exch_rem[found]=1; } }else{ found= armci_region_loc_found(start,size); if(found>-1){ if(!exch_loc[found]){ reg = (&loc_regions_arr)->list+found; exch_loc[found]=1; } } } bzero(exch_list,2*armci_nclus*sizeof(void *)); if( reg && (armci_me == armci_master)){ exch_list[2*armci_clus_me] = reg->start; exch_list[2*armci_clus_me+1] = reg->end; #ifdef REGIONS_REQUIRE_MEMHDL armci_copy(®->memhdl,&hdlarr[armci_clus_me],sizeof(ARMCI_MEMHDL_T)); #endif } /* exchange info on new regions with other nodes */ armci_exchange_address(exch_list,2*armci_nclus); #ifdef REGIONS_REQUIRE_MEMHDL i = armci_nclus*sizeof(ARMCI_MEMHDL_T)/sizeof(int); armci_msg_gop_scope(SCOPE_ALL,hdlarr,i,"+",ARMCI_INT); #endif for(i=0; i<armci_nclus; i++){ armci_reglist_t *r=clus_regions+i; if(i==armci_clus_me) continue; if(exch_list[2*i]){ #if 0 printf("%d recording clus=%d mem %p-%p n=%d\n",armci_me,i,exch_list[2*i], exch_list[2*i+1],r->n); fflush(stdout); #endif #ifdef REGIONS_REQUIRE_MEMHDL armci_copy(&hdlarr[i],&(r->list+r->n)->memhdl,sizeof(ARMCI_MEMHDL_T)); #endif armci_region_record(exch_list[2*i],exch_list[2*i+1], r); } } }
void armci_msg_dgop(double *x, int n, char *op) { armci_msg_gop_scope(SCOPE_ALL, x, n, op, ARMCI_DOUBLE); }
void armci_msg_fgop(float *x, int n, char *op) { armci_msg_gop_scope(SCOPE_ALL, x, n, op, ARMCI_FLOAT); }
void armci_msg_llgop(long long *x, int n, char *op) { armci_msg_gop_scope(SCOPE_ALL, x, n, op, ARMCI_LONG_LONG); }
void armci_msg_igop(int *x, int n, char *op) { armci_msg_gop_scope(SCOPE_ALL, x, n, op, ARMCI_INT); }
int PARMCI_Malloc(void *ptr_arr[], armci_size_t bytes) { int mynslave = armci_clus_info[armci_clus_me].nslave; void *servptr,*mynodeptrs[mynslave]; int id,nodeids[mynslave],mynodeid=armci_me-armci_master; ARMCI_PR_DBG("enter",0); #ifdef DEBUG_MEM fprintf(stderr,"%d bytes in armci_malloc %d\n",armci_me, (int)bytes); fflush(stderr); armci_msg_barrier(); #endif if(bytes>0){ if(mynslave>1){ #ifdef DEBUG_MEM printf("\n%d:%s:mynslave is %d",armci_me,__FUNCTION__,mynslave);fflush(stdout); #endif bzero((void *)nodeids,sizeof(int)*mynslave); id =nodeids[mynodeid]= armci_shmget(bytes,"PARMCI_Malloc"); armci_msg_gop_scope(SCOPE_NODE,nodeids,mynslave,"+",ARMCI_INT); for(int i=0;i<mynslave;i++){ if((long)((mynodeptrs[i] = shmat(nodeids[i],mynodeptrs[i],0))) == -1L){ char command[64]; CLEANUP_CMD(command); if(system(command) == -1) printf("clean shared memory (id=%d): see man ipcrm\n",nodeids[i]); armci_die("allocate: failed to attach to shared id=",nodeids[i]); } } /* mark my id for rem so OS cleans when all attached processes vanish*/ if(shmctl( id, IPC_RMID, (struct shmid_ds *)NULL)) fprintf(stderr,"failed to remove shm id=%d\n",id); #ifdef DEBUG_MEM printf("%d:attach:id=%d paddr=%p size=%ld\n",armci_me,id,mynodeptrs[mynodeid],bytes); fflush(stdout); #endif if(armci_nclus>1){ servptr = armci_server_ptr(id); } else servptr = mynodeptrs[mynodeid]; } else{ #ifdef DEBUG_MEM printf("\n%d:%s:mynslave is %d, doing malloc",armci_me,__FUNCTION__,mynslave);fflush(stdout); #endif mynodeptrs[mynodeid] = servptr = malloc(bytes); } } else{ mynodeptrs[mynodeid] = servptr = NULL; } bzero((char*)ptr_arr,armci_nproc*sizeof(void*)); /*ptr_arr[armci_me] = servptr;*/ ptr_arr[armci_me] = mynodeptrs[mynodeid]; armci_exchange_address(ptr_arr,armci_nproc); if(mynslave>1)for(int i=0;i<mynslave;i++){ ptr_arr[armci_master+i] = mynodeptrs[i]; } if(armci_nclus>1){ armci_portals_memsetup((long)servptr-(long)ptr_arr[armci_me]); } ARMCI_PR_DBG("exit",0); return(0); }
/*\ * for server thread to know which region is what in cases where there is one * process per node \*/ void armci_global_region_exchange(void *start, long size) { #ifdef REGIONS_REQUIRE_MEMHDL ARMCI_MEMHDL_T *hdlarr; hdlarr = calloc(armci_nclus,sizeof(ARMCI_MEMHDL_T)); #endif if(!allow_pin)return; if(armci_nclus<=1)return; armci_region_exchange(start,size); #ifdef REGIONS_REQUIRE_MEMHDL { int foundclus=0, foundserv=0, i,loc=0; armci_reglist_t *reglist=NULL,*clreglist=NULL; armci_region_t *reg=NULL; foundclus=armci_region_clus_found(armci_clus_me, start,size); foundserv=armci_region_serv_found(armci_clus_me, start,size); if(foundclus==-1){ foundclus = armci_region_loc_found(start,size); loc=1; } if(foundclus!=-1 && foundserv==-1){ reglist = (serv_regions+armci_clus_me); if(loc) clreglist = &(loc_regions_arr); else clreglist = (clus_regions+armci_clus_me); #if defined(DATA_SERVER) || defined(PORTALS) # if defined(PORTALS) ((reglist->list+reglist->n)->memhdl).regid=(reglist->n); # endif armci_serv_register_req((clreglist->list+foundclus)->start,((char *)(clreglist->list+foundclus)->end-(char *)((clreglist->list+foundclus)->start)),&((reglist->list+reglist->n)->memhdl)); #endif (void)armci_region_record((clreglist->list+foundclus)->start,(clreglist->list+foundclus)->end,reglist); #ifdef LAPI_RDMA armci_copy(&(clreglist->list+foundclus)->memhdl, &(reglist->list+foundclus)->memhdl, sizeof(ARMCI_MEMHDL_T)); #endif #if DEBUG printf("\n%d:serv recording st=%p end=%p sz=%d from %d n=%d sz=%d\n",armci_me,(clreglist->list+foundclus)->start,(clreglist->list+foundclus)->end,(clreglist->list+foundclus)->end-(clreglist->list+foundclus)->start,armci_clus_me,reglist->n,sizeof(ARMCI_MEMHDL_T));fflush(stdout); #endif foundserv=armci_region_serv_found(armci_clus_me, start,size); reg = (serv_regions+armci_clus_me)->list+foundserv; } if(reg) armci_copy(®->memhdl,&hdlarr[armci_clus_me],sizeof(ARMCI_MEMHDL_T)); i = armci_nclus*sizeof(ARMCI_MEMHDL_T)/sizeof(int); armci_msg_gop_scope(SCOPE_ALL,hdlarr,i,"+",ARMCI_INT); for(i=0; i<armci_nclus; i++){ armci_reglist_t *r=serv_regions+i; armci_reglist_t *rc=clus_regions+i; if(i==armci_clus_me) continue; if((rc->list+r->n)->start){ #if 0 printf("\n%d:serv recording %p from %d n=%d \n",armci_me,(rc->list+r->n)->start,i,r->n);fflush(stdout); #endif armci_copy(&hdlarr[i],&(r->list+r->n)->memhdl,sizeof(ARMCI_MEMHDL_T)); armci_region_record((rc->list+r->n)->start,(rc->list+r->n)->end,r); } } } #endif }