int PIOc_InitDecomp_bc(const int iosysid, const int basetype,const int ndims, const int dims[],
const long int start[], const long int count[], int *ioidp)
{
iosystem_desc_t *ios;
io_desc_t *iodesc;
int mpierr;
int ierr;
int iosize;
int ndisp;
for(int i=0;i<ndims;i++){
if(dims[i]<=0){
piodie("Invalid dims argument",__FILE__,__LINE__);
}
if(start[i]<0 || count[i]< 0 || (start[i]+count[i])>dims[i]){
piodie("Invalid start or count argument ",__FILE__,__LINE__);
}
}
ios = pio_get_iosystem_from_id(iosysid);
if(ios == NULL)
return PIO_EBADID;
int n, i, maplen=1;
for( i=0;i<ndims;i++){
maplen*=count[i];
}
PIO_Offset compmap[maplen], prod[ndims], loc[ndims];
prod[ndims-1]=1;
loc[ndims-1]=0;
for(n=ndims-2;n>=0;n--){
prod[n]=prod[n+1]*dims[n+1];
loc[n]=0;
}
for(i=0;i<maplen;i++){
compmap[i]=0;
for(n=ndims-1;n>=0;n--){
compmap[i]+=(start[n]+loc[n])*prod[n];
}
n=ndims-1;
loc[n]=(loc[n]+1)%count[n];
while(loc[n]==0 && n>0){
n--;
loc[n]=(loc[n]+1)%count[n];
}
}
int rearr = PIO_REARR_SUBSET;
PIOc_InitDecomp( iosysid, basetype,ndims, dims,
maplen, compmap, ioidp, &rearr, NULL, NULL);
return PIO_NOERR;
}
file_desc_t *pio_get_file_from_id(int ncid)
{
file_desc_t *cfile;
cfile = NULL;
if(current_file != NULL && current_file->fh == ncid)
cfile=current_file;
for(cfile=pio_file_list; cfile != NULL; cfile=cfile->next){
if(cfile->fh == ncid){
current_file = cfile;
break;
}
}
if(cfile==NULL){
printf("ERROR: Operation on undefined file %s %d\n",__FILE__,__LINE__);
}else if(cfile->nreq > PIO_MAX_REQUESTS || cfile->nreq < 0){
printf("%s %d %d %d\n",__FILE__,__LINE__,ncid,cfile->nreq);
piodie("Bad request count",__FILE__,__LINE__);
}
return cfile;
}
int PIOc_InitDecomp(const int iosysid, const int basetype,const int ndims, const int dims[],
const int maplen, const PIO_Offset *compmap, int *ioidp,const int *rearranger,
const PIO_Offset *iostart,const PIO_Offset *iocount)
{
iosystem_desc_t *ios;
io_desc_t *iodesc;
int mpierr;
int ierr;
int iosize;
int ndisp;
for(int i=0;i<ndims;i++){
if(dims[i]<=0){
piodie("Invalid dims argument",__FILE__,__LINE__);
}
}
ios = pio_get_iosystem_from_id(iosysid);
if(ios == NULL)
return PIO_EBADID;
if(PIO_Save_Decomps){
char filename[30];
if(ios->num_comptasks < 100) {
sprintf(filename, "piodecomp%2.2dtasks%2.2ddims%2.2d.dat",ios->num_comptasks,ndims,counter);
}else if(ios->num_comptasks < 10000) {
sprintf(filename, "piodecomp%4.4dtasks%2.2ddims%2.2d.dat",ios->num_comptasks,ndims,counter);
}else{
sprintf(filename, "piodecomp%6.6dtasks%2.2ddims%2.2d.dat",ios->num_comptasks,ndims,counter);
}
PIOc_writemap(filename,ndims,dims,maplen,compmap,ios->comp_comm);
counter++;
}
iodesc = malloc_iodesc(basetype, ndims);
if(rearranger == NULL)
iodesc->rearranger = ios->default_rearranger;
else
iodesc->rearranger = *rearranger;
if(iodesc->rearranger==PIO_REARR_SUBSET){
if((iostart != NULL) && (iocount != NULL)){
fprintf(stderr,"%s %s\n","Iostart and iocount arguments to PIOc_InitDecomp",
"are incompatable with subset rearrange method and will be ignored");
}
iodesc->num_aiotasks = ios->num_iotasks;
ierr = subset_rearrange_create( *ios, maplen, compmap, dims, ndims, iodesc);
}else{
if(ios->ioproc){
// Unless the user specifies the start and count for each IO task compute it.
if((iostart != NULL) && (iocount != NULL)){
// printf("iocount[0] = %ld %ld\n",iocount[0], iocount);
iodesc->maxiobuflen=1;
for(int i=0;i<ndims;i++){
iodesc->firstregion->start[i] = iostart[i];
iodesc->firstregion->count[i] = iocount[i];
compute_maxIObuffersize(ios->io_comm, iodesc);
}
iodesc->num_aiotasks = ios->num_iotasks;
}else{
iodesc->num_aiotasks = CalcStartandCount(basetype, ndims, dims,
ios->num_iotasks, ios->io_rank,
iodesc->firstregion->start, iodesc->firstregion->count);
}
compute_maxIObuffersize(ios->io_comm, iodesc);
}
// Depending on array size and io-blocksize the actual number of io tasks used may vary
CheckMPIReturn(MPI_Bcast(&(iodesc->num_aiotasks), 1, MPI_INT, ios->ioroot,
ios->my_comm),__FILE__,__LINE__);
// Compute the communications pattern for this decomposition
if(iodesc->rearranger==PIO_REARR_BOX){
ierr = box_rearrange_create( *ios, maplen, compmap, dims, ndims, iodesc);
}
/*
if(ios->ioproc){
io_region *ioregion = iodesc->firstregion;
while(ioregion != NULL){
for(int i=0;i<ndims;i++)
printf("%s %d i %d dim %d start %ld count %ld\n",__FILE__,__LINE__,i,dims[i],ioregion->start[i],ioregion->count[i]);
ioregion = ioregion->next;
}
}
*/
}
*ioidp = pio_add_to_iodesc_list(iodesc);
performance_tune_rearranger(*ios, iodesc);
return PIO_NOERR;
}
int CalcStartandCount(const int basetype, const int ndims,const int *gdims, const int num_io_procs,
const int myiorank, PIO_Offset *start, PIO_Offset *kount)
{
int minbytes;
int maxbytes;
int minblocksize;
int basesize;
int use_io_procs;
int i;
long int p;
long int pgdims;
bool converged;
int iorank;
int ldims;
int tiorank;
int ioprocs;
int tioprocs;
int mystart[ndims], mykount[ndims];
long int pknt;
long int tpsize=0;
minbytes = blocksize-256;
maxbytes = blocksize+256;
switch (basetype){
case PIO_INT:
basesize = sizeof(int);
break;
case PIO_REAL:
basesize = sizeof(float);
break;
case PIO_DOUBLE:
basesize = sizeof(double);
break;
default:
#ifndef TESTCALCDECOMP
piodie("Invalid basetype ",__FILE__,__LINE__);
#endif
break;
}
minblocksize = minbytes/basesize;
pgdims = 1;
for( i=0;i<ndims;i++)
pgdims *= (long int) gdims[i];
p = pgdims;
use_io_procs = max(1, min((int) ((float) p/ (float) minblocksize + 0.5),num_io_procs));
converged = 0;
for( i=0;i<ndims;i++){
mystart[i]=0;
mykount[i]=0;
}
while(! converged){
for(iorank=0;iorank<use_io_procs;iorank++){
for( i=0;i<ndims;i++){
start[i]=0;
kount[i]=gdims[i];
}
ldims = ndims-1;
p=basesize;
for(i=ndims-1;i>=0;i--){
p=p*gdims[i];
if(p/use_io_procs > maxbytes){
ldims = i;
break;
}
}
if(gdims[ldims]< use_io_procs){
if(ldims>0 && gdims[ldims-1] > use_io_procs)
ldims--;
else
use_io_procs -= (use_io_procs % gdims[ldims]);
}
ioprocs = use_io_procs;
tiorank = iorank;
#ifdef TESTCALCDECOMP
if(myiorank==0) printf("%d use_io_procs %d ldims %d\n",__LINE__,use_io_procs,ldims);
#endif
for(i=0;i<=ldims;i++){
if(gdims[i]>= ioprocs){
computestartandcount(gdims[i],ioprocs,tiorank,start+i,kount+i);
#ifdef TESTCALCDECOMP
if(myiorank==0) printf("%d tiorank %d i %d start %d count %d\n",__LINE__,tiorank,i,start[i],kount[i]);
#endif
if(start[i]+kount[i]>gdims[i]+1){
#ifndef TESTCALCDECOMP
piodie("Start plus count exceeds dimension bound",__FILE__,__LINE__);
#endif
}
}else if(gdims[i]>1){
tioprocs=gdims[i];
tiorank = (iorank*tioprocs)/ioprocs;
computestartandcount(gdims[i],tioprocs,tiorank,start+i,kount+i);
ioprocs = ioprocs/tioprocs;
tiorank = iorank % ioprocs;
}
}
if(myiorank==iorank){
for(i=0;i<ndims;i++){
mystart[i]=start[i];
mykount[i]=kount[i];
}
}
pknt = 1;
for(i=0;i<ndims;i++)
pknt*=kount[i];
tpsize+=pknt;
// printf("%d myiorank %d tpsize %ld pknt %ld kount %ld %ld\n",__LINE__,myiorank,tpsize,pknt, kount[0],kount[1]);
if(tpsize==pgdims && use_io_procs==iorank+1){
converged = true;
break;
}else if(tpsize>=pgdims) {
break;
}
}
if(! converged) {
#ifdef TESTCALCDECOMP
printf("%d start %d %d count %d %d tpsize %ld\n",__LINE__,mystart[0],mystart[1],mykount[0],mykount[1],tpsize);
#endif
tpsize = 0;
use_io_procs--;
}
}
if(myiorank<use_io_procs){
for(i=0;i<ndims;i++){
start[i]=mystart[i];
kount[i]=mykount[i];
// printf("%s %d %ld %ld\n",__FILE__,__LINE__,start[i],kount[i]);
}
}else{
for(i=0;i<ndims;i++){
start[i]=0;
kount[i]=0;
}
}
return use_io_procs;
}