int PIOc_InitDecomp_bc(const int iosysid, const int basetype,const int ndims, const int dims[], const long int start[], const long int count[], int *ioidp) { iosystem_desc_t *ios; io_desc_t *iodesc; int mpierr; int ierr; int iosize; int ndisp; for(int i=0;i<ndims;i++){ if(dims[i]<=0){ piodie("Invalid dims argument",__FILE__,__LINE__); } if(start[i]<0 || count[i]< 0 || (start[i]+count[i])>dims[i]){ piodie("Invalid start or count argument ",__FILE__,__LINE__); } } ios = pio_get_iosystem_from_id(iosysid); if(ios == NULL) return PIO_EBADID; int n, i, maplen=1; for( i=0;i<ndims;i++){ maplen*=count[i]; } PIO_Offset compmap[maplen], prod[ndims], loc[ndims]; prod[ndims-1]=1; loc[ndims-1]=0; for(n=ndims-2;n>=0;n--){ prod[n]=prod[n+1]*dims[n+1]; loc[n]=0; } for(i=0;i<maplen;i++){ compmap[i]=0; for(n=ndims-1;n>=0;n--){ compmap[i]+=(start[n]+loc[n])*prod[n]; } n=ndims-1; loc[n]=(loc[n]+1)%count[n]; while(loc[n]==0 && n>0){ n--; loc[n]=(loc[n]+1)%count[n]; } } int rearr = PIO_REARR_SUBSET; PIOc_InitDecomp( iosysid, basetype,ndims, dims, maplen, compmap, ioidp, &rearr, NULL, NULL); return PIO_NOERR; }
/** ** @brief return a logical indicating whether this task is an iotask */ int PIOc_iam_iotask(const int iosysid, bool *ioproc) { iosystem_desc_t *ios; ios = pio_get_iosystem_from_id(iosysid); if(ios == NULL) return PIO_EBADID; *ioproc = ios->ioproc; return PIO_NOERR; }
/** * @ingroup PIO_def_var * * Set chunk cache netCDF files to be opened/created. * * This function only applies to netCDF-4 files. When used with netCDF * classic files, the error PIO_ENOTNC4 will be returned. * * The file chunk cache for HDF5 can be set, and will apply for any * files opened or created until the program ends, or the settings are * changed again. The cache settings apply only to the open file. They * do not persist with the file, and must be set each time the file is * opened, before it is opened, if they are to have effect. * * See the <a * href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF * variable documentation</a> for details about the operation of this * function. * * @param iotype the iotype of files to be created or opened. * @param size size of file cache. * @param nelems number of elements in file cache. * @param preemption preemption setting for file cache. * * @return PIO_NOERR for success, otherwise an error code. */ int PIOc_set_chunk_cache(int iosysid, int iotype, PIO_Offset size, PIO_Offset nelems, float preemption) { int ierr; int msg; int mpierr; iosystem_desc_t *ios; file_desc_t *file; char *errstr; errstr = NULL; ierr = PIO_NOERR; msg = PIO_MSG_SET_CHUNK_CACHE; ios = pio_get_iosystem_from_id(iosysid); if(ios == NULL) return PIO_EBADID; /* if (ios->async_interface && ! ios->ioproc){ */ /* if (ios->compmaster) */ /* mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); */ /* mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); */ /* } */ switch (iotype) { #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_set_chunk_cache(size, nelems, preemption); break; case PIO_IOTYPE_NETCDF4C: if (!ios->io_rank) ierr = nc_set_chunk_cache(size, nelems, preemption); break; #endif case PIO_IOTYPE_NETCDF: ierr = PIO_ENOTNC4; break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: ierr = PIO_ENOTNC4; break; #endif default: ierr = iotype_error(file->iotype,__FILE__,__LINE__); } /* Propogate error code to all processes. */ MPI_Bcast(&ierr, 1, MPI_INTEGER, ios->ioroot, ios->my_comm); return ierr; }
/** ** @brief Get the number of IO tasks set. */ int PIOc_get_numiotasks(int iosysid, int *numiotasks) { iosystem_desc_t *ios; ios = pio_get_iosystem_from_id(iosysid); if(ios == NULL) return PIO_EBADID; *numiotasks = ios->num_iotasks; return PIO_NOERR; }
/** ** @brief return the rank of this task in the io comm or ** -1 if this task is not in the comm */ int PIOc_iotask_rank(const int iosysid, int *iorank) { iosystem_desc_t *ios; ios = pio_get_iosystem_from_id(iosysid); if(ios == NULL) return PIO_EBADID; *iorank = ios->io_rank; return PIO_NOERR; }
/** ** @brief Send a hint to the MPI-IO library ** */ int PIOc_set_hint(const int iosysid, char hint[], const char hintval[]) { iosystem_desc_t *ios; ios = pio_get_iosystem_from_id(iosysid); if(ios == NULL) return PIO_EBADID; if(ios->ioproc) CheckMPIReturn( MPI_Info_set(ios->info, hint, hintval), __FILE__,__LINE__); return PIO_NOERR; }
int PIOc_finalize(const int iosysid) { iosystem_desc_t *ios, *nios; int msg; int mpierr; ios = pio_get_iosystem_from_id(iosysid); if(ios == NULL) return PIO_EBADID; /* If asynch IO is in use, send the PIO_MSG_EXIT message from the * comp master to the IO processes. */ if (ios->async_interface && !ios->comp_rank) { msg = PIO_MSG_EXIT; mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); CheckMPIReturn(mpierr, __FILE__, __LINE__); } /* Free this memory that was allocated in init_intracomm. */ if (ios->ioranks) free(ios->ioranks); /* Free the buffer pool. */ free_cn_buffer_pool(*ios); /* Free the MPI groups. */ if (ios->compgroup != MPI_GROUP_NULL) MPI_Group_free(&ios->compgroup); if (ios->iogroup != MPI_GROUP_NULL) MPI_Group_free(&(ios->iogroup)); /* Free the MPI communicators. my_comm is just a copy (but not an * MPI copy), so does not have to have an MPI_Comm_free() call. */ if(ios->intercomm != MPI_COMM_NULL){ MPI_Comm_free(&(ios->intercomm)); } if(ios->io_comm != MPI_COMM_NULL){ MPI_Comm_free(&(ios->io_comm)); } if(ios->comp_comm != MPI_COMM_NULL){ MPI_Comm_free(&(ios->comp_comm)); } if(ios->union_comm != MPI_COMM_NULL){ MPI_Comm_free(&(ios->union_comm)); } /* Delete the iosystem_desc_t data associated with this id. */ return pio_delete_iosystem_from_list(iosysid); }
/** ** @brief Check to see if PIO has been initialized. */ int PIOc_iosystem_is_active(const int iosysid, bool *active) { iosystem_desc_t *ios; ios = pio_get_iosystem_from_id(iosysid); if(ios == NULL) return PIO_EBADID; if(ios->comp_comm == MPI_COMM_NULL && ios->io_comm == MPI_COMM_NULL){ *active = false; }else{ *active = true; } return PIO_NOERR; }
int PIOc_Set_IOSystem_Error_Handling(int iosysid, int method) { iosystem_desc_t *ios; int oldmethod; ios = pio_get_iosystem_from_id(iosysid); if(ios==NULL){ fprintf(stderr,"%s %d Error setting eh method\n",__FILE__,__LINE__); print_trace(stderr); return PIO_EBADID; } oldmethod = ios->error_handler; ios->error_handler = method; return(oldmethod); }
/** ** @ingroup PIO_deletefile ** @brief Delete a file ** @param iosysid : a pio system handle ** @param filename : a filename */ int PIOc_deletefile(const int iosysid, const char filename[]) { int ierr; int msg; int mpierr; int chkerr; iosystem_desc_t *ios; ierr = PIO_NOERR; ios = pio_get_iosystem_from_id(iosysid); if(ios == NULL) return PIO_EBADID; msg = 0; if(ios->async_interface && ! ios->ioproc){ if(ios->comp_rank==0) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); // mpierr = MPI_Bcast(iosysid,1, MPI_INT, ios->compmaster, ios->intercomm); } // The barriers are needed to assure that no task is trying to operate on the file while it is being deleted. if(ios->ioproc){ MPI_Barrier(ios->io_comm); #ifdef _NETCDF if(ios->io_rank==0) ierr = nc_delete(filename); #else #ifdef _PNETCDF ierr = ncmpi_delete(filename, ios->info); #endif #endif MPI_Barrier(ios->io_comm); } // Special case - always broadcast the return from the MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm); return ierr; }
int PIOc_finalize(const int iosysid) { iosystem_desc_t *ios, *nios; ios = pio_get_iosystem_from_id(iosysid); if(ios == NULL) return PIO_EBADID; /* FIXME: The memory for ioranks is allocated in C only for intracomms * Remove this check once mem allocs for ioranks completely moves to the * C code */ if(ios->intercomm == MPI_COMM_NULL){ if(ios->ioranks != NULL){ free(ios->ioranks); } } free_cn_buffer_pool(*ios); /* Free the MPI groups. */ MPI_Group_free(&(ios->compgroup)); MPI_Group_free(&(ios->iogroup)); /* Free the MPI communicators. */ if(ios->intercomm != MPI_COMM_NULL){ MPI_Comm_free(&(ios->intercomm)); } if(ios->io_comm != MPI_COMM_NULL){ MPI_Comm_free(&(ios->io_comm)); } if(ios->comp_comm != MPI_COMM_NULL){ MPI_Comm_free(&(ios->comp_comm)); } if(ios->union_comm != MPI_COMM_NULL){ MPI_Comm_free(&(ios->union_comm)); } return pio_delete_iosystem_from_list(iosysid); }
/** * Test the decomp read/write functionality. * * @param iosysid the IO system ID. * @param ioid the ID of the decomposition. * @param num_flavors the number of IOTYPES available in this build. * @param flavor array of available iotypes. * @param my_rank rank of this task. * @param rearranger the rearranger to use (PIO_REARR_BOX or * PIO_REARR_SUBSET). * @param test_comm the MPI communicator for this test. * @returns 0 for success, error code otherwise. */ int test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank, int rearranger, MPI_Comm test_comm) { char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */ int ioid2; /* ID for decomposition we will create from file. */ char title_in[PIO_MAX_NAME + 1]; /* Optional title. */ char history_in[PIO_MAX_NAME + 1]; /* Optional history. */ int fortran_order_in; /* Indicates fortran vs. c order. */ int ret; /* Return code. */ /* Use PIO to create the decomp file in each of the four * available ways. */ for (int fmt = 0; fmt < num_flavors; fmt++) { /* Create the filename. */ sprintf(filename, "decomp_%s_iotype_%d.nc", TEST_NAME, flavor[fmt]); if ((ret = PIOc_write_nc_decomp(iosysid, filename, 0, ioid, NULL, NULL, 0))) return ret; /* Read the data. */ if ((ret = PIOc_read_nc_decomp(iosysid, filename, &ioid2, test_comm, PIO_INT, title_in, history_in, &fortran_order_in))) return ret; /* Check the results. */ { iosystem_desc_t *ios; io_desc_t *iodesc; /* Get the IO system info. */ if (!(ios = pio_get_iosystem_from_id(iosysid))) return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__); /* Get the IO desc, which describes the decomposition. */ if (!(iodesc = pio_get_iodesc_from_id(ioid2))) return pio_err(ios, NULL, PIO_EBADID, __FILE__, __LINE__); if (iodesc->ioid != ioid2 || iodesc->maplen != EXPECTED_MAPLEN || iodesc->ndims != NDIM3 || iodesc->ndof != EXPECTED_MAPLEN) return ERR_WRONG; if (iodesc->rearranger != rearranger || iodesc->maxregions != 1 || iodesc->needsfill || iodesc->mpitype != MPI_INT) return ERR_WRONG; for (int e = 0; e < iodesc->maplen; e++) if (iodesc->map[e] != my_rank * iodesc->maplen + e + 1) return ERR_WRONG; if (iodesc->dimlen[0] != X_DIM_LEN || iodesc->dimlen[1] != Y_DIM_LEN || iodesc->dimlen[2] != Z_DIM_LEN) return ERR_WRONG; if (rearranger == PIO_REARR_SUBSET) { if (iodesc->nrecvs != 1 || iodesc->num_aiotasks != TARGET_NTASKS) return ERR_WRONG; } else { /* I haven't figured out yet what these should be for * the box rearranger. */ /* printf("iodesc->nrecv = %d iodesc->num_aiotasks = %d\n", iodesc->nrecvs, */ /* iodesc->num_aiotasks); */ } } /* Free the PIO decomposition. */ if ((ret = PIOc_freedecomp(iosysid, ioid2))) ERR(ret); } return PIO_NOERR; }
int PIOc_createfile(const int iosysid, int *ncidp, int *iotype, const char filename[], const int mode) { int ierr; int msg; int mpierr; size_t len; iosystem_desc_t *ios; file_desc_t *file; ierr = PIO_NOERR; ios = pio_get_iosystem_from_id(iosysid); file = (file_desc_t *) malloc(sizeof(file_desc_t)); file->next = NULL; file->iosystem = ios; file->iotype = *iotype; file->buffer.validvars=0; file->buffer.data=NULL; file->buffer.next=NULL; file->buffer.vid=NULL; file->buffer.ioid=-1; file->buffer.frame=NULL; file->buffer.fillvalue=NULL; for(int i=0; i<PIO_MAX_VARS;i++){ file->varlist[i].record = -1; file->varlist[i].ndims = -1; #ifdef _PNETCDF file->varlist[i].request = NULL; file->varlist[i].nreqs=0; #endif file->varlist[i].fillbuf = NULL; file->varlist[i].iobuf = NULL; } msg = PIO_MSG_CREATE_FILE; file->mode = mode; if(ios->async_interface && ! ios->ioproc){ if(ios->comp_rank==0) mpierr = MPI_Send( &msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); len = strlen(filename); mpierr = MPI_Bcast((void *) filename,len, MPI_CHAR, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast(&(file->iotype), 1, MPI_INT, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->compmaster, ios->intercomm); } if(ios->ioproc){ switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: // The 64 bit options are not compatable with hdf5 format files // printf("%d %d %d %d %d \n",__LINE__,file->mode,PIO_64BIT_DATA, PIO_64BIT_OFFSET, NC_MPIIO); file->mode = file->mode | NC_MPIIO | NC_NETCDF4; //printf("%s %d %d %d\n",__FILE__,__LINE__,file->mode, NC_MPIIO| NC_NETCDF4); ierr = nc_create_par(filename, file->mode, ios->io_comm,ios->info , &(file->fh)); break; case PIO_IOTYPE_NETCDF4C: file->mode = file->mode | NC_NETCDF4; #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ ierr = nc_create(filename, file->mode, &(file->fh)); } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: ierr = ncmpi_create(ios->io_comm, filename, file->mode, ios->info, &(file->fh)); if(ierr == PIO_NOERR){ if(ios->io_rank==0){ printf("%d Setting IO buffer size on all iotasks to %ld\n",ios->io_rank,PIO_BUFFER_SIZE_LIMIT); } int oldfill; ierr = ncmpi_buffer_attach(file->fh, PIO_BUFFER_SIZE_LIMIT ); // ierr = ncmpi_set_fill(file->fh, NC_FILL, &oldfill); } break; #endif default: ierr = iotype_error(file->iotype,__FILE__,__LINE__); } } ierr = check_netcdf(file, ierr, __FILE__,__LINE__); if(ierr == PIO_NOERR){ mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT, ios->ioroot, ios->union_comm); file->mode = file->mode | PIO_WRITE; // This flag is implied by netcdf create functions but we need to know if its set pio_add_to_file_list(file); *ncidp = file->fh; } if(ios->io_rank==0){ printf("Create file %s %d\n",filename,file->fh); //,file->fh,file->id,ios->io_rank,ierr); // if(file->fh==5) print_trace(stdout); } return ierr; }
/** * @ingroup PIO_def_var * Get current file chunk cache settings from HDF5. * * This function has no effect on netCDF classic files. Calling this * function with iotype of PIO_IOTYPE_PNETCDF or PIO_IOTYPE_NETCDF * returns an error. * * The file chunk cache for HDF5 can be set, and will apply for any * files opened or created until the program ends, or the settings are * changed again. The cache settings apply only to the open file. They * do not persist with the file, and must be set each time the file is * opened, before it is opened, if they are to have effect. * * See the <a * href="http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html">netCDF * variable documentation</a> for details about the operation of this * function. * * Chunksizes have important performance repercussions. NetCDF * attempts to choose sensible chunk sizes by default, but for best * performance check chunking against access patterns. * * @param iotype the iotype of files to be created or opened. * @param sizep gets the size of file cache. * @param nelemsp gets the number of elements in file cache. * @param preemptionp gets the preemption setting for file cache. * * @return PIO_NOERR for success, otherwise an error code. */ int PIOc_get_chunk_cache(int iosysid, int iotype, PIO_Offset *sizep, PIO_Offset *nelemsp, float *preemptionp) { int ierr; int msg; int mpierr; iosystem_desc_t *ios; char *errstr; errstr = NULL; ierr = PIO_NOERR; ios = pio_get_iosystem_from_id(iosysid); if(ios == NULL) return PIO_EBADID; /* Since this is a property of the running HDF5 instance, not the * file, it's not clear if this message passing will apply. For * now, comment it out. EJH */ /* msg = PIO_MSG_INQ_VAR_FLETCHER32; */ /* if (ios->async_interface && ! ios->ioproc){ */ /* if (ios->compmaster) */ /* mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); */ /* mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); */ /* } */ switch (iotype) { #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: ierr = nc_get_chunk_cache((size_t *)sizep, (size_t *)nelemsp, preemptionp); break; case PIO_IOTYPE_NETCDF4C: if (!ios->io_rank) ierr = nc_get_chunk_cache((size_t *)sizep, (size_t *)nelemsp, preemptionp); break; #endif case PIO_IOTYPE_NETCDF: ierr = PIO_ENOTNC4; break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: ierr = PIO_ENOTNC4; break; #endif default: ierr = iotype_error(iotype,__FILE__,__LINE__); } /* Check for netCDF error. */ MPI_Bcast(&ierr, 1, MPI_INTEGER, ios->ioroot, ios->my_comm); if (!ierr) { if (sizep) if ((ierr = MPI_Bcast(sizep, 1, MPI_OFFSET, ios->ioroot, ios->my_comm))) ierr = PIO_EIO; if (nelemsp && !ierr) if ((ierr = MPI_Bcast(nelemsp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm))) ierr = PIO_EIO; if (preemptionp && !ierr) if ((ierr = MPI_Bcast(preemptionp, 1, MPI_FLOAT, ios->ioroot, ios->my_comm))) ierr = PIO_EIO; } return ierr; }
/** @ingroup PIO_finalize * Clean up internal data structures, free MPI resources, and exit the * pio library. * * @param iosysid: the io system ID provided by PIOc_Init_Intracomm(). * * @returns 0 for success or non-zero for error. */ int PIOc_finalize(const int iosysid) { iosystem_desc_t *ios, *nios; int mpierr = MPI_SUCCESS, mpierr2; /* Return code from MPI function codes. */ int ierr = PIO_NOERR; LOG((1, "PIOc_finalize iosysid = %d MPI_COMM_NULL = %d", iosysid, MPI_COMM_NULL)); /* Find the IO system information. */ if (!(ios = pio_get_iosystem_from_id(iosysid))) return PIO_EBADID; LOG((3, "found iosystem info comproot = %d union_comm = %d comp_idx = %d", ios->comproot, ios->union_comm, ios->comp_idx)); /* If asynch IO is in use, send the PIO_MSG_EXIT message from the * comp master to the IO processes. This may be called by * componets for other components iosysid. So don't send unless * there is a valid union_comm. */ if (ios->async_interface && ios->union_comm != MPI_COMM_NULL) { int msg = PIO_MSG_EXIT; LOG((3, "async")); if (!ios->ioproc) { LOG((2, "sending msg = %d ioroot = %d union_comm = %d", msg, ios->ioroot, ios->union_comm)); /* Send the message to the message handler. */ if (ios->compmaster) mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm); LOG((2, "sending iosysid = %d", iosysid)); /* Send the parameters of the function call. */ if (!mpierr) mpierr = MPI_Bcast((int *)&iosysid, 1, MPI_INT, ios->compmaster, ios->intercomm); } /* Handle MPI errors. */ LOG((3, "handling async errors mpierr = %d my_comm = %d", mpierr, ios->my_comm)); if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm))) return check_mpi(NULL, mpierr2, __FILE__, __LINE__); if (mpierr) return check_mpi(NULL, mpierr, __FILE__, __LINE__); LOG((3, "async errors bcast")); } /* Free this memory that was allocated in init_intracomm. */ if (ios->ioranks) free(ios->ioranks); LOG((3, "Freed ioranks.")); /* Free the buffer pool. */ free_cn_buffer_pool(*ios); LOG((2, "Freed buffer pool.")); /* Free the MPI groups. */ if (ios->compgroup != MPI_GROUP_NULL) MPI_Group_free(&ios->compgroup); if (ios->iogroup != MPI_GROUP_NULL) { MPI_Group_free(&(ios->iogroup)); LOG((2, "Freed MPI groups.")); } /* Free the MPI communicators. my_comm is just a copy (but not an * MPI copy), so does not have to have an MPI_Comm_free() * call. comp_comm and io_comm are MPI duplicates of the comms * handed into init_intercomm. So they need to be freed by MPI. */ if (ios->intercomm != MPI_COMM_NULL) { LOG((3, "freeing intercomm %d", ios->intercomm)); MPI_Comm_free(&ios->intercomm); } if (ios->union_comm != MPI_COMM_NULL) { LOG((3, "freeing union_comm %d", ios->union_comm)); MPI_Comm_free(&ios->union_comm); } if (ios->io_comm != MPI_COMM_NULL) { LOG((3, "freeing io_comm %d", ios->io_comm)); MPI_Comm_free(&ios->io_comm); } if (ios->comp_comm != MPI_COMM_NULL) { LOG((3, "freeing comp_comm %d", ios->comp_comm)); MPI_Comm_free(&ios->comp_comm); } if (ios->my_comm != MPI_COMM_NULL) ios->my_comm = MPI_COMM_NULL; /* Delete the iosystem_desc_t data associated with this id. */ LOG((2, "About to delete iosysid %d.", iosysid)); ierr = pio_delete_iosystem_from_list(iosysid); LOG((2, "Deleted iosysid %d ierr = %d", iosysid, ierr)); LOG((2, "PIOc_finalize completed successfully")); return ierr; }
int PIOc_openfile(const int iosysid, int *ncidp, int *iotype, const char filename[], const int mode) { int ierr; int msg; int mpierr; size_t len; iosystem_desc_t *ios; file_desc_t *file; ierr = PIO_NOERR; msg = PIO_MSG_OPEN_FILE; ios = pio_get_iosystem_from_id(iosysid); if(ios==NULL){ printf("bad iosysid %d\n",iosysid); return PIO_EBADID; } file = (file_desc_t *) malloc(sizeof(*file)); if(file==NULL){ return PIO_ENOMEM; } file->iotype = *iotype; file->next = NULL; file->iosystem = ios; file->mode = mode; for(int i=0; i<PIO_MAX_VARS;i++){ file->varlist[i].record = -1; file->varlist[i].ndims = -1; #ifdef _PNETCDF file->varlist[i].request = NULL; file->varlist[i].nreqs=0; #endif file->varlist[i].fillbuf = NULL; file->varlist[i].iobuf = NULL; } file->buffer.validvars=0; file->buffer.vid=NULL; file->buffer.data=NULL; file->buffer.next=NULL; file->buffer.frame=NULL; file->buffer.fillvalue=NULL; if(ios->async_interface && ! ios->ioproc){ if(ios->comp_rank==0) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); len = strlen(filename); mpierr = MPI_Bcast((void *) filename,len, MPI_CHAR, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast(&(file->iotype), 1, MPI_INT, ios->compmaster, ios->intercomm); mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT, ios->compmaster, ios->intercomm); } if(ios->ioproc){ switch(file->iotype){ #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: #ifdef _MPISERIAL ierr = nc_open(filename, file->mode, &(file->fh)); #else file->mode = file->mode | NC_MPIIO; ierr = nc_open_par(filename, file->mode, ios->io_comm,ios->info, &(file->fh)); #endif break; case PIO_IOTYPE_NETCDF4C: file->mode = file->mode | NC_NETCDF4; // *** Note the INTENTIONAL FALLTHROUGH *** #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ ierr = nc_open(filename, file->mode, &(file->fh)); } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: ierr = ncmpi_open(ios->io_comm, filename, file->mode, ios->info, &(file->fh)); // This should only be done with a file opened to append if(ierr == PIO_NOERR && (file->mode & PIO_WRITE)){ if(ios->iomaster) printf("%d Setting IO buffer %ld\n",__LINE__,PIO_BUFFER_SIZE_LIMIT); ierr = ncmpi_buffer_attach(file->fh, PIO_BUFFER_SIZE_LIMIT ); } break; #endif default: ierr = iotype_error(file->iotype,__FILE__,__LINE__); break; } // If we failed to open a file due to an incompatible type of NetCDF, try it // once with just plain old basic NetCDF #ifdef _NETCDF if(ierr == NC_ENOTNC && (file->iotype != PIO_IOTYPE_NETCDF)) { if(ios->iomaster) printf("PIO2 pio_file.c retry NETCDF\n"); // reset ierr on all tasks ierr = PIO_NOERR; // reset file markers for NETCDF on all tasks file->iotype = PIO_IOTYPE_NETCDF; // open netcdf file serially on main task if(ios->io_rank==0){ ierr = nc_open(filename, file->mode, &(file->fh)); } } #endif } ierr = check_netcdf(file, ierr, __FILE__,__LINE__); if(ierr==PIO_NOERR){ mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT, ios->ioroot, ios->union_comm); pio_add_to_file_list(file); *ncidp = file->fh; } if(ios->io_rank==0){ printf("Open file %s %d\n",filename,file->fh); //,file->fh,file->id,ios->io_rank,ierr); // if(file->fh==5) print_trace(stdout); } return ierr; }
int PIOc_InitDecomp(const int iosysid, const int basetype,const int ndims, const int dims[], const int maplen, const PIO_Offset *compmap, int *ioidp,const int *rearranger, const PIO_Offset *iostart,const PIO_Offset *iocount) { iosystem_desc_t *ios; io_desc_t *iodesc; int mpierr; int ierr; int iosize; int ndisp; for(int i=0;i<ndims;i++){ if(dims[i]<=0){ piodie("Invalid dims argument",__FILE__,__LINE__); } } ios = pio_get_iosystem_from_id(iosysid); if(ios == NULL) return PIO_EBADID; if(PIO_Save_Decomps){ char filename[30]; if(ios->num_comptasks < 100) { sprintf(filename, "piodecomp%2.2dtasks%2.2ddims%2.2d.dat",ios->num_comptasks,ndims,counter); }else if(ios->num_comptasks < 10000) { sprintf(filename, "piodecomp%4.4dtasks%2.2ddims%2.2d.dat",ios->num_comptasks,ndims,counter); }else{ sprintf(filename, "piodecomp%6.6dtasks%2.2ddims%2.2d.dat",ios->num_comptasks,ndims,counter); } PIOc_writemap(filename,ndims,dims,maplen,compmap,ios->comp_comm); counter++; } iodesc = malloc_iodesc(basetype, ndims); if(rearranger == NULL) iodesc->rearranger = ios->default_rearranger; else iodesc->rearranger = *rearranger; if(iodesc->rearranger==PIO_REARR_SUBSET){ if((iostart != NULL) && (iocount != NULL)){ fprintf(stderr,"%s %s\n","Iostart and iocount arguments to PIOc_InitDecomp", "are incompatable with subset rearrange method and will be ignored"); } iodesc->num_aiotasks = ios->num_iotasks; ierr = subset_rearrange_create( *ios, maplen, compmap, dims, ndims, iodesc); }else{ if(ios->ioproc){ // Unless the user specifies the start and count for each IO task compute it. if((iostart != NULL) && (iocount != NULL)){ // printf("iocount[0] = %ld %ld\n",iocount[0], iocount); iodesc->maxiobuflen=1; for(int i=0;i<ndims;i++){ iodesc->firstregion->start[i] = iostart[i]; iodesc->firstregion->count[i] = iocount[i]; compute_maxIObuffersize(ios->io_comm, iodesc); } iodesc->num_aiotasks = ios->num_iotasks; }else{ iodesc->num_aiotasks = CalcStartandCount(basetype, ndims, dims, ios->num_iotasks, ios->io_rank, iodesc->firstregion->start, iodesc->firstregion->count); } compute_maxIObuffersize(ios->io_comm, iodesc); } // Depending on array size and io-blocksize the actual number of io tasks used may vary CheckMPIReturn(MPI_Bcast(&(iodesc->num_aiotasks), 1, MPI_INT, ios->ioroot, ios->my_comm),__FILE__,__LINE__); // Compute the communications pattern for this decomposition if(iodesc->rearranger==PIO_REARR_BOX){ ierr = box_rearrange_create( *ios, maplen, compmap, dims, ndims, iodesc); } /* if(ios->ioproc){ io_region *ioregion = iodesc->firstregion; while(ioregion != NULL){ for(int i=0;i<ndims;i++) printf("%s %d i %d dim %d start %ld count %ld\n",__FILE__,__LINE__,i,dims[i],ioregion->start[i],ioregion->count[i]); ioregion = ioregion->next; } } */ } *ioidp = pio_add_to_iodesc_list(iodesc); performance_tune_rearranger(*ios, iodesc); return PIO_NOERR; }