bool Excn::ExodusFile::initialize(const SystemInterface& si, int start_part, int part_count)
{
processorCount_ = si.processor_count(); // Total number processors
partCount_ = part_count; // Total parts we are processing
startPart_ = start_part; // Which one to start with
SMART_ASSERT(partCount_ + startPart_ <= processorCount_)(partCount_)(startPart_)(processorCount_);
// EPU always wants entity (block, set, map) ids as 64-bit quantities...
mode64bit_ = EX_IDS_INT64_API;
if (si.int64()) {
mode64bit_ |= EX_ALL_INT64_API;
// For output...
mode64bit_ |= EX_ALL_INT64_DB;
}
// See if we can keep files open
int max_files = get_free_descriptor_count();
if (partCount_ <= max_files) {
keepOpen_ = true;
if (si.debug() & 1)
std::cout << "Files kept open... (Max open = " << max_files << ")\n\n";
} else {
keepOpen_ = false;
std::cout << "Single file mode... (Max open = " << max_files << ")\n"
<< "Consider using the -subcycle option for faster execution...\n\n";
}
fileids_.resize(processorCount_);
filenames_.resize(processorCount_);
std::string curdir = si.cwd();
std::string file_prefix = si.basename();
std::string exodus_suffix = si.exodus_suffix();
std::string root_dir = si.root_dir();
std::string sub_dir = si.sub_dir();
ParallelDisks disks;
disks.Raid_Offset(si.raid_offset());
disks.Number_of_Raids(si.raid_count());
float version = 0.0;
// create exo names
for(int p = 0; p < partCount_; p++) {
std::string name = file_prefix;
if (!exodus_suffix.empty()) {
name += "." + exodus_suffix;
}
int proc = p + startPart_;
disks.rename_file_for_mp(root_dir, sub_dir, name, proc,
processorCount_);
filenames_[p] = name;
if (p == 0) {
int cpu_word_size = sizeof(float);
int io_word_size_var = 0;
int mode = EX_READ;
mode |= mode64bit_;
int exoid = ex_open(filenames_[p].c_str(),
mode, &cpu_word_size,
&io_word_size_var, &version);
if (exoid < 0) {
std::cerr << "Cannot open file '" << filenames_[p] << "'" << std::endl;
return false;
}
int int64db = ex_int64_status(exoid) & EX_ALL_INT64_DB;
if (int64db) {
// If anything stored on input db as 64-bit int, then output db will have
// everything stored as 64-bit ints and all API functions will use 64-bit
mode64bit_ |= EX_ALL_INT64_API;
mode64bit_ |= EX_ALL_INT64_DB;
}
int max_name_length = ex_inquire_int(exoid, EX_INQ_DB_MAX_USED_NAME_LENGTH);
if (max_name_length > maximumNameLength_)
maximumNameLength_ = max_name_length;
ex_close(exoid);
if (io_word_size_var < (int)sizeof(float))
io_word_size_var = sizeof(float);
ioWordSize_ = io_word_size_var;
cpuWordSize_ = io_word_size_var;
}
if (keepOpen_ || p == 0) {
int io_word_size_var = 0;
int mode = EX_READ;
// All entity ids (block, sets) are read/written as 64-bit...
mode |= mode64bit_;
fileids_[p] = ex_open(filenames_[p].c_str(),
mode, &cpuWordSize_,
&io_word_size_var, &version);
if (fileids_[p] < 0) {
std::cerr << "Cannot open file '" << filenames_[p] << "'" << std::endl;
return false;
}
ex_set_max_name_length(fileids_[p], maximumNameLength_);
SMART_ASSERT(ioWordSize_ == io_word_size_var)(ioWordSize_)(io_word_size_var);
}
if (si.debug()&64 || p==0 || p==partCount_-1) {
std::cout << "Input(" << p << "): '" << name.c_str() << "'" << std::endl;
if (!(si.debug()&64) && p==0)
std::cout << "..." << std::endl;
}
}
if (mode64bit_ & EX_ALL_INT64_DB) {
std::cout << "Input files contain 8-byte integers.\n";
si.set_int64();
}
return true;
}
void NemSpread<T,INT>::read_restart_data ()
/* Function which reads the restart variable data from the EXODUS II
* database which contains the results information. Then distribute
* it to the processors, and write it to the parallel exodus files.
*
*----------------------------------------------------------------------------
*
* Functions called:
*
* read_vars -- function which reads the variable values from the restart
* file, and then distributes them to the processors
*
* write_var_timestep -- function which writes out the variables for a
* to a parallel ExodusII file.
*
*----------------------------------------------------------------------------
*/
{
const char *yo="read_restart_data";
/* need to get the element block ids and counts */
std::vector<INT> eb_ids_global(globals.Num_Elem_Blk);
std::vector<INT> eb_cnts_global(globals.Num_Elem_Blk);
std::vector<INT> ss_ids_global(globals.Num_Side_Set);
std::vector<INT> ss_cnts_global(globals.Num_Side_Set);
std::vector<INT> ns_ids_global(globals.Num_Node_Set);
std::vector<INT> ns_cnts_global(globals.Num_Node_Set);
INT ***eb_map_ptr = NULL, **eb_cnts_local = NULL;
int exoid=0, *par_exoid = NULL;
float vers;
char cTemp[512];
/* computing precision should be the same as the database precision
*
* EXCEPTION: if the io_ws is smaller than the machine precision,
* ie - database with io_ws == 4 on a Cray (sizeof(float) == 8),
* then the cpu_ws must be the machine precision.
*/
int cpu_ws;
if (io_ws < (int)sizeof(float)) cpu_ws = sizeof(float);
else cpu_ws = io_ws;
/* Open the ExodusII file */
{
cpu_ws = io_ws;
int mode = EX_READ | int64api;
if ((exoid=ex_open(Exo_Res_File, mode, &cpu_ws, &io_ws, &vers)) < 0) {
fprintf(stderr, "%s: Could not open file %s for restart info\n",
yo, Exo_Res_File);
exit(1);
}
}
/* allocate space for the global variables */
Restart_Info.Glob_Vals.resize(Restart_Info.NVar_Glob);
if (Restart_Info.NVar_Elem > 0 ) {
/* allocate storage space */
Restart_Info.Elem_Vals.resize(Proc_Info[2]);
/* now allocate storage for the values */
for (int iproc = 0; iproc <Proc_Info[2]; iproc++) {
size_t array_size = Restart_Info.NVar_Elem *
(globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc]);
Restart_Info.Elem_Vals[iproc].resize(array_size);
}
/*
* at this point, I need to broadcast the global element block ids
* and counts to the processors. I know that this is redundant data
* since they will all receive this information in read_mesh, but
* the variables which contain that information are static in
* el_exoII_io.c, and cannot be used here. So, take a second and
* broadcast all of this out.
*
* I want to do this here so that it is done only once no matter
* how many time steps are retrieved
*/
/* Get the Element Block IDs from the input file */
if (ex_get_ids (exoid, EX_ELEM_BLOCK, TOPTR(eb_ids_global)) < 0)
{
fprintf(stderr, "%s: unable to get element block IDs", yo);
exit(1);
}
/* Get the count of elements in each element block */
for (int cnt = 0; cnt < globals.Num_Elem_Blk; cnt++) {
if (ex_get_block(exoid, EX_ELEM_BLOCK, eb_ids_global[cnt], cTemp,
&(eb_cnts_global[cnt]), NULL, NULL, NULL, NULL) < 0) {
fprintf(stderr, "%s: unable to get element count for block id "ST_ZU"",
yo, (size_t)eb_ids_global[cnt]);
exit(1);
}
}
/*
* in order to speed up finding matches in the global element
* number map, set up an array of pointers to the start of
* each element block's global element number map. That way
* only entries for the current element block have to be searched
*/
eb_map_ptr = (INT ***) array_alloc (__FILE__, __LINE__, 2,Proc_Info[2],
globals.Num_Elem_Blk, sizeof(INT *));
if (!eb_map_ptr) {
fprintf(stderr, "[%s]: ERROR, insufficient memory!\n", yo);
exit(1);
}
eb_cnts_local = (INT **) array_alloc (__FILE__, __LINE__, 2,Proc_Info[2],
globals.Num_Elem_Blk, sizeof(INT));
if (!eb_cnts_local) {
fprintf(stderr, "[%s]: ERROR, insufficient memory!\n", yo);
exit(1);
}
/*
* for now, assume that element blocks have been
* stored in the same order as the global blocks
*/
for (int iproc = 0; iproc <Proc_Info[2]; iproc++) {
int ifound = 0;
size_t offset = 0;
int ilocal;
for (int cnt = 0; cnt < globals.Num_Elem_Blk; cnt++) {
for (ilocal = ifound; ilocal < globals.Proc_Num_Elem_Blk[iproc]; ilocal++) {
if (globals.Proc_Elem_Blk_Ids[iproc][ilocal] == eb_ids_global[cnt])
break;
}
if (ilocal < globals.Proc_Num_Elem_Blk[iproc]) {
eb_map_ptr[iproc][cnt] = &globals.GElems[iproc][offset];
eb_cnts_local[iproc][cnt] = globals.Proc_Num_Elem_In_Blk[iproc][ilocal];
offset += globals.Proc_Num_Elem_In_Blk[iproc][ilocal];
ifound = ilocal; /* don't search the same part of the list over */
}
else {
eb_map_ptr[iproc][cnt] = NULL;
eb_cnts_local[iproc][cnt] = 0;
}
}
}
} /* End: "if (Restart_Info.NVar_Elem > 0 )" */
if (Restart_Info.NVar_Node > 0 ) {
/* allocate storage space */
Restart_Info.Node_Vals.resize(Proc_Info[2]);
/* now allocate storage for the values */
for (int iproc = 0; iproc <Proc_Info[2]; iproc++) {
size_t array_size = Restart_Info.NVar_Node * (globals.Num_Internal_Nodes[iproc] +
globals.Num_Border_Nodes[iproc] + globals.Num_External_Nodes[iproc]);
Restart_Info.Node_Vals[iproc].resize(array_size);
}
}
if (Restart_Info.NVar_Sset > 0 ) {
/* allocate storage space */
Restart_Info.Sset_Vals.resize(Proc_Info[2]);
/* now allocate storage for the values */
for (int iproc = 0; iproc <Proc_Info[2]; iproc++) {
size_t array_size = Restart_Info.NVar_Sset * globals.Proc_SS_Elem_List_Length[iproc];
Restart_Info.Sset_Vals[iproc].resize(array_size);
}
/*
* at this point, I need to broadcast the ids and counts to the
* processors. I know that this is redundant data since they will
* all receive this information in read_mesh, but the variables
* which contain that information are static in el_exoII_io.c, and
* cannot be used here. So, take a second and broadcast all of
* this out.
*
* I want to do this here so that it is done only once no matter
* how many time steps are retrieved
*/
/* Get the Sideset IDs from the input file */
if (ex_get_ids (exoid, EX_SIDE_SET, TOPTR(ss_ids_global)) < 0) {
fprintf(stderr, "%s: unable to get sideset IDs", yo);
exit(1);
}
/* Get the count of elements in each sideset */
for (int cnt = 0; cnt < globals.Num_Side_Set; cnt++) {
if (ex_get_set_param(exoid, EX_SIDE_SET,
ss_ids_global[cnt],
&(ss_cnts_global[cnt]), NULL) < 0) {
fprintf(stderr, "%s: unable to get element count for sideset id "ST_ZU"",
yo, (size_t)ss_ids_global[cnt]);
exit(1);
}
}
} /* End: "if (Restart_Info.NVar_Sset > 0 )" */
if (Restart_Info.NVar_Nset > 0 ) {
/* allocate storage space */
Restart_Info.Nset_Vals.resize(Proc_Info[2]);
/* now allocate storage for the values */
for (int iproc = 0; iproc <Proc_Info[2]; iproc++) {
size_t array_size = Restart_Info.NVar_Nset * globals.Proc_NS_List_Length[iproc];
Restart_Info.Nset_Vals[iproc].resize(array_size);
}
/*
* at this point, I need to broadcast the ids and counts to the
* processors. I know that this is redundant data since they will
* all receive this information in read_mesh, but the variables
* which contain that information are static in el_exoII_io.c, and
* cannot be used here. So, take a second and broadcast all of
* this out.
*
* I want to do this here so that it is done only once no matter
* how many time steps are retrieved
*/
/* Get the Nodeset IDs from the input file */
if (ex_get_ids (exoid, EX_NODE_SET, TOPTR(ns_ids_global)) < 0) {
fprintf(stderr, "%s: unable to get nodeset IDs", yo);
exit(1);
}
/* Get the count of elements in each nodeset */
for (int cnt = 0; cnt < globals.Num_Node_Set; cnt++) {
if (ex_get_set_param(exoid, EX_NODE_SET,
ns_ids_global[cnt],
&(ns_cnts_global[cnt]), NULL) < 0) {
fprintf(stderr, "%s: unable to get element count for nodeset id "ST_ZU"",
yo, (size_t)ns_ids_global[cnt]);
exit(1);
}
}
} /* End: "if (Restart_Info.NVar_Nset > 0 )" */
/*
* NOTE: A possible place to speed this up would be to
* get the global node and element lists here, and broadcast
* them out only once.
*/
par_exoid = (int*)malloc(Proc_Info[2] * sizeof(int));
if(!par_exoid) {
fprintf(stderr, "[%s]: ERROR, insufficient memory!\n",
yo);
exit(1);
}
/* See if any '/' in the name. IF present, isolate the basename of the file */
if (strrchr(PIO_Info.Scalar_LB_File_Name, '/') != NULL) {
/* There is a path separator. Get the portion after the
* separator
*/
strcpy(cTemp, strrchr(PIO_Info.Scalar_LB_File_Name, '/')+1);
} else {
/* No separator; this is already just the basename... */
strcpy(cTemp, PIO_Info.Scalar_LB_File_Name);
}
if (strlen(PIO_Info.Exo_Extension) == 0)
add_fname_ext(cTemp, ".par");
else
add_fname_ext(cTemp, PIO_Info.Exo_Extension);
int open_file_count = get_free_descriptor_count();
if (open_file_count >Proc_Info[5]) {
printf("All output files opened simultaneously.\n");
for (int iproc=Proc_Info[4]; iproc <Proc_Info[4]+Proc_Info[5]; iproc++) {
gen_par_filename(cTemp, Par_Nem_File_Name, Proc_Ids[iproc],
Proc_Info[0]);
/* Open the parallel Exodus II file for writing */
cpu_ws = io_ws;
int mode = EX_WRITE | int64api | int64db;
if ((par_exoid[iproc]=ex_open(Par_Nem_File_Name, mode, &cpu_ws,
&io_ws, &vers)) < 0) {
fprintf(stderr,"[%d] %s Could not open parallel Exodus II file: %s\n",
iproc, yo, Par_Nem_File_Name);
exit(1);
}
}
} else {
printf("All output files opened one-at-a-time.\n");
}
/* Now loop over the number of time steps */
for (int time_idx = 0; time_idx < Restart_Info.Num_Times; time_idx++) {
double start_t = second ();
/* read and distribute the variables for this time step */
if (read_vars(exoid, Restart_Info.Time_Idx[time_idx],
TOPTR(eb_ids_global), TOPTR(eb_cnts_global), eb_map_ptr,
eb_cnts_local,
TOPTR(ss_ids_global), TOPTR(ss_cnts_global),
TOPTR(ns_ids_global), TOPTR(ns_cnts_global)) < 0) {
fprintf(stderr, "%s: Error occured while reading variables\n",
yo);
exit(1);
}
double end_t = second () - start_t;
printf ("\tTime to read vars for timestep %d: %f (sec.)\n", (time_idx+1), end_t);
start_t = second ();
for (int iproc=Proc_Info[4]; iproc <Proc_Info[4]+Proc_Info[5]; iproc++) {
if (open_file_count <Proc_Info[5]) {
gen_par_filename(cTemp, Par_Nem_File_Name, Proc_Ids[iproc],
Proc_Info[0]);
/* Open the parallel Exodus II file for writing */
cpu_ws = io_ws;
int mode = EX_WRITE | int64api | int64db;
if ((par_exoid[iproc]=ex_open(Par_Nem_File_Name, mode, &cpu_ws,
&io_ws, &vers)) < 0) {
fprintf(stderr,"[%d] %s Could not open parallel Exodus II file: %s\n",
iproc, yo, Par_Nem_File_Name);
exit(1);
}
}
/*
* Write out the variable data for the time steps in this
* block to each parallel file.
*/
write_var_timestep(par_exoid[iproc], iproc, (time_idx+1),
TOPTR(eb_ids_global), TOPTR(ss_ids_global), TOPTR(ns_ids_global));
if (iproc%10 == 0 || iproc ==Proc_Info[2]-1)
printf("%d", iproc);
else
printf(".");
if (open_file_count <Proc_Info[5]) {
if (ex_close(par_exoid[iproc]) == -1) {
fprintf(stderr, "[%d] %s Could not close the parallel Exodus II file.\n",
iproc, yo);
exit(1);
}
}
} /* End "for (iproc=0; iproc <Proc_Info[2]; iproc++)" */
end_t = second () - start_t;
printf ("\n\tTime to write vars for timestep %d: %f (sec.)\n", (time_idx+1), end_t);
}
if (Restart_Info.NVar_Elem > 0 ) {
safe_free((void **) &eb_map_ptr);
safe_free((void **) &eb_cnts_local);
}
/* Close the restart exodus II file */
if (ex_close(exoid) == -1) {
fprintf(stderr, "%sCould not close the restart Exodus II file\n",
yo);
exit(1);
}
if (open_file_count >Proc_Info[5]) {
for (int iproc=Proc_Info[4]; iproc <Proc_Info[4]+Proc_Info[5]; iproc++) {
/* Close the parallel exodus II file */
if (ex_close(par_exoid[iproc]) == -1) {
fprintf(stderr, "[%d] %s Could not close the parallel Exodus II file.\n",
iproc, yo);
exit(1);
}
}
}
if (par_exoid != NULL) {
free(par_exoid);
par_exoid = NULL;
}
}
bool Excn::ExodusFile::initialize(const SystemInterface& si)
{
// See if we can keep files open
size_t max_files = get_free_descriptor_count();
if (si.inputFiles_.size() <= max_files) {
keepOpen_ = true;
if (si.debug() & 1)
std::cout << "Files kept open... (Max open = " << max_files << ")\n\n";
} else {
keepOpen_ = false;
std::cout << "Single file mode... (Max open = " << max_files << ")\n"
<< "Consider using the -subcycle option for faster execution...\n\n";
}
float version = 0.0;
// create exo names
filenames_.resize(si.inputFiles_.size());
fileids_.resize(si.inputFiles_.size());
int int_byte_size_api = 4;
if (si.ints_64_bit())
int_byte_size_api = 8;
int overall_max_name_length = 0;
for(size_t p = 0; p < si.inputFiles_.size(); p++) {
std::string name = si.inputFiles_[p];
filenames_[p] = name;
if (p == 0) {
int cpu_word_size = sizeof(float);
int io_wrd_size = 0;
int exoid = ex_open(filenames_[p].c_str(),
EX_READ, &cpu_word_size,
&io_wrd_size, &version);
if (exoid < 0) {
std::cerr << "Cannot open file '" << filenames_[p] << "'" << std::endl;
return false;
}
int max_name_length = ex_inquire_int(exoid, EX_INQ_DB_MAX_USED_NAME_LENGTH);
if (max_name_length > overall_max_name_length)
overall_max_name_length = max_name_length;
ex_close(exoid);
if (io_wrd_size < (int)sizeof(float))
io_wrd_size = sizeof(float);
ioWordSize_ = io_wrd_size;
cpuWordSize_ = io_wrd_size;
if (ex_int64_status(exoid & EX_ALL_INT64_DB) || si.ints_64_bit()) {
exodusMode_ = EX_ALL_INT64_API;
}
}
if (keepOpen_ || p == 0) {
int io_wrd_size = 0;
int mode = EX_READ | exodusMode_;
fileids_[p] = ex_open(filenames_[p].c_str(),
mode, &cpuWordSize_,
&io_wrd_size, &version);
if (fileids_[p] < 0) {
std::cerr << "Cannot open file '" << filenames_[p] << "'" << std::endl;
return false;
}
SMART_ASSERT(ioWordSize_ == io_wrd_size)(ioWordSize_)(io_wrd_size);
}
std::cout << "Part " << p+1 << ": '" << name.c_str() << "'" << std::endl;
}
maximumNameLength_ = overall_max_name_length;
for(size_t p = 0; p < si.inputFiles_.size(); p++) {
ex_set_max_name_length(fileids_[p], maximumNameLength_);
}
return true;
}
