int ne_test_gem(int fileid)
{
int iproc, j, error, j1=0;
int elem_mapi[NINTE], elem_mapb[NBORE];
/*-----------------------------Execution Begins-----------------------------*/
for(iproc=0; iproc < NPROCF; iproc++) {
error = ex_get_processor_elem_maps(fileid, elem_mapi, elem_mapb, iproc);
if (error < 0) return error;
for(j=0; j < NINTE; j++) {
if (elem_mapi[j] != j1++) return -1;
}
for(j=0; j < NBORE; j++) {
if (elem_mapb[j] != j1++) return -1;
}
j1 = 0;
}
return 0;
}
int ne_get_elem_map(int neid, /* NetCDF/Exodus file ID */
void_int *elem_mapi, /* Internal element IDs */
void_int *elem_mapb, /* Border element IDs */
int processor /* Processor ID */
)
{
return ex_get_processor_elem_maps(neid, elem_mapi, elem_mapb, processor);
}
void NemSpread<T,INT>::load_lb_info(void)
/*
* load_lb_info:
*
* This function reads and distributes the load balance information.
* This information is defined as the following scalars, which are specific
* to each processor:
*
* Num_Internal_Nodes
* Num_Border_Nodes
* Num_External_Nodes
* globals.Num_Internal_Elems
* globals.Num_Border_Elems
*
* and the following vectors, also specific to each processor:
*
* globals.GNodes [Num_Internal_Nodes+Num_Border_Nodes+Num_External_Nodes]
* globals.GElems [globals.Num_Internal_Elems+globals.Num_Border_Elems]
*
* For the 1 processor case, this routine fills in appropriate values
* for these quantities.
*/
{
int lb_exoid=0;
INT cmap_max_size=0, *comm_vec;
const char *yo = "load_lb_info";
char Title[MAX_LINE_LENGTH+1];
float version;
int cpu_ws=0;
/******************************** START EXECUTION ****************************/
if(Debug_Flag)
printf ("\nStart to read in and distribute the load balance info\n");
/* Open the Load Balance exoII file for reading */
printf ("EXODUS II load-balance file: %s\n", Exo_LB_File);
cpu_ws = io_ws;
int mode = EX_READ | int64api;
int iio_ws = 0; // Don't interfere with exodus files; this is the nemesis file.
if((lb_exoid = ex_open(Exo_LB_File, mode, &cpu_ws,
&iio_ws, &version)) == -1) {
fprintf(stderr, "%sERROR: Couldn\'t open lb file, %s\n", yo,
Exo_LB_File);
exit(1);
}
/* Read information about the processor configuration */
read_proc_init(lb_exoid,Proc_Info, &Proc_Ids);
/* Allocate space for the counts */
globals.Num_Internal_Nodes = (INT *)array_alloc(__FILE__, __LINE__, 1,
7*Proc_Info[2], sizeof(INT));
globals.Num_Border_Nodes = globals.Num_Internal_Nodes +Proc_Info[2];
globals.Num_External_Nodes = globals.Num_Border_Nodes +Proc_Info[2];
globals.Num_Internal_Elems = globals.Num_External_Nodes +Proc_Info[2];
globals.Num_Border_Elems = globals.Num_Internal_Elems +Proc_Info[2];
globals.Num_N_Comm_Maps = globals.Num_Border_Elems +Proc_Info[2];
globals.Num_E_Comm_Maps = globals.Num_N_Comm_Maps +Proc_Info[2];
/* Allocate space for each processor entity */
globals.GNodes = (INT **)array_alloc(__FILE__, __LINE__, 1, 3*Proc_Info[2],
sizeof(INT *));
globals.GElems = globals.GNodes +Proc_Info[2];
globals.Elem_Map = globals.GElems +Proc_Info[2];
/* Allocate contiguous space for the pointer vectors on all processors */
INT *Int_Space = (INT *)array_alloc(__FILE__, __LINE__, 1,
(7*Proc_Info[0] + 1), sizeof(INT));
INT *Int_Node_Num = Int_Space + 1;
INT *Bor_Node_Num = Int_Node_Num +Proc_Info[0];
INT *Ext_Node_Num = Bor_Node_Num +Proc_Info[0];
INT *Int_Elem_Num = Ext_Node_Num +Proc_Info[0];
INT *Bor_Elem_Num = Int_Elem_Num +Proc_Info[0];
INT *Node_Comm_Num = Bor_Elem_Num +Proc_Info[0];
INT *Elem_Comm_Num = Node_Comm_Num +Proc_Info[0];
/* Read the initial information contained in the load balance file */
read_lb_init(lb_exoid, Int_Space, Int_Node_Num,
Bor_Node_Num, Ext_Node_Num, Int_Elem_Num,
Bor_Elem_Num, Node_Comm_Num, Elem_Comm_Num,
Title);
/* Allocate memory for the communication map arrays */
globals.N_Comm_Map = (NODE_COMM_MAP<INT>**)malloc(Proc_Info[2]*sizeof(NODE_COMM_MAP<INT> *));
globals.E_Comm_Map = (ELEM_COMM_MAP<INT>**)malloc(Proc_Info[2]*sizeof(ELEM_COMM_MAP<INT> *));
if(!globals.N_Comm_Map || !globals.E_Comm_Map) {
fprintf(stderr, "ERROR: Insufficient memory!\n");
exit(1);
}
for (int iproc=0; iproc <Proc_Info[2]; iproc++) {
/*
* Error check:
* Currently a maximum of one nodal communication map and one
* elemental communication map is supported.
*/
if(globals.Num_N_Comm_Maps[iproc] > 1 || globals.Num_E_Comm_Maps[iproc] > 1) {
fprintf(stderr, "%s: ERROR. Only 1 nodal and elemental comm map "
"is supported\n", yo);
exit(1);
}
else {
/* Always allocate at least one and initialize the counts to 0 */
globals.N_Comm_Map[iproc] = (NODE_COMM_MAP<INT> *)malloc(PEX_MAX(1,
globals.Num_N_Comm_Maps[iproc]) *
sizeof(NODE_COMM_MAP<INT>));
if(globals.N_Comm_Map[iproc] == NULL && globals.Num_N_Comm_Maps[iproc] > 0) {
fprintf(stderr,
"%s: ERROR. Insufficient memory for nodal comm. map!\n",
yo);
exit(1);
}
for(size_t ijump=0; ijump < PEX_MAX(1, globals.Num_N_Comm_Maps[iproc]); ijump++)
((globals.N_Comm_Map[iproc])+ijump)->node_cnt = 0;
globals.E_Comm_Map[iproc] = (ELEM_COMM_MAP<INT> *)malloc(PEX_MAX(1,
globals.Num_E_Comm_Maps[iproc]) *
sizeof(ELEM_COMM_MAP<INT>));
if(globals.E_Comm_Map[iproc] == NULL && globals.Num_E_Comm_Maps[iproc] > 0) {
fprintf(stderr,
"%s: ERROR. Insufficient memory for elemental comm. map!\n",
yo);
exit(1);
}
for(size_t ijump=0; ijump < PEX_MAX(1, globals.Num_E_Comm_Maps[iproc]); ijump++)
((globals.E_Comm_Map[iproc])+ijump)->elem_cnt = 0;
}
} /* End "for (int iproc=0; iproc <Proc_Info[2]; iproc++)" */
/* Set up each processor for the communication map parameters */
read_cmap_params(lb_exoid, Node_Comm_Num, Elem_Comm_Num,
globals.Num_N_Comm_Maps, globals.Num_E_Comm_Maps,
globals.E_Comm_Map, globals.N_Comm_Map, &cmap_max_size,
&comm_vec);
/* Allocate enough space to read the LB_data for one processor */
INT *Integer_Vector = (INT *)array_alloc(__FILE__, __LINE__, 1,
Int_Space[0] + cmap_max_size,
sizeof (INT));
/*
* loop through the processors, one at a time, to read
* their load balance information
*
* NOTE: From here on there are no provisions for multiple nodal
* or elemental communication maps.
*/
size_t ijump = 0; /* keep track of where in comm_vec we are */
for (int iproc = 0; iproc <Proc_Info[0]; iproc++) {
/* Get the node map for processor "iproc" */
if(ex_get_processor_node_maps(lb_exoid, &Integer_Vector[0],
&Integer_Vector[Int_Node_Num[iproc]],
&Integer_Vector[Int_Node_Num[iproc]+
Bor_Node_Num[iproc]],
iproc) < 0) {
fprintf(stderr, "%s: ERROR, failed to get node map for Proc %d!\n",
yo, iproc);
exit(1);
}
size_t vec_indx = Int_Node_Num[iproc] + Bor_Node_Num[iproc] +
Ext_Node_Num[iproc];
/* Get the element map for processor number "iproc" */
if(ex_get_processor_elem_maps(lb_exoid, &Integer_Vector[vec_indx],
&Integer_Vector[vec_indx+Int_Elem_Num[iproc]],
iproc) < 0) {
fprintf(stderr, "%s: ERROR, failed to get element map for Proc %d!\n",
yo, iproc);
exit(1);
}
if(Node_Comm_Num[iproc] > 0) {
vec_indx += Int_Elem_Num[iproc] + Bor_Elem_Num[iproc];
if(ex_get_node_cmap(lb_exoid, comm_vec[ijump],
&Integer_Vector[vec_indx],
&Integer_Vector[vec_indx+comm_vec[ijump+1]],
iproc) < 0) {
/*
* If there are disconnected mesh pieces, then it is
* possible that there is no comminication between the
* pieces and there will be no communication maps. Normally
* this is a problem, so output a warning, but don't abort.
*/
fprintf(stderr,
"%s: WARNING. Failed to get nodal comm map for Proc %d!\n",
yo, iproc);
}
}
if(Elem_Comm_Num[iproc] > 0) {
vec_indx += 2*comm_vec[ijump+1];
if(ex_get_elem_cmap(lb_exoid, comm_vec[ijump+2],
&Integer_Vector[vec_indx],
&Integer_Vector[vec_indx+comm_vec[ijump+3]],
&Integer_Vector[vec_indx+2*comm_vec[ijump+3]],
iproc) < 0) {
fprintf(stderr,
"%s: ERROR. Failed to get elemental comm map for Proc %d!\n",
yo, iproc);
exit(1);
}
}
/*
* Communicate load balance information to the correct processor
* - if iproc = Proc_Ids[*] then process the data instead.
*/
assert(Proc_Ids[iproc] == iproc);
process_lb_data (Integer_Vector, iproc);
/*
* now move ijump to the next communications map
* make sure to check if there are any for this processor
*/
if (Node_Comm_Num[iproc] > 0) ijump += 2;
if (Elem_Comm_Num[iproc] > 0) ijump += 2;
}
/* Close the load balance file - we are finished with it */
if(ex_close (lb_exoid) == -1) {
fprintf (stderr, "%sERROR: Error in closing load balance file\n", yo);
exit(1);
}
/************************* Cleanup and Printout Phase ***********************/
/* Free temporary memory */
safe_free((void **) &Integer_Vector);
if(num_qa_rec > 0) {
for(int i = 0; i < length_qa; i++) safe_free((void **) &(qa_record_ptr[i]));
safe_free((void **) &qa_record_ptr);
}
if(num_inf_rec > 0) {
for(int i = 0; i < num_inf_rec; i++) safe_free((void **) &(inf_record_ptr[i]));
safe_free((void **) &inf_record_ptr);
}
safe_free((void **) &Int_Space);
safe_free((void **) &comm_vec);
for (int iproc=0; iproc <Proc_Info[2]; iproc++) {
if (globals.Num_Internal_Nodes[iproc] == 0 &&
globals.Num_Border_Nodes[iproc] == 0 &&
globals.Num_External_Nodes[iproc] == 0) {
fprintf(stderr, "\n%s: WARNING, Processor %d has no nodes!\n",
yo, iproc);
}
if (globals.Num_Internal_Elems[iproc] == 0 &&
globals.Num_Border_Elems[iproc] == 0) {
fprintf(stderr, "\n%s: WARNING, Processor %d has no elements!\n",
yo, iproc);
}
}
/*========================================================================*/
if(Debug_Flag)
printf ("\nFinished distributing load balance info\n");
/* Output Detailed timing information for the progam */
/*
* Print out a Large table of Load Balance Information if the debug_flag
* setting is large enough
*/
if(Debug_Flag >= 7) {
printf ("\n\n");
print_line ("=", 79);
for (int iproc=0; iproc <Proc_Info[2]; iproc++) {
printf("\n\t***For Processor %d***\n", Proc_Ids[iproc]);
printf("\tInternal nodes owned by the current processor\n\t");
for(INT i = 0; i < globals.Num_Internal_Nodes[iproc]; i++)
printf(" " ST_ZU "", (size_t)globals.GNodes[iproc][i]);
printf("\n");
printf("\tBorder nodes owned by the current processor\n\t");
for(INT i = 0; i < globals.Num_Border_Nodes[iproc]; i++)
printf(" " ST_ZU "", (size_t)globals.GNodes[iproc][i + globals.Num_Internal_Nodes[iproc]]);
printf("\n");
if(globals.Num_External_Nodes[iproc] > 0) {
printf("\tExternal nodes needed by the current processor\n\t");
for(INT i = 0; i < globals.Num_External_Nodes[iproc]; i++)
printf(" " ST_ZU "", (size_t)globals.GNodes[iproc][i + globals.Num_Internal_Nodes[iproc] +
globals.Num_Border_Nodes[iproc]]);
printf("\n");
}
printf("\tInternal elements owned by the current processor\n\t");
for(INT i = 0; i < globals.Num_Internal_Elems[iproc]; i++)
printf(" " ST_ZU "", (size_t)globals.GElems[iproc][i]);
printf("\n");
if(globals.Num_Border_Elems[iproc] > 0) {
printf("\tBorder elements owned by the current processor\n\t");
for(INT i=0; i < globals.Num_Border_Elems[iproc]; i++)
printf(" " ST_ZU "", (size_t)globals.GElems[iproc][i+globals.Num_Internal_Elems[iproc]]);
printf("\n");
}
if(globals.Num_N_Comm_Maps[iproc] > 0) {
printf("\tNodal Comm Map for the current processor\n");
printf("\t\tnode IDs:");
for(size_t i=0; i < globals.N_Comm_Map[iproc]->node_cnt; i++)
printf(" " ST_ZU "", (size_t)globals.N_Comm_Map[iproc]->node_ids[i]);
printf("\n\t\tproc IDs:");
for(size_t i=0; i < globals.N_Comm_Map[iproc]->node_cnt; i++)
printf(" " ST_ZU "", (size_t)globals.N_Comm_Map[iproc]->proc_ids[i]);
printf("\n");
}
if(globals.Num_E_Comm_Maps[iproc] > 0) {
printf("\tElemental Comm Map for the current processor\n");
printf("\t\telement IDs:");
for(size_t i=0; i < globals.E_Comm_Map[iproc]->elem_cnt; i++)
printf(" " ST_ZU "", (size_t)globals.E_Comm_Map[iproc]->elem_ids[i]);
printf("\n\t\tside IDs:");
for(size_t i=0; i < globals.E_Comm_Map[iproc]->elem_cnt; i++)
printf(" " ST_ZU "", (size_t)globals.E_Comm_Map[iproc]->side_ids[i]);
printf("\n\t\tproc IDs:");
for(size_t i=0; i < globals.E_Comm_Map[iproc]->elem_cnt; i++)
printf(" " ST_ZU "", (size_t)globals.E_Comm_Map[iproc]->proc_ids[i]);
printf("\n");
}
}
printf("\n");
print_line ("=", 79);
}
} /* END of routine load_lb_info () ******************************************/
