int ne_test_gnm(int fileid) { int iproc, j, error, j1=0; int node_mapi[NINTN], node_mapb[NBORN], node_mape[NEXTN]; /*-----------------------------Execution Begins-----------------------------*/ for(iproc=0; iproc < NPROCF; iproc++) { error = ex_get_processor_node_maps(fileid, node_mapi, node_mapb, node_mape, iproc); if (error < 0) return error; for(j=0; j < NINTN; j++) { if (node_mapi[j] != j1++) return -1; } for(j=0; j < NBORN; j++) { if (node_mapb[j] != j1++) return -1; } for(j=0; j < NEXTN; j++) { if (node_mape[j] != j1++) return -1; } j1 = 0; } return 0; }
int ne_get_node_map(int neid, /* NetCDF/Exodus file ID */ void_int *node_mapi, /* Internal FEM node IDs */ void_int *node_mapb, /* Border FEM node IDs */ void_int *node_mape, /* External FEM node IDs */ int processor /* Processor IDs */ ) { return ex_get_processor_node_maps(neid, node_mapi, node_mapb, node_mape, processor); }
void NemSpread<T,INT>::load_lb_info(void) /* * load_lb_info: * * This function reads and distributes the load balance information. * This information is defined as the following scalars, which are specific * to each processor: * * Num_Internal_Nodes * Num_Border_Nodes * Num_External_Nodes * globals.Num_Internal_Elems * globals.Num_Border_Elems * * and the following vectors, also specific to each processor: * * globals.GNodes [Num_Internal_Nodes+Num_Border_Nodes+Num_External_Nodes] * globals.GElems [globals.Num_Internal_Elems+globals.Num_Border_Elems] * * For the 1 processor case, this routine fills in appropriate values * for these quantities. */ { int lb_exoid=0; INT cmap_max_size=0, *comm_vec; const char *yo = "load_lb_info"; char Title[MAX_LINE_LENGTH+1]; float version; int cpu_ws=0; /******************************** START EXECUTION ****************************/ if(Debug_Flag) printf ("\nStart to read in and distribute the load balance info\n"); /* Open the Load Balance exoII file for reading */ printf ("EXODUS II load-balance file: %s\n", Exo_LB_File); cpu_ws = io_ws; int mode = EX_READ | int64api; int iio_ws = 0; // Don't interfere with exodus files; this is the nemesis file. if((lb_exoid = ex_open(Exo_LB_File, mode, &cpu_ws, &iio_ws, &version)) == -1) { fprintf(stderr, "%sERROR: Couldn\'t open lb file, %s\n", yo, Exo_LB_File); exit(1); } /* Read information about the processor configuration */ read_proc_init(lb_exoid,Proc_Info, &Proc_Ids); /* Allocate space for the counts */ globals.Num_Internal_Nodes = (INT *)array_alloc(__FILE__, __LINE__, 1, 7*Proc_Info[2], sizeof(INT)); globals.Num_Border_Nodes = globals.Num_Internal_Nodes +Proc_Info[2]; globals.Num_External_Nodes = globals.Num_Border_Nodes +Proc_Info[2]; globals.Num_Internal_Elems = globals.Num_External_Nodes +Proc_Info[2]; globals.Num_Border_Elems = globals.Num_Internal_Elems +Proc_Info[2]; globals.Num_N_Comm_Maps = globals.Num_Border_Elems +Proc_Info[2]; globals.Num_E_Comm_Maps = globals.Num_N_Comm_Maps +Proc_Info[2]; /* Allocate space for each processor entity */ globals.GNodes = (INT **)array_alloc(__FILE__, __LINE__, 1, 3*Proc_Info[2], sizeof(INT *)); globals.GElems = globals.GNodes +Proc_Info[2]; globals.Elem_Map = globals.GElems +Proc_Info[2]; /* Allocate contiguous space for the pointer vectors on all processors */ INT *Int_Space = (INT *)array_alloc(__FILE__, __LINE__, 1, (7*Proc_Info[0] + 1), sizeof(INT)); INT *Int_Node_Num = Int_Space + 1; INT *Bor_Node_Num = Int_Node_Num +Proc_Info[0]; INT *Ext_Node_Num = Bor_Node_Num +Proc_Info[0]; INT *Int_Elem_Num = Ext_Node_Num +Proc_Info[0]; INT *Bor_Elem_Num = Int_Elem_Num +Proc_Info[0]; INT *Node_Comm_Num = Bor_Elem_Num +Proc_Info[0]; INT *Elem_Comm_Num = Node_Comm_Num +Proc_Info[0]; /* Read the initial information contained in the load balance file */ read_lb_init(lb_exoid, Int_Space, Int_Node_Num, Bor_Node_Num, Ext_Node_Num, Int_Elem_Num, Bor_Elem_Num, Node_Comm_Num, Elem_Comm_Num, Title); /* Allocate memory for the communication map arrays */ globals.N_Comm_Map = (NODE_COMM_MAP<INT>**)malloc(Proc_Info[2]*sizeof(NODE_COMM_MAP<INT> *)); globals.E_Comm_Map = (ELEM_COMM_MAP<INT>**)malloc(Proc_Info[2]*sizeof(ELEM_COMM_MAP<INT> *)); if(!globals.N_Comm_Map || !globals.E_Comm_Map) { fprintf(stderr, "ERROR: Insufficient memory!\n"); exit(1); } for (int iproc=0; iproc <Proc_Info[2]; iproc++) { /* * Error check: * Currently a maximum of one nodal communication map and one * elemental communication map is supported. */ if(globals.Num_N_Comm_Maps[iproc] > 1 || globals.Num_E_Comm_Maps[iproc] > 1) { fprintf(stderr, "%s: ERROR. Only 1 nodal and elemental comm map " "is supported\n", yo); exit(1); } else { /* Always allocate at least one and initialize the counts to 0 */ globals.N_Comm_Map[iproc] = (NODE_COMM_MAP<INT> *)malloc(PEX_MAX(1, globals.Num_N_Comm_Maps[iproc]) * sizeof(NODE_COMM_MAP<INT>)); if(globals.N_Comm_Map[iproc] == NULL && globals.Num_N_Comm_Maps[iproc] > 0) { fprintf(stderr, "%s: ERROR. Insufficient memory for nodal comm. map!\n", yo); exit(1); } for(size_t ijump=0; ijump < PEX_MAX(1, globals.Num_N_Comm_Maps[iproc]); ijump++) ((globals.N_Comm_Map[iproc])+ijump)->node_cnt = 0; globals.E_Comm_Map[iproc] = (ELEM_COMM_MAP<INT> *)malloc(PEX_MAX(1, globals.Num_E_Comm_Maps[iproc]) * sizeof(ELEM_COMM_MAP<INT>)); if(globals.E_Comm_Map[iproc] == NULL && globals.Num_E_Comm_Maps[iproc] > 0) { fprintf(stderr, "%s: ERROR. Insufficient memory for elemental comm. map!\n", yo); exit(1); } for(size_t ijump=0; ijump < PEX_MAX(1, globals.Num_E_Comm_Maps[iproc]); ijump++) ((globals.E_Comm_Map[iproc])+ijump)->elem_cnt = 0; } } /* End "for (int iproc=0; iproc <Proc_Info[2]; iproc++)" */ /* Set up each processor for the communication map parameters */ read_cmap_params(lb_exoid, Node_Comm_Num, Elem_Comm_Num, globals.Num_N_Comm_Maps, globals.Num_E_Comm_Maps, globals.E_Comm_Map, globals.N_Comm_Map, &cmap_max_size, &comm_vec); /* Allocate enough space to read the LB_data for one processor */ INT *Integer_Vector = (INT *)array_alloc(__FILE__, __LINE__, 1, Int_Space[0] + cmap_max_size, sizeof (INT)); /* * loop through the processors, one at a time, to read * their load balance information * * NOTE: From here on there are no provisions for multiple nodal * or elemental communication maps. */ size_t ijump = 0; /* keep track of where in comm_vec we are */ for (int iproc = 0; iproc <Proc_Info[0]; iproc++) { /* Get the node map for processor "iproc" */ if(ex_get_processor_node_maps(lb_exoid, &Integer_Vector[0], &Integer_Vector[Int_Node_Num[iproc]], &Integer_Vector[Int_Node_Num[iproc]+ Bor_Node_Num[iproc]], iproc) < 0) { fprintf(stderr, "%s: ERROR, failed to get node map for Proc %d!\n", yo, iproc); exit(1); } size_t vec_indx = Int_Node_Num[iproc] + Bor_Node_Num[iproc] + Ext_Node_Num[iproc]; /* Get the element map for processor number "iproc" */ if(ex_get_processor_elem_maps(lb_exoid, &Integer_Vector[vec_indx], &Integer_Vector[vec_indx+Int_Elem_Num[iproc]], iproc) < 0) { fprintf(stderr, "%s: ERROR, failed to get element map for Proc %d!\n", yo, iproc); exit(1); } if(Node_Comm_Num[iproc] > 0) { vec_indx += Int_Elem_Num[iproc] + Bor_Elem_Num[iproc]; if(ex_get_node_cmap(lb_exoid, comm_vec[ijump], &Integer_Vector[vec_indx], &Integer_Vector[vec_indx+comm_vec[ijump+1]], iproc) < 0) { /* * If there are disconnected mesh pieces, then it is * possible that there is no comminication between the * pieces and there will be no communication maps. Normally * this is a problem, so output a warning, but don't abort. */ fprintf(stderr, "%s: WARNING. Failed to get nodal comm map for Proc %d!\n", yo, iproc); } } if(Elem_Comm_Num[iproc] > 0) { vec_indx += 2*comm_vec[ijump+1]; if(ex_get_elem_cmap(lb_exoid, comm_vec[ijump+2], &Integer_Vector[vec_indx], &Integer_Vector[vec_indx+comm_vec[ijump+3]], &Integer_Vector[vec_indx+2*comm_vec[ijump+3]], iproc) < 0) { fprintf(stderr, "%s: ERROR. Failed to get elemental comm map for Proc %d!\n", yo, iproc); exit(1); } } /* * Communicate load balance information to the correct processor * - if iproc = Proc_Ids[*] then process the data instead. */ assert(Proc_Ids[iproc] == iproc); process_lb_data (Integer_Vector, iproc); /* * now move ijump to the next communications map * make sure to check if there are any for this processor */ if (Node_Comm_Num[iproc] > 0) ijump += 2; if (Elem_Comm_Num[iproc] > 0) ijump += 2; } /* Close the load balance file - we are finished with it */ if(ex_close (lb_exoid) == -1) { fprintf (stderr, "%sERROR: Error in closing load balance file\n", yo); exit(1); } /************************* Cleanup and Printout Phase ***********************/ /* Free temporary memory */ safe_free((void **) &Integer_Vector); if(num_qa_rec > 0) { for(int i = 0; i < length_qa; i++) safe_free((void **) &(qa_record_ptr[i])); safe_free((void **) &qa_record_ptr); } if(num_inf_rec > 0) { for(int i = 0; i < num_inf_rec; i++) safe_free((void **) &(inf_record_ptr[i])); safe_free((void **) &inf_record_ptr); } safe_free((void **) &Int_Space); safe_free((void **) &comm_vec); for (int iproc=0; iproc <Proc_Info[2]; iproc++) { if (globals.Num_Internal_Nodes[iproc] == 0 && globals.Num_Border_Nodes[iproc] == 0 && globals.Num_External_Nodes[iproc] == 0) { fprintf(stderr, "\n%s: WARNING, Processor %d has no nodes!\n", yo, iproc); } if (globals.Num_Internal_Elems[iproc] == 0 && globals.Num_Border_Elems[iproc] == 0) { fprintf(stderr, "\n%s: WARNING, Processor %d has no elements!\n", yo, iproc); } } /*========================================================================*/ if(Debug_Flag) printf ("\nFinished distributing load balance info\n"); /* Output Detailed timing information for the progam */ /* * Print out a Large table of Load Balance Information if the debug_flag * setting is large enough */ if(Debug_Flag >= 7) { printf ("\n\n"); print_line ("=", 79); for (int iproc=0; iproc <Proc_Info[2]; iproc++) { printf("\n\t***For Processor %d***\n", Proc_Ids[iproc]); printf("\tInternal nodes owned by the current processor\n\t"); for(INT i = 0; i < globals.Num_Internal_Nodes[iproc]; i++) printf(" " ST_ZU "", (size_t)globals.GNodes[iproc][i]); printf("\n"); printf("\tBorder nodes owned by the current processor\n\t"); for(INT i = 0; i < globals.Num_Border_Nodes[iproc]; i++) printf(" " ST_ZU "", (size_t)globals.GNodes[iproc][i + globals.Num_Internal_Nodes[iproc]]); printf("\n"); if(globals.Num_External_Nodes[iproc] > 0) { printf("\tExternal nodes needed by the current processor\n\t"); for(INT i = 0; i < globals.Num_External_Nodes[iproc]; i++) printf(" " ST_ZU "", (size_t)globals.GNodes[iproc][i + globals.Num_Internal_Nodes[iproc] + globals.Num_Border_Nodes[iproc]]); printf("\n"); } printf("\tInternal elements owned by the current processor\n\t"); for(INT i = 0; i < globals.Num_Internal_Elems[iproc]; i++) printf(" " ST_ZU "", (size_t)globals.GElems[iproc][i]); printf("\n"); if(globals.Num_Border_Elems[iproc] > 0) { printf("\tBorder elements owned by the current processor\n\t"); for(INT i=0; i < globals.Num_Border_Elems[iproc]; i++) printf(" " ST_ZU "", (size_t)globals.GElems[iproc][i+globals.Num_Internal_Elems[iproc]]); printf("\n"); } if(globals.Num_N_Comm_Maps[iproc] > 0) { printf("\tNodal Comm Map for the current processor\n"); printf("\t\tnode IDs:"); for(size_t i=0; i < globals.N_Comm_Map[iproc]->node_cnt; i++) printf(" " ST_ZU "", (size_t)globals.N_Comm_Map[iproc]->node_ids[i]); printf("\n\t\tproc IDs:"); for(size_t i=0; i < globals.N_Comm_Map[iproc]->node_cnt; i++) printf(" " ST_ZU "", (size_t)globals.N_Comm_Map[iproc]->proc_ids[i]); printf("\n"); } if(globals.Num_E_Comm_Maps[iproc] > 0) { printf("\tElemental Comm Map for the current processor\n"); printf("\t\telement IDs:"); for(size_t i=0; i < globals.E_Comm_Map[iproc]->elem_cnt; i++) printf(" " ST_ZU "", (size_t)globals.E_Comm_Map[iproc]->elem_ids[i]); printf("\n\t\tside IDs:"); for(size_t i=0; i < globals.E_Comm_Map[iproc]->elem_cnt; i++) printf(" " ST_ZU "", (size_t)globals.E_Comm_Map[iproc]->side_ids[i]); printf("\n\t\tproc IDs:"); for(size_t i=0; i < globals.E_Comm_Map[iproc]->elem_cnt; i++) printf(" " ST_ZU "", (size_t)globals.E_Comm_Map[iproc]->proc_ids[i]); printf("\n"); } } printf("\n"); print_line ("=", 79); } } /* END of routine load_lb_info () ******************************************/