void gds_qsort(INT v[], size_t N)
{
if (N <= 1) return;
gds_qsort(v, 0, N-1);
gds_isort(v, N);
#if defined(DEBUG_QSORT)
fprintf(stderr, "Checking sort of %d values\n", N+1);
for (size_t i=1; i < N; i++) {
assert(v[i-1] <= v[i]);
}
#endif
}
void NemSpread<T,INT>::process_lb_data (INT *Integer_Vector, int indx)
/*
* Function which reads the load balance information from the processor's
* communication buffer.
*
* Author: Scott Hutchinson (1421)
* Date: 11 March 1993
* Revised: 11 March 1993
*
* int *Integer_Vector; Data structure for sending int data to processors
* (ptr to vector of length, length_int)
*
*/
{
/* Local variables */
int icount = 0, itotal_nodes, itotal_elems, ig_count = 0;
/* Function declarations */
extern void sort_int ( int n, int ra[] );
/***************************** execution begins ******************************/
/* Calculate the length of the globals.GNodes and globals.GElems global variable */
itotal_nodes = globals.Num_Internal_Nodes[indx] + globals.Num_Border_Nodes[indx] +
globals.Num_External_Nodes[indx];
itotal_elems = globals.Num_Internal_Elems[indx] + globals.Num_Border_Elems[indx];
/* Allocate Permament Arrays on the current processor */
globals.GNodes[indx] = (INT *)
array_alloc(__FILE__, __LINE__, 1,
itotal_nodes + 2*itotal_elems + 2*(globals.N_Comm_Map[indx]->node_cnt) +
3*(globals.E_Comm_Map[indx]->elem_cnt), sizeof(INT));
globals.GElems[indx] = globals.GNodes[indx] + itotal_nodes;
globals.Elem_Map[indx] = globals.GElems[indx] + itotal_elems;
globals.N_Comm_Map[indx]->node_ids = globals.Elem_Map[indx] + itotal_elems;
globals.N_Comm_Map[indx]->proc_ids = globals.N_Comm_Map[indx]->node_ids +
globals.N_Comm_Map[indx]->node_cnt;
globals.E_Comm_Map[indx]->elem_ids = globals.N_Comm_Map[indx]->proc_ids +
globals.N_Comm_Map[indx]->node_cnt;
globals.E_Comm_Map[indx]->side_ids = globals.E_Comm_Map[indx]->elem_ids +
globals.E_Comm_Map[indx]->elem_cnt;
globals.E_Comm_Map[indx]->proc_ids = globals.E_Comm_Map[indx]->side_ids +
globals.E_Comm_Map[indx]->elem_cnt;
/*
* Extract the load balance information, and store it in
* permanent vectors:
* - globals.GNodes[0] - Internal_Nodes
* globals.GNodes[+] - Border_Nodes
* globals.GNodes[+] - External_Nodes
* globals.GElems[0] - Internal_Elems
* globals.GElems[+] - Border_Elems
*/
for(INT i = 0; i < globals.Num_Internal_Nodes[indx]; i++)
globals.GNodes[indx][ig_count++] = Integer_Vector[icount++];
for(INT i = 0; i < globals.Num_Border_Nodes[indx]; i++)
globals.GNodes[indx][ig_count++] = Integer_Vector[icount++];
for(INT i = 0; i < globals.Num_External_Nodes[indx]; i++)
globals.GNodes[indx][ig_count++] = Integer_Vector[icount++];
ig_count = 0;
for(INT i = 0; i < globals.Num_Internal_Elems[indx]; i++) {
globals.GElems[indx][ig_count] = Integer_Vector[icount++];
globals.Elem_Map[indx][ig_count] = globals.GElems[indx][ig_count];
ig_count++;
}
for(INT i = 0; i < globals.Num_Border_Elems[indx]; i++) {
globals.GElems[indx][ig_count] = Integer_Vector[icount++];
globals.Elem_Map[indx][ig_count] = globals.GElems[indx][ig_count];
ig_count++;
}
for(size_t i = 0; i < globals.N_Comm_Map[indx]->node_cnt; i++)
(globals.N_Comm_Map[indx]->node_ids)[i] = Integer_Vector[icount++];
for(size_t i = 0; i < globals.N_Comm_Map[indx]->node_cnt; i++)
(globals.N_Comm_Map[indx]->proc_ids)[i] = Integer_Vector[icount++];
for(size_t i = 0; i < globals.E_Comm_Map[indx]->elem_cnt; i++)
(globals.E_Comm_Map[indx]->elem_ids)[i] = Integer_Vector[icount++];
for(size_t i = 0; i < globals.E_Comm_Map[indx]->elem_cnt; i++)
(globals.E_Comm_Map[indx]->side_ids)[i] = Integer_Vector[icount++];
for(size_t i = 0; i < globals.E_Comm_Map[indx]->elem_cnt; i++)
(globals.E_Comm_Map[indx]->proc_ids)[i] = Integer_Vector[icount++];
/*
* Sort the local element numbers in ascending global element numbers.
* This means that globals.GElems will be monotonic.
*/
gds_qsort(globals.GElems[indx], globals.Num_Internal_Elems[indx]);
gds_qsort(globals.Elem_Map[indx], globals.Num_Internal_Elems[indx]);
/* Check that globals.GNodes is monotonic, from i = 0 to Num_Internal_Nodes */
#ifdef DEBUG
assert(check_monot (globals.GNodes[indx], globals.Num_Internal_Nodes[indx]));
/*
* Check that globals.GNodes is monotonic, from i = Num_Internal_Nodes to
* (Num_Internal_Nodes + Num_Border_Nodes)
*/
assert(check_monot(&(globals.GNodes[indx][globals.Num_Internal_Nodes[indx]]),
globals.Num_Border_Nodes[indx]));
#endif
} /* END of process_lb_data () ***********************************************/
int write_nemesis(std::string &nemI_out_file,
Machine_Description* machine,
Problem_Description* problem,
Mesh_Description<INT>* mesh,
LB_Description<INT>* lb,
Sphere_Info* sphere)
{
int exoid;
char title[MAX_LINE_LENGTH+1], method1[MAX_LINE_LENGTH+1];
char method2[MAX_LINE_LENGTH+1];
int cpu_ws = sizeof(float);
int io_ws = sizeof(float);
printf("Outputting load balance to file %s\n", nemI_out_file.c_str());
/* Create the load balance file */
/* Attempt to create a netcdf4-format file; if it fails, then assume
that the netcdf library does not support that mode and fall back
to classic netcdf3 format. If that fails, issue an error and
return failure.
*/
int mode3 = EX_CLOBBER;
int mode4 = mode3|EX_NETCDF4|EX_NOCLASSIC|problem->int64db|problem->int64api;
ex_opts(EX_DEFAULT); // Eliminate misleading error if the first ex_create fails, but the second succeeds.
if((exoid=ex_create(nemI_out_file.c_str(), mode4, &cpu_ws, &io_ws)) < 0) {
/* If int64api or int64db non-zero, then netcdf-4 format is required, so
fail now...
*/
if (problem->int64db|problem->int64api) {
Gen_Error(0, "fatal: failed to create Nemesis netcdf-4 file");
return 0;
}
if((exoid=ex_create(nemI_out_file.c_str(), mode3, &cpu_ws, &io_ws)) < 0) {
Gen_Error(0, "fatal: failed to create Nemesis file");
return 0;
}
}
ON_BLOCK_EXIT(ex_close, exoid);
/* Set the error reporting value */
if (error_lev > 1)
ex_opts(EX_VERBOSE | EX_DEBUG);
else
ex_opts(EX_VERBOSE);
/* Enable compression (if netcdf-4) */
ex_set_option(exoid, EX_OPT_COMPRESSION_LEVEL, 1);
ex_set_option(exoid, EX_OPT_COMPRESSION_SHUFFLE, 1);
/* Create the title */
if(problem->type == NODAL)
strcpy(method1, "nodal");
else
strcpy(method1, "elemental");
sprintf(title, "nem_slice %s load balance file", method1);
strcpy(method1, "method1: ");
strcpy(method2, "method2: ");
switch(lb->type)
{
case MULTIKL:
strcat(method1, "Multilevel-KL decomposition");
strcat(method2, "With Kernighan-Lin refinement");
break;
case SPECTRAL:
strcat(method1, "Spectral decomposition");
break;
case INERTIAL:
strcat(method1, "Inertial decomposition");
break;
case ZPINCH:
strcat(method1, "ZPINCH decomposition");
break;
case BRICK:
strcat(method1, "BRICK decomposition");
break;
case ZOLTAN_RCB:
strcat(method1, "RCB decomposition");
break;
case ZOLTAN_RIB:
strcat(method1, "RIB decomposition");
break;
case ZOLTAN_HSFC:
strcat(method1, "HSFC decomposition");
break;
case LINEAR:
strcat(method1, "Linear decomposition");
break;
case RANDOM:
strcat(method1, "Random decomposition");
break;
case SCATTERED:
strcat(method1, "Scattered decomposition");
break;
}
if(lb->refine == KL_REFINE && lb->type != MULTIKL)
strcat(method2, "with Kernighan-Lin refinement");
else if(lb->type != MULTIKL)
strcat(method2, "no refinement");
switch(lb->num_sects)
{
case 1:
strcat(method1, " via bisection");
break;
case 2:
strcat(method1, " via quadrasection");
break;
case 3:
strcat(method1, " via octasection");
break;
}
/* Do some sorting */
for(int proc=0; proc < machine->num_procs; proc++) {
/* Sort node maps */
gds_qsort(TOPTR(lb->int_nodes[proc]), lb->int_nodes[proc].size());
if(problem->type == NODAL) {
sort2(lb->ext_nodes[proc].size(), TOPTR(lb->ext_nodes[proc]),
TOPTR(lb->ext_procs[proc]));
}
/* Sort element maps */
gds_qsort(TOPTR(lb->int_elems[proc]), lb->int_elems[proc].size());
}
/* Output the info records */
char *info[3];
info[0] = title;
info[1] = method1;
info[2] = method2;
if(ex_put_info(exoid, 3, info) < 0)
Gen_Error(0, "warning: output of info records failed");
/* Generate a QA record for the utility */
time_t time_val = time(nullptr);
char *ct_ptr = asctime(localtime(&time_val));
char tm_date[30];
strcpy(tm_date, ct_ptr);
/* Break string with null characters */
tm_date[3] = '\0';
tm_date[7] = '\0';
tm_date[10] = '\0';
tm_date[19] = '\0';
char qa_date[15], qa_time[10], qa_name[MAX_STR_LENGTH];
char qa_vers[10];
sprintf(qa_date, "%s %s %s", &tm_date[8], &tm_date[4], &tm_date[20]);
sprintf(qa_time, "%s", &tm_date[11]);
strcpy(qa_name, UTIL_NAME);
strcpy(qa_vers, ELB_VERSION);
if(qa_date[strlen(qa_date)-1] == '\n')
qa_date[strlen(qa_date)-1] = '\0';
char **lqa_record = (char **)array_alloc(1, 4, sizeof(char *));
for(int i2=0; i2 < 4; i2++)
lqa_record[i2] = (char *)array_alloc(1, MAX_STR_LENGTH+1, sizeof(char));
strcpy(lqa_record[0], qa_name);
strcpy(lqa_record[1], qa_vers);
strcpy(lqa_record[2], qa_date);
strcpy(lqa_record[3], qa_time);
printf("QA Record:\n");
for(int i2=0; i2 < 4; i2++) {
printf("\t%s\n", lqa_record[i2]);
}
if(ex_put_qa(exoid, 1, (char *(*)[4]) &lqa_record[0]) < 0) {
Gen_Error(0, "fatal: unable to output QA records");
return 0;
}
/* free up memory */
for(int i2=0; i2 < 4; i2++)
free(lqa_record[i2]);
free(lqa_record);
/* Output the the initial Nemesis global information */
if(ex_put_init_global(exoid, mesh->num_nodes, mesh->num_elems,
mesh->num_el_blks, 0, 0) < 0) {
Gen_Error(0, "fatal: failed to output initial Nemesis parameters");
return 0;
}
/* Set up dummy arrays for ouput */
std::vector<INT> num_nmap_cnts(machine->num_procs);
std::vector<INT> num_emap_cnts(machine->num_procs);
if(problem->type == NODAL) {
/* need to check and make sure that there really are comm maps */
for(int cnt=0; cnt < machine->num_procs; cnt++) {
if (!lb->bor_nodes[cnt].empty())
num_nmap_cnts[cnt] = 1;
}
}
else { /* Elemental load balance */
if(((problem->num_vertices)-(sphere->num)) > 0) {
/* need to check and make sure that there really are comm maps */
for(int cnt=0; cnt < machine->num_procs; cnt++) {
if (!lb->bor_nodes[cnt].empty()) num_nmap_cnts[cnt] = 1;
}
for(int cnt=0; cnt < machine->num_procs; cnt++) {
if (!lb->bor_elems[cnt].empty()) num_emap_cnts[cnt] = 1;
}
}
}
if(ex_put_init_info(exoid, machine->num_procs, machine->num_procs, (char*)"s") < 0) {
Gen_Error(0, "fatal: unable to output init info");
return 0;
}
// Need to create 5 arrays with the sizes of lb->int_nodes[i].size()...
{
std::vector<INT> ins(machine->num_procs);
std::vector<INT> bns(machine->num_procs);
std::vector<INT> ens(machine->num_procs);
std::vector<INT> ies(machine->num_procs);
std::vector<INT> bes(machine->num_procs);
for (int iproc = 0; iproc < machine->num_procs; iproc++) {
ins[iproc] = lb->int_nodes[iproc].size();
bns[iproc] = lb->bor_nodes[iproc].size();
ens[iproc] = lb->ext_nodes[iproc].size();
ies[iproc] = lb->int_elems[iproc].size();
bes[iproc] = lb->bor_elems[iproc].size();
}
if(ex_put_loadbal_param_cc(exoid,
TOPTR(ins), TOPTR(bns), TOPTR(ens),
TOPTR(ies), TOPTR(bes), TOPTR(num_nmap_cnts),
TOPTR(num_emap_cnts)) < 0)
{
Gen_Error(0, "fatal: unable to output load-balance parameters");
return 0;
}
}
if(problem->type == NODAL) /* Nodal load balance output */
{
/* Set up for the concatenated communication map parameters */
std::vector<INT> node_proc_ptr(machine->num_procs+1);
std::vector<INT> node_cmap_ids_cc(machine->num_procs);
std::vector<INT> node_cmap_cnts_cc(machine->num_procs);
node_proc_ptr[0] = 0;
for(int proc=0; proc < machine->num_procs; proc++) {
node_proc_ptr[proc+1] = node_proc_ptr[proc] + 1;
node_cmap_cnts_cc[proc] = lb->ext_nodes[proc].size();
node_cmap_ids_cc[proc] = 1;
}
/* Output the communication map parameters */
if(ex_put_cmap_params_cc(exoid, TOPTR(node_cmap_ids_cc),
TOPTR(node_cmap_cnts_cc),
TOPTR(node_proc_ptr), nullptr, nullptr, nullptr) < 0)
{
Gen_Error(0, "fatal: unable to output communication map parameters");
return 0;
}
/* Output the node and element maps */
for(int proc=0; proc < machine->num_procs; proc++) {
/* Output the nodal map */
if(ex_put_processor_node_maps(exoid,
TOPTR(lb->int_nodes[proc]),
TOPTR(lb->bor_nodes[proc]),
TOPTR(lb->ext_nodes[proc]), proc) < 0)
{
Gen_Error(0, "fatal: failed to output node map");
return 0;
}
/* Output the elemental map */
if(ex_put_processor_elem_maps(exoid, TOPTR(lb->int_elems[proc]), nullptr, proc) < 0)
{
Gen_Error(0, "fatal: failed to output element map");
return 0;
}
/*
* Reorder the nodal communication maps so that they are ordered
* by processor and then by global ID.
*/
/* This is a 2-key sort */
qsort2(TOPTR(lb->ext_procs[proc]), TOPTR(lb->ext_nodes[proc]), lb->ext_nodes[proc].size());
/* Output the nodal communication map */
if(ex_put_node_cmap(exoid, 1,
TOPTR(lb->ext_nodes[proc]),
TOPTR(lb->ext_procs[proc]), proc) < 0)
{
Gen_Error(0, "fatal: failed to output nodal communication map");
return 0;
}
} /* End "for(proc=0; proc < machine->num_procs; proc++)" */
}
else if(problem->type == ELEMENTAL) /* Elemental load balance output */
{
std::vector<INT> node_proc_ptr(machine->num_procs+1);
std::vector<INT> node_cmap_ids_cc(machine->num_procs);
std::vector<INT> node_cmap_cnts_cc(machine->num_procs);
node_proc_ptr[0] = 0;
for(int proc=0; proc < machine->num_procs; proc++) {
node_proc_ptr[proc+1] = node_proc_ptr[proc] + 1;
node_cmap_cnts_cc[proc] = 0;
for(size_t cnt=0; cnt < lb->bor_nodes[proc].size(); cnt++)
node_cmap_cnts_cc[proc] += lb->born_procs[proc][cnt].size();
node_cmap_ids_cc[proc] = 1;
}
std::vector<INT> elem_proc_ptr(machine->num_procs+1);
std::vector<INT> elem_cmap_ids_cc(machine->num_procs);
std::vector<INT> elem_cmap_cnts_cc(machine->num_procs);
elem_proc_ptr[0] = 0;
for(int proc=0; proc < machine->num_procs; proc++) {
elem_proc_ptr[proc+1] = elem_proc_ptr[proc] + 1;
elem_cmap_cnts_cc[proc] = lb->e_cmap_elems[proc].size();
elem_cmap_ids_cc[proc] = 1;
}
/* Output the communication map parameters */
if(ex_put_cmap_params_cc(exoid, TOPTR(node_cmap_ids_cc), TOPTR(node_cmap_cnts_cc),
TOPTR(node_proc_ptr), TOPTR(elem_cmap_ids_cc),
TOPTR(elem_cmap_cnts_cc), TOPTR(elem_proc_ptr)) < 0)
{
Gen_Error(0, "fatal: unable to output communication map parameters");
return 0;
}
/* Output the node and element maps */
for(int proc=0; proc < machine->num_procs; proc++)
{
/* Output the nodal map */
if(ex_put_processor_node_maps(exoid,
TOPTR(lb->int_nodes[proc]),
TOPTR(lb->bor_nodes[proc]),
nullptr, proc) < 0)
{
Gen_Error(0, "fatal: failed to output node map");
return 0;
}
/* Output the elemental map */
if(ex_put_processor_elem_maps(exoid,
TOPTR(lb->int_elems[proc]),
TOPTR(lb->bor_elems[proc]),
proc) < 0)
{
Gen_Error(0, "fatal: failed to output element map");
return 0;
}
/*
* Build a nodal communication map from the list of border nodes
* and their associated processors and side IDs.
*/
size_t nsize = 0;
for(size_t cnt=0; cnt < lb->bor_nodes[proc].size(); cnt++)
nsize += lb->born_procs[proc][cnt].size();
if (nsize > 0) {
std::vector<INT> n_cmap_nodes(nsize);
std::vector<INT> n_cmap_procs(nsize);
size_t cnt3 = 0;
for(size_t cnt=0; cnt < lb->bor_nodes[proc].size(); cnt++) {
for(size_t cnt2=0; cnt2 < lb->born_procs[proc][cnt].size(); cnt2++) {
n_cmap_nodes[cnt3] = lb->bor_nodes[proc][cnt];
n_cmap_procs[cnt3++] = lb->born_procs[proc][cnt][cnt2];
}
}
/*
* Reorder the nodal communication maps so that they are ordered
* by processor and then by global ID.
*/
/* This is a 2-key sort */
qsort2(TOPTR(n_cmap_procs), TOPTR(n_cmap_nodes), cnt3);
/* Output the nodal communication map */
if(ex_put_node_cmap(exoid, 1, TOPTR(n_cmap_nodes), TOPTR(n_cmap_procs), proc) < 0) {
Gen_Error(0, "fatal: unable to output nodal communication map");
return 0;
}
} /* End "if (nsize > 0)" */
/* Output the elemental communication map */
if(!lb->e_cmap_elems[proc].empty()) {
if(ex_put_elem_cmap(exoid, 1,
TOPTR(lb->e_cmap_elems[proc]),
TOPTR(lb->e_cmap_sides[proc]),
TOPTR(lb->e_cmap_procs[proc]), proc) < 0)
{
Gen_Error(0, "fatal: unable to output elemental communication map");
return 0;
}
}
} /* End "for(proc=0; proc < machine->num_procs; proc++)" */
}
return 1;
} /*------------------------End write_nemesis()------------------------------*/
