void print_distributed_mesh(
int Proc,
int Num_Proc,
MESH_INFO_PTR mesh)
{
int i, j, k;
int elem;
int offset;
ELEM_INFO_PTR current_elem;
/*
* Print the distributed mesh description for each processor. This routine
* is useful for debugging the input meshes (Nemesis or Chaco). It is
* serial, so it should not be used for production runs.
*/
print_sync_start(Proc, 1);
printf ("############## Mesh information for processor %d ##############\n",
Proc);
printf ("Number Dimensions: %d\n", mesh->num_dims);
printf ("Number Nodes: %d\n", mesh->num_nodes);
printf ("Number Elements: %d\n", mesh->num_elems);
printf ("Number Element Blocks: %d\n", mesh->num_el_blks);
printf ("Number Node Sets: %d\n", mesh->num_node_sets);
printf ("Number Side Sets: %d\n", mesh->num_side_sets);
for (i = 0; i < mesh->num_el_blks; i++) {
printf("\nElement block #%d\n", (i+1));
printf("\tID: %d\n", mesh->eb_ids[i]);
printf("\tElement Type: %s\n", mesh->eb_names[i]);
printf("\tElement Count: %d\n", mesh->eb_cnts[i]);
printf("\tNodes Per Element: %d\n", mesh->eb_nnodes[i]);
printf("\tAttrib Per Element: %d\n", mesh->eb_nattrs[i]);
}
printf("\nElement connect table, partition, weights and coordinates:\n");
for (i = 0; i < mesh->elem_array_len; i++) {
current_elem = &(mesh->elements[i]);
if (current_elem->globalID == -1) continue;
printf("%d in part %d (%f):\n", current_elem->globalID,
current_elem->my_part, current_elem->cpu_wgt[0]);
for (j = 0; j < mesh->eb_nnodes[current_elem->elem_blk]; j++) {
printf("\t%d |", current_elem->connect[j]);
for (k = 0; k < mesh->num_dims; k++) {
printf(" %f", current_elem->coord[j][k]);
}
printf("\n");
}
}
/* now print the adjacencies */
printf("\nElement adjacencies:\n");
printf("elem\tnadj(adj_len)\tadj,proc\n");
for (i = 0; i < mesh->elem_array_len; i++) {
current_elem = &(mesh->elements[i]);
if (current_elem->globalID == -1) continue;
printf("%d\t", current_elem->globalID);
printf("%d(%d)\t", current_elem->nadj, current_elem->adj_len);
for (j = 0; j < current_elem->adj_len; j++) {
/* Skip NULL adjacencies (sides that are not adjacent to another elem). */
if (current_elem->adj[j] == -1) continue;
if (current_elem->adj_proc[j] == Proc)
elem = mesh->elements[current_elem->adj[j]].globalID;
else
elem = current_elem->adj[j];
printf("%d,%d ", elem, current_elem->adj_proc[j]);
}
printf("\n");
}
printf("\nCommunication maps\n");
printf("Number of maps: %d\n", mesh->necmap);
printf("Map Ids(and Counts):");
for (i = 0; i < mesh->necmap; i++)
printf(" %d(%d)", mesh->ecmap_id[i], mesh->ecmap_cnt[i]);
printf("\n");
offset = 0;
for (i = 0; i < mesh->necmap; i++) {
printf("Map %d:\n", mesh->ecmap_id[i]);
printf(" elem side globalID neighID\n");
for (j = 0; j < mesh->ecmap_cnt[i]; j++) {
k = j + offset;
printf(" %d %d %d %d\n", mesh->ecmap_elemids[k],
mesh->ecmap_sideids[k],
mesh->elements[mesh->ecmap_elemids[k]].globalID,
mesh->ecmap_neighids[k]);
}
offset += mesh->ecmap_cnt[i];
}
printf("\nHyperedges\n");
printf("Number of global hyperedges: %d\n", mesh->gnhedges);
if (mesh->format == ZOLTAN_COMPRESSED_EDGE){
printf("Number of rows (edges): %d\n", mesh->nhedges);
}
else{
printf("Number of columns (vertices): %d\n", mesh->nhedges);
}
for (i = 0; i < mesh->nhedges; i++) {
printf(" %d (%d): (", mesh->hgid[i], i);
for (j = mesh->hindex[i]; j < mesh->hindex[i+1]; j++)
printf("%d ", mesh->hvertex[j]);
printf(")\n");
}
if (mesh->hewgt_dim && (mesh->heNumWgts > 0)){
printf("\nHyperedge Weights\n");
for (i=0; i<mesh->heNumWgts; i++){
if (mesh->heWgtId){
printf("Hyperedge %d (%d): (", mesh->heWgtId[i], i);
}
else{
printf("Hyperedge %d (%d): (", mesh->hgid[i], i);
}
for (j = 0; j < mesh->hewgt_dim; j++)
printf("%f ", mesh->hewgts[i*mesh->hewgt_dim + j]);
printf(")\n");
}
}
print_sync_end(Proc, Num_Proc, 1);
}
void
bc_matrl_index(Exo_DB *exo)
/********************************************************************
*
* bc_matrl_index():
*
* Find out what materials are on each side of a boundary
* condition. Note, some boundary conditions require one
* to specify this information in the form of an element block
* id. However, some others do not. This procedure attempts to
* calculate this for all boundary conditions and then fill in
* the BC_matrl_index_# elements of the Boundary_Condition
* structure.
*
* The basic algorithm involves finding some information about
* the side or node set on which the boundary condition is
* applied. Then, given the bc name and this information,
* a decision is made as to what the BC_matrl_index_#'s should be
* set to.
*
* We gather the following information to make this decision:
* 1) element block indecises input from deck
* 2) Number of nodes in the bc set in each material
* 3) Number of elements, whoses sides are in the side set,
* in each material
* 4) Node in the bc set which contains the minimum
* number of materials
* 5) Node in the bc set which contains the maximum
* number of materials
* 6) A random node in the bc set which contains a
* specific number of materials (1, 2, 3, 4)
*
*
* HKM NOTE:
* This routine is a work in progress. Calculation of EDGE's and
* VERTICES are not done yet. Also, mp aspects haven't been
* figured out yet.
*
*******************************************************************/
{
int ibc, ss_index, side_index, k, node_num, i;
int i_apply_meth, num_matrl_needed = -1;
int found = FALSE, min_node_matrl, max_node_matrl, min_matrl,
max_matrl, node_matrl_1, node_matrl_2, node_matrl_3,
node_matrl_4, matrl_first;
int *bin_matrl, *ind_matrl, *bin_matrl_elem,
*node_flag_1ss, node_count;
int mat_index, success, ielem;
NODE_INFO_STRUCT *node_ptr;
UMI_LIST_STRUCT *matrlLP;
struct Boundary_Condition *bc_ptr;
static char *yo = "bc_matrl_index :";
bin_matrl = alloc_int_1(upd->Num_Mat * 4, INT_NOINIT);
ind_matrl = bin_matrl + upd->Num_Mat;
bin_matrl_elem = ind_matrl + upd->Num_Mat;
node_flag_1ss = alloc_int_1(exo->num_nodes, INT_NOINIT);
for (ibc = 0; ibc < Num_BC; ibc++) {
bc_ptr = BC_Types + ibc;
found = FALSE;
/*
* Some boundary condition specifications already require
* you to specify the element blocks on either side of the
* side set.
*/
switch (bc_ptr->BC_Name) {
/*
* For these boundary conditions, the element block ID numbers
* are in the firest two integer slots
*/
case POROUS_PRESSURE_BC:
case DARCY_CONTINUOUS_BC:
case Y_DISCONTINUOUS_BC:
case POROUS_GAS_BC:
case VP_EQUIL_BC:
case VN_POROUS_BC:
case FLUID_SOLID_BC:
case SOLID_FLUID_BC:
case NO_SLIP_BC:
case FLUID_SOLID_CONTACT_BC:
case SOLID_FLUID_CONTACT_BC:
case T_CONTACT_RESIS_BC:
case T_CONTACT_RESIS_2_BC:
case LIGHTP_JUMP_BC:
case LIGHTM_JUMP_BC:
case LIGHTP_JUMP_2_BC:
case LIGHTM_JUMP_2_BC:
bc_ptr->BC_matrl_index_1 = map_mat_index(bc_ptr->BC_Data_Int[0]);
bc_ptr->BC_matrl_index_2 = map_mat_index(bc_ptr->BC_Data_Int[1]);
break;
/*
* For this boundary condition the element block numbers
* are in the second and third integer slots
*/
case VL_EQUIL_BC:
case YFLUX_DISC_RXN_BC:
case DISCONTINUOUS_VELO_BC:
bc_ptr->BC_matrl_index_1 = map_mat_index(bc_ptr->BC_Data_Int[1]);
bc_ptr->BC_matrl_index_2 = map_mat_index(bc_ptr->BC_Data_Int[2]);
break;
}
/*
* Initialize quantities
*/
for (i = 0; i < 4 * upd->Num_Mat; i++) bin_matrl[i] = 0;
for (i = 0; i < exo->num_nodes; i++) node_flag_1ss[i] = 0;
max_node_matrl = min_node_matrl = -1;
min_matrl = 4000000;
max_matrl = -1;
node_matrl_1 = node_matrl_2 = node_matrl_3 = node_matrl_4 = -1;
/*
* Determine how many materials each boundary condition needs
* based on the type of the boundary condition
*/
i_apply_meth = BC_Types[ibc].desc->i_apply;
if (i_apply_meth == CROSS_PHASE_DISCONTINUOUS ||
i_apply_meth == CROSS_PHASE) {
num_matrl_needed = 2;
} else if (i_apply_meth == SINGLE_PHASE) {
num_matrl_needed = 1;
}
/*
* Loop over the nodes in the side set looking up what
* materials are located at each node
* -> this will work for side sets. Need to do node sets
* as well.
*/
if (!strcmp(bc_ptr->Set_Type, "SS")) {
for (ss_index = 0; ss_index < exo->num_side_sets; ss_index++) {
/*
* This logic works for one side set specifications. For
* two side set specifications (EDGES), we will have to go
* with a calculation of the union of side sets.
*/
if (bc_ptr->BC_ID == exo->ss_id[ss_index]) {
found = TRUE;
for (side_index = 0;
side_index < exo->ss_num_sides[ss_index];
side_index++) {
/*
* Locate the element number, find the material index,
* then bin the result.
*/
ielem = exo->ss_elem_list[exo->ss_elem_index[ss_index]+side_index];
mat_index = find_mat_number(ielem, exo);
bin_matrl_elem[mat_index]++;
for (k = exo->ss_node_side_index[ss_index][side_index];
k < exo->ss_node_side_index[ss_index][side_index+1];
k++) {
node_num = exo->ss_node_list[ss_index][k];
if (!node_flag_1ss[node_num]) {
node_flag_1ss[node_num] = 1;
node_ptr = Nodes[node_num];
matrlLP = &(node_ptr->Mat_List);
/*
* Bin the materials at this node for later usage.
*/
for (i = 0; i < matrlLP->Length; i++) {
#ifdef DEBUG_IGNORE_ELEMENT_BLOCK_CAPABILITY
if (matrlLP->List[i] < 0) {
fprintf(stderr,"Material list contains negative number\n");
EH(-1,"logic error in ignoring an element block");
}
#endif
bin_matrl[matrlLP->List[i]]++;
}
/*
* Find the max and min number of materials for a
* node in this side set
*/
if (matrlLP->Length > max_matrl) {
max_matrl = matrlLP->Length;
max_node_matrl = node_num;
}
if (matrlLP->Length < min_matrl) {
min_matrl = matrlLP->Length;
min_node_matrl = node_num;
}
/*
* Find representative nodes with specific
* numbers of materials
*/
if (matrlLP->Length == 1) node_matrl_1 = node_num;
if (matrlLP->Length == 2) node_matrl_2 = node_num;
if (matrlLP->Length == 3) node_matrl_3 = node_num;
if (matrlLP->Length == 4) node_matrl_4 = node_num;
}
}
}
}
} /* End of side set loop */
} /* if SS */
/*
* Node Sets
*/
if (!strcmp(bc_ptr->Set_Type, "NS")) {
for (ss_index = 0; ss_index < exo->num_node_sets; ss_index++) {
/*
* This logic works for one side set specifications. For
* two side set specifications (EDGES), we will have to go
* with a calculation of the union of side sets.
*/
if (bc_ptr->BC_ID == exo->ns_id[ss_index]) {
found = TRUE;
/*
* Loop over the number of nodes
*/
for (k = 0; k < exo->ns_num_nodes[ss_index]; k++) {
node_num = exo->ns_node_list[exo->ns_node_index[ss_index]+k];
if (!node_flag_1ss[node_num]) {
node_flag_1ss[node_num] = 1;
node_ptr = Nodes[node_num];
matrlLP = &(node_ptr->Mat_List);
/*
* Bin the materials at this node for later usage.
*/
for (i = 0; i < matrlLP->Length; i++) {
bin_matrl[matrlLP->List[i]]++;
}
/*
* Find the max and min number of materials for a
* node in this side set
*/
if (matrlLP->Length > max_matrl) {
max_matrl = matrlLP->Length;
max_node_matrl = node_num;
}
if (matrlLP->Length < min_matrl) {
min_matrl = matrlLP->Length;
min_node_matrl = node_num;
}
/*
* Find representative nodes with specific
* numbers of materials
*/
if (matrlLP->Length == 1) node_matrl_1 = node_num;
if (matrlLP->Length == 2) node_matrl_2 = node_num;
if (matrlLP->Length == 3) node_matrl_3 = node_num;
if (matrlLP->Length == 4) node_matrl_4 = node_num;
}
}
}
} /* End of side set loop */
} /* if NS */
/*
* Ok, we have obtained statistics on the side and node sets
* let's make a decision
*/
if (found) {
matrl_first = find_next_max(bin_matrl, ind_matrl, upd->Num_Mat);
success = assign_matrl_2(bc_ptr, matrl_first);
if (success < 0) {
printf("%s P_%d: problem in assigning first matrl index in ibc %d, %d:\n",
yo, ProcID, ibc, matrl_first);
bc_matrl_index_print(bc_ptr, bin_matrl, bin_matrl_elem,
min_node_matrl, max_node_matrl,
node_matrl_1, node_matrl_2, node_matrl_3,
node_matrl_4, ibc);
}
matrl_first = find_next_max(bin_matrl, ind_matrl, upd->Num_Mat);
if (matrl_first >= 0) {
success = assign_matrl_2(bc_ptr, matrl_first);
if (success < 0) {
printf("%s P_%d: problem in assigning second matrl index in ibc %d, %d:\n",
yo, ProcID, ibc, matrl_first);
bc_matrl_index_print(bc_ptr, bin_matrl, bin_matrl_elem,
min_node_matrl, max_node_matrl,
node_matrl_1, node_matrl_2, node_matrl_3,
node_matrl_4, ibc);
}
} else {
if (num_matrl_needed > 1) {
printf("%s P_%d: problem in finding a needed second matrl index:\n",
yo, ProcID);
EH(-1,"bc_matrl_index ERROR");
}
}
}
/*
* For debug purposes, print out everything that we have found
* out and decided about this bc on all of the processors.
*/
#ifdef DEBUG_HKM
print_sync_start(FALSE);
bc_matrl_index_print(bc_ptr, bin_matrl, bin_matrl_elem,
min_node_matrl, max_node_matrl,
node_matrl_1, node_matrl_2, node_matrl_3,
node_matrl_4, ibc);
print_sync_end(FALSE);
#endif
/*
* MP Fix: We may not get the same results on all processors
* In this case, just take the processor with the most
* nodes in this bc and with a valid result, and use
* that. Broadcast that result to all nodes. Cross your
* fingers.
*/
node_count = 0;
for (i = 0; i < exo->num_nodes; i++) {
node_count += node_flag_1ss[i];
}
#ifdef PARALLEL
k = ProcWithMaxInt(node_count, &i);
MPI_Bcast(&(bc_ptr->BC_matrl_index_1), 1, MPI_INT, k,
MPI_COMM_WORLD);
MPI_Bcast(&(bc_ptr->BC_matrl_index_2), 1, MPI_INT, k,
MPI_COMM_WORLD);
MPI_Bcast(&(bc_ptr->BC_matrl_index_3), 1, MPI_INT, k,
MPI_COMM_WORLD);
MPI_Bcast(&(bc_ptr->BC_matrl_index_4), 1, MPI_INT, k,
MPI_COMM_WORLD);
#ifdef DEBUG_HKM
print_sync_start(FALSE);
if ( !ProcID ) {
printf("Final matrl_index's for ibc = %d:\n", ibc);
printf("\tBC_matrl_index_1 = %d\n", bc_ptr->BC_matrl_index_1);
printf("\tBC_matrl_index_2 = %d\n", bc_ptr->BC_matrl_index_2);
printf("\tBC_matrl_index_3 = %d\n", bc_ptr->BC_matrl_index_3);
printf("\tBC_matrl_index_4 = %d\n", bc_ptr->BC_matrl_index_4);
fflush(stdout);
}
print_sync_end(FALSE);
#endif
#endif
}
safer_free((void **) &bin_matrl);
safer_free((void **) &node_flag_1ss);
}
int
coordinate_discontinuous_variables(Exo_DB *exo, Dpi *dpi)
/********************************************************************
*
* coordinate_discontinuous_variables():
*
* -> Make sure we have the correct designations for the v field
* in the problem description structure for each material.
*
* -> Make sure that we have the same v field for all material
* types on all processors.
*
*
*******************************************************************/
{
int ibc, eqn_type, ss_index, side_index, k, node_num, imat;
int num_mat, mat_index, var_type, *ivec;
UMI_LIST_STRUCT *curr_mat_list;
NODE_INFO_STRUCT *node_ptr;
PROBLEM_DESCRIPTION_STRUCT *curr_pd;
/*
* Loop over the boundary conditions. If we have a cross
* phase discontinuous boundary condition, then we need to set the
* v field for the appropriate variable types on both sides of the
* interface to denote a discontinuous interpolation at the
* interface.
*/
for (ibc = 0; ibc < Num_BC; ibc++) {
if (BC_Types[ibc].desc->i_apply == CROSS_PHASE_DISCONTINUOUS) {
eqn_type = BC_Types[ibc].desc->equation;
/*
* If we are applying a bc on the momentum equations
* let's assign it a base equation type
*/
if (eqn_type == R_MOMENTUM1 || eqn_type == R_MOMENTUM2 ||
eqn_type == R_MOMENTUM3 ||
eqn_type == R_MOM_NORMAL || eqn_type == R_MOM_TANG1 ||
eqn_type == R_MOM_TANG2 ) {
eqn_type = R_MOMENTUM1;
}
/*
* If we are applying a discontinuous bc on one species
* equation, then we must apply it to all species equations.
*/
if (eqn_type == R_MASS ||
(eqn_type >= R_SPECIES_UNK_0 && eqn_type <= R_SPECIES_UNK_LAST)
) {
eqn_type = R_SPECIES_UNK_0;
}
for (ss_index = 0; ss_index < exo->num_side_sets; ss_index++) {
if (BC_Types[ibc].BC_ID == exo->ss_id[ss_index]) {
for (side_index = 0; side_index < exo->ss_num_sides[ss_index];
side_index++) {
for (k = exo->ss_node_side_index[ss_index][side_index];
k < exo->ss_node_side_index[ss_index][side_index+1]; k++) {
node_num = exo->ss_node_list[ss_index][k];
node_ptr = Nodes[node_num];
curr_mat_list = &(node_ptr->Mat_List);
num_mat = curr_mat_list->Length;
/*
* Now make sure that we have the discontinuous var turned
* on
*/
for (imat = 0; imat < num_mat; imat++) {
mat_index= (curr_mat_list->List)[imat];
curr_pd = pd_glob[mat_index];
if (eqn_type == R_MOMENTUM1) {
turn_on_discontinuous(curr_pd, R_MOMENTUM1);
turn_on_discontinuous(curr_pd, R_MOMENTUM2);
turn_on_discontinuous(curr_pd, R_MOMENTUM3);
turn_on_discontinuous(curr_pd, PRESSURE);
} else if (eqn_type == R_SPECIES_UNK_0) {
turn_on_discontinuous(curr_pd, R_MASS);
for (var_type = R_SPECIES_UNK_0;
var_type < R_SPECIES_UNK_LAST; var_type++) {
turn_on_discontinuous(curr_pd, var_type);
}
} else {
turn_on_discontinuous(curr_pd, eqn_type);
}
}
}
}
}
}
}
}
/*
* Just to dot the eyes, make sure that v fields are uniform on
* distributed processor problems. We will use the MPI_BOR
* operation on a Reduce operation to processor zero, followed
* by a broadcast from zero, to accomplish this.
*/
#ifdef PARALLEL
ivec = alloc_int_1(V_LAST, 0);
for (imat = 0; imat < upd->Num_Mat; imat++) {
curr_pd = pd_glob[imat];
for (k = 0; k < V_LAST; k++) {
ivec[k] = curr_pd->v[k];
}
ReduceBcast_BOR(ivec, V_LAST);
#ifdef DEBUG_HKM
print_sync_start(TRUE);
for (k = 0; k < V_LAST; k++) {
if (curr_pd->v[k] != ivec[k]) {
printf("P_%d: v field for var_type %d changed from %d to %d\n",
ProcID, k, curr_pd->v[k], ivec[k]);
}
}
print_sync_end(TRUE);
#endif
for (k = 0; k < V_LAST; k++) {
curr_pd->v[k] = ivec[k];
}
}
safer_free((void **) &ivec);
#endif
return 0;
}
static void compare_maps_with_ddirectory_results(
int proc,
MESH_INFO_PTR mesh
)
{
/*
* Routine to demonstrate the use of the Zoltan Distributed Directory
* to build communication maps. This functionality essentially duplicates
* that in build_elem_comm_maps. It provides a test of the DDirectory,
* as the maps generated by the directory should match those generated
* by build_elem_comm_maps.
* This test is probably more complicated than necessary, but perhaps it
* will be adapted to become a communication maps tool within Zoltan.
*/
static const int want_size = 4;
int num_elems = mesh->num_elems;
Zoltan_DD *dd = NULL;
ZOLTAN_ID_PTR gids = NULL;
ZOLTAN_ID_PTR lids = NULL;
ZOLTAN_ID_PTR my_gids = NULL;
ZOLTAN_ID_PTR nbor_gids = NULL;
ZOLTAN_ID_PTR nbor_lids = NULL;
ZOLTAN_ID_PTR i_want = NULL;
ZOLTAN_ID_PTR others_want = NULL;
int *ownerlist = NULL;
int *sindex = NULL;
int num_nbor = 0; /* Number of neighboring elements not on this processor. */
/* This counter counts duplicate entries when an element */
/* is a neighbor of > 1 element on this processor. */
int cnt;
int num_others = 0;
int num_maps = 0;
int map_size = 0;
int max_map_size = 0;
int nbor_proc;
int i, j, k, ierr, error = 0, gerror = 0;
ELEM_INFO_PTR current;
struct map_list_head map;
Zoltan_Comm *comm;
/* Load array of element globalIDs for elements on this processor. */
/* Count number of neighboring elements not on this processor. */
gids = (ZOLTAN_ID_PTR) malloc(sizeof(ZOLTAN_ID_TYPE) * 2 * num_elems);
if (num_elems > 0 && gids == NULL) {
Gen_Error(0, "Fatal: insufficient memory");
error = 1;
}
lids = gids + num_elems;
for (i = 0; i < num_elems; i++) {
current = &(mesh->elements[i]);
gids[i] = current->globalID;
lids[i] = i;
for (j = 0; j < current->adj_len; j++) {
/* Skip NULL adjacencies (sides that are not adjacent to another elem). */
if (current->adj[j] == ZOLTAN_ID_INVALID) continue;
if (current->adj_proc[j] != proc)
num_nbor++;
}
}
/*
* Create DDirectory and register all owned elements.
*/
dd = new Zoltan_DD(MPI_COMM_WORLD, 1, 1, 0, 0, 0);
ierr = dd->Update(gids, lids, NULL, NULL, num_elems);
if (ierr) {
Gen_Error(0, "Fatal: Error returned by Zoltan_DD::Update");
error = 1;
}
free(gids);
/*
* Test copy operator and constructor
*/
Zoltan_DD ddCopy(*dd); // uses copy constructor
Zoltan_DD ddNew; // uses ordinary constructor
delete dd;
ddNew = ddCopy; // uses copy operator
dd = &ddNew;
/*
* Use the DDirectory to find owners of off-processor neighboring elements.
* Of course, we have this info in ELEM_INFO, but we do the find to test
* the DDirectory.
*/
nbor_gids = (ZOLTAN_ID_PTR) malloc(sizeof(ZOLTAN_ID_TYPE) * 3 * num_nbor);
if (num_nbor > 0 && nbor_gids == NULL) {
Gen_Error(0, "Fatal: insufficient memory");
error = 1;
}
nbor_lids = nbor_gids + num_nbor;
my_gids = nbor_lids + num_nbor;
ownerlist = (int *) malloc(sizeof(int) * num_nbor);
/*
* Get list of elements whose info is needed.
*/
cnt = 0;
for (i = 0; i < num_elems; i++) {
current = &(mesh->elements[i]);
for (j = 0; j < current->adj_len; j++) {
/* Skip NULL adjacencies (sides that are not adjacent to another elem). */
if (current->adj[j] == ZOLTAN_ID_INVALID) continue;
if (current->adj_proc[j] != proc) {
nbor_gids[cnt] = current->adj[j];
my_gids[cnt] = current->globalID;
cnt++;
}
}
}
/* Sanity check */
if (cnt != num_nbor) {
Gen_Error(0, "Fatal: cnt != num_nbor");
error = 1;
}
ierr = dd->Find( nbor_gids, nbor_lids, NULL, NULL, num_nbor, ownerlist);
if (ierr) {
Gen_Error(0, "Fatal: Error returned by Zoltan_DD::Find");
error = 1;
}
/*
* Check for errors
*/
MPI_Allreduce(&error, &gerror, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
if (gerror) {
Gen_Error(0, "Fatal: Error returned by DDirectory Test");
error_report(proc);
free(nbor_gids);
free(ownerlist);
return;
}
/*
* Use the Communication library to invert this information and build
* communication maps.
* We know what to receive and from where; compute what to send and
* to whom.
* Again, this info was computed by build_elem_com_maps, but we are
* testing DDirectory here.
*/
/*
* Build list of off-proc elements (and their owners)
* that this proc wants.
* This list includes duplicate entries when an element
* is a neighbor of > 1 element on this processor.
*/
i_want = (ZOLTAN_ID_PTR) malloc(sizeof(ZOLTAN_ID_TYPE) * want_size * num_nbor);
if (num_nbor > 0 && i_want == NULL) {
Gen_Error(0, "Fatal: insufficient memory");
return;
}
j = 0;
for (i = 0; i < num_nbor; i++) {
i_want[j++] = proc;
i_want[j++] = nbor_gids[i];
i_want[j++] = nbor_lids[i];
i_want[j++] = my_gids[i];
}
comm = new Zoltan_Comm(num_nbor, ownerlist, MPI_COMM_WORLD, 747,
&num_others);
/*
* Test copy operator and constructor
*/
printf("Test comm copy functions\n");
Zoltan_Comm commCopy(*comm); // uses copy constructor
Zoltan_Comm commNew; // uses ordinary constructor
delete comm;
commNew = commCopy; // uses copy operator
comm = &commNew;
/*
* Do communication to determine which of this proc's data is wanted by
* other procs.
* This info will determine what is put in this proc's communication maps.
*/
others_want = (ZOLTAN_ID_PTR) malloc(sizeof(ZOLTAN_ID_TYPE)*want_size*(num_others+1));
if (others_want == NULL) {
Gen_Error(0, "Fatal: insufficient memory");
return;
}
ierr = comm->Do(757, (char *) i_want, want_size * sizeof(ZOLTAN_ID_TYPE),
(char *) others_want);
if (ierr) {
Gen_Error(0, "Fatal: Error returned from Zoltan_Comm::Do");
return;
}
free(i_want);
/*
* Find number of maps and the size of the largest map.
* The maps should be grouped by neighboring processor
* in others_want.
*/
num_maps = 0;
map_size = max_map_size = 0;
for (i = 0, j = 0; i < num_others; i++, j += want_size) {
nbor_proc = (int)others_want[j];
map_size++;
if (i == (num_others - 1) || nbor_proc != (int) others_want[j+want_size]) {
/* End of map reached */
num_maps++;
if (map_size > max_map_size) max_map_size = map_size;
map_size = 0;
}
}
if (num_maps != mesh->necmap) {
printf("%d DDirectory Test: Different number of maps: "
"%d != %d\n", proc, num_maps, mesh->necmap);
}
/*
* For each map,
* build a map_list_head for the map;
* sort the map_list_head appropriately (as in build_elem_comm_maps);
* compare sorted lists with actually communication maps.
*/
sindex = (int *) malloc(max_map_size * sizeof(int));
if (max_map_size > 0 && sindex == NULL) {
Gen_Error(0, "Fatal: insufficient memory");
return;
}
map.map_alloc_size = max_map_size;
map.glob_id = (ZOLTAN_ID_TYPE *) malloc(max_map_size * sizeof(ZOLTAN_ID_TYPE));
map.elem_id = (int *) malloc(max_map_size * sizeof(int));
map.side_id = (int *) malloc(max_map_size * sizeof(int));
map.neigh_id = (ZOLTAN_ID_TYPE *) malloc(max_map_size * sizeof(ZOLTAN_ID_TYPE));
if (max_map_size > 0 && map.neigh_id == NULL) {
Gen_Error(0, "Fatal: insufficient memory");
return;
}
if (Debug_Driver > 3) {
/* For high debug levels, serialize the following section so that
* output of generated map is serialized (and not junked up).
*/
print_sync_start(proc, 1);
}
map_size = 0;
for (i = 0, j = 0; i < num_others; i++, j += want_size) {
nbor_proc = (int)others_want[j];
map.glob_id[map_size] = others_want[j+1];
map.elem_id[map_size] = others_want[j+2];
current = &(mesh->elements[map.elem_id[map_size]]);
for (k = 0; k < current->adj_len; k++) {
/* Skip NULL adjacencies (sides that are not adjacent to another elem). */
if (current->adj[k] == ZOLTAN_ID_INVALID) continue;
if (current->adj_proc[k] == nbor_proc &&
current->adj[k] == others_want[j+3]) {
map.side_id[map_size] = k + 1;
map.neigh_id[map_size] = current->adj[k];
break;
}
}
map_size++;
if (i == (num_others-1) || nbor_proc != (int) others_want[j+want_size]) {
/*
* End of map has been reached.
* Sort and compare the current map.
*/
sort_and_compare_maps(proc, nbor_proc, mesh, &map, map_size, sindex);
/*
* Reinitialize data structures for new map.
*/
map_size = 0;
}
}
if (Debug_Driver > 3) {
/* For high debug levels, serialize the previous section so that
* output of generated map is serialized (and not junked up).
*/
int nprocs = 0;
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
print_sync_end(proc, nprocs, 1);
}
free(map.glob_id);
free(map.elem_id);
free(map.neigh_id);
free(map.side_id);
free(sindex);
free(nbor_gids);
free(ownerlist);
free(others_want);
}
int
ns_data_print(pp_Data * p,
double x[],
const Exo_DB * exo,
const double time_value,
const double time_step_size)
{
const int quantity = p->data_type;
int mat_num = p->mat_num;
const int elemBlock_id = p->elem_blk_id;
const int node_set_id = p->ns_id;
const int species_id = p->species_number;
const char * filenm = p->data_filenm;
const char * qtity_str = p->data_type_name;
const char * format_flag = p->format_flag;
int * first_time = &(p->first_time);
static int err=0;
int num_nodes_on_side;
int ebIndex_first = -1;
int local_side[2];
int side_nodes[3]; /* Assume quad has no more than 3 per side. */
int elem_list[4], elem_ct=0, face, ielem, node2;
int local_node[4];
int node = -1;
int idx, idy, idz, id_var;
int iprint;
int nsp; /* node set pointer for this node set */
dbl x_pos, y_pos, z_pos;
int j, wspec;
int doPressure = 0;
#ifdef PARALLEL
double some_time=0.0;
#endif
double abscissa=0;
double ordinate=0;
double n1[3], n2[3];
double xi[3];
/*
* Find an element block that has the desired material id.
*/
if (elemBlock_id != -1) {
for (j = 0; j < exo->num_elem_blocks; j++) {
if (elemBlock_id == exo->eb_id[j]) {
ebIndex_first = j;
break;
}
}
if (ebIndex_first == -1) {
sprintf(err_msg, "Can't find an element block with the elem Block id %d\n", elemBlock_id);
if (Num_Proc == 1) {
EH(-1, err_msg);
}
}
mat_num = Matilda[ebIndex_first];
p->mat_num = mat_num;
pd = pd_glob[mat_num];
} else {
mat_num = -1;
p->mat_num = -1;
pd = pd_glob[0];
}
nsp = match_nsid(node_set_id);
if( nsp != -1 )
{
node = Proc_NS_List[Proc_NS_Pointers[nsp]];
}
else
{
sprintf(err_msg, "Node set ID %d not found.", node_set_id);
if( Num_Proc == 1 ) EH(-1,err_msg);
}
/* first right time stamp or run stamp to separate the sets */
print_sync_start(FALSE);
if (*first_time)
{
if ( format_flag[0] != '\0' )
{
if (ProcID == 0)
{
uf = fopen(filenm,"a");
if (uf != NULL)
{
fprintf(uf,"# %s %s @ nsid %d node (%d) \n",
format_flag, qtity_str, node_set_id, node );
*first_time = FALSE;
fclose(uf);
}
}
}
}
if (format_flag[0] == '\0')
{
if (ProcID == 0)
{
if ((uf = fopen(filenm,"a")) != NULL)
{
fprintf(uf,"Time/iteration = %e \n", time_value);
fprintf(uf," %s Node_Set %d Species %d\n", qtity_str,node_set_id,species_id);
fflush(uf);
fclose(uf);
}
}
}
if (nsp != -1 ) {
for (j = 0; j < Proc_NS_Count[nsp]; j++) {
node = Proc_NS_List[Proc_NS_Pointers[nsp]+j];
if (node < num_internal_dofs + num_boundary_dofs ) {
idx = Index_Solution(node, MESH_DISPLACEMENT1, 0, 0, -1);
if (idx == -1) {
x_pos = Coor[0][node];
WH(idx, "Mesh variable not found. May get undeformed coords.");
} else {
x_pos = Coor[0][node] + x[idx];
}
idy = Index_Solution(node, MESH_DISPLACEMENT2, 0, 0, -1);
if (idy == -1) {
y_pos = Coor[1][node];
} else {
y_pos = Coor[1][node] + x[idy];
}
z_pos = 0.;
if(pd->Num_Dim == 3) {
idz = Index_Solution(node, MESH_DISPLACEMENT3, 0, 0, -1);
if (idz == -1) {
z_pos = Coor[2][node];
} else{
z_pos = Coor[2][node] + x[idz];
}
}
if (quantity == MASS_FRACTION) {
id_var = Index_Solution(node, quantity, species_id, 0, mat_num);
} else if (quantity < 0) {
id_var = -1;
} else {
id_var = Index_Solution(node, quantity, 0, 0, mat_num);
}
/*
* In the easy case, the variable can be found somewhere in the
* big vector of unknowns. But sometimes we want a derived quantity
* that requires a little more work than an array reference.
*
* For now, save the good result if we have it.
*/
if ( id_var != -1 )
{
ordinate = x[id_var];
iprint = 1;
}
else
{
/*
* If we have an element based interpolation, let's calculate the interpolated value
*/
if (quantity == PRESSURE) {
if ((pd->i[PRESSURE] == I_P1) || ( (pd->i[PRESSURE] > I_PQ1) && (pd->i[PRESSURE] < I_Q2_HVG) )) {
doPressure = 1;
}
}
iprint = 0;
}
/*
* If the quantity is "theta", an interior angle that only
* makes sense at a point, in 2D, we'll need to compute it.
*/
if ( strncasecmp(qtity_str, "theta", 5 ) == 0 || doPressure)
{
/*
* Look for the two sides connected to this node...?
*
* Premise:
* 1. The node appears in only one element(removed-RBS,6/14/06)
* 2. Exactly two sides emanate from the node.
* 3. Quadrilateral.
*
* Apologies to people who wish to relax premise 1. I know
* there are some obtuse angles out there that benefit from
* having more than one element at a vertex. With care, this
* procedure could be extended to cover that case as well.
*/
if ( ! exo->node_elem_conn_exists )
{
EH(-1, "Cannot compute angle without node_elem_conn.");
}
elem_list[0] = exo->node_elem_list[exo->node_elem_pntr[node]];
/*
* Find out where this node appears in the elements local
* node ordering scheme...
*/
local_node[0] = in_list(node, exo->elem_node_pntr[elem_list[0]],
exo->elem_node_pntr[elem_list[0]+1],
exo->elem_node_list);
EH(local_node[0], "Can not find node in elem node connectivity!?! ");
local_node[0] -= exo->elem_node_pntr[elem_list[0]];
/* check for neighbors*/
if( mat_num == find_mat_number(elem_list[0], exo))
{elem_ct = 1;}
else
{WH(-1,"block id doesn't match first element");}
for (face=0 ; face<ei->num_sides ; face++)
{
ielem = exo->elem_elem_list[exo->elem_elem_pntr[elem_list[0]]+face];
if (ielem != -1)
{
node2 = in_list(node, exo->elem_node_pntr[ielem],
exo->elem_node_pntr[ielem+1],
exo->elem_node_list);
if (node2 != -1 && (mat_num == find_mat_number(ielem, exo)))
{
elem_list[elem_ct] = ielem;
local_node[elem_ct] = node2;
local_node[elem_ct] -= exo->elem_node_pntr[ielem];
elem_ct++;
}
}
}
/*
* Note that indeces are zero based!
*/
ordinate = 0.0;
for (ielem = 0 ; ielem < elem_ct ; ielem++)
{
if ( local_node[ielem] < 0 || local_node[ielem] > 3 )
{
if (strncasecmp(qtity_str, "theta", 5 ) == 0) {
EH(-1, "Node out of bounds.");
}
}
/*
* Now, determine the local name of the sides adjacent to this
* node...this works for the exo patran convention for quads...
*
* Again, local_node and local_side are zero based...
*/
local_side[0] = (local_node[ielem]+3)%4;
local_side[1] = local_node[ielem];
/*
* With the side names, we can find the normal vector.
* Again, assume the sides live on the same element.
*/
load_ei(elem_list[ielem], exo, 0);
/*
* We abuse the argument list under the conditions that
* we're going to do read-only operations and that
* we're not interested in old time steps, time derivatives
* etc.
*/
if (x == x_static) /* be the least disruptive possible */
{
err = load_elem_dofptr(elem_list[ielem], exo, x_static, x_old_static,
xdot_static, xdot_old_static, x_static, 1);
}
else
{
err = load_elem_dofptr(elem_list[ielem], exo, x, x, x, x, x, 1);
}
/*
* What are the local coordinates of the nodes in a quadrilateral?
*/
find_nodal_stu(local_node[ielem], ei->ielem_type, xi, xi+1, xi+2);
err = load_basis_functions(xi, bfd);
EH( err, "problem from load_basis_functions");
err = beer_belly();
EH( err, "beer_belly");
err = load_fv();
EH( err, "load_fv");
err = load_bf_grad();
EH( err, "load_bf_grad");
err = load_bf_mesh_derivs();
EH(err, "load_bf_mesh_derivs");
if (doPressure) {
ordinate = fv->P;
iprint = 1;
} else {
/* First, one side... */
get_side_info(ei->ielem_type, local_side[0]+1, &num_nodes_on_side,
side_nodes);
surface_determinant_and_normal(elem_list[ielem],
exo->elem_node_pntr[elem_list[ielem]],
ei->num_local_nodes,
ei->ielem_dim-1,
local_side[0]+1,
num_nodes_on_side,
side_nodes);
n1[0] = fv->snormal[0];
n1[1] = fv->snormal[1];
/* Second, the adjacent side of the quad... */
get_side_info(ei->ielem_type, local_side[1]+1, &num_nodes_on_side,
side_nodes);
surface_determinant_and_normal(elem_list[ielem],
exo->elem_node_pntr[elem_list[ielem]],
ei->num_local_nodes,
ei->ielem_dim-1,
local_side[1]+1,
num_nodes_on_side,
side_nodes);
n2[0] = fv->snormal[0];
n2[1] = fv->snormal[1];
/* cos (theta) = n1.n2 / ||n1|| ||n2|| */
ordinate += 180. - (180./M_PI)*acos((n1[0]*n2[0] + n1[1]*n2[1])/
(sqrt(n1[0]*n1[0]+n1[1]*n1[1])*
sqrt(n2[0]*n2[0]+n2[1]*n2[1])));
}
iprint = 1;
} /*ielem loop */
}
else if ( strncasecmp(qtity_str, "timestepsize", 12 ) == 0 )
{
ordinate = time_step_size;
iprint = 1;
}
else if ( strncasecmp(qtity_str, "cputime", 7 ) == 0 )
{
ordinate = ut();
iprint = 1;
}
else if ( strncasecmp(qtity_str, "wallclocktime", 13 ) == 0 )
{
/* Get these from extern via main...*/
#ifdef PARALLEL
some_time = MPI_Wtime();
ordinate = some_time - time_goma_started;
#endif
#ifndef PARALLEL
time_t now=0;
(void)time(&now);
ordinate = (double)(now) - time_goma_started;
#endif
iprint = 1;
}
else if ( strncasecmp(qtity_str, "speed", 5 ) == 0 )
{
id_var = Index_Solution(node, VELOCITY1, 0, 0, mat_num);
ordinate = SQUARE(x[id_var]);
id_var = Index_Solution(node, VELOCITY2, 0, 0, mat_num);
ordinate += SQUARE(x[id_var]);
id_var = Index_Solution(node, VELOCITY3, 0, 0, mat_num);
ordinate += SQUARE(x[id_var]);
ordinate = sqrt(ordinate);
iprint = 1;
}
else if ( strncasecmp(qtity_str, "ac_pres", 7 ) == 0 )
{
id_var = Index_Solution(node, ACOUS_PREAL, 0, 0, mat_num);
ordinate = SQUARE(x[id_var]);
id_var = Index_Solution(node, ACOUS_PIMAG, 0, 0, mat_num);
ordinate += SQUARE(x[id_var]);
ordinate = sqrt(ordinate);
iprint = 1;
}
else if ( strncasecmp(qtity_str, "light_comp", 10 ) == 0 )
{
id_var = Index_Solution(node, LIGHT_INTP, 0, 0, mat_num);
ordinate = x[id_var];
id_var = Index_Solution(node, LIGHT_INTM, 0, 0, mat_num);
ordinate += x[id_var];
iprint = 1;
}
else if ( strncasecmp(qtity_str, "nonvolatile", 11 ) == 0 )
{
ordinate = 1.0;
for(wspec = 0 ; wspec < pd->Num_Species_Eqn ; wspec++)
{
id_var = Index_Solution(node, MASS_FRACTION, wspec, 0, mat_num);
ordinate -= x[id_var]*mp_glob[mat_num]->molar_volume[wspec];
}
iprint = 1;
}
else
{
WH(id_var,
"Requested print variable is not defined at all nodes. May get 0.");
if(id_var == -1) iprint = 0;
}
if ((uf=fopen(filenm,"a")) != NULL)
{
if ( format_flag[0] == '\0' )
{
if (iprint)
{
fprintf(uf," %e %e %e %e \n", x_pos, y_pos, z_pos, ordinate);
}
}
else
{
if ( strncasecmp(format_flag, "t", 1) == 0 )
{
abscissa = time_value;
}
else if ( strncasecmp(format_flag, "x", 1) == 0 )
{
abscissa = x_pos;
}
else if ( strncasecmp(format_flag, "y", 1) == 0 )
{
abscissa = y_pos;
}
else if ( strncasecmp(format_flag, "z", 1) == 0 )
{
abscissa = z_pos;
}
else
{
abscissa = 0;
}
if (iprint)
{
fprintf(uf, "%.16g\t%.16g\n", abscissa, ordinate);
}
}
fclose(uf);
}
}
}
}
print_sync_end(FALSE);
return(1);
} /* END of routine ns_data_print */
int
ns_data_sens_print(const struct Post_Processing_Data_Sens *p,
const double x[], /* solution vector */
double **x_sens, /* solution sensitivity vector */
const double time_value) /* current time */
{
const int node_set_id = p->ns_id;
const int quantity = p->data_type;
const int mat_id = p->mat_id;
const int species_id = p->species_number;
const int sens_type = p->sens_type;
const int sens_id = p->sens_id;
const int sens_flt = p->sens_flt;
const int sens_flt2 = p->sens_flt2;
const char *filenm = p->data_filenm;
const char *qtity_str = p->data_type_name;
const int sens_ct = p->vector_id;
int node;
int idx, idy, idz, id_var;
int nsp; /* node set pointer for this node set */
dbl x_pos, y_pos, z_pos;
int j;
nsp = match_nsid(node_set_id);
if( nsp != -1 ) {
node = Proc_NS_List[Proc_NS_Pointers[nsp]];
}
else
{
sprintf(err_msg, "Node set ID %d not found.", node_set_id);
if( Num_Proc == 1 ) EH(-1,err_msg);
}
/* first right time stamp or run stamp to separate the sets */
print_sync_start(TRUE);
if (ProcID == 0 && (uf=fopen(filenm,"a")) != NULL)
{
fprintf(uf,"Time/iteration = %e \n", time_value);
fprintf(uf," %s Node_Set %d \n", qtity_str,node_set_id);
if(sens_type == 1)
{
fprintf(uf,"Sensitivity_type BC ID %d Float %d\n",sens_id,sens_flt);
}
else
if(sens_type == 2)
{
fprintf(uf,"Sensitivity_type MT NO %d Prop. %d\n",sens_id,sens_flt);
}
else
if(sens_type == 3)
{
fprintf(uf,"Sensitivity_type AC ID %d Float %d\n",sens_id,sens_flt);
}
else
if(sens_type == 4)
{
fprintf(uf,"Sensitivity_type UM NO %d Prop. %d %d\n",sens_id,sens_flt,sens_flt2);
}
else
if(sens_type == 5)
{
fprintf(uf,"Sensitivity_type UF ID %d Float %d\n",sens_id,sens_flt);
}
else
if(sens_type == 6)
{
fprintf(uf,"Sensitivity_type AN ID %d Float %d\n",sens_id,sens_flt);
}
fflush(uf);
fclose(uf);
}
if( nsp != -1 ) {
for (j = 0; j < Proc_NS_Count[nsp]; j++) {
node = Proc_NS_List[Proc_NS_Pointers[nsp]+j];
if( node < num_internal_dofs + num_boundary_dofs ) {
idx = Index_Solution (node, MESH_DISPLACEMENT1, 0, 0, -1);
if (idx == -1) {
x_pos = Coor[0][node];
WH(idx, "Mesh variable not found. May get undeformed coords.");
} else {
x_pos = Coor[0][node] + x[idx];
}
idy = Index_Solution (node, MESH_DISPLACEMENT2, 0, 0, -1);
if (idy == -1) {
y_pos = Coor[1][node];
} else {
y_pos = Coor[1][node] + x[idy];
}
z_pos = 0.;
if (pd->Num_Dim == 3) {
idz = Index_Solution(node, MESH_DISPLACEMENT3, 0, 0, -1);
if (idz == -1) {
z_pos = Coor[2][node];
} else {
z_pos = Coor[2][node] + x[idz];
}
}
if(quantity == MASS_FRACTION) {
id_var = Index_Solution(node, quantity, species_id, 0, mat_id);
} else {
id_var = Index_Solution(node, quantity, 0, 0, mat_id);
}
WH(id_var,
"Requested print variable is not defined at all nodes. May get 0.");
if ((uf=fopen(filenm,"a")) != NULL) {
if (id_var != -1) {
fprintf(uf, " %e %e %e %e \n",
x_pos, y_pos, z_pos, x_sens[sens_ct][id_var]);
}
fclose(uf);
}
}
}
}
print_sync_end(TRUE);
return(1);
} /* END of routine ns_data_sens_print */
