void print_distributed_mesh( int Proc, int Num_Proc, MESH_INFO_PTR mesh) { int i, j, k; int elem; int offset; ELEM_INFO_PTR current_elem; /* * Print the distributed mesh description for each processor. This routine * is useful for debugging the input meshes (Nemesis or Chaco). It is * serial, so it should not be used for production runs. */ print_sync_start(Proc, 1); printf ("############## Mesh information for processor %d ##############\n", Proc); printf ("Number Dimensions: %d\n", mesh->num_dims); printf ("Number Nodes: %d\n", mesh->num_nodes); printf ("Number Elements: %d\n", mesh->num_elems); printf ("Number Element Blocks: %d\n", mesh->num_el_blks); printf ("Number Node Sets: %d\n", mesh->num_node_sets); printf ("Number Side Sets: %d\n", mesh->num_side_sets); for (i = 0; i < mesh->num_el_blks; i++) { printf("\nElement block #%d\n", (i+1)); printf("\tID: %d\n", mesh->eb_ids[i]); printf("\tElement Type: %s\n", mesh->eb_names[i]); printf("\tElement Count: %d\n", mesh->eb_cnts[i]); printf("\tNodes Per Element: %d\n", mesh->eb_nnodes[i]); printf("\tAttrib Per Element: %d\n", mesh->eb_nattrs[i]); } printf("\nElement connect table, partition, weights and coordinates:\n"); for (i = 0; i < mesh->elem_array_len; i++) { current_elem = &(mesh->elements[i]); if (current_elem->globalID == -1) continue; printf("%d in part %d (%f):\n", current_elem->globalID, current_elem->my_part, current_elem->cpu_wgt[0]); for (j = 0; j < mesh->eb_nnodes[current_elem->elem_blk]; j++) { printf("\t%d |", current_elem->connect[j]); for (k = 0; k < mesh->num_dims; k++) { printf(" %f", current_elem->coord[j][k]); } printf("\n"); } } /* now print the adjacencies */ printf("\nElement adjacencies:\n"); printf("elem\tnadj(adj_len)\tadj,proc\n"); for (i = 0; i < mesh->elem_array_len; i++) { current_elem = &(mesh->elements[i]); if (current_elem->globalID == -1) continue; printf("%d\t", current_elem->globalID); printf("%d(%d)\t", current_elem->nadj, current_elem->adj_len); for (j = 0; j < current_elem->adj_len; j++) { /* Skip NULL adjacencies (sides that are not adjacent to another elem). */ if (current_elem->adj[j] == -1) continue; if (current_elem->adj_proc[j] == Proc) elem = mesh->elements[current_elem->adj[j]].globalID; else elem = current_elem->adj[j]; printf("%d,%d ", elem, current_elem->adj_proc[j]); } printf("\n"); } printf("\nCommunication maps\n"); printf("Number of maps: %d\n", mesh->necmap); printf("Map Ids(and Counts):"); for (i = 0; i < mesh->necmap; i++) printf(" %d(%d)", mesh->ecmap_id[i], mesh->ecmap_cnt[i]); printf("\n"); offset = 0; for (i = 0; i < mesh->necmap; i++) { printf("Map %d:\n", mesh->ecmap_id[i]); printf(" elem side globalID neighID\n"); for (j = 0; j < mesh->ecmap_cnt[i]; j++) { k = j + offset; printf(" %d %d %d %d\n", mesh->ecmap_elemids[k], mesh->ecmap_sideids[k], mesh->elements[mesh->ecmap_elemids[k]].globalID, mesh->ecmap_neighids[k]); } offset += mesh->ecmap_cnt[i]; } printf("\nHyperedges\n"); printf("Number of global hyperedges: %d\n", mesh->gnhedges); if (mesh->format == ZOLTAN_COMPRESSED_EDGE){ printf("Number of rows (edges): %d\n", mesh->nhedges); } else{ printf("Number of columns (vertices): %d\n", mesh->nhedges); } for (i = 0; i < mesh->nhedges; i++) { printf(" %d (%d): (", mesh->hgid[i], i); for (j = mesh->hindex[i]; j < mesh->hindex[i+1]; j++) printf("%d ", mesh->hvertex[j]); printf(")\n"); } if (mesh->hewgt_dim && (mesh->heNumWgts > 0)){ printf("\nHyperedge Weights\n"); for (i=0; i<mesh->heNumWgts; i++){ if (mesh->heWgtId){ printf("Hyperedge %d (%d): (", mesh->heWgtId[i], i); } else{ printf("Hyperedge %d (%d): (", mesh->hgid[i], i); } for (j = 0; j < mesh->hewgt_dim; j++) printf("%f ", mesh->hewgts[i*mesh->hewgt_dim + j]); printf(")\n"); } } print_sync_end(Proc, Num_Proc, 1); }
void bc_matrl_index(Exo_DB *exo) /******************************************************************** * * bc_matrl_index(): * * Find out what materials are on each side of a boundary * condition. Note, some boundary conditions require one * to specify this information in the form of an element block * id. However, some others do not. This procedure attempts to * calculate this for all boundary conditions and then fill in * the BC_matrl_index_# elements of the Boundary_Condition * structure. * * The basic algorithm involves finding some information about * the side or node set on which the boundary condition is * applied. Then, given the bc name and this information, * a decision is made as to what the BC_matrl_index_#'s should be * set to. * * We gather the following information to make this decision: * 1) element block indecises input from deck * 2) Number of nodes in the bc set in each material * 3) Number of elements, whoses sides are in the side set, * in each material * 4) Node in the bc set which contains the minimum * number of materials * 5) Node in the bc set which contains the maximum * number of materials * 6) A random node in the bc set which contains a * specific number of materials (1, 2, 3, 4) * * * HKM NOTE: * This routine is a work in progress. Calculation of EDGE's and * VERTICES are not done yet. Also, mp aspects haven't been * figured out yet. * *******************************************************************/ { int ibc, ss_index, side_index, k, node_num, i; int i_apply_meth, num_matrl_needed = -1; int found = FALSE, min_node_matrl, max_node_matrl, min_matrl, max_matrl, node_matrl_1, node_matrl_2, node_matrl_3, node_matrl_4, matrl_first; int *bin_matrl, *ind_matrl, *bin_matrl_elem, *node_flag_1ss, node_count; int mat_index, success, ielem; NODE_INFO_STRUCT *node_ptr; UMI_LIST_STRUCT *matrlLP; struct Boundary_Condition *bc_ptr; static char *yo = "bc_matrl_index :"; bin_matrl = alloc_int_1(upd->Num_Mat * 4, INT_NOINIT); ind_matrl = bin_matrl + upd->Num_Mat; bin_matrl_elem = ind_matrl + upd->Num_Mat; node_flag_1ss = alloc_int_1(exo->num_nodes, INT_NOINIT); for (ibc = 0; ibc < Num_BC; ibc++) { bc_ptr = BC_Types + ibc; found = FALSE; /* * Some boundary condition specifications already require * you to specify the element blocks on either side of the * side set. */ switch (bc_ptr->BC_Name) { /* * For these boundary conditions, the element block ID numbers * are in the firest two integer slots */ case POROUS_PRESSURE_BC: case DARCY_CONTINUOUS_BC: case Y_DISCONTINUOUS_BC: case POROUS_GAS_BC: case VP_EQUIL_BC: case VN_POROUS_BC: case FLUID_SOLID_BC: case SOLID_FLUID_BC: case NO_SLIP_BC: case FLUID_SOLID_CONTACT_BC: case SOLID_FLUID_CONTACT_BC: case T_CONTACT_RESIS_BC: case T_CONTACT_RESIS_2_BC: case LIGHTP_JUMP_BC: case LIGHTM_JUMP_BC: case LIGHTP_JUMP_2_BC: case LIGHTM_JUMP_2_BC: bc_ptr->BC_matrl_index_1 = map_mat_index(bc_ptr->BC_Data_Int[0]); bc_ptr->BC_matrl_index_2 = map_mat_index(bc_ptr->BC_Data_Int[1]); break; /* * For this boundary condition the element block numbers * are in the second and third integer slots */ case VL_EQUIL_BC: case YFLUX_DISC_RXN_BC: case DISCONTINUOUS_VELO_BC: bc_ptr->BC_matrl_index_1 = map_mat_index(bc_ptr->BC_Data_Int[1]); bc_ptr->BC_matrl_index_2 = map_mat_index(bc_ptr->BC_Data_Int[2]); break; } /* * Initialize quantities */ for (i = 0; i < 4 * upd->Num_Mat; i++) bin_matrl[i] = 0; for (i = 0; i < exo->num_nodes; i++) node_flag_1ss[i] = 0; max_node_matrl = min_node_matrl = -1; min_matrl = 4000000; max_matrl = -1; node_matrl_1 = node_matrl_2 = node_matrl_3 = node_matrl_4 = -1; /* * Determine how many materials each boundary condition needs * based on the type of the boundary condition */ i_apply_meth = BC_Types[ibc].desc->i_apply; if (i_apply_meth == CROSS_PHASE_DISCONTINUOUS || i_apply_meth == CROSS_PHASE) { num_matrl_needed = 2; } else if (i_apply_meth == SINGLE_PHASE) { num_matrl_needed = 1; } /* * Loop over the nodes in the side set looking up what * materials are located at each node * -> this will work for side sets. Need to do node sets * as well. */ if (!strcmp(bc_ptr->Set_Type, "SS")) { for (ss_index = 0; ss_index < exo->num_side_sets; ss_index++) { /* * This logic works for one side set specifications. For * two side set specifications (EDGES), we will have to go * with a calculation of the union of side sets. */ if (bc_ptr->BC_ID == exo->ss_id[ss_index]) { found = TRUE; for (side_index = 0; side_index < exo->ss_num_sides[ss_index]; side_index++) { /* * Locate the element number, find the material index, * then bin the result. */ ielem = exo->ss_elem_list[exo->ss_elem_index[ss_index]+side_index]; mat_index = find_mat_number(ielem, exo); bin_matrl_elem[mat_index]++; for (k = exo->ss_node_side_index[ss_index][side_index]; k < exo->ss_node_side_index[ss_index][side_index+1]; k++) { node_num = exo->ss_node_list[ss_index][k]; if (!node_flag_1ss[node_num]) { node_flag_1ss[node_num] = 1; node_ptr = Nodes[node_num]; matrlLP = &(node_ptr->Mat_List); /* * Bin the materials at this node for later usage. */ for (i = 0; i < matrlLP->Length; i++) { #ifdef DEBUG_IGNORE_ELEMENT_BLOCK_CAPABILITY if (matrlLP->List[i] < 0) { fprintf(stderr,"Material list contains negative number\n"); EH(-1,"logic error in ignoring an element block"); } #endif bin_matrl[matrlLP->List[i]]++; } /* * Find the max and min number of materials for a * node in this side set */ if (matrlLP->Length > max_matrl) { max_matrl = matrlLP->Length; max_node_matrl = node_num; } if (matrlLP->Length < min_matrl) { min_matrl = matrlLP->Length; min_node_matrl = node_num; } /* * Find representative nodes with specific * numbers of materials */ if (matrlLP->Length == 1) node_matrl_1 = node_num; if (matrlLP->Length == 2) node_matrl_2 = node_num; if (matrlLP->Length == 3) node_matrl_3 = node_num; if (matrlLP->Length == 4) node_matrl_4 = node_num; } } } } } /* End of side set loop */ } /* if SS */ /* * Node Sets */ if (!strcmp(bc_ptr->Set_Type, "NS")) { for (ss_index = 0; ss_index < exo->num_node_sets; ss_index++) { /* * This logic works for one side set specifications. For * two side set specifications (EDGES), we will have to go * with a calculation of the union of side sets. */ if (bc_ptr->BC_ID == exo->ns_id[ss_index]) { found = TRUE; /* * Loop over the number of nodes */ for (k = 0; k < exo->ns_num_nodes[ss_index]; k++) { node_num = exo->ns_node_list[exo->ns_node_index[ss_index]+k]; if (!node_flag_1ss[node_num]) { node_flag_1ss[node_num] = 1; node_ptr = Nodes[node_num]; matrlLP = &(node_ptr->Mat_List); /* * Bin the materials at this node for later usage. */ for (i = 0; i < matrlLP->Length; i++) { bin_matrl[matrlLP->List[i]]++; } /* * Find the max and min number of materials for a * node in this side set */ if (matrlLP->Length > max_matrl) { max_matrl = matrlLP->Length; max_node_matrl = node_num; } if (matrlLP->Length < min_matrl) { min_matrl = matrlLP->Length; min_node_matrl = node_num; } /* * Find representative nodes with specific * numbers of materials */ if (matrlLP->Length == 1) node_matrl_1 = node_num; if (matrlLP->Length == 2) node_matrl_2 = node_num; if (matrlLP->Length == 3) node_matrl_3 = node_num; if (matrlLP->Length == 4) node_matrl_4 = node_num; } } } } /* End of side set loop */ } /* if NS */ /* * Ok, we have obtained statistics on the side and node sets * let's make a decision */ if (found) { matrl_first = find_next_max(bin_matrl, ind_matrl, upd->Num_Mat); success = assign_matrl_2(bc_ptr, matrl_first); if (success < 0) { printf("%s P_%d: problem in assigning first matrl index in ibc %d, %d:\n", yo, ProcID, ibc, matrl_first); bc_matrl_index_print(bc_ptr, bin_matrl, bin_matrl_elem, min_node_matrl, max_node_matrl, node_matrl_1, node_matrl_2, node_matrl_3, node_matrl_4, ibc); } matrl_first = find_next_max(bin_matrl, ind_matrl, upd->Num_Mat); if (matrl_first >= 0) { success = assign_matrl_2(bc_ptr, matrl_first); if (success < 0) { printf("%s P_%d: problem in assigning second matrl index in ibc %d, %d:\n", yo, ProcID, ibc, matrl_first); bc_matrl_index_print(bc_ptr, bin_matrl, bin_matrl_elem, min_node_matrl, max_node_matrl, node_matrl_1, node_matrl_2, node_matrl_3, node_matrl_4, ibc); } } else { if (num_matrl_needed > 1) { printf("%s P_%d: problem in finding a needed second matrl index:\n", yo, ProcID); EH(-1,"bc_matrl_index ERROR"); } } } /* * For debug purposes, print out everything that we have found * out and decided about this bc on all of the processors. */ #ifdef DEBUG_HKM print_sync_start(FALSE); bc_matrl_index_print(bc_ptr, bin_matrl, bin_matrl_elem, min_node_matrl, max_node_matrl, node_matrl_1, node_matrl_2, node_matrl_3, node_matrl_4, ibc); print_sync_end(FALSE); #endif /* * MP Fix: We may not get the same results on all processors * In this case, just take the processor with the most * nodes in this bc and with a valid result, and use * that. Broadcast that result to all nodes. Cross your * fingers. */ node_count = 0; for (i = 0; i < exo->num_nodes; i++) { node_count += node_flag_1ss[i]; } #ifdef PARALLEL k = ProcWithMaxInt(node_count, &i); MPI_Bcast(&(bc_ptr->BC_matrl_index_1), 1, MPI_INT, k, MPI_COMM_WORLD); MPI_Bcast(&(bc_ptr->BC_matrl_index_2), 1, MPI_INT, k, MPI_COMM_WORLD); MPI_Bcast(&(bc_ptr->BC_matrl_index_3), 1, MPI_INT, k, MPI_COMM_WORLD); MPI_Bcast(&(bc_ptr->BC_matrl_index_4), 1, MPI_INT, k, MPI_COMM_WORLD); #ifdef DEBUG_HKM print_sync_start(FALSE); if ( !ProcID ) { printf("Final matrl_index's for ibc = %d:\n", ibc); printf("\tBC_matrl_index_1 = %d\n", bc_ptr->BC_matrl_index_1); printf("\tBC_matrl_index_2 = %d\n", bc_ptr->BC_matrl_index_2); printf("\tBC_matrl_index_3 = %d\n", bc_ptr->BC_matrl_index_3); printf("\tBC_matrl_index_4 = %d\n", bc_ptr->BC_matrl_index_4); fflush(stdout); } print_sync_end(FALSE); #endif #endif } safer_free((void **) &bin_matrl); safer_free((void **) &node_flag_1ss); }
int coordinate_discontinuous_variables(Exo_DB *exo, Dpi *dpi) /******************************************************************** * * coordinate_discontinuous_variables(): * * -> Make sure we have the correct designations for the v field * in the problem description structure for each material. * * -> Make sure that we have the same v field for all material * types on all processors. * * *******************************************************************/ { int ibc, eqn_type, ss_index, side_index, k, node_num, imat; int num_mat, mat_index, var_type, *ivec; UMI_LIST_STRUCT *curr_mat_list; NODE_INFO_STRUCT *node_ptr; PROBLEM_DESCRIPTION_STRUCT *curr_pd; /* * Loop over the boundary conditions. If we have a cross * phase discontinuous boundary condition, then we need to set the * v field for the appropriate variable types on both sides of the * interface to denote a discontinuous interpolation at the * interface. */ for (ibc = 0; ibc < Num_BC; ibc++) { if (BC_Types[ibc].desc->i_apply == CROSS_PHASE_DISCONTINUOUS) { eqn_type = BC_Types[ibc].desc->equation; /* * If we are applying a bc on the momentum equations * let's assign it a base equation type */ if (eqn_type == R_MOMENTUM1 || eqn_type == R_MOMENTUM2 || eqn_type == R_MOMENTUM3 || eqn_type == R_MOM_NORMAL || eqn_type == R_MOM_TANG1 || eqn_type == R_MOM_TANG2 ) { eqn_type = R_MOMENTUM1; } /* * If we are applying a discontinuous bc on one species * equation, then we must apply it to all species equations. */ if (eqn_type == R_MASS || (eqn_type >= R_SPECIES_UNK_0 && eqn_type <= R_SPECIES_UNK_LAST) ) { eqn_type = R_SPECIES_UNK_0; } for (ss_index = 0; ss_index < exo->num_side_sets; ss_index++) { if (BC_Types[ibc].BC_ID == exo->ss_id[ss_index]) { for (side_index = 0; side_index < exo->ss_num_sides[ss_index]; side_index++) { for (k = exo->ss_node_side_index[ss_index][side_index]; k < exo->ss_node_side_index[ss_index][side_index+1]; k++) { node_num = exo->ss_node_list[ss_index][k]; node_ptr = Nodes[node_num]; curr_mat_list = &(node_ptr->Mat_List); num_mat = curr_mat_list->Length; /* * Now make sure that we have the discontinuous var turned * on */ for (imat = 0; imat < num_mat; imat++) { mat_index= (curr_mat_list->List)[imat]; curr_pd = pd_glob[mat_index]; if (eqn_type == R_MOMENTUM1) { turn_on_discontinuous(curr_pd, R_MOMENTUM1); turn_on_discontinuous(curr_pd, R_MOMENTUM2); turn_on_discontinuous(curr_pd, R_MOMENTUM3); turn_on_discontinuous(curr_pd, PRESSURE); } else if (eqn_type == R_SPECIES_UNK_0) { turn_on_discontinuous(curr_pd, R_MASS); for (var_type = R_SPECIES_UNK_0; var_type < R_SPECIES_UNK_LAST; var_type++) { turn_on_discontinuous(curr_pd, var_type); } } else { turn_on_discontinuous(curr_pd, eqn_type); } } } } } } } } /* * Just to dot the eyes, make sure that v fields are uniform on * distributed processor problems. We will use the MPI_BOR * operation on a Reduce operation to processor zero, followed * by a broadcast from zero, to accomplish this. */ #ifdef PARALLEL ivec = alloc_int_1(V_LAST, 0); for (imat = 0; imat < upd->Num_Mat; imat++) { curr_pd = pd_glob[imat]; for (k = 0; k < V_LAST; k++) { ivec[k] = curr_pd->v[k]; } ReduceBcast_BOR(ivec, V_LAST); #ifdef DEBUG_HKM print_sync_start(TRUE); for (k = 0; k < V_LAST; k++) { if (curr_pd->v[k] != ivec[k]) { printf("P_%d: v field for var_type %d changed from %d to %d\n", ProcID, k, curr_pd->v[k], ivec[k]); } } print_sync_end(TRUE); #endif for (k = 0; k < V_LAST; k++) { curr_pd->v[k] = ivec[k]; } } safer_free((void **) &ivec); #endif return 0; }
static void compare_maps_with_ddirectory_results( int proc, MESH_INFO_PTR mesh ) { /* * Routine to demonstrate the use of the Zoltan Distributed Directory * to build communication maps. This functionality essentially duplicates * that in build_elem_comm_maps. It provides a test of the DDirectory, * as the maps generated by the directory should match those generated * by build_elem_comm_maps. * This test is probably more complicated than necessary, but perhaps it * will be adapted to become a communication maps tool within Zoltan. */ static const int want_size = 4; int num_elems = mesh->num_elems; Zoltan_DD *dd = NULL; ZOLTAN_ID_PTR gids = NULL; ZOLTAN_ID_PTR lids = NULL; ZOLTAN_ID_PTR my_gids = NULL; ZOLTAN_ID_PTR nbor_gids = NULL; ZOLTAN_ID_PTR nbor_lids = NULL; ZOLTAN_ID_PTR i_want = NULL; ZOLTAN_ID_PTR others_want = NULL; int *ownerlist = NULL; int *sindex = NULL; int num_nbor = 0; /* Number of neighboring elements not on this processor. */ /* This counter counts duplicate entries when an element */ /* is a neighbor of > 1 element on this processor. */ int cnt; int num_others = 0; int num_maps = 0; int map_size = 0; int max_map_size = 0; int nbor_proc; int i, j, k, ierr, error = 0, gerror = 0; ELEM_INFO_PTR current; struct map_list_head map; Zoltan_Comm *comm; /* Load array of element globalIDs for elements on this processor. */ /* Count number of neighboring elements not on this processor. */ gids = (ZOLTAN_ID_PTR) malloc(sizeof(ZOLTAN_ID_TYPE) * 2 * num_elems); if (num_elems > 0 && gids == NULL) { Gen_Error(0, "Fatal: insufficient memory"); error = 1; } lids = gids + num_elems; for (i = 0; i < num_elems; i++) { current = &(mesh->elements[i]); gids[i] = current->globalID; lids[i] = i; for (j = 0; j < current->adj_len; j++) { /* Skip NULL adjacencies (sides that are not adjacent to another elem). */ if (current->adj[j] == ZOLTAN_ID_INVALID) continue; if (current->adj_proc[j] != proc) num_nbor++; } } /* * Create DDirectory and register all owned elements. */ dd = new Zoltan_DD(MPI_COMM_WORLD, 1, 1, 0, 0, 0); ierr = dd->Update(gids, lids, NULL, NULL, num_elems); if (ierr) { Gen_Error(0, "Fatal: Error returned by Zoltan_DD::Update"); error = 1; } free(gids); /* * Test copy operator and constructor */ Zoltan_DD ddCopy(*dd); // uses copy constructor Zoltan_DD ddNew; // uses ordinary constructor delete dd; ddNew = ddCopy; // uses copy operator dd = &ddNew; /* * Use the DDirectory to find owners of off-processor neighboring elements. * Of course, we have this info in ELEM_INFO, but we do the find to test * the DDirectory. */ nbor_gids = (ZOLTAN_ID_PTR) malloc(sizeof(ZOLTAN_ID_TYPE) * 3 * num_nbor); if (num_nbor > 0 && nbor_gids == NULL) { Gen_Error(0, "Fatal: insufficient memory"); error = 1; } nbor_lids = nbor_gids + num_nbor; my_gids = nbor_lids + num_nbor; ownerlist = (int *) malloc(sizeof(int) * num_nbor); /* * Get list of elements whose info is needed. */ cnt = 0; for (i = 0; i < num_elems; i++) { current = &(mesh->elements[i]); for (j = 0; j < current->adj_len; j++) { /* Skip NULL adjacencies (sides that are not adjacent to another elem). */ if (current->adj[j] == ZOLTAN_ID_INVALID) continue; if (current->adj_proc[j] != proc) { nbor_gids[cnt] = current->adj[j]; my_gids[cnt] = current->globalID; cnt++; } } } /* Sanity check */ if (cnt != num_nbor) { Gen_Error(0, "Fatal: cnt != num_nbor"); error = 1; } ierr = dd->Find( nbor_gids, nbor_lids, NULL, NULL, num_nbor, ownerlist); if (ierr) { Gen_Error(0, "Fatal: Error returned by Zoltan_DD::Find"); error = 1; } /* * Check for errors */ MPI_Allreduce(&error, &gerror, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); if (gerror) { Gen_Error(0, "Fatal: Error returned by DDirectory Test"); error_report(proc); free(nbor_gids); free(ownerlist); return; } /* * Use the Communication library to invert this information and build * communication maps. * We know what to receive and from where; compute what to send and * to whom. * Again, this info was computed by build_elem_com_maps, but we are * testing DDirectory here. */ /* * Build list of off-proc elements (and their owners) * that this proc wants. * This list includes duplicate entries when an element * is a neighbor of > 1 element on this processor. */ i_want = (ZOLTAN_ID_PTR) malloc(sizeof(ZOLTAN_ID_TYPE) * want_size * num_nbor); if (num_nbor > 0 && i_want == NULL) { Gen_Error(0, "Fatal: insufficient memory"); return; } j = 0; for (i = 0; i < num_nbor; i++) { i_want[j++] = proc; i_want[j++] = nbor_gids[i]; i_want[j++] = nbor_lids[i]; i_want[j++] = my_gids[i]; } comm = new Zoltan_Comm(num_nbor, ownerlist, MPI_COMM_WORLD, 747, &num_others); /* * Test copy operator and constructor */ printf("Test comm copy functions\n"); Zoltan_Comm commCopy(*comm); // uses copy constructor Zoltan_Comm commNew; // uses ordinary constructor delete comm; commNew = commCopy; // uses copy operator comm = &commNew; /* * Do communication to determine which of this proc's data is wanted by * other procs. * This info will determine what is put in this proc's communication maps. */ others_want = (ZOLTAN_ID_PTR) malloc(sizeof(ZOLTAN_ID_TYPE)*want_size*(num_others+1)); if (others_want == NULL) { Gen_Error(0, "Fatal: insufficient memory"); return; } ierr = comm->Do(757, (char *) i_want, want_size * sizeof(ZOLTAN_ID_TYPE), (char *) others_want); if (ierr) { Gen_Error(0, "Fatal: Error returned from Zoltan_Comm::Do"); return; } free(i_want); /* * Find number of maps and the size of the largest map. * The maps should be grouped by neighboring processor * in others_want. */ num_maps = 0; map_size = max_map_size = 0; for (i = 0, j = 0; i < num_others; i++, j += want_size) { nbor_proc = (int)others_want[j]; map_size++; if (i == (num_others - 1) || nbor_proc != (int) others_want[j+want_size]) { /* End of map reached */ num_maps++; if (map_size > max_map_size) max_map_size = map_size; map_size = 0; } } if (num_maps != mesh->necmap) { printf("%d DDirectory Test: Different number of maps: " "%d != %d\n", proc, num_maps, mesh->necmap); } /* * For each map, * build a map_list_head for the map; * sort the map_list_head appropriately (as in build_elem_comm_maps); * compare sorted lists with actually communication maps. */ sindex = (int *) malloc(max_map_size * sizeof(int)); if (max_map_size > 0 && sindex == NULL) { Gen_Error(0, "Fatal: insufficient memory"); return; } map.map_alloc_size = max_map_size; map.glob_id = (ZOLTAN_ID_TYPE *) malloc(max_map_size * sizeof(ZOLTAN_ID_TYPE)); map.elem_id = (int *) malloc(max_map_size * sizeof(int)); map.side_id = (int *) malloc(max_map_size * sizeof(int)); map.neigh_id = (ZOLTAN_ID_TYPE *) malloc(max_map_size * sizeof(ZOLTAN_ID_TYPE)); if (max_map_size > 0 && map.neigh_id == NULL) { Gen_Error(0, "Fatal: insufficient memory"); return; } if (Debug_Driver > 3) { /* For high debug levels, serialize the following section so that * output of generated map is serialized (and not junked up). */ print_sync_start(proc, 1); } map_size = 0; for (i = 0, j = 0; i < num_others; i++, j += want_size) { nbor_proc = (int)others_want[j]; map.glob_id[map_size] = others_want[j+1]; map.elem_id[map_size] = others_want[j+2]; current = &(mesh->elements[map.elem_id[map_size]]); for (k = 0; k < current->adj_len; k++) { /* Skip NULL adjacencies (sides that are not adjacent to another elem). */ if (current->adj[k] == ZOLTAN_ID_INVALID) continue; if (current->adj_proc[k] == nbor_proc && current->adj[k] == others_want[j+3]) { map.side_id[map_size] = k + 1; map.neigh_id[map_size] = current->adj[k]; break; } } map_size++; if (i == (num_others-1) || nbor_proc != (int) others_want[j+want_size]) { /* * End of map has been reached. * Sort and compare the current map. */ sort_and_compare_maps(proc, nbor_proc, mesh, &map, map_size, sindex); /* * Reinitialize data structures for new map. */ map_size = 0; } } if (Debug_Driver > 3) { /* For high debug levels, serialize the previous section so that * output of generated map is serialized (and not junked up). */ int nprocs = 0; MPI_Comm_size(MPI_COMM_WORLD, &nprocs); print_sync_end(proc, nprocs, 1); } free(map.glob_id); free(map.elem_id); free(map.neigh_id); free(map.side_id); free(sindex); free(nbor_gids); free(ownerlist); free(others_want); }
int ns_data_print(pp_Data * p, double x[], const Exo_DB * exo, const double time_value, const double time_step_size) { const int quantity = p->data_type; int mat_num = p->mat_num; const int elemBlock_id = p->elem_blk_id; const int node_set_id = p->ns_id; const int species_id = p->species_number; const char * filenm = p->data_filenm; const char * qtity_str = p->data_type_name; const char * format_flag = p->format_flag; int * first_time = &(p->first_time); static int err=0; int num_nodes_on_side; int ebIndex_first = -1; int local_side[2]; int side_nodes[3]; /* Assume quad has no more than 3 per side. */ int elem_list[4], elem_ct=0, face, ielem, node2; int local_node[4]; int node = -1; int idx, idy, idz, id_var; int iprint; int nsp; /* node set pointer for this node set */ dbl x_pos, y_pos, z_pos; int j, wspec; int doPressure = 0; #ifdef PARALLEL double some_time=0.0; #endif double abscissa=0; double ordinate=0; double n1[3], n2[3]; double xi[3]; /* * Find an element block that has the desired material id. */ if (elemBlock_id != -1) { for (j = 0; j < exo->num_elem_blocks; j++) { if (elemBlock_id == exo->eb_id[j]) { ebIndex_first = j; break; } } if (ebIndex_first == -1) { sprintf(err_msg, "Can't find an element block with the elem Block id %d\n", elemBlock_id); if (Num_Proc == 1) { EH(-1, err_msg); } } mat_num = Matilda[ebIndex_first]; p->mat_num = mat_num; pd = pd_glob[mat_num]; } else { mat_num = -1; p->mat_num = -1; pd = pd_glob[0]; } nsp = match_nsid(node_set_id); if( nsp != -1 ) { node = Proc_NS_List[Proc_NS_Pointers[nsp]]; } else { sprintf(err_msg, "Node set ID %d not found.", node_set_id); if( Num_Proc == 1 ) EH(-1,err_msg); } /* first right time stamp or run stamp to separate the sets */ print_sync_start(FALSE); if (*first_time) { if ( format_flag[0] != '\0' ) { if (ProcID == 0) { uf = fopen(filenm,"a"); if (uf != NULL) { fprintf(uf,"# %s %s @ nsid %d node (%d) \n", format_flag, qtity_str, node_set_id, node ); *first_time = FALSE; fclose(uf); } } } } if (format_flag[0] == '\0') { if (ProcID == 0) { if ((uf = fopen(filenm,"a")) != NULL) { fprintf(uf,"Time/iteration = %e \n", time_value); fprintf(uf," %s Node_Set %d Species %d\n", qtity_str,node_set_id,species_id); fflush(uf); fclose(uf); } } } if (nsp != -1 ) { for (j = 0; j < Proc_NS_Count[nsp]; j++) { node = Proc_NS_List[Proc_NS_Pointers[nsp]+j]; if (node < num_internal_dofs + num_boundary_dofs ) { idx = Index_Solution(node, MESH_DISPLACEMENT1, 0, 0, -1); if (idx == -1) { x_pos = Coor[0][node]; WH(idx, "Mesh variable not found. May get undeformed coords."); } else { x_pos = Coor[0][node] + x[idx]; } idy = Index_Solution(node, MESH_DISPLACEMENT2, 0, 0, -1); if (idy == -1) { y_pos = Coor[1][node]; } else { y_pos = Coor[1][node] + x[idy]; } z_pos = 0.; if(pd->Num_Dim == 3) { idz = Index_Solution(node, MESH_DISPLACEMENT3, 0, 0, -1); if (idz == -1) { z_pos = Coor[2][node]; } else{ z_pos = Coor[2][node] + x[idz]; } } if (quantity == MASS_FRACTION) { id_var = Index_Solution(node, quantity, species_id, 0, mat_num); } else if (quantity < 0) { id_var = -1; } else { id_var = Index_Solution(node, quantity, 0, 0, mat_num); } /* * In the easy case, the variable can be found somewhere in the * big vector of unknowns. But sometimes we want a derived quantity * that requires a little more work than an array reference. * * For now, save the good result if we have it. */ if ( id_var != -1 ) { ordinate = x[id_var]; iprint = 1; } else { /* * If we have an element based interpolation, let's calculate the interpolated value */ if (quantity == PRESSURE) { if ((pd->i[PRESSURE] == I_P1) || ( (pd->i[PRESSURE] > I_PQ1) && (pd->i[PRESSURE] < I_Q2_HVG) )) { doPressure = 1; } } iprint = 0; } /* * If the quantity is "theta", an interior angle that only * makes sense at a point, in 2D, we'll need to compute it. */ if ( strncasecmp(qtity_str, "theta", 5 ) == 0 || doPressure) { /* * Look for the two sides connected to this node...? * * Premise: * 1. The node appears in only one element(removed-RBS,6/14/06) * 2. Exactly two sides emanate from the node. * 3. Quadrilateral. * * Apologies to people who wish to relax premise 1. I know * there are some obtuse angles out there that benefit from * having more than one element at a vertex. With care, this * procedure could be extended to cover that case as well. */ if ( ! exo->node_elem_conn_exists ) { EH(-1, "Cannot compute angle without node_elem_conn."); } elem_list[0] = exo->node_elem_list[exo->node_elem_pntr[node]]; /* * Find out where this node appears in the elements local * node ordering scheme... */ local_node[0] = in_list(node, exo->elem_node_pntr[elem_list[0]], exo->elem_node_pntr[elem_list[0]+1], exo->elem_node_list); EH(local_node[0], "Can not find node in elem node connectivity!?! "); local_node[0] -= exo->elem_node_pntr[elem_list[0]]; /* check for neighbors*/ if( mat_num == find_mat_number(elem_list[0], exo)) {elem_ct = 1;} else {WH(-1,"block id doesn't match first element");} for (face=0 ; face<ei->num_sides ; face++) { ielem = exo->elem_elem_list[exo->elem_elem_pntr[elem_list[0]]+face]; if (ielem != -1) { node2 = in_list(node, exo->elem_node_pntr[ielem], exo->elem_node_pntr[ielem+1], exo->elem_node_list); if (node2 != -1 && (mat_num == find_mat_number(ielem, exo))) { elem_list[elem_ct] = ielem; local_node[elem_ct] = node2; local_node[elem_ct] -= exo->elem_node_pntr[ielem]; elem_ct++; } } } /* * Note that indeces are zero based! */ ordinate = 0.0; for (ielem = 0 ; ielem < elem_ct ; ielem++) { if ( local_node[ielem] < 0 || local_node[ielem] > 3 ) { if (strncasecmp(qtity_str, "theta", 5 ) == 0) { EH(-1, "Node out of bounds."); } } /* * Now, determine the local name of the sides adjacent to this * node...this works for the exo patran convention for quads... * * Again, local_node and local_side are zero based... */ local_side[0] = (local_node[ielem]+3)%4; local_side[1] = local_node[ielem]; /* * With the side names, we can find the normal vector. * Again, assume the sides live on the same element. */ load_ei(elem_list[ielem], exo, 0); /* * We abuse the argument list under the conditions that * we're going to do read-only operations and that * we're not interested in old time steps, time derivatives * etc. */ if (x == x_static) /* be the least disruptive possible */ { err = load_elem_dofptr(elem_list[ielem], exo, x_static, x_old_static, xdot_static, xdot_old_static, x_static, 1); } else { err = load_elem_dofptr(elem_list[ielem], exo, x, x, x, x, x, 1); } /* * What are the local coordinates of the nodes in a quadrilateral? */ find_nodal_stu(local_node[ielem], ei->ielem_type, xi, xi+1, xi+2); err = load_basis_functions(xi, bfd); EH( err, "problem from load_basis_functions"); err = beer_belly(); EH( err, "beer_belly"); err = load_fv(); EH( err, "load_fv"); err = load_bf_grad(); EH( err, "load_bf_grad"); err = load_bf_mesh_derivs(); EH(err, "load_bf_mesh_derivs"); if (doPressure) { ordinate = fv->P; iprint = 1; } else { /* First, one side... */ get_side_info(ei->ielem_type, local_side[0]+1, &num_nodes_on_side, side_nodes); surface_determinant_and_normal(elem_list[ielem], exo->elem_node_pntr[elem_list[ielem]], ei->num_local_nodes, ei->ielem_dim-1, local_side[0]+1, num_nodes_on_side, side_nodes); n1[0] = fv->snormal[0]; n1[1] = fv->snormal[1]; /* Second, the adjacent side of the quad... */ get_side_info(ei->ielem_type, local_side[1]+1, &num_nodes_on_side, side_nodes); surface_determinant_and_normal(elem_list[ielem], exo->elem_node_pntr[elem_list[ielem]], ei->num_local_nodes, ei->ielem_dim-1, local_side[1]+1, num_nodes_on_side, side_nodes); n2[0] = fv->snormal[0]; n2[1] = fv->snormal[1]; /* cos (theta) = n1.n2 / ||n1|| ||n2|| */ ordinate += 180. - (180./M_PI)*acos((n1[0]*n2[0] + n1[1]*n2[1])/ (sqrt(n1[0]*n1[0]+n1[1]*n1[1])* sqrt(n2[0]*n2[0]+n2[1]*n2[1]))); } iprint = 1; } /*ielem loop */ } else if ( strncasecmp(qtity_str, "timestepsize", 12 ) == 0 ) { ordinate = time_step_size; iprint = 1; } else if ( strncasecmp(qtity_str, "cputime", 7 ) == 0 ) { ordinate = ut(); iprint = 1; } else if ( strncasecmp(qtity_str, "wallclocktime", 13 ) == 0 ) { /* Get these from extern via main...*/ #ifdef PARALLEL some_time = MPI_Wtime(); ordinate = some_time - time_goma_started; #endif #ifndef PARALLEL time_t now=0; (void)time(&now); ordinate = (double)(now) - time_goma_started; #endif iprint = 1; } else if ( strncasecmp(qtity_str, "speed", 5 ) == 0 ) { id_var = Index_Solution(node, VELOCITY1, 0, 0, mat_num); ordinate = SQUARE(x[id_var]); id_var = Index_Solution(node, VELOCITY2, 0, 0, mat_num); ordinate += SQUARE(x[id_var]); id_var = Index_Solution(node, VELOCITY3, 0, 0, mat_num); ordinate += SQUARE(x[id_var]); ordinate = sqrt(ordinate); iprint = 1; } else if ( strncasecmp(qtity_str, "ac_pres", 7 ) == 0 ) { id_var = Index_Solution(node, ACOUS_PREAL, 0, 0, mat_num); ordinate = SQUARE(x[id_var]); id_var = Index_Solution(node, ACOUS_PIMAG, 0, 0, mat_num); ordinate += SQUARE(x[id_var]); ordinate = sqrt(ordinate); iprint = 1; } else if ( strncasecmp(qtity_str, "light_comp", 10 ) == 0 ) { id_var = Index_Solution(node, LIGHT_INTP, 0, 0, mat_num); ordinate = x[id_var]; id_var = Index_Solution(node, LIGHT_INTM, 0, 0, mat_num); ordinate += x[id_var]; iprint = 1; } else if ( strncasecmp(qtity_str, "nonvolatile", 11 ) == 0 ) { ordinate = 1.0; for(wspec = 0 ; wspec < pd->Num_Species_Eqn ; wspec++) { id_var = Index_Solution(node, MASS_FRACTION, wspec, 0, mat_num); ordinate -= x[id_var]*mp_glob[mat_num]->molar_volume[wspec]; } iprint = 1; } else { WH(id_var, "Requested print variable is not defined at all nodes. May get 0."); if(id_var == -1) iprint = 0; } if ((uf=fopen(filenm,"a")) != NULL) { if ( format_flag[0] == '\0' ) { if (iprint) { fprintf(uf," %e %e %e %e \n", x_pos, y_pos, z_pos, ordinate); } } else { if ( strncasecmp(format_flag, "t", 1) == 0 ) { abscissa = time_value; } else if ( strncasecmp(format_flag, "x", 1) == 0 ) { abscissa = x_pos; } else if ( strncasecmp(format_flag, "y", 1) == 0 ) { abscissa = y_pos; } else if ( strncasecmp(format_flag, "z", 1) == 0 ) { abscissa = z_pos; } else { abscissa = 0; } if (iprint) { fprintf(uf, "%.16g\t%.16g\n", abscissa, ordinate); } } fclose(uf); } } } } print_sync_end(FALSE); return(1); } /* END of routine ns_data_print */
int ns_data_sens_print(const struct Post_Processing_Data_Sens *p, const double x[], /* solution vector */ double **x_sens, /* solution sensitivity vector */ const double time_value) /* current time */ { const int node_set_id = p->ns_id; const int quantity = p->data_type; const int mat_id = p->mat_id; const int species_id = p->species_number; const int sens_type = p->sens_type; const int sens_id = p->sens_id; const int sens_flt = p->sens_flt; const int sens_flt2 = p->sens_flt2; const char *filenm = p->data_filenm; const char *qtity_str = p->data_type_name; const int sens_ct = p->vector_id; int node; int idx, idy, idz, id_var; int nsp; /* node set pointer for this node set */ dbl x_pos, y_pos, z_pos; int j; nsp = match_nsid(node_set_id); if( nsp != -1 ) { node = Proc_NS_List[Proc_NS_Pointers[nsp]]; } else { sprintf(err_msg, "Node set ID %d not found.", node_set_id); if( Num_Proc == 1 ) EH(-1,err_msg); } /* first right time stamp or run stamp to separate the sets */ print_sync_start(TRUE); if (ProcID == 0 && (uf=fopen(filenm,"a")) != NULL) { fprintf(uf,"Time/iteration = %e \n", time_value); fprintf(uf," %s Node_Set %d \n", qtity_str,node_set_id); if(sens_type == 1) { fprintf(uf,"Sensitivity_type BC ID %d Float %d\n",sens_id,sens_flt); } else if(sens_type == 2) { fprintf(uf,"Sensitivity_type MT NO %d Prop. %d\n",sens_id,sens_flt); } else if(sens_type == 3) { fprintf(uf,"Sensitivity_type AC ID %d Float %d\n",sens_id,sens_flt); } else if(sens_type == 4) { fprintf(uf,"Sensitivity_type UM NO %d Prop. %d %d\n",sens_id,sens_flt,sens_flt2); } else if(sens_type == 5) { fprintf(uf,"Sensitivity_type UF ID %d Float %d\n",sens_id,sens_flt); } else if(sens_type == 6) { fprintf(uf,"Sensitivity_type AN ID %d Float %d\n",sens_id,sens_flt); } fflush(uf); fclose(uf); } if( nsp != -1 ) { for (j = 0; j < Proc_NS_Count[nsp]; j++) { node = Proc_NS_List[Proc_NS_Pointers[nsp]+j]; if( node < num_internal_dofs + num_boundary_dofs ) { idx = Index_Solution (node, MESH_DISPLACEMENT1, 0, 0, -1); if (idx == -1) { x_pos = Coor[0][node]; WH(idx, "Mesh variable not found. May get undeformed coords."); } else { x_pos = Coor[0][node] + x[idx]; } idy = Index_Solution (node, MESH_DISPLACEMENT2, 0, 0, -1); if (idy == -1) { y_pos = Coor[1][node]; } else { y_pos = Coor[1][node] + x[idy]; } z_pos = 0.; if (pd->Num_Dim == 3) { idz = Index_Solution(node, MESH_DISPLACEMENT3, 0, 0, -1); if (idz == -1) { z_pos = Coor[2][node]; } else { z_pos = Coor[2][node] + x[idz]; } } if(quantity == MASS_FRACTION) { id_var = Index_Solution(node, quantity, species_id, 0, mat_id); } else { id_var = Index_Solution(node, quantity, 0, 0, mat_id); } WH(id_var, "Requested print variable is not defined at all nodes. May get 0."); if ((uf=fopen(filenm,"a")) != NULL) { if (id_var != -1) { fprintf(uf, " %e %e %e %e \n", x_pos, y_pos, z_pos, x_sens[sens_ct][id_var]); } fclose(uf); } } } } print_sync_end(TRUE); return(1); } /* END of routine ns_data_sens_print */