static void write_q_formatted (char *qfile)
{
int nz, i, j, n, k, nk, np;
FILE *fp;
printf ("\nwriting formatted Q file to %s\n", qfile);
if (NULL == (fp = fopen (qfile, "w+"))) {
fprintf (stderr, "couldn't open <%s> for writing\n", qfile);
exit (1);
}
if (mblock || nblocks > 1)
fprintf (fp, "%d\n", nblocks);
for (nz = 0; nz < nZones; nz++) {
if (Zones[nz].type == CGNS_ENUMV(Structured))
fprintf (fp, "%d %d %d\n", (int)Zones[nz].dim[0],
(int)Zones[nz].dim[1], (int)Zones[nz].dim[2]);
}
for (nz = 0; nz < nZones; nz++) {
if (Zones[nz].type == CGNS_ENUMV(Structured)) {
printf (" zone %d ... ", nz+1);
fflush (stdout);
fprintf (fp, "%#g %#g %#g %#g\n", reference[0], reference[1],
reference[2], reference[3]);
compute_solution (nz);
if (whole) {
np = (int)Zones[nz].nverts;
nk = 1;
}
else {
np = (int)(Zones[nz].dim[0] * Zones[nz].dim[1]);
nk = (int)Zones[nz].dim[2];
}
for (k = 0; k < nk; k++) {
i = k * np;
for (j = 0; j < 5; j++) {
for (n = 0; n < np; n++) {
if (n) putc ((n % 5) == 0 ? '\n' : ' ', fp);
fprintf (fp, "%#g", q[j][i+n]);
}
putc ('\n', fp);
}
}
puts ("done");
}
}
fclose (fp);
}
int main(int argc, char *argv[]) {
int my_rank, num_procs;
const int max_iters = 10000; /// maximum number of iteration to perform
/** Simulation parameters parsed from the input datasets */
int nintci, nintcf; /// internal cells start and end index
/// external cells start and end index. The external cells are only ghost cells.
/// They are accessed only through internal cells
int nextci, nextcf;
int **lcc; /// link cell-to-cell array - stores neighboring information
int *lcc_local;
/// Boundary coefficients for each volume cell (South, East, North, West, High, Low)
double *bs, *be, *bn, *bw, *bl, *bh;
double *bp; /// Pole coefficient
double *su; /// Source values
double residual_ratio; /// the ratio between the reference and the current residual
double *var; /// the variation vector -> keeps the result in the end
/** Additional vectors required for the computation */
double *cgup, *oc, *cnorm;
/** Geometry data */
int points_count; /// total number of points that define the geometry
int** points; /// coordinates of the points that define the cells - size [points_cnt][3]
int* elems; /// definition of the cells using their nodes (points) - each cell has 8 points
int num_elems;
/** Mapping between local and remote cell indices */
int* local_global_index; /// local to global index mapping
int* global_local_index; /// global to local index mapping
int* local_global_index_full;
/** Lists of cells requires for the communication */
int neighbors_count = 0; /// total number of neighbors to communicate with
int* send_count; /// number of elements to send to each neighbor (size: neighbors_count)
/// send lists for the other neighbors(cell ids which should be sent)(size:[#neighbors][#cells]
int** send_list;
int* recv_count; /// how many elements are in the recv lists for each neighbor
int** recv_list; /// send lists for the other neighbor (see send_list)
/** Metis Results */
int* epart; /// partition vector for the elements of the mesh
int* npart; /// partition vector for the points (nodes) of the mesh
int objval; /// resulting edgecut of total communication volume (classical distrib->zeros)
MPI_Init(&argc, &argv); /// Start MPI
SCOREP_USER_REGION_DEFINE(OA_Phase);
MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); /// Get current process id
MPI_Comm_size(MPI_COMM_WORLD, &num_procs); /// get number of processes
double elapsed_time, elapsed_time_max;
FILE *pFile;
if ( argc < 3 ) {
fprintf(stderr, "Usage: ./gccg <input_file> <output_prefix> <partition_type>\n");
MPI_Abort(MPI_COMM_WORLD, -1);
}
char *file_in = argv[1];
char *out_prefix = argv[2];
char *part_type = (argc == 3 ? "classical" : argv[3]);
char file_vtk_out[50];
char measure_out[20];
/********** START INITIALIZATION **********/
// read-in the input file
elapsed_time = - MPI_Wtime();
SCOREP_USER_OA_PHASE_BEGIN(OA_Phase, "OA_Phase", SCOREP_USER_REGION_TYPE_COMMON);
int init_status = initialization(file_in, part_type, &nintci, &nintcf, &nextci, &nextcf, &lcc,
&bs, &be, &bn, &bw, &bl, &bh, &bp, &su, &points_count, &points,
&elems, &var, &cgup, &oc, &cnorm, &local_global_index,
&global_local_index, &local_global_index_full, &lcc_local,
&neighbors_count, &send_count, &send_list,
&recv_count, &recv_list, &epart, &npart, &objval);
elapsed_time += MPI_Wtime();
MPI_Reduce(&elapsed_time, &elapsed_time_max, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
if (my_rank == 0) {
sprintf(measure_out, "time--%d.txt", num_procs);
pFile = fopen(measure_out, "a");
fprintf(pFile, "%s Elapsed max --initialization time-init_status = %d: %f secs\n",
argv[2], init_status, elapsed_time_max);
}
if (init_status != 0 && init_status != 1) {
fprintf(stderr, "Failed to initialize data!\n");
MPI_Abort(MPI_COMM_WORLD, my_rank);
}
num_elems = nintcf - nintci + 1;
// test distribution
/*if (my_rank == 2) {
sprintf(file_vtk_out, "%s_cgup.vtk", out_prefix);
test_distribution(file_in, file_vtk_out, local_global_index,
num_elems, cgup);
}*/
// Implement this function in test_functions.c and call it here
/*if (my_rank == 2) {
sprintf(file_vtk_out, "%s_commlist.vtk", out_prefix);
test_communication(file_in, file_vtk_out, local_global_index, num_elems,
neighbors_count, send_count, send_list, recv_count, recv_list);
}*/
/********** END INITIALIZATION **********/
/********** START COMPUTATIONAL LOOP **********/
elapsed_time = - MPI_Wtime();
int total_iters = compute_solution(max_iters, nintci, nintcf, nextcf, lcc, bp, bs, bw, bl, bn,
be, bh, cnorm, var, su, cgup, &residual_ratio,
local_global_index, global_local_index,
lcc_local, neighbors_count,
send_count, send_list, recv_count, recv_list);
elapsed_time += MPI_Wtime();
MPI_Reduce(&elapsed_time, &elapsed_time_max, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
if (my_rank == 0) {
pFile = fopen(measure_out, "a");
fprintf(pFile, "%s Elapsed max --computation time-init_status = %d: %f secs\n",
argv[2], init_status, elapsed_time_max);
}
/********** END COMPUTATIONAL LOOP **********/
/********** START FINALIZATION **********/
elapsed_time = - MPI_Wtime();
finalization(file_in, out_prefix, total_iters, residual_ratio, nintci, nintcf, points_count,
points, elems, var, cgup, su, local_global_index_full, init_status);
elapsed_time += MPI_Wtime();
MPI_Reduce(&elapsed_time, &elapsed_time_max, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
if (my_rank == 0) {
pFile = fopen(measure_out, "a");
fprintf(pFile, "%s Elapsed max --finalization time-init_status = %d: %f secs\n",
argv[2], init_status, elapsed_time_max);
}
SCOREP_USER_OA_PHASE_END(OA_Phase);
/********** END FINALIZATION **********/
free(var);
free(cgup);
free(su);
free(bp);
free(bh);
free(bl);
free(bw);
free(bn);
free(be);
free(bs);
free(lcc_local);
if (my_rank == 0) {
free(cnorm);
free(oc);
free(elems);
free(local_global_index);
int i;
for (i = 0; i < nintcf + 1; i++) {
free(lcc[i]);
}
free(lcc);
for (i = 0; i < points_count; i++) {
free(points[i]);
}
free(points);
}
MPI_Finalize(); /// Cleanup MPI
return 0;
}
int main(int argc, char *argv[]) {
int my_rank, num_procs;
const int max_iters = 10000; // maximum number of iteration to perform
/** Simulation parameters parsed from the input datasets */
int nintci, nintcf; // internal cells start and end index
// external cells start and end index. The external cells are only ghost cells.
// They are accessed only through internal cells
int nextci, nextcf;
int **lcc; // link cell-to-cell array - stores neighboring information
// Boundary coefficients for each volume cell (South, East, North, West, High, Low)
double *bs, *be, *bn, *bw, *bl, *bh;
double *bp; // Pole coefficient
double *su; // Source values
double residual_ratio; // the ratio between the reference and the current residual
double *var; // the variation vector -> keeps the result in the end
/* Additional vectors required for the computation */
double *cgup, *oc, *cnorm;
/* Geometry data */
int points_count; // total number of points that define the geometry
int** points; // coordinates of the points that define the cells - size [points_cnt][3]
int* elems; // definition of the cells using their nodes (points) - each cell has 8 points
int num_elems_local; // number of elemens in each processor
/* Mapping between local and remote cell indices */
int* local_global_index; // local to global index mapping
int* global_local_index; // global to local index mapping
/* Lists of cells requires for the communication */
int neighbors_count = 0; // total number of neighbors to communicate with
int* send_count; // number of elements to send to each neighbor (size: neighbors_count)
/// send lists for the other neighbors(cell ids which should be sent)(size:[#neighbors][#cells]
int** send_list;
int* recv_count; // how many elements are in the recv lists for each neighbor
int** recv_list; // send lists for the other neighbor (see send_list)
/* Metis Results */
int* epart; // partition vector for the elements of the mesh
int* npart; // partition vector for the points (nodes) of the mesh
int* objval; /// resulting edgecut of total communication volume (classical distrib->zeros)
MPI_Init(&argc, &argv); // Start MPI
MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); // Get current process id
MPI_Comm_size(MPI_COMM_WORLD, &num_procs); // get number of processes
if ( argc < 3 ) {
fprintf(stderr, "Usage: ./gccg <input_file> <output_prefix> <partition_type>\n");
MPI_Abort(MPI_COMM_WORLD, -1);
}
char *file_in = argv[1];
char *out_prefix = argv[2];
char *part_type = (argc == 3 ? "classical" : argv[3]);
/********** START INITIALIZATION **********/
// read-in the input file
int init_status = initialization(file_in, part_type, &nintci, &nintcf, &nextci, &nextcf, &lcc,
&bs, &be, &bn, &bw, &bl, &bh, &bp, &su, &points_count, &points,
&elems, &var, &cgup, &oc, &cnorm, &local_global_index,
&global_local_index, &neighbors_count, &send_count, &send_list,
&recv_count, &recv_list, &epart, &npart,
&objval, &num_elems_local);
if ( init_status != 0 ) {
fprintf(stderr, "Failed to initialize data!\n");
MPI_Abort(MPI_COMM_WORLD, my_rank);
}
// Implement test function to test the results from initialization
/* char file_vtk_out[100];
sprintf(file_vtk_out, "%s.vtk", out_prefix);
sprintf(file_vtk_out_com, "%scom.vtk", out_prefix);
if ( my_rank == 0 ) {
test_distribution( file_in, file_vtk_out, local_global_index,
num_elems_local, cgup, epart, npart, objval );
test_communication( file_in, file_vtk_out, local_global_index, num_elems_local,
neighbors_count, send_count, send_list, recv_count, recv_list );
}*/
/********** END INITIALIZATION **********/
/********** START COMPUTATIONAL LOOP **********/
int total_iters = compute_solution(max_iters, nintci, nintcf, nextcf, lcc, bp, bs, bw, bl, bn,
be, bh, cnorm, var, su, cgup, &residual_ratio,
local_global_index, global_local_index, neighbors_count,
send_count, send_list, recv_count, recv_list,
num_elems_local, epart);
/********** END COMPUTATIONAL LOOP **********/
/********** START FINALIZATION **********/
finalization(file_in, out_prefix, total_iters, residual_ratio, nintci, nintcf, points_count,
points, elems, var, cgup, su, local_global_index, num_elems_local);
/********** END FINALIZATION **********/
free(cnorm);
free(oc);
free(var);
free(cgup);
free(su);
free(bp);
free(bh);
free(bl);
free(bw);
free(bn);
free(be);
free(bs);
MPI_Finalize(); /// Cleanup MPI
return 0;
}
int main(int argc, char *argv[]) {
int my_rank, num_procs, i;
const int max_iters = 10000; /// maximum number of iteration to perform
/** Simulation parameters parsed from the input datasets */
int nintci, nintcf; /// internal cells start and end index
/// external cells start and end index. The external cells are only ghost cells.
/// They are accessed only through internal cells
int nextci, nextcf;
int **lcc; /// link cell-to-cell array - stores neighboring information
/// Boundary coefficients for each volume cell (South, East, North, West, High, Low)
double *bs, *be, *bn, *bw, *bh, *bl;
double *bp; /// Pole coefficient
double *su; /// Source values
double residual_ratio; /// the ratio between the reference and the current residual
double *var; /// the variation vector -> keeps the result in the end
/** Additional vectors required for the computation */
double *cgup, *oc, *cnorm;
/** Geometry data */
int points_count; /// total number of points that define the geometry
int** points; /// coordinates of the points that define the cells - size [points_cnt][3]
int* elems; /// definition of the cells using their nodes (points) - each cell has 8 points
/** Mapping between local and remote cell indices */
int* local_global_index; /// local to global index mapping
int* global_local_index; /// global to local index mapping
int** l2g_g;
/** Lists for neighbouring information */
int nghb_cnt = 0; /// total number of neighbors of the current process
int *nghb_to_rank; /// mapping of the neighbour index to the corresponding process rank
int *send_cnt; /// number of cells to be sent to each neighbour (size: nghb_cnt)
int **send_lst; /// lists of cells to be sent to each neighbour (size: nghb_cnt x send_cnt[*])
int *recv_cnt; /// number of cells to be received from each neighbour (size: nghb_cnt)
int **recv_lst; /// lists of cells to be received from each neighbour (size: nghb_cnt x recv_cnt[*])
double start_time, end_time;
MPI_Init(&argc, &argv); /// Start MPI
MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); /// get current process id
MPI_Comm_size(MPI_COMM_WORLD, &num_procs); /// get number of processes
int int_cells_per_proc[num_procs];
/** process call arguments **/
if ( argc < 4 ) {
fprintf(stderr, "Usage: ./gccg <input_file> <partition_type> <algorithm_type>\n");
MPI_Abort(MPI_COMM_WORLD, -1);
}
char *file_in = argv[1];
char *part_type = argv[2];
if ( strcmp( part_type, "classic" ) && strcmp( part_type, "dual" )
&& strcmp( part_type, "nodal" ) ) {
printf(
" Wrong partition type selected. Valid values are classic, nodal and dual \n" );
MPI_Abort(MPI_COMM_WORLD, -1);
}
char *read_type = argv[3];
if ( strcmp( read_type, "oneread" ) && strcmp( read_type, "allread" ) ) {
printf(
" Wrong read-in algorithm selected. Valid values are oneread and allread. \n" );
MPI_Abort(MPI_COMM_WORLD, -1);
}
if(my_rank==0) {
/********** START INITIALIZATION **********/
// read-in the input file
start_time = MPI_Wtime();
int init_status = initialization(file_in, part_type, read_type, num_procs, my_rank,
&nintci, &nintcf, &nextci, &nextcf,
&lcc, &bs, &be, &bn, &bw, &bl, &bh, &bp, &su,
&points_count, &points, &elems, &var, &cgup, &oc, &cnorm,
&local_global_index, &l2g_g, int_cells_per_proc);
end_time = MPI_Wtime();
printf("Initialization ET (secs): %f\n", end_time-start_time);
/** LOCAL DATA FROM HERE ON **/
// at this point, all initialized vectors should contain only the locally needed data
// and all variables representing the number of elements, cells, points, etc. should
// reflect the local setup, e.g. nintcf-nintci+1 is the local number of internal cells
if ( init_status != 0 ) {
fprintf(stderr, "Failed to initialize data!\n");
MPI_Abort(MPI_COMM_WORLD, my_rank);
}
/********** END INITIALIZATION **********/
/********** START COMPUTATIONAL LOOP **********/
start_time = MPI_Wtime();
int total_iters = compute_solution(num_procs, my_rank, max_iters, nintci, nintcf, nextcf,
lcc, bp, bs, bw, bl, bn, be, bh,
cnorm, var, su, cgup, &residual_ratio,
local_global_index, global_local_index, nghb_cnt,
nghb_to_rank, send_cnt, send_lst, recv_cnt, recv_lst,
file_in, points_count, points, elems, part_type, read_type,
l2g_g, int_cells_per_proc);
end_time = MPI_Wtime();
printf("Computation ET (secs): %f\n", end_time-start_time);
/********** END COMPUTATIONAL LOOP **********/
/********** START FINALIZATION **********/
finalization(file_in, num_procs, my_rank, total_iters, residual_ratio, nintci, nintcf, var);
/********** END FINALIZATION **********/
// cleanup allocated memory
free(cnorm);
free(var);
free(cgup);
free(su);
free(bp);
free(bh);
free(bl);
free(bw);
free(bn);
free(be);
free(bs);
free(elems);
for ( i = 0; i < nintcf + 1; i++ ) {
free(lcc[i]);
}
free(lcc);
for ( i = 0; i < points_count; i++ ) {
free(points[i]);
}
free(points);
}
MPI_Finalize(); /// cleanup MPI
return 0;
}
void Detector2D::comp_weight_potential() {
compute_solution(potential_solver_weight, *solution_weight_potential);
}
void Detector2D::comp_potential() {
compute_solution(potential_solver, *solution_potential);
}
static void write_q_unformatted (char *qfile)
{
int np, nk, nz, nq, ierr;
int i, j, k, n, *indices;
char buff[129];
void *qdata;
float *qf = 0;
double *qd = 0;
printf ("\nwriting unformatted Q file to %s\n", qfile);
printf (" in %s-precision\n", use_double ? "double" : "single");
for (np = 0, nz = 0; nz < nZones; nz++) {
if (Zones[nz].type == CGNS_ENUMV(Structured)) {
nk = whole ? (int)Zones[nz].nverts :
(int)(Zones[nz].dim[0] * Zones[nz].dim[1]);
if (np < nk) np = nk;
}
}
indices = (int *) malloc (3 * nblocks * sizeof(int));
if (use_double) {
qdata = (void *) malloc (5 * np * sizeof(double));
qd = (double *)qdata;
}
else {
qdata = (void *) malloc (5 * np * sizeof(float));
qf = (float *)qdata;
}
if (NULL == indices || NULL == qdata)
FATAL ("write_q_unformatted", "malloc failed for working arrays");
unlink (qfile);
strcpy (buff, qfile);
for (n = (int)strlen(buff); n < 128; n++)
buff[n] = ' ';
buff[128] = 0;
n = 0;
OPENF (&n, buff, 128);
if (mblock || nblocks > 1) {
n = 1;
WRITEIF (&n, &nblocks, &ierr);
}
for (np = 0, nz = 0; nz < nZones; nz++) {
if (Zones[nz].type == CGNS_ENUMV(Structured)) {
for (n = 0; n < 3; n++)
indices[np++] = (int)Zones[nz].dim[n];
}
}
WRITEIF (&np, indices, &ierr);
for (nz = 0; nz < nZones; nz++) {
if (Zones[nz].type == CGNS_ENUMV(Structured)) {
printf (" zone %d ... ", nz+1);
fflush (stdout);
compute_solution (nz);
if (whole) {
np = (int)Zones[nz].nverts;
nk = 1;
}
else {
np = (int)(Zones[nz].dim[0] * Zones[nz].dim[1]);
nk = (int)Zones[nz].dim[2];
}
if (use_double) {
n = 4;
WRITEDF (&n, reference, &ierr);
for (k = 0; k < nk; k++) {
for (nq = 0, j = 0; j < 5; j++) {
i = k * np;
for (n = 0; n < np; n++, i++)
qd[nq++] = q[j][i];
}
WRITEDF (&nq, qd, &ierr);
}
}
else {
for (n = 0; n < 4; n++)
qf[n] = (float)reference[n];
WRITEFF (&n, qf, &ierr);
for (k = 0; k < nk; k++) {
for (nq = 0, j = 0; j < 5; j++) {
i = k * np;
for (n = 0; n < np; n++, i++)
qf[nq++] = (float)q[j][i];
}
WRITEFF (&nq, qf, &ierr);
}
}
puts ("done");
}
}
CLOSEF ();
free (indices);
free (qdata);
}
static void write_q_binary (char *qfile)
{
int i, j, n, k, nk, nz, np, dim[3];
float qf[4];
FILE *fp;
printf ("\nwriting binary Q file to %s\n", qfile);
printf (" in %s-precision\n", use_double ? "double" : "single");
if (NULL == (fp = fopen (qfile, "w+b"))) {
fprintf (stderr, "couldn't open <%s> for writing\n", qfile);
exit (1);
}
if (mblock || nblocks > 1)
fwrite (&nblocks, sizeof(int), 1, fp);
for (nz = 0; nz < nZones; nz++) {
if (Zones[nz].type == CGNS_ENUMV(Structured)) {
for (i = 0; i < 3; i++)
dim[i] = (int)Zones[nz].dim[i];
fwrite (dim, sizeof(int), 3, fp);
}
}
for (nz = 0; nz < nZones; nz++) {
if (Zones[nz].type == CGNS_ENUMV(Structured)) {
printf (" zone %d ... ", nz+1);
fflush (stdout);
compute_solution (nz);
if (whole) {
np = (int)Zones[nz].nverts;
nk = 1;
}
else {
np = (int)(Zones[nz].dim[0] * Zones[nz].dim[1]);
nk = (int)Zones[nz].dim[2];
}
if (use_double) {
fwrite (reference, sizeof(double), 4, fp);
for (k = 0; k < nk; k++) {
for (i = 0; i < 5; i++)
fwrite (&q[i][k*np], sizeof(double), np, fp);
}
}
else {
for (n = 0; n < 4; n++)
qf[n] = (float)reference[n];
fwrite (qf, sizeof(float), 4, fp);
for (k = 0; k < nk; k++) {
for (i = 0; i < 5; i++) {
j = k * np;
for (n = 0; n < np; n++, j++) {
qf[0] = (float)q[i][j];
fwrite (qf, sizeof(float), 1, fp);
}
}
}
}
puts ("done");
}
}
fclose (fp);
}
