int test_distribution( char *file_in, char *file_vtk_out, int *local_global_index,
int local_num_elems, double *cgup_local ) {
// Global variables for reading from file
int nintci, nintcf, nextci, nextcf;
int **lcc;
double *bs, *be, *bn, *bw, *bh, *bl, *bp, *su;
int points_count, **points, *elems;
double *distr;
int i;
// read-in the input file
int f_status = read_binary_geo( file_in, &nintci, &nintcf, &nextci, &nextcf, &lcc,
&bs, &be, &bn, &bw, &bl, &bh, &bp, &su,
&points_count, &points, &elems );
if( f_status != 0 ) {
return f_status;
}
// Free unnecessary global data
for( i = nintci; i <= nintcf; i++ ) {
free( lcc[i] );
}
free( lcc );
free( bs );
free( be );
free( bn );
free( bw );
free( bh );
free( bl );
free( bp );
free( su );
// Allocate and fill distr
if( ( distr = ( double * ) calloc( nintcf - nintci + 1, sizeof( double ) ) ) == NULL ) {
fprintf( stderr, "malloc(distr) failed\n" );
return -1;
}
for( i = 0; i < local_num_elems; i++ ) {
distr[local_global_index[i]] = cgup_local[i];
}
// Write vtk file
vtk_write_unstr_grid_header( "test_distribution", file_vtk_out, nintci, nintcf, points_count,
points, elems );
vtk_append_double( file_vtk_out, "CGUP", nintci, nintcf, distr );
// Free the rest of global data
for( i = 0; i < points_count; i++ ) {
free( points[i] );
}
free( points );
free( elems );
free( distr );
return 0;
}
int test_distribution(char *file_in, char *file_vtk_out, int *local_global_index, int num_elems,
double *cgup) {
int i;
// Read the geometry from the input file
/** Simulation parameters parsed from the input datasets */
int nintci, nintcf; /// internal cells start and end index
/// external cells start and end index. The external cells are only ghost cells.
/// They are accessed only through internal cells
int nextci, nextcf;
int **lcc; /// link cell-to-cell array - stores neighboring information
/// Boundary coefficients for each volume cell (South, East, North, West, High, Low)
double *bs, *be, *bn, *bw, *bl, *bh;
double *bp; /// Pole coefficient
double *su; /// Source values
/** Geometry data */
int points_count; /// total number of points that define the geometry
int** points; /// coordinates of the points that define the cells - size [points_cnt][3]
int* elems; /// definition of the cells using their nodes (points) - each cell has 8 points
double* distr;
int read_input = read_binary_geo(file_in, &nintci, &nintcf, &nextci, &nextcf, &lcc, &bs, &be,
&bn, &bw, &bl, &bh, &bp, &su, &points_count, &points, &elems);
if (read_input != 0)
printf("Could not read input file in test distribution");
// allocate the distr vector and initialize it with zeros
if ((distr = (double*) calloc(sizeof(double), nintcf - nintci + 1)) == NULL ) {
fprintf(stderr, "calloc failed to allocate distr");
return -1;
}
// copy the locally initialized CGUP vector to the right place in the distr array
int ind;
for (i = 0; i < num_elems; i++) {
ind = local_global_index[i];
distr[ind] = cgup[i];
}
// write the vtk header
const char experiment_name[] = "test for distribution";
vtk_write_unstr_grid_header(experiment_name, file_vtk_out, nintci, nintcf, points_count, points,
elems);
// write the values to the vtk file
vtk_append_double(file_vtk_out, "CGUP", nintci, nintcf, distr);
return 0;
}
int test_distribution(char *file_in, char *file_vtk_out, int *local_global_index, int num_elems,
double *cgup) {
int nintci;
int nintcf;
int nextci;
int nextcf;
int** lcc;
double* bs;
double* be;
double* bn;
double* bw;
double* bl;
double* bh;
double* bp;
double* su;
int points_count;
int** points;
int* elems;
double* v = NULL;
int f_status = read_binary_geo(file_in, &nintci, &nintcf, &nextci, &nextcf, &lcc,
&bs, &be, &bn, &bw,
&bl, &bh, &bp,
&su, &points_count,
&points, &elems);
int my_rank;
MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); /// Get current process id
vtk_write_unstr_grid_header(file_in, file_vtk_out, 0, nintcf, points_count, points, elems);
v = (double*) malloc( (size_t)(nintcf+1) * sizeof(double));
int i;
for ( i = 0; i <= num_elems; ++i ) {
v[local_global_index[i]] = cgup[i];
}
vtk_append_double(file_vtk_out, "partition", 0, nintcf, v);
free(v);
return 0;
}
int read_init_data(char* file_in, int read_key, int myrank, int *nintci, int *nintcf,
int *nextci, int *nextcf, int ***lcc, double **bs, double **be, double **bn, double **bw,
double **bl, double **bh, double **bp, double **su, int* points_count, int***points,
int** elems) {
int f_status=0;
if (read_key == POSL_INIT_ONE_READ) {
if (myrank == 0) {
f_status = read_binary_geo(file_in, &*nintci, &*nintcf, &*nextci, &*nextcf, &*lcc, &*bs,
&*be, &*bn, &*bw, &*bl, &*bh, &*bp, &*su, &*points_count, &*points, &*elems);
}
} else {
//TODO:implement sheer geometry reading
// f_status = read_lcc_boundary(file_in, &*nintci, &*nintcf, &*nextci, &*nextcf, &*lcc, &*bs,
// &*be, &*bn, &*bw, &*bl, &*bh, &*bp, &*su);
f_status = read_geometry(file_in, &*nintci, &*nintcf, &*nextci, &*nextcf,
&*points_count, &*points, &*elems);
}
return f_status;
}
int initialization(char* file_in, char* part_type, int* nintci, int* nintcf, int* nextci,
int* nextcf, int*** lcc, double** bs, double** be, double** bn, double** bw,
double** bl, double** bh, double** bp, double** su, int* points_count,
int*** points, int** elems, double** var, double** cgup, double** oc,
double** cnorm, int** local_global_index, int** global_local_index,
int* neighbors_count, int** send_count, int*** send_list, int** recv_count,
int*** recv_list, int** epart, int** npart, int** objval, int* num_elems_local) {
/********** START INITIALIZATION **********/
int i = 0;
int j = 0;
int num_elems_pro; // number of elements in each processor
int my_rank, num_procs;
/// Boundary coefficients for each volume cell (South, East, North, West, High, Low)
double *bs_a, *be_a, *bn_a, *bw_a, *bl_a, *bh_a;
double *bp_a; /// Pole coefficient
double *su_a; /// Source values
int** lcc_a; /// link cell-to-cell array - stores neighboring information
int** lcc_b;
MPI_Status status;
MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); /// Get current process id
MPI_Comm_size(MPI_COMM_WORLD, &num_procs); /// get number of processe
// read-in the input file by one processor
if ( my_rank == 0 ) {
int f_status = read_binary_geo(file_in, &*nintci, &*nintcf, &*nextci, &*nextcf,
&lcc_a, &bs_a, &be_a, &bn_a, &bw_a, &bl_a, &bh_a,
&bp_a, &su_a, &*points_count, &*points, &*elems);
if ( f_status != 0 ) return f_status;
}
// Send the common information to other processors
MPI_Bcast(nintci, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(nintcf, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(nextci, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(nextcf, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(points_count, 1, MPI_INT, 0, MPI_COMM_WORLD);
// local arrays and share parameters
int num_elems = *nintcf - *nintci + 1;
if (my_rank != 0) {
*elems = (int*) calloc(sizeof(int), num_elems * 8);
}
MPI_Bcast(*elems, num_elems * 8, MPI_INT, 0, MPI_COMM_WORLD);
int points_num = *points_count;
int npro = num_elems / num_procs;
int exter = *nextcf - *nextci + 1;
int remain = 0;
int *k = (int*) calloc(sizeof(int), num_procs);
int *k_sum = (int*) calloc(sizeof(int), num_procs);
int last_proc = num_procs - 1;
if (my_rank == last_proc) {
remain = num_elems % num_procs;
}
int local_array_size = npro + remain + exter;
*local_global_index = (int*) calloc(sizeof(int), npro + remain + exter);
*global_local_index = (int*) calloc(sizeof(int), num_elems);
*bs = (double*) calloc(sizeof(double), (local_array_size));
*bn = (double*) calloc(sizeof(double), (local_array_size));
*bw = (double*) calloc(sizeof(double), (local_array_size));
*be = (double*) calloc(sizeof(double), (local_array_size));
*bl = (double*) calloc(sizeof(double), (local_array_size));
*bh = (double*) calloc(sizeof(double), (local_array_size));
*bp = (double*) calloc(sizeof(double), (local_array_size));
*su = (double*) calloc(sizeof(double), (local_array_size));
*var = (double*) calloc(sizeof(double), (local_array_size));
*cgup = (double*) calloc(sizeof(double), (local_array_size));
*oc = (double*) calloc(sizeof(double), (npro + remain));
*cnorm = (double*) calloc(sizeof(double), (npro + remain));
*lcc = (int**) calloc(sizeof(int*), (local_array_size));
for ( i = 0; i < local_array_size; i++ ) {
(*lcc)[i] = (int *) calloc(sizeof(int), (6));
}
int *data = (int *) calloc(sizeof(int), (num_elems*6));
lcc_b = (int **) calloc(sizeof(int*), (num_elems));
for (i = 0; i < num_elems; i++) {
lcc_b[i] = &(data[6 * i]);
}
if ( my_rank == 0 ) {
for ( i = 0; i< num_elems; i++ ) {
for ( j = 0; j < 6; j++ ) {
lcc_b[i][j] = lcc_a[i][j];
}
}
}
MPI_Bcast(&(lcc_b[0][0]), num_elems*6, MPI_INT, 0, MPI_COMM_WORLD);
// choose part type
if (strcmp(part_type, "classical") == 0) {
k[my_rank] = npro + remain;
(num_elems_pro) = npro + remain;
int p = 0;
for (p = 0; p < num_procs; p++) {
MPI_Bcast(&k[p], 1, MPI_INT, p, MPI_COMM_WORLD);
}
if (my_rank == 0) {
for (i = 1; i < num_procs; i++) {
k_sum[i] = k_sum[i-1] + k[i-1];
}
}
// ditribute all B* array
MPI_Scatterv(bs_a, k, k_sum, MPI_DOUBLE, *bs, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD);
MPI_Scatterv(bn_a, k, k_sum, MPI_DOUBLE, *bn, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD);
MPI_Scatterv(bw_a, k, k_sum, MPI_DOUBLE, *bw, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD);
MPI_Scatterv(be_a, k, k_sum, MPI_DOUBLE, *be, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD);
MPI_Scatterv(bl_a, k, k_sum, MPI_DOUBLE, *bl, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD);
MPI_Scatterv(bh_a, k, k_sum, MPI_DOUBLE, *bh, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD);
MPI_Scatterv(bp_a, k, k_sum, MPI_DOUBLE, *bp, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD);
MPI_Scatterv(su_a, k, k_sum, MPI_DOUBLE, *su, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD);
for (i = 0; i < num_elems_pro; i++) {
(*local_global_index)[i] = my_rank * npro + i;
for (j = 0; j < 6; j++) {
(*lcc)[i][j] = lcc_b[my_rank*npro+i][j];
}
}
for (i = 0; i < num_elems_pro; i++) {
if (i > npro) {
(*global_local_index)[my_rank*npro+i] = (my_rank*npro+i) % npro + npro;
} else {
(*global_local_index)[my_rank*npro+i] = (my_rank*npro+i) % npro;
}
}
// part type is not classics but metis
} else {
*epart = (int*) calloc(sizeof(int), num_elems);
*npart = (int*) calloc(sizeof(int), num_elems*8);
// if ( my_rank == 0 ) {
// parametes and array for metis partition libary
idx_t ne = (idx_t) num_elems;
idx_t nn = (idx_t) points_num;
idx_t ncommon = 4;
idx_t nparts = num_procs;
int node_num = ne * 8;
idx_t *eptr = (idx_t*) calloc(sizeof(idx_t), num_elems + 1);
idx_t *eind = (idx_t*) calloc(sizeof(idx_t), node_num);
idx_t objval_METIS;
idx_t *epart_METIS = (idx_t*) calloc(sizeof(idx_t), num_elems);
idx_t *npart_METIS = (idx_t*) calloc(sizeof(idx_t), node_num);
int metis_final;
for (i = (*nintci); i <= (*nintcf + 1) ; i++) {
eptr[i] = (idx_t) i * 8;
}
for (i = 0; i < node_num; i++) {
eind[i] = (idx_t) (*elems)[i];
}
if (strcmp(part_type, "dual") == 0) {
metis_final = METIS_PartMeshDual(&ne, &nn, eptr, eind, NULL, NULL,
&ncommon, &nparts, NULL, NULL, &objval_METIS,
epart_METIS, npart_METIS);
} else if (strcmp(part_type, "noda") == 0) {
metis_final = METIS_PartMeshNodal(&ne, &nn, eptr, eind, NULL, NULL,
&nparts, NULL, NULL, &objval_METIS,
epart_METIS, npart_METIS);
}
if (metis_final != METIS_OK) {
printf("Metis part fails\n");
}
(*objval) = (int*) calloc(sizeof(int), 1);
(*objval)[0] = (int) objval_METIS;
for (i = 0; i < num_elems; i++) {
(*epart)[i] = (int) epart_METIS[i];
}
for (i = 0; i < node_num; i++) {
(*npart)[i] = (int) npart_METIS[i];
}
// ditribute data according to METIS Partition
int p = 0;
// store local to global mapping
for (p = 0; p < num_procs; p++) {
if (my_rank == p) {
for (j = 0; j < num_elems; j++) {
if ((*epart)[j] == my_rank) {
(*local_global_index)[k[my_rank]] = j;
for (i = 0; i < 6; i++) {
(*lcc)[k[my_rank]][i] = lcc_b[j][i];
}
(*global_local_index)[j] = k[my_rank];
k[my_rank] = k[my_rank] + 1;
}
}
}
MPI_Bcast(&k[p], 1, MPI_INT, p, MPI_COMM_WORLD); /// send k[p] to other processors
} /// finish storing local to global mapping
(num_elems_pro) = k[my_rank];
int *local_global_index_sum = (int*) calloc(sizeof(int), num_elems);
if (my_rank == 0) {
for (i = 1; i < num_procs; i++) {
k_sum[i] = k_sum[i-1] + k[i-1];
}
}
MPI_Gatherv(*local_global_index, k[my_rank], MPI_INT,
local_global_index_sum, k, k_sum,
MPI_INT, 0, MPI_COMM_WORLD);
// copy B* array into new array accoring to order from metis partition
double *bs_b = (double*) calloc(sizeof(double), (num_elems));
double *bn_b = (double*) calloc(sizeof(double), (num_elems));
double *bw_b = (double*) calloc(sizeof(double), (num_elems));
double *be_b = (double*) calloc(sizeof(double), (num_elems));
double *bl_b = (double*) calloc(sizeof(double), (num_elems));
double *bh_b = (double*) calloc(sizeof(double), (num_elems));
double *bp_b = (double*) calloc(sizeof(double), (num_elems));
double *su_b = (double*) calloc(sizeof(double), (num_elems));
if (my_rank == 0) {
for (i= 0; i < num_elems; i++) {
j = local_global_index_sum[i];
bs_b[i] = bs_a[j];
bn_b[i] = bn_a[j];
bw_b[i] = bw_a[j];
be_b[i] = be_a[j];
bl_b[i] = bl_a[j];
bh_b[i] = bh_a[j];
bp_b[i] = bp_a[j];
su_b[i] = su_a[j];
}
}
MPI_Scatterv(bs_b, k, k_sum , MPI_DOUBLE, *bs, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD);
MPI_Scatterv(bn_b, k, k_sum , MPI_DOUBLE, *bn, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD);
MPI_Scatterv(bw_b, k, k_sum , MPI_DOUBLE, *bw, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD);
MPI_Scatterv(be_b, k, k_sum , MPI_DOUBLE, *be, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD);
MPI_Scatterv(bl_b, k, k_sum , MPI_DOUBLE, *bl, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD);
MPI_Scatterv(bh_b, k, k_sum , MPI_DOUBLE, *bh, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD);
MPI_Scatterv(bp_b, k, k_sum , MPI_DOUBLE, *bp, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD);
MPI_Scatterv(su_b, k, k_sum , MPI_DOUBLE, *su, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD);
free(bp_b);
free(bh_b);
free(bl_b);
free(bw_b);
free(bn_b);
free(be_b);
free(bs_b);
free(su_b);
free(local_global_index_sum);
} // finish choose part type section and all local array are stored
// initialization computational array
for (i = 0; i <= 10; i++) {
(*oc)[i] = 0.0;
(*cnorm)[i] = 1.0;
}
for (i = 0; i < num_elems_pro; i++) {
(*cgup)[i] = 0.0;
(*var)[i] = 0.0;
}
for (i = num_elems_pro; i < local_array_size; i++) {
(*var)[i] = 0.0;
(*cgup)[i] = 0.0;
(*bs)[i] = 0.0;
(*be)[i] = 0.0;
(*bn)[i] = 0.0;
(*bw)[i] = 0.0;
(*bl)[i] = 0.0;
(*bh)[i] = 0.0;
}
for (i = 0; i < num_elems_pro; i++) {
(*cgup)[i] = 1.0 / ((*bp)[i]);
}
// ************Comunication List********* //
*num_elems_local = num_elems_pro;
*neighbors_count = num_procs-1;
*send_count = (int*) calloc(sizeof(int), (num_procs));
*recv_count = (int*) calloc(sizeof(int), (num_procs));
*send_list = (int **) calloc(sizeof(int*), (*neighbors_count + 1));
for (i = 0; i < *neighbors_count + 1; i++) {
(*send_list)[i] = (int *) calloc(sizeof(int), (6 * num_elems_pro));
}
int num_elems_global = 0;
int* rank = (int*) calloc(sizeof(int), (num_procs));
int m = 0;
int** count_time_local = (int**) calloc(sizeof(int*), (num_procs));
for (i = 0; i < num_procs; i++) {
count_time_local[i] = (int *) calloc(sizeof(int), (num_elems));
}
int* count_time = (int*) calloc(sizeof(int), (num_elems));
for (i = 0; i < num_elems_pro; i++) {
for (j = 0; j < 6; j++) {
num_elems_global = (*lcc)[i][j];
// choose only ghost cell
if (num_elems_global < num_elems) {
// choose part type
if (strcmp(part_type, "classical") == 0) {
if (num_elems_global >= npro * num_procs) {
rank[my_rank]= num_elems_global / npro - 1;
} else {
rank[my_rank]= num_elems_global / npro;
}
} else {
rank[my_rank]=(*epart)[num_elems_global];
} /// end choosing part type
// record times of this elems occur
if (rank[my_rank] != my_rank) {
count_time_local[rank[my_rank]][i] = count_time_local[rank[my_rank]][i] + 1;
if (count_time_local[rank[my_rank]][i] == 1) {
(*send_list)[rank[my_rank]][(*send_count)[rank[my_rank]]] = (*local_global_index)[i];
(*send_count)[rank[my_rank]]=(*send_count)[rank[my_rank]] + 1;
}
}
} /// choose ghost cell
} /// end j for loop
} /// end i for loop
// Set the order in send_list and recv_list same
for (i = 0; i < num_procs; i++) {
MPI_Sendrecv(&((*send_count)[i]), 1, MPI_INT, i, i * 1000,
&((*recv_count)[i]), 1, MPI_INT, i,
my_rank * 1000, MPI_COMM_WORLD, &status);
}
*recv_list = (int **) calloc(sizeof(int*), (*neighbors_count+1));
for (i = 0; i < *neighbors_count+1; i++) {
(*recv_list)[i] = (int *) calloc(sizeof(int), ((*recv_count)[i]));
}
for (i = 0; i < num_procs; i++) {
if (my_rank == i) {
for (j = 0; j < num_procs; j++) {
if (j != my_rank) {
MPI_Send((*send_list)[j], (*send_count)[j], MPI_INT, j, 100,
MPI_COMM_WORLD);
}
}
} else {
MPI_Recv((*recv_list)[i], (*recv_count)[i], MPI_INT, i, 100,
MPI_COMM_WORLD, &status);
}
}
free(lcc_b);
free(count_time_local);
return 0;
}
int test_communication( char *file_in, char *file_vtk_out, int *local_global_index,
int local_num_elems, int neighbors_count, int *send_count, int **send_list,
int *recv_count, int **recv_list ) {
// Global variables for reading from file
int nintci, nintcf, nextci, nextcf;
int **lcc;
double *bs, *be, *bn, *bw, *bh, *bl, *bp, *su;
int points_count, **points, *elems;
double *commlist;
int i, j;
// read-in the input file
int f_status = read_binary_geo( file_in, &nintci, &nintcf, &nextci, &nextcf, &lcc,
&bs, &be, &bn, &bw, &bl, &bh, &bp, &su,
&points_count, &points, &elems );
if( f_status != 0 ) {
return f_status;
}
// Free unnecessary global data
for( i = nintci; i <= nintcf; i++ ) {
free( lcc[i] );
}
free( lcc );
free( bs );
free( be );
free( bn );
free( bw );
free( bh );
free( bl );
free( bp );
free( su );
// Allocate and fill commlist
if( ( commlist = ( double * ) calloc( nintcf - nintci + 1, sizeof( double ) ) ) == NULL ) {
fprintf( stderr, "malloc(commlist) failed\n" );
return -1;
}
for( i = 0; i < local_num_elems; i++ ) {
// internal cells
commlist[local_global_index[i]] = 15;
}
for( i = 0; i < neighbors_count; i++ ) {
for( j = 0; j < send_count[i]; j++ ) {
// ghost cell to be sent
commlist[local_global_index[send_list[i][j]]] = 10;
}
}
for( i = 0; i < neighbors_count; i++ ) {
for( j = 0; j < recv_count[i]; j++ ) {
// ghost cell to be received
commlist[local_global_index[recv_list[i][j]]] = 5;
}
}
// Write vtk file
vtk_write_unstr_grid_header( "test_communication", file_vtk_out, nintci, nintcf, points_count,
points, elems );
vtk_append_double( file_vtk_out, "commlist", nintci, nintcf, commlist );
// Free the rest of global data
for( i = 0; i < points_count; i++ ) {
free( points[i] );
}
free( points );
free( elems );
free( commlist );
return 0;
}
int test_communication(char *file_in, char *file_vtk_out, int *local_global_index, int num_elems,
int neighbors_count, int* send_count, int** send_list, int* recv_count,
int** recv_list) {
int nintci;
int nintcf;
int nextci;
int nextcf;
int** lcc;
double* bs;
double* be;
double* bn;
double* bw;
double* bl;
double* bh;
double* bp;
double* su;
int points_count;
int** points;
int* elems;
int f_status = read_binary_geo(file_in, &nintci, &nintcf, &nextci, &nextcf, &lcc,
&bs, &be, &bn, &bw,
&bl, &bh, &bp,
&su, &points_count,
&points, &elems);
int my_rank;
int num_procs;
MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); /// Get current process id
MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
int* commlist = NULL;
commlist = (int*) calloc((nintcf + 1), sizeof(int));
int i;
for ( i = 0; i < (nintcf +1); ++i )
commlist[i] = 0;
// 5 received
// 10 send
// 15 other cells
int j;
for ( i = 0; i <= num_elems; ++i ) {
commlist[local_global_index[i]] = 15;
}
for ( i = 0; i < num_procs; ++i ) {
for ( j = 0; j < send_count[i]; ++j ) {
commlist[send_list[i][j]] = 10;
}
for ( j = 0; j < recv_count[i]; ++j ) {
commlist[recv_list[i][j]] = 5;
}
}
vtk_append_integer(file_vtk_out, "communication", 0, nintcf, commlist);
free(commlist);
return 0;
}
int initialization_classic(char* file_in, char* part_type, int* nintci, int* nintcf, int* nextci,
int* nextcf, int*** lcc, double** bs, double** be, double** bn, double** bw,
double** bl, double** bh, double** bp, double** su, int* points_count,
int*** points, int** elems, double** var, double** cgup, double** oc,
double** cnorm, int** local_global_index, int** global_local_index,
int* neighbors_count, int** send_count, int*** send_list, int** recv_count,
int*** recv_list, int** epart, int** npart, int* objval) {
int i = 0;
int my_rank, num_procs;
MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); /// get current process id
MPI_Comm_size(MPI_COMM_WORLD, &num_procs);
// read-in the input file
int f_status = read_binary_geo(file_in, &*nintci, &*nintcf, &*nextci, &*nextcf, &*lcc, &*bs,
&*be, &*bn, &*bw, &*bl, &*bh, &*bp, &*su, &*points_count,
&*points, &*elems);
if ( f_status != 0 ) return f_status;
// size of internal and external junks
int CHUNKSIZE_INT = (int)ceil((double)(*nintcf-*nintci+1) / (double)num_procs); // ceil
int REMAINING_INT = (*nintcf-*nintci +1) - (num_procs-1) * CHUNKSIZE_INT;
int CHUNKSIZE_EXT = (int)ceil((double)(*nextcf-*nextci+1) / (double)num_procs); // ceil
int REMAINING_EXT = (*nextcf-*nextci +1) - (num_procs-1) * CHUNKSIZE_EXT;
*var = (double*) calloc(sizeof(double), (CHUNKSIZE_INT + CHUNKSIZE_EXT));
*cgup = (double*) calloc(sizeof(double), (CHUNKSIZE_INT + CHUNKSIZE_EXT));
*cnorm = (double*) calloc(sizeof(double), (CHUNKSIZE_INT));
int i_end_int = CHUNKSIZE_INT;
int i_end_ext = CHUNKSIZE_EXT;
if (my_rank == num_procs - 1) {
i_end_int = REMAINING_INT;
i_end_ext = REMAINING_EXT;
}
// initialize the arrays
for ( i = 0; i <= 10; i++ ) {
(*cnorm)[i] = 1.0;
}
for ( i = 0; i < i_end_int; i++ ) {
(*var)[i] = 0.0;
}
for ( i = 0; i < i_end_int; i++ ) {
(*cgup)[i] = 1.0 / ((*bp)[i]);
}
printf("i_end_int=%d , i_end_ext=%d, CHUNKSIZE_INT = %d", i_end_int, i_end_ext, CHUNKSIZE_INT);
MPI_Barrier( MPI_COMM_WORLD );
for ( i = CHUNKSIZE_INT; i < CHUNKSIZE_INT + i_end_ext; i++ ) {
(*var)[i] = 0.0;
(*cgup)[i] = 0.0;
(*bs)[i] = 0.0;
(*be)[i] = 0.0;
(*bn)[i] = 0.0;
(*bw)[i] = 0.0;
(*bh)[i] = 0.0;
(*bl)[i] = 0.0;
}
// initialize local to global mapping
*local_global_index = (int*) malloc(sizeof(int) * (CHUNKSIZE_INT+CHUNKSIZE_EXT));
for ( i = 0; i < CHUNKSIZE_INT; i++ ) {
(*local_global_index)[i] = i + my_rank * CHUNKSIZE_INT;
}
for ( i = CHUNKSIZE_INT; i < CHUNKSIZE_INT + CHUNKSIZE_EXT; i++ ) {
(*local_global_index)[i] = *nintcf-*nintci + i + my_rank * CHUNKSIZE_EXT;
}
return 0;
}
int initialization(char* file_in, char* part_type, int* nintci, int* nintcf, int* nextci,
int* nextcf, int*** lcc, double** bs, double** be, double** bn, double** bw,
double** bl, double** bh, double** bp, double** su, int* points_count,
int*** points, int** elems, double** var, double** cgup, double** oc,
double** cnorm, int** local_global_index, int** global_local_index,
int* neighbors_count, int** send_count, int*** send_list, int** recv_count,
int*** recv_list, int** epart, int** npart, int** objval, int* num_elems_local) {
/********** START INITIALIZATION **********/
int i = 0;
int j = 0;
int num_elems_pro;//number of elements in each processor
int my_rank, num_procs;
/// Boundary coefficients for each volume cell (South, East, North, West, High, Low)
double *bs_a, *be_a, *bn_a, *bw_a, *bl_a, *bh_a;
double *bp_a; /// Pole coefficient
double *su_a; /// Source values
int** lcc_a; /// link cell-to-cell array - stores neighboring information
int** lcc_b;
MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); /// Get current process id
MPI_Comm_size(MPI_COMM_WORLD, &num_procs); /// get number of processe
// read-in the input file by one processor
if ( my_rank == 0 ) {
int f_status = read_binary_geo(file_in, &*nintci, &*nintcf, &*nextci, &*nextcf, &lcc_a, &bs_a,
&be_a, &bn_a, &bw_a, &bl_a, &bh_a, &bp_a, &su_a, &*points_count,
&*points, &*elems);
if ( f_status != 0 ) return f_status;
}
//Send the common information to other processors
MPI_Bcast (nintci,1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast (nintcf,1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast (nextci,1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast (nextcf,1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast (points_count,1, MPI_INT, 0, MPI_COMM_WORLD);
//local arrays and share parameters
int num_elems = *nintcf-*nintci+1;
int points_num = *points_count;
int npro = num_elems/num_procs;
int exter = *nextcf - *nextci + 1;
int remain = 0;
int *k = (int*) calloc(sizeof(int), num_procs);
int *k_sum = (int*) calloc(sizeof(int), num_procs);
if (my_rank == (num_procs-1) ) {
remain = num_elems % num_procs;
}
int local_array_size = npro + remain + exter;
*local_global_index = (int*) calloc(sizeof(int), npro+remain+exter);
*bs = (double*) calloc(sizeof(double), (local_array_size));
*bn = (double*) calloc(sizeof(double), (local_array_size));
*bw = (double*) calloc(sizeof(double), (local_array_size));
*be = (double*) calloc(sizeof(double), (local_array_size));
*bl = (double*) calloc(sizeof(double), (local_array_size));
*bh = (double*) calloc(sizeof(double), (local_array_size));
*bp = (double*) calloc(sizeof(double), (local_array_size));
*su = (double*) calloc(sizeof(double), (local_array_size));
*var = (double*) calloc(sizeof(double), (local_array_size));
*cgup = (double*) calloc(sizeof(double), (local_array_size));
*oc = (double*) calloc(sizeof(double), (npro+remain));
*cnorm = (double*) calloc(sizeof(double), (npro+remain));
*lcc = (int**) calloc(sizeof(int*),(local_array_size));
for ( i = 0; i < local_array_size; i++ ) {
(*lcc)[i] = (int *) calloc(sizeof(int),(6));
}
int *data = (int *)calloc(sizeof(int),(num_elems*6));
lcc_b = (int **)calloc(sizeof(int*),(num_elems));
for ( i=0; i<num_elems; i++){
lcc_b[i] = &(data[6*i]);
}
if ( my_rank == 0 ) {
for ( i = 0; i< num_elems; i++ ) {
for ( j = 0; j < 6; j++ ) {
lcc_b[i][j]=lcc_a[i][j];
}
}
}
MPI_Bcast (&(lcc_b[0][0]),num_elems*6, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Barrier(MPI_COMM_WORLD);
//choose part type
if (strcmp(part_type,"classical") == 0) {
//int *k_c = (int*) calloc(sizeof(int), num_procs);
k[my_rank] = npro + remain;
(num_elems_pro) = npro + remain;
int p = 0;
for ( p = 0; p < num_procs; p++ ) {
MPI_Bcast(&k[p],1,MPI_INT,p,MPI_COMM_WORLD);
}
//int *k_c_sum = (int*) calloc(sizeof(int), num_procs);
if ( my_rank == 0 ) {
for (i = 1; i < num_procs; i++ ) {
k_sum[i]=k_sum[i-1]+k[i-1];
}
}
//ditribute all B* array
MPI_Scatterv(bs_a, k, k_sum, MPI_DOUBLE, *bs, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD);
MPI_Scatterv(bn_a, k, k_sum, MPI_DOUBLE, *bn, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD);
MPI_Scatterv(bw_a, k, k_sum, MPI_DOUBLE, *bw, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD);
MPI_Scatterv(be_a, k, k_sum, MPI_DOUBLE, *be, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD);
MPI_Scatterv(bl_a, k, k_sum, MPI_DOUBLE, *bl, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD);
MPI_Scatterv(bh_a, k, k_sum, MPI_DOUBLE, *bh, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD);
MPI_Scatterv(bp_a, k, k_sum, MPI_DOUBLE, *bp, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD);
MPI_Scatterv(su_a, k, k_sum, MPI_DOUBLE, *su, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD);
/* create a datatype to describe the subarrays of the global array */
/*int sizes[2] = {num_elems, 6};
int subsizes[2] = {npro, 6};
int starts[2] = {0,0};
MPI_Datatype type, subarrtype;
MPI_Type_create_subarray(2, sizes, subsizes, starts, MPI_ORDER_C, MPI_INT, &type);
MPI_Type_create_resized(type, 0, npro*sizeof(int), &subarrtype);
MPI_Type_commit(&subarrtype);*/
//*lcc = (int**) calloc(sizeof(int*),(npro));
//for ( i = 0; i < npro; i++ ) {
// (*lcc)[i] = &(data[6*i]);
//(int *) calloc(sizeof(int),(6));
//}
/*int *globalptr;
if (my_rank == 0) globalptr = &(lcc_a[0][0]);
// scatter the array to all processors
int sendcounts[num_procs];
int displs[num_procs];
if (my_rank == 0) {
for ( i = 0; i < num_procs; i++) sendcounts[i] = 1;
// int disp = 0;
// for (int i=0; i<; i++) {
// for (int j=0; j<; j++) {
// displs[i*procgridsize+j] = disp;
// disp += 1;
// }
// disp += ((gridsize/procgridsize)-1)*procgridsize;
// }
}
MPI_Scatterv(&(lcc_a[0][0]),sendcounts, k_c_sum, subarrtype, &((*lcc)[0][0]),
npro*6, MPI_INT,
0, MPI_COMM_WORLD);*/
//initialization of computational array
for ( i = 0; i < num_elems_pro; i++ ) {
(*local_global_index)[i] = my_rank * npro + i;
for (j = 0;j < 6;j++){
(*lcc)[i][j]=lcc_b[my_rank*npro+i][j];
}
}
for ( i = 0; i <= 10; i++ ) {
(*oc)[i] = 0.0;
(*cnorm)[i] = 1.0;
}
for ( i = 0; i < num_elems_pro; i++ ) {
(*cgup)[i] = 0.0;
(*var)[i] = 0.0;
}
for ( i = num_elems_pro; i < local_array_size; i++ ) {
(*var)[i] = 0.0;
(*cgup)[i] = 0.0;
(*bs)[i] = 0.0;
(*be)[i] = 0.0;
(*bn)[i] = 0.0;
(*bw)[i] = 0.0;
(*bl)[i] = 0.0;
(*bh)[i] = 0.0;
}
for ( i = 0; i < num_elems_pro; i++ ) {
(*cgup)[i] = 1.0 / ((*bp)[i]);
}
//part type is not classics but metis
}else{
*epart = (int*) calloc(sizeof(int), num_elems);
*npart = (int*) calloc(sizeof(int), num_elems*8);
if ( my_rank == 0 ) {
//parametes and array for metis partition libary
idx_t ne = (idx_t) num_elems;
idx_t nn = (idx_t) points_num;
idx_t ncommon = 4;
idx_t nparts = num_procs;
int node_num = ne*8;
idx_t *eptr = (idx_t*) calloc(sizeof(idx_t), num_elems + 1);
idx_t *eind = (idx_t*) calloc(sizeof(idx_t), node_num);
idx_t objval_METIS;
idx_t *epart_METIS = (idx_t*) calloc(sizeof(idx_t), num_elems);
idx_t *npart_METIS = (idx_t*) calloc(sizeof(idx_t), node_num);
int metis_final;
for ( i = (*nintci); i <= (*nintcf + 1) ; i++ ) {
eptr[i] = (idx_t) i*8;
}
for( i = 0; i < node_num; i++ ) {
eind[i] = (idx_t) (*elems)[i];
}
if ( strcmp(part_type,"dual") == 0 ) {
metis_final = METIS_PartMeshDual(&ne,&nn,eptr, eind, NULL, NULL,
&ncommon, &nparts, NULL,NULL, &objval_METIS, epart_METIS, npart_METIS);
} else if ( strcmp(part_type,"noda") == 0 ) {
metis_final = METIS_PartMeshNodal(&ne,&nn,eptr, eind, NULL, NULL,
&nparts, NULL,NULL, &objval_METIS, epart_METIS, npart_METIS);
}
if ( metis_final != METIS_OK ) {
printf("Metis part fails\n");
}
(*objval)=(int*) calloc(sizeof(int), 1);
(*objval)[0]=(int) objval_METIS;
for ( i = 0; i < num_elems; i++ ) {
(*epart)[i] = (int) epart_METIS[i];
}
for ( i = 0; i < node_num; i++ ) {
(*npart)[i] = (int) npart_METIS[i];
}
}//single processor
//Full METIS arrary should be avaible for every processor
MPI_Bcast(*epart,num_elems,MPI_INT,0,MPI_COMM_WORLD);
MPI_Bcast(*npart,num_elems*8,MPI_INT,0,MPI_COMM_WORLD);
//ditribute data according to METIS Partition
MPI_Barrier(MPI_COMM_WORLD);
int p = 0;
//int *k = (int*) calloc(sizeof(int), num_procs);
//store local to global mapping
for ( p = 0; p < num_procs; p++ ) {
if (my_rank == p ) {
for (j = 0; j < num_elems; j++ ) {
if ( (*epart)[j] == my_rank ) {
(*local_global_index)[k[my_rank]] = j ;
for (i=0;i<6;i++){
(*lcc)[k[my_rank]][i]=lcc_b[j][i];
}
k[my_rank] = k[my_rank] + 1;
}
}
}
MPI_Bcast(&k[p],1,MPI_INT,p,MPI_COMM_WORLD);/// send k[p] to other processors
}///finish storing local to global mapping
(num_elems_pro) = k[my_rank];
//int *k_sum = (int*) calloc(sizeof(int), num_procs);
int *local_global_index_sum = (int*) calloc(sizeof(int), num_elems);
if ( my_rank == 0 ) {
for (i = 1; i < num_procs; i++ ) {
k_sum[i]=k_sum[i-1]+k[i-1];
}
}
MPI_Gatherv( *local_global_index, k[my_rank], MPI_INT,
local_global_index_sum, k, k_sum,
MPI_INT, 0, MPI_COMM_WORLD);
//copy B* array into new array accoring to order from metis partition
double *bs_b = (double*) calloc(sizeof(double), (num_elems));
double *bn_b = (double*) calloc(sizeof(double), (num_elems));
double *bw_b = (double*) calloc(sizeof(double), (num_elems));
double *be_b = (double*) calloc(sizeof(double), (num_elems));
double *bl_b = (double*) calloc(sizeof(double), (num_elems));
double *bh_b = (double*) calloc(sizeof(double), (num_elems));
double *bp_b = (double*) calloc(sizeof(double), (num_elems));
double *su_b = (double*) calloc(sizeof(double), (num_elems));
if (my_rank==0){
for (i= 0; i<num_elems; i++){
j=local_global_index_sum[i];
bs_b[i]=bs_a[j];
bn_b[i]=bn_a[j];
bw_b[i]=bw_a[j];
be_b[i]=be_a[j];
bl_b[i]=bl_a[j];
bh_b[i]=bh_a[j];
bp_b[i]=bp_a[j];
su_b[i]=su_a[j];
}
}
MPI_Scatterv( bs_b, k, k_sum , MPI_DOUBLE, *bs, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD);
MPI_Scatterv( bn_b, k, k_sum , MPI_DOUBLE, *bn, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD);
MPI_Scatterv( bw_b, k, k_sum , MPI_DOUBLE, *bw, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD);
MPI_Scatterv( be_b, k, k_sum , MPI_DOUBLE, *be, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD);
MPI_Scatterv( bl_b, k, k_sum , MPI_DOUBLE, *bl, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD);
MPI_Scatterv( bh_b, k, k_sum , MPI_DOUBLE, *bh, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD);
MPI_Scatterv( bp_b, k, k_sum , MPI_DOUBLE, *bp, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD);
MPI_Scatterv( su_b, k, k_sum , MPI_DOUBLE, *su, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD);
//initialization computational array
for ( i = 0; i <= 10; i++ ) {
(*oc)[i] = 0.0;
(*cnorm)[i] = 1.0;
}
for ( i = 0; i < num_elems_pro; i++ ) {
(*cgup)[i] = 0.0;
(*var)[i] = 0.0;
}
for ( i = num_elems_pro; i < local_array_size; i++ ) {
(*var)[i] = 0.0;
(*cgup)[i] = 0.0;
(*bs)[i] = 0.0;
(*be)[i] = 0.0;
(*bn)[i] = 0.0;
(*bw)[i] = 0.0;
(*bl)[i] = 0.0;
(*bh)[i] = 0.0;
}
for ( i = 0; i < num_elems_pro; i++ ) {
(*cgup)[i] = 1.0 / ((*bp)[i]);
}
MPI_Barrier(MPI_COMM_WORLD);
free(bp_b);
free(bh_b);
free(bl_b);
free(bw_b);
free(bn_b);
free(be_b);
free(bs_b);
free(su_b);
free(local_global_index_sum);
}//finish choose part type section and all local array are stored
MPI_Barrier(MPI_COMM_WORLD);
//************Comunication List*********//
*num_elems_local = num_elems_pro;
*neighbors_count = num_procs-1;
*send_count = (int*) calloc(sizeof(int), (num_procs));
*recv_count = (int*) calloc(sizeof(int), (num_procs));
*send_list = (int **) calloc(*neighbors_count+1, sizeof(int*));
for ( i = 0; i < *neighbors_count+1; i++ ) {
(*send_list)[i] = (int *) calloc(6*num_elems_pro, sizeof(int));
}
*recv_list = (int **) calloc(*neighbors_count+1, sizeof(int*));
for ( i = 0; i < *neighbors_count+1; i++ ) {
(*recv_list)[i] = (int *) calloc(6*num_elems_pro, sizeof(int));
}
//MPI_Barrier(MPI_COMM_WORLD);
int num_elems_global=0;
int* rank = (int*) calloc(sizeof(int), (num_procs));
int m = 0;
for (i = 0; i < num_elems_pro; i++) {
for (j = 0; j < 6; j++ ) {
num_elems_global=(*lcc)[i][j];
// choose only ghost cell
if (num_elems_global < num_elems){
// choose part type
if (strcmp(part_type,"classical") == 0) {
if (num_elems_global >= npro * num_procs){
rank[my_rank]= num_elems_global/npro-1;
}else{
rank[my_rank]= num_elems_global/npro;
}
} else {
rank[my_rank]=(*epart)[num_elems_global];
}///end choosing part type
if (rank[my_rank] != my_rank ) {
(*send_list)[rank[my_rank]][(*send_count)[rank[my_rank]]] = (*local_global_index)[i];
(*send_count)[rank[my_rank]]=(*send_count)[rank[my_rank]]+1;
(*recv_list)[rank[my_rank]][(*recv_count)[rank[my_rank]]] = num_elems_global;
(*recv_count)[rank[my_rank]]=(*recv_count)[rank[my_rank]]+1;
}
}///choose ghost cell
}///end j for loop
}///end i for loop
free(lcc_b);
if (my_rank == 0) {
printf("Initializition finished successfully!\n");
}
return 0;
}
int test_distribution(char *file_in, char *file_vtk_out, int *local_global_index,
int local_num_elems, double *scalars) {
// global sized variables, for reading the input file
int nintci_m, nintcf_m;
int nextci_m, nextcf_m;
int **lcc_m;
double *bs_m, *be_m, *bn_m, *bw_m, *bh_m, *bl_m;
double *bp_m;
double *su_m;
int points_count_m;
int** points_m;
int* elems_m;
int i;
// read the entire file
int f_status = read_binary_geo( file_in, &nintci_m, &nintcf_m, &nextci_m,
&nextcf_m, &lcc_m, &bs_m, &be_m, &bn_m, &bw_m, &bl_m, &bh_m, &bp_m,
&su_m, &points_count_m, &points_m, &elems_m );
if ( f_status != 0 ) {
printf( "Error in reading input file \n" );
return -1;
}
// allocate distribution vector
double *distr;
if ( ( distr = (double *) malloc( ( nintcf_m + 1 ) * sizeof(double) ) )
== NULL ) {
printf( "malloc failed to allocate distr array" );
return -1;
}
for ( i = nintci_m; i < ( nintcf_m + 1 ); i++ ) {
distr[i] = 0.0;
}
// copy the local values using the generated map
for ( i = 0; i < local_num_elems; i++ ) {
distr[local_global_index[i]] = scalars[i];
}
// write vtk file
vtk_write_unstr_grid_header( file_in, file_vtk_out, nintci_m, nintcf_m,
points_count_m, points_m, elems_m );
vtk_append_double( file_vtk_out, "SCALARS", nintci_m, nintcf_m, distr );
printf( "Distribution VTK file succesfully generated! \n" );
// free the allocated memory
free( su_m );
free( bp_m );
free( bh_m );
free( bl_m );
free( bw_m );
free( bn_m );
free( be_m );
free( bs_m );
free( elems_m );
for ( i = 0; i < nintcf_m + 1; i++ ) {
free( lcc_m[i] );
}
free( lcc_m );
for ( i = 0; i < points_count_m; i++ ) {
free( points_m[i] );
}
free( points_m );
free( distr );
return 0;
}
int initialization(char* file_in, char* part_type, char* read_type, int nprocs, int myrank,
int* nintci, int* nintcf, int* nextci,
int* nextcf, int*** lcc, double** bs, double** be, double** bn, double** bw,
double** bl, double** bh, double** bp, double** su, int* points_count,
int*** points, int** elems, double** var, double** cgup, double** oc,
double** cnorm, int** local_global_index) {
/********** START INITIALIZATION **********/
printf("INIT!\n");
int i = 0;
// read-in the input file
int f_status = read_binary_geo(file_in, &*nintci, &*nintcf, &*nextci, &*nextcf, &*lcc, &*bs,
&*be, &*bn, &*bw, &*bl, &*bh, &*bp, &*su, &*points_count,
&*points, &*elems);
// ====================== For testing purposes ======================
// For allread
// int *loc_global_index;
// int *rank = (int*) malloc(sizeof(int)*(*nintcf + 1));
// int nintci_loc, nintcf_loc, nextci_loc, nextcf_loc;
// int r;
//
// int numproc = 3;
//
// for (r=0; r<numproc; r++)
// {
//
// allread_calc_global_idx(&loc_global_index, &nintci_loc, &nintcf_loc, &nextci_loc,
// &nextcf_loc, part_type, read_type, numproc, r,
// *nintci, *nintcf, *nextci,
// *nextcf, *lcc, *elems, *points_count);
//
// for (i=nintci_loc; i <= nintcf_loc; i++) {
// rank[loc_global_index[i - nintci_loc]] = r;
// }
//
// free(loc_global_index);
// }
//
// For oneread
int **loc_global_index;
int *rank = (int*) malloc(sizeof(int)*(*nintcf + 1));
int *nintci_loc, *nintcf_loc, *nextci_loc, *nextcf_loc;
int r;
int numproc = 4;
oneread_calc_global_idx(&loc_global_index, &nintci_loc, &nintcf_loc, &nextci_loc,
&nextcf_loc, part_type, read_type, numproc,
*nintci, *nintcf, *nextci,
*nextcf, *lcc, *elems, *points_count);
for (r=0;r<numproc;r++)
{
printf("%d\t%d\n", r, nintcf_loc[r]);
for (i=nintci_loc[r]; i <= nintcf_loc[r]; i++) {
rank[(loc_global_index[r][i - nintci_loc[r]])] = r;
}
}
// for (r=0; r<numproc; r++)
// {
// free(loc_global_index[r]);
// }
// free(loc_global_index);
// free(nintcf_loc);
// free(nintci_loc);
// free(nextci_loc);
// free(nextcf_loc);
// vtk_write_unstr_grid_header("a", "b.vtk", *nintci, *nintcf, *points_count, *points, *elems);
// vtk_append_integer("b.vtk", "rank", *nintci, *nintcf, rank);
double *scalars = (double*) calloc(nextci_loc[2], sizeof(double));
for (i=0; i<nextci_loc[2]; i++)
{
scalars[i] = 1.0;
}
test_distribution(file_in, "bb.vtk", loc_global_index[2], nextci_loc[2], scalars);
free(scalars);
free(rank);
for (r=0; r<numproc; r++)
{
free(loc_global_index[r]);
}
free(loc_global_index);
free(nintcf_loc);
free(nintci_loc);
free(nextci_loc);
free(nextcf_loc);
// ====================== For testing purposes ======================
if ( f_status != 0 ) return f_status;
*var = (double*) calloc(sizeof(double), (*nextcf + 1));
*cgup = (double*) calloc(sizeof(double), (*nextcf + 1));
*cnorm = (double*) calloc(sizeof(double), (*nintcf + 1));
// initialize the arrays
for ( i = 0; i <= 10; i++ ) {
(*cnorm)[i] = 1.0;
}
for ( i = (*nintci); i <= (*nintcf); i++ ) {
(*var)[i] = 0.0;
}
for ( i = (*nintci); i <= (*nintcf); i++ ) {
(*cgup)[i] = 1.0 / ((*bp)[i]);
}
for ( i = (*nextci); i <= (*nextcf); i++ ) {
(*var)[i] = 0.0;
(*cgup)[i] = 0.0;
(*bs)[i] = 0.0;
(*be)[i] = 0.0;
(*bn)[i] = 0.0;
(*bw)[i] = 0.0;
(*bh)[i] = 0.0;
(*bl)[i] = 0.0;
}
return 0;
}
int test_communication(char *file_in, char *file_vtk_out, int *local_global_index, int num_elems,
int neighbors_count, int* send_count, int** send_list, int* recv_count, int** recv_list) {
int i, j, id;
int my_rank, num_procs;
MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); // Get current process id
MPI_Comm_size(MPI_COMM_WORLD, &num_procs); // get number of processes
// Read the geometry from the input file
/** Simulation parameters parsed from the input datasets */
int nintci, nintcf; /// internal cells start and end index
/// external cells start and end index. The external cells are only ghost cells.
/// They are accessed only through internal cells
int nextci, nextcf;
int **lcc; /// link cell-to-cell array - stores neighboring information
/// Boundary coefficients for each volume cell (South, East, North, West, High, Low)
double *bs, *be, *bn, *bw, *bl, *bh;
double *bp; /// Pole coefficient
double *su; /// Source values
/** Geometry data */
int points_count; /// total number of points that define the geometry
int** points; /// coordinates of the points that define the cells - size [points_cnt][3]
int* elems; /// definition of the cells using their nodes (points) - each cell has 8 points
double* commlist;
int ne_g;
int read_input = read_binary_geo(file_in, &nintci, &nintcf, &nextci, &nextcf, &lcc, &bs, &be,
&bn, &bw, &bl, &bh, &bp, &su, &points_count, &points, &elems);
if (read_input != 0)
printf("Could not read input file in test distribution");
ne_g = nintcf - nintci + 1;
// allocate the commlist vector and initialize it with zeros
if ((commlist = (double*) calloc(sizeof(double), ne_g)) == NULL ) {
fprintf(stderr, "calloc failed to allocate distr");
return -1;
}
// set all internal cells
for (i = 0; i < num_elems; i++) {
id = local_global_index[i]; // get global id of the element
commlist[id] = 15.0;
}
// set the elements which are to be sent
for (i = 0; i < num_procs; i++) { // for all neighbors in of this process
for (j = 0; j < send_count[i]; j++) { // for all elements in the list
id = send_list[i][j]; // get global id of the element to be sent
commlist[id] = 10.0;
}
}
// set the elements which are to be received
for (i = 0; i < num_procs; i++) { // for all neighbors in of this process
for (j = 0; j < recv_count[i]; j++) { // for all elements in the list
id = recv_list[i][j]; // get global id of the element to be received
commlist[id] = 5.0;
}
}
// write the vtk header
const char experiment_name[] = "test for communication";
vtk_write_unstr_grid_header(experiment_name, file_vtk_out, nintci, nintcf, points_count, points,
elems);
// write the values to the vtk file
vtk_append_double(file_vtk_out, "commlist", nintci, nintcf, commlist);
return 0;
}
