int test_distribution( char *file_in, char *file_vtk_out, int *local_global_index, int local_num_elems, double *cgup_local ) { // Global variables for reading from file int nintci, nintcf, nextci, nextcf; int **lcc; double *bs, *be, *bn, *bw, *bh, *bl, *bp, *su; int points_count, **points, *elems; double *distr; int i; // read-in the input file int f_status = read_binary_geo( file_in, &nintci, &nintcf, &nextci, &nextcf, &lcc, &bs, &be, &bn, &bw, &bl, &bh, &bp, &su, &points_count, &points, &elems ); if( f_status != 0 ) { return f_status; } // Free unnecessary global data for( i = nintci; i <= nintcf; i++ ) { free( lcc[i] ); } free( lcc ); free( bs ); free( be ); free( bn ); free( bw ); free( bh ); free( bl ); free( bp ); free( su ); // Allocate and fill distr if( ( distr = ( double * ) calloc( nintcf - nintci + 1, sizeof( double ) ) ) == NULL ) { fprintf( stderr, "malloc(distr) failed\n" ); return -1; } for( i = 0; i < local_num_elems; i++ ) { distr[local_global_index[i]] = cgup_local[i]; } // Write vtk file vtk_write_unstr_grid_header( "test_distribution", file_vtk_out, nintci, nintcf, points_count, points, elems ); vtk_append_double( file_vtk_out, "CGUP", nintci, nintcf, distr ); // Free the rest of global data for( i = 0; i < points_count; i++ ) { free( points[i] ); } free( points ); free( elems ); free( distr ); return 0; }
int test_distribution(char *file_in, char *file_vtk_out, int *local_global_index, int num_elems, double *cgup) { int i; // Read the geometry from the input file /** Simulation parameters parsed from the input datasets */ int nintci, nintcf; /// internal cells start and end index /// external cells start and end index. The external cells are only ghost cells. /// They are accessed only through internal cells int nextci, nextcf; int **lcc; /// link cell-to-cell array - stores neighboring information /// Boundary coefficients for each volume cell (South, East, North, West, High, Low) double *bs, *be, *bn, *bw, *bl, *bh; double *bp; /// Pole coefficient double *su; /// Source values /** Geometry data */ int points_count; /// total number of points that define the geometry int** points; /// coordinates of the points that define the cells - size [points_cnt][3] int* elems; /// definition of the cells using their nodes (points) - each cell has 8 points double* distr; int read_input = read_binary_geo(file_in, &nintci, &nintcf, &nextci, &nextcf, &lcc, &bs, &be, &bn, &bw, &bl, &bh, &bp, &su, &points_count, &points, &elems); if (read_input != 0) printf("Could not read input file in test distribution"); // allocate the distr vector and initialize it with zeros if ((distr = (double*) calloc(sizeof(double), nintcf - nintci + 1)) == NULL ) { fprintf(stderr, "calloc failed to allocate distr"); return -1; } // copy the locally initialized CGUP vector to the right place in the distr array int ind; for (i = 0; i < num_elems; i++) { ind = local_global_index[i]; distr[ind] = cgup[i]; } // write the vtk header const char experiment_name[] = "test for distribution"; vtk_write_unstr_grid_header(experiment_name, file_vtk_out, nintci, nintcf, points_count, points, elems); // write the values to the vtk file vtk_append_double(file_vtk_out, "CGUP", nintci, nintcf, distr); return 0; }
int test_distribution(char *file_in, char *file_vtk_out, int *local_global_index, int num_elems, double *cgup) { int nintci; int nintcf; int nextci; int nextcf; int** lcc; double* bs; double* be; double* bn; double* bw; double* bl; double* bh; double* bp; double* su; int points_count; int** points; int* elems; double* v = NULL; int f_status = read_binary_geo(file_in, &nintci, &nintcf, &nextci, &nextcf, &lcc, &bs, &be, &bn, &bw, &bl, &bh, &bp, &su, &points_count, &points, &elems); int my_rank; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); /// Get current process id vtk_write_unstr_grid_header(file_in, file_vtk_out, 0, nintcf, points_count, points, elems); v = (double*) malloc( (size_t)(nintcf+1) * sizeof(double)); int i; for ( i = 0; i <= num_elems; ++i ) { v[local_global_index[i]] = cgup[i]; } vtk_append_double(file_vtk_out, "partition", 0, nintcf, v); free(v); return 0; }
int read_init_data(char* file_in, int read_key, int myrank, int *nintci, int *nintcf, int *nextci, int *nextcf, int ***lcc, double **bs, double **be, double **bn, double **bw, double **bl, double **bh, double **bp, double **su, int* points_count, int***points, int** elems) { int f_status=0; if (read_key == POSL_INIT_ONE_READ) { if (myrank == 0) { f_status = read_binary_geo(file_in, &*nintci, &*nintcf, &*nextci, &*nextcf, &*lcc, &*bs, &*be, &*bn, &*bw, &*bl, &*bh, &*bp, &*su, &*points_count, &*points, &*elems); } } else { //TODO:implement sheer geometry reading // f_status = read_lcc_boundary(file_in, &*nintci, &*nintcf, &*nextci, &*nextcf, &*lcc, &*bs, // &*be, &*bn, &*bw, &*bl, &*bh, &*bp, &*su); f_status = read_geometry(file_in, &*nintci, &*nintcf, &*nextci, &*nextcf, &*points_count, &*points, &*elems); } return f_status; }
int initialization(char* file_in, char* part_type, int* nintci, int* nintcf, int* nextci, int* nextcf, int*** lcc, double** bs, double** be, double** bn, double** bw, double** bl, double** bh, double** bp, double** su, int* points_count, int*** points, int** elems, double** var, double** cgup, double** oc, double** cnorm, int** local_global_index, int** global_local_index, int* neighbors_count, int** send_count, int*** send_list, int** recv_count, int*** recv_list, int** epart, int** npart, int** objval, int* num_elems_local) { /********** START INITIALIZATION **********/ int i = 0; int j = 0; int num_elems_pro; // number of elements in each processor int my_rank, num_procs; /// Boundary coefficients for each volume cell (South, East, North, West, High, Low) double *bs_a, *be_a, *bn_a, *bw_a, *bl_a, *bh_a; double *bp_a; /// Pole coefficient double *su_a; /// Source values int** lcc_a; /// link cell-to-cell array - stores neighboring information int** lcc_b; MPI_Status status; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); /// Get current process id MPI_Comm_size(MPI_COMM_WORLD, &num_procs); /// get number of processe // read-in the input file by one processor if ( my_rank == 0 ) { int f_status = read_binary_geo(file_in, &*nintci, &*nintcf, &*nextci, &*nextcf, &lcc_a, &bs_a, &be_a, &bn_a, &bw_a, &bl_a, &bh_a, &bp_a, &su_a, &*points_count, &*points, &*elems); if ( f_status != 0 ) return f_status; } // Send the common information to other processors MPI_Bcast(nintci, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(nintcf, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(nextci, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(nextcf, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(points_count, 1, MPI_INT, 0, MPI_COMM_WORLD); // local arrays and share parameters int num_elems = *nintcf - *nintci + 1; if (my_rank != 0) { *elems = (int*) calloc(sizeof(int), num_elems * 8); } MPI_Bcast(*elems, num_elems * 8, MPI_INT, 0, MPI_COMM_WORLD); int points_num = *points_count; int npro = num_elems / num_procs; int exter = *nextcf - *nextci + 1; int remain = 0; int *k = (int*) calloc(sizeof(int), num_procs); int *k_sum = (int*) calloc(sizeof(int), num_procs); int last_proc = num_procs - 1; if (my_rank == last_proc) { remain = num_elems % num_procs; } int local_array_size = npro + remain + exter; *local_global_index = (int*) calloc(sizeof(int), npro + remain + exter); *global_local_index = (int*) calloc(sizeof(int), num_elems); *bs = (double*) calloc(sizeof(double), (local_array_size)); *bn = (double*) calloc(sizeof(double), (local_array_size)); *bw = (double*) calloc(sizeof(double), (local_array_size)); *be = (double*) calloc(sizeof(double), (local_array_size)); *bl = (double*) calloc(sizeof(double), (local_array_size)); *bh = (double*) calloc(sizeof(double), (local_array_size)); *bp = (double*) calloc(sizeof(double), (local_array_size)); *su = (double*) calloc(sizeof(double), (local_array_size)); *var = (double*) calloc(sizeof(double), (local_array_size)); *cgup = (double*) calloc(sizeof(double), (local_array_size)); *oc = (double*) calloc(sizeof(double), (npro + remain)); *cnorm = (double*) calloc(sizeof(double), (npro + remain)); *lcc = (int**) calloc(sizeof(int*), (local_array_size)); for ( i = 0; i < local_array_size; i++ ) { (*lcc)[i] = (int *) calloc(sizeof(int), (6)); } int *data = (int *) calloc(sizeof(int), (num_elems*6)); lcc_b = (int **) calloc(sizeof(int*), (num_elems)); for (i = 0; i < num_elems; i++) { lcc_b[i] = &(data[6 * i]); } if ( my_rank == 0 ) { for ( i = 0; i< num_elems; i++ ) { for ( j = 0; j < 6; j++ ) { lcc_b[i][j] = lcc_a[i][j]; } } } MPI_Bcast(&(lcc_b[0][0]), num_elems*6, MPI_INT, 0, MPI_COMM_WORLD); // choose part type if (strcmp(part_type, "classical") == 0) { k[my_rank] = npro + remain; (num_elems_pro) = npro + remain; int p = 0; for (p = 0; p < num_procs; p++) { MPI_Bcast(&k[p], 1, MPI_INT, p, MPI_COMM_WORLD); } if (my_rank == 0) { for (i = 1; i < num_procs; i++) { k_sum[i] = k_sum[i-1] + k[i-1]; } } // ditribute all B* array MPI_Scatterv(bs_a, k, k_sum, MPI_DOUBLE, *bs, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD); MPI_Scatterv(bn_a, k, k_sum, MPI_DOUBLE, *bn, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD); MPI_Scatterv(bw_a, k, k_sum, MPI_DOUBLE, *bw, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD); MPI_Scatterv(be_a, k, k_sum, MPI_DOUBLE, *be, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD); MPI_Scatterv(bl_a, k, k_sum, MPI_DOUBLE, *bl, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD); MPI_Scatterv(bh_a, k, k_sum, MPI_DOUBLE, *bh, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD); MPI_Scatterv(bp_a, k, k_sum, MPI_DOUBLE, *bp, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD); MPI_Scatterv(su_a, k, k_sum, MPI_DOUBLE, *su, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD); for (i = 0; i < num_elems_pro; i++) { (*local_global_index)[i] = my_rank * npro + i; for (j = 0; j < 6; j++) { (*lcc)[i][j] = lcc_b[my_rank*npro+i][j]; } } for (i = 0; i < num_elems_pro; i++) { if (i > npro) { (*global_local_index)[my_rank*npro+i] = (my_rank*npro+i) % npro + npro; } else { (*global_local_index)[my_rank*npro+i] = (my_rank*npro+i) % npro; } } // part type is not classics but metis } else { *epart = (int*) calloc(sizeof(int), num_elems); *npart = (int*) calloc(sizeof(int), num_elems*8); // if ( my_rank == 0 ) { // parametes and array for metis partition libary idx_t ne = (idx_t) num_elems; idx_t nn = (idx_t) points_num; idx_t ncommon = 4; idx_t nparts = num_procs; int node_num = ne * 8; idx_t *eptr = (idx_t*) calloc(sizeof(idx_t), num_elems + 1); idx_t *eind = (idx_t*) calloc(sizeof(idx_t), node_num); idx_t objval_METIS; idx_t *epart_METIS = (idx_t*) calloc(sizeof(idx_t), num_elems); idx_t *npart_METIS = (idx_t*) calloc(sizeof(idx_t), node_num); int metis_final; for (i = (*nintci); i <= (*nintcf + 1) ; i++) { eptr[i] = (idx_t) i * 8; } for (i = 0; i < node_num; i++) { eind[i] = (idx_t) (*elems)[i]; } if (strcmp(part_type, "dual") == 0) { metis_final = METIS_PartMeshDual(&ne, &nn, eptr, eind, NULL, NULL, &ncommon, &nparts, NULL, NULL, &objval_METIS, epart_METIS, npart_METIS); } else if (strcmp(part_type, "noda") == 0) { metis_final = METIS_PartMeshNodal(&ne, &nn, eptr, eind, NULL, NULL, &nparts, NULL, NULL, &objval_METIS, epart_METIS, npart_METIS); } if (metis_final != METIS_OK) { printf("Metis part fails\n"); } (*objval) = (int*) calloc(sizeof(int), 1); (*objval)[0] = (int) objval_METIS; for (i = 0; i < num_elems; i++) { (*epart)[i] = (int) epart_METIS[i]; } for (i = 0; i < node_num; i++) { (*npart)[i] = (int) npart_METIS[i]; } // ditribute data according to METIS Partition int p = 0; // store local to global mapping for (p = 0; p < num_procs; p++) { if (my_rank == p) { for (j = 0; j < num_elems; j++) { if ((*epart)[j] == my_rank) { (*local_global_index)[k[my_rank]] = j; for (i = 0; i < 6; i++) { (*lcc)[k[my_rank]][i] = lcc_b[j][i]; } (*global_local_index)[j] = k[my_rank]; k[my_rank] = k[my_rank] + 1; } } } MPI_Bcast(&k[p], 1, MPI_INT, p, MPI_COMM_WORLD); /// send k[p] to other processors } /// finish storing local to global mapping (num_elems_pro) = k[my_rank]; int *local_global_index_sum = (int*) calloc(sizeof(int), num_elems); if (my_rank == 0) { for (i = 1; i < num_procs; i++) { k_sum[i] = k_sum[i-1] + k[i-1]; } } MPI_Gatherv(*local_global_index, k[my_rank], MPI_INT, local_global_index_sum, k, k_sum, MPI_INT, 0, MPI_COMM_WORLD); // copy B* array into new array accoring to order from metis partition double *bs_b = (double*) calloc(sizeof(double), (num_elems)); double *bn_b = (double*) calloc(sizeof(double), (num_elems)); double *bw_b = (double*) calloc(sizeof(double), (num_elems)); double *be_b = (double*) calloc(sizeof(double), (num_elems)); double *bl_b = (double*) calloc(sizeof(double), (num_elems)); double *bh_b = (double*) calloc(sizeof(double), (num_elems)); double *bp_b = (double*) calloc(sizeof(double), (num_elems)); double *su_b = (double*) calloc(sizeof(double), (num_elems)); if (my_rank == 0) { for (i= 0; i < num_elems; i++) { j = local_global_index_sum[i]; bs_b[i] = bs_a[j]; bn_b[i] = bn_a[j]; bw_b[i] = bw_a[j]; be_b[i] = be_a[j]; bl_b[i] = bl_a[j]; bh_b[i] = bh_a[j]; bp_b[i] = bp_a[j]; su_b[i] = su_a[j]; } } MPI_Scatterv(bs_b, k, k_sum , MPI_DOUBLE, *bs, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD); MPI_Scatterv(bn_b, k, k_sum , MPI_DOUBLE, *bn, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD); MPI_Scatterv(bw_b, k, k_sum , MPI_DOUBLE, *bw, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD); MPI_Scatterv(be_b, k, k_sum , MPI_DOUBLE, *be, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD); MPI_Scatterv(bl_b, k, k_sum , MPI_DOUBLE, *bl, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD); MPI_Scatterv(bh_b, k, k_sum , MPI_DOUBLE, *bh, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD); MPI_Scatterv(bp_b, k, k_sum , MPI_DOUBLE, *bp, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD); MPI_Scatterv(su_b, k, k_sum , MPI_DOUBLE, *su, k[my_rank], MPI_DOUBLE, 0, MPI_COMM_WORLD); free(bp_b); free(bh_b); free(bl_b); free(bw_b); free(bn_b); free(be_b); free(bs_b); free(su_b); free(local_global_index_sum); } // finish choose part type section and all local array are stored // initialization computational array for (i = 0; i <= 10; i++) { (*oc)[i] = 0.0; (*cnorm)[i] = 1.0; } for (i = 0; i < num_elems_pro; i++) { (*cgup)[i] = 0.0; (*var)[i] = 0.0; } for (i = num_elems_pro; i < local_array_size; i++) { (*var)[i] = 0.0; (*cgup)[i] = 0.0; (*bs)[i] = 0.0; (*be)[i] = 0.0; (*bn)[i] = 0.0; (*bw)[i] = 0.0; (*bl)[i] = 0.0; (*bh)[i] = 0.0; } for (i = 0; i < num_elems_pro; i++) { (*cgup)[i] = 1.0 / ((*bp)[i]); } // ************Comunication List********* // *num_elems_local = num_elems_pro; *neighbors_count = num_procs-1; *send_count = (int*) calloc(sizeof(int), (num_procs)); *recv_count = (int*) calloc(sizeof(int), (num_procs)); *send_list = (int **) calloc(sizeof(int*), (*neighbors_count + 1)); for (i = 0; i < *neighbors_count + 1; i++) { (*send_list)[i] = (int *) calloc(sizeof(int), (6 * num_elems_pro)); } int num_elems_global = 0; int* rank = (int*) calloc(sizeof(int), (num_procs)); int m = 0; int** count_time_local = (int**) calloc(sizeof(int*), (num_procs)); for (i = 0; i < num_procs; i++) { count_time_local[i] = (int *) calloc(sizeof(int), (num_elems)); } int* count_time = (int*) calloc(sizeof(int), (num_elems)); for (i = 0; i < num_elems_pro; i++) { for (j = 0; j < 6; j++) { num_elems_global = (*lcc)[i][j]; // choose only ghost cell if (num_elems_global < num_elems) { // choose part type if (strcmp(part_type, "classical") == 0) { if (num_elems_global >= npro * num_procs) { rank[my_rank]= num_elems_global / npro - 1; } else { rank[my_rank]= num_elems_global / npro; } } else { rank[my_rank]=(*epart)[num_elems_global]; } /// end choosing part type // record times of this elems occur if (rank[my_rank] != my_rank) { count_time_local[rank[my_rank]][i] = count_time_local[rank[my_rank]][i] + 1; if (count_time_local[rank[my_rank]][i] == 1) { (*send_list)[rank[my_rank]][(*send_count)[rank[my_rank]]] = (*local_global_index)[i]; (*send_count)[rank[my_rank]]=(*send_count)[rank[my_rank]] + 1; } } } /// choose ghost cell } /// end j for loop } /// end i for loop // Set the order in send_list and recv_list same for (i = 0; i < num_procs; i++) { MPI_Sendrecv(&((*send_count)[i]), 1, MPI_INT, i, i * 1000, &((*recv_count)[i]), 1, MPI_INT, i, my_rank * 1000, MPI_COMM_WORLD, &status); } *recv_list = (int **) calloc(sizeof(int*), (*neighbors_count+1)); for (i = 0; i < *neighbors_count+1; i++) { (*recv_list)[i] = (int *) calloc(sizeof(int), ((*recv_count)[i])); } for (i = 0; i < num_procs; i++) { if (my_rank == i) { for (j = 0; j < num_procs; j++) { if (j != my_rank) { MPI_Send((*send_list)[j], (*send_count)[j], MPI_INT, j, 100, MPI_COMM_WORLD); } } } else { MPI_Recv((*recv_list)[i], (*recv_count)[i], MPI_INT, i, 100, MPI_COMM_WORLD, &status); } } free(lcc_b); free(count_time_local); return 0; }
int test_communication( char *file_in, char *file_vtk_out, int *local_global_index, int local_num_elems, int neighbors_count, int *send_count, int **send_list, int *recv_count, int **recv_list ) { // Global variables for reading from file int nintci, nintcf, nextci, nextcf; int **lcc; double *bs, *be, *bn, *bw, *bh, *bl, *bp, *su; int points_count, **points, *elems; double *commlist; int i, j; // read-in the input file int f_status = read_binary_geo( file_in, &nintci, &nintcf, &nextci, &nextcf, &lcc, &bs, &be, &bn, &bw, &bl, &bh, &bp, &su, &points_count, &points, &elems ); if( f_status != 0 ) { return f_status; } // Free unnecessary global data for( i = nintci; i <= nintcf; i++ ) { free( lcc[i] ); } free( lcc ); free( bs ); free( be ); free( bn ); free( bw ); free( bh ); free( bl ); free( bp ); free( su ); // Allocate and fill commlist if( ( commlist = ( double * ) calloc( nintcf - nintci + 1, sizeof( double ) ) ) == NULL ) { fprintf( stderr, "malloc(commlist) failed\n" ); return -1; } for( i = 0; i < local_num_elems; i++ ) { // internal cells commlist[local_global_index[i]] = 15; } for( i = 0; i < neighbors_count; i++ ) { for( j = 0; j < send_count[i]; j++ ) { // ghost cell to be sent commlist[local_global_index[send_list[i][j]]] = 10; } } for( i = 0; i < neighbors_count; i++ ) { for( j = 0; j < recv_count[i]; j++ ) { // ghost cell to be received commlist[local_global_index[recv_list[i][j]]] = 5; } } // Write vtk file vtk_write_unstr_grid_header( "test_communication", file_vtk_out, nintci, nintcf, points_count, points, elems ); vtk_append_double( file_vtk_out, "commlist", nintci, nintcf, commlist ); // Free the rest of global data for( i = 0; i < points_count; i++ ) { free( points[i] ); } free( points ); free( elems ); free( commlist ); return 0; }
int test_communication(char *file_in, char *file_vtk_out, int *local_global_index, int num_elems, int neighbors_count, int* send_count, int** send_list, int* recv_count, int** recv_list) { int nintci; int nintcf; int nextci; int nextcf; int** lcc; double* bs; double* be; double* bn; double* bw; double* bl; double* bh; double* bp; double* su; int points_count; int** points; int* elems; int f_status = read_binary_geo(file_in, &nintci, &nintcf, &nextci, &nextcf, &lcc, &bs, &be, &bn, &bw, &bl, &bh, &bp, &su, &points_count, &points, &elems); int my_rank; int num_procs; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); /// Get current process id MPI_Comm_size(MPI_COMM_WORLD, &num_procs); int* commlist = NULL; commlist = (int*) calloc((nintcf + 1), sizeof(int)); int i; for ( i = 0; i < (nintcf +1); ++i ) commlist[i] = 0; // 5 received // 10 send // 15 other cells int j; for ( i = 0; i <= num_elems; ++i ) { commlist[local_global_index[i]] = 15; } for ( i = 0; i < num_procs; ++i ) { for ( j = 0; j < send_count[i]; ++j ) { commlist[send_list[i][j]] = 10; } for ( j = 0; j < recv_count[i]; ++j ) { commlist[recv_list[i][j]] = 5; } } vtk_append_integer(file_vtk_out, "communication", 0, nintcf, commlist); free(commlist); return 0; }
int initialization_classic(char* file_in, char* part_type, int* nintci, int* nintcf, int* nextci, int* nextcf, int*** lcc, double** bs, double** be, double** bn, double** bw, double** bl, double** bh, double** bp, double** su, int* points_count, int*** points, int** elems, double** var, double** cgup, double** oc, double** cnorm, int** local_global_index, int** global_local_index, int* neighbors_count, int** send_count, int*** send_list, int** recv_count, int*** recv_list, int** epart, int** npart, int* objval) { int i = 0; int my_rank, num_procs; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); /// get current process id MPI_Comm_size(MPI_COMM_WORLD, &num_procs); // read-in the input file int f_status = read_binary_geo(file_in, &*nintci, &*nintcf, &*nextci, &*nextcf, &*lcc, &*bs, &*be, &*bn, &*bw, &*bl, &*bh, &*bp, &*su, &*points_count, &*points, &*elems); if ( f_status != 0 ) return f_status; // size of internal and external junks int CHUNKSIZE_INT = (int)ceil((double)(*nintcf-*nintci+1) / (double)num_procs); // ceil int REMAINING_INT = (*nintcf-*nintci +1) - (num_procs-1) * CHUNKSIZE_INT; int CHUNKSIZE_EXT = (int)ceil((double)(*nextcf-*nextci+1) / (double)num_procs); // ceil int REMAINING_EXT = (*nextcf-*nextci +1) - (num_procs-1) * CHUNKSIZE_EXT; *var = (double*) calloc(sizeof(double), (CHUNKSIZE_INT + CHUNKSIZE_EXT)); *cgup = (double*) calloc(sizeof(double), (CHUNKSIZE_INT + CHUNKSIZE_EXT)); *cnorm = (double*) calloc(sizeof(double), (CHUNKSIZE_INT)); int i_end_int = CHUNKSIZE_INT; int i_end_ext = CHUNKSIZE_EXT; if (my_rank == num_procs - 1) { i_end_int = REMAINING_INT; i_end_ext = REMAINING_EXT; } // initialize the arrays for ( i = 0; i <= 10; i++ ) { (*cnorm)[i] = 1.0; } for ( i = 0; i < i_end_int; i++ ) { (*var)[i] = 0.0; } for ( i = 0; i < i_end_int; i++ ) { (*cgup)[i] = 1.0 / ((*bp)[i]); } printf("i_end_int=%d , i_end_ext=%d, CHUNKSIZE_INT = %d", i_end_int, i_end_ext, CHUNKSIZE_INT); MPI_Barrier( MPI_COMM_WORLD ); for ( i = CHUNKSIZE_INT; i < CHUNKSIZE_INT + i_end_ext; i++ ) { (*var)[i] = 0.0; (*cgup)[i] = 0.0; (*bs)[i] = 0.0; (*be)[i] = 0.0; (*bn)[i] = 0.0; (*bw)[i] = 0.0; (*bh)[i] = 0.0; (*bl)[i] = 0.0; } // initialize local to global mapping *local_global_index = (int*) malloc(sizeof(int) * (CHUNKSIZE_INT+CHUNKSIZE_EXT)); for ( i = 0; i < CHUNKSIZE_INT; i++ ) { (*local_global_index)[i] = i + my_rank * CHUNKSIZE_INT; } for ( i = CHUNKSIZE_INT; i < CHUNKSIZE_INT + CHUNKSIZE_EXT; i++ ) { (*local_global_index)[i] = *nintcf-*nintci + i + my_rank * CHUNKSIZE_EXT; } return 0; }
int initialization(char* file_in, char* part_type, int* nintci, int* nintcf, int* nextci, int* nextcf, int*** lcc, double** bs, double** be, double** bn, double** bw, double** bl, double** bh, double** bp, double** su, int* points_count, int*** points, int** elems, double** var, double** cgup, double** oc, double** cnorm, int** local_global_index, int** global_local_index, int* neighbors_count, int** send_count, int*** send_list, int** recv_count, int*** recv_list, int** epart, int** npart, int** objval, int* num_elems_local) { /********** START INITIALIZATION **********/ int i = 0; int j = 0; int num_elems_pro;//number of elements in each processor int my_rank, num_procs; /// Boundary coefficients for each volume cell (South, East, North, West, High, Low) double *bs_a, *be_a, *bn_a, *bw_a, *bl_a, *bh_a; double *bp_a; /// Pole coefficient double *su_a; /// Source values int** lcc_a; /// link cell-to-cell array - stores neighboring information int** lcc_b; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); /// Get current process id MPI_Comm_size(MPI_COMM_WORLD, &num_procs); /// get number of processe // read-in the input file by one processor if ( my_rank == 0 ) { int f_status = read_binary_geo(file_in, &*nintci, &*nintcf, &*nextci, &*nextcf, &lcc_a, &bs_a, &be_a, &bn_a, &bw_a, &bl_a, &bh_a, &bp_a, &su_a, &*points_count, &*points, &*elems); if ( f_status != 0 ) return f_status; } //Send the common information to other processors MPI_Bcast (nintci,1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast (nintcf,1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast (nextci,1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast (nextcf,1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast (points_count,1, MPI_INT, 0, MPI_COMM_WORLD); //local arrays and share parameters int num_elems = *nintcf-*nintci+1; int points_num = *points_count; int npro = num_elems/num_procs; int exter = *nextcf - *nextci + 1; int remain = 0; int *k = (int*) calloc(sizeof(int), num_procs); int *k_sum = (int*) calloc(sizeof(int), num_procs); if (my_rank == (num_procs-1) ) { remain = num_elems % num_procs; } int local_array_size = npro + remain + exter; *local_global_index = (int*) calloc(sizeof(int), npro+remain+exter); *bs = (double*) calloc(sizeof(double), (local_array_size)); *bn = (double*) calloc(sizeof(double), (local_array_size)); *bw = (double*) calloc(sizeof(double), (local_array_size)); *be = (double*) calloc(sizeof(double), (local_array_size)); *bl = (double*) calloc(sizeof(double), (local_array_size)); *bh = (double*) calloc(sizeof(double), (local_array_size)); *bp = (double*) calloc(sizeof(double), (local_array_size)); *su = (double*) calloc(sizeof(double), (local_array_size)); *var = (double*) calloc(sizeof(double), (local_array_size)); *cgup = (double*) calloc(sizeof(double), (local_array_size)); *oc = (double*) calloc(sizeof(double), (npro+remain)); *cnorm = (double*) calloc(sizeof(double), (npro+remain)); *lcc = (int**) calloc(sizeof(int*),(local_array_size)); for ( i = 0; i < local_array_size; i++ ) { (*lcc)[i] = (int *) calloc(sizeof(int),(6)); } int *data = (int *)calloc(sizeof(int),(num_elems*6)); lcc_b = (int **)calloc(sizeof(int*),(num_elems)); for ( i=0; i<num_elems; i++){ lcc_b[i] = &(data[6*i]); } if ( my_rank == 0 ) { for ( i = 0; i< num_elems; i++ ) { for ( j = 0; j < 6; j++ ) { lcc_b[i][j]=lcc_a[i][j]; } } } MPI_Bcast (&(lcc_b[0][0]),num_elems*6, MPI_INT, 0, MPI_COMM_WORLD); MPI_Barrier(MPI_COMM_WORLD); //choose part type if (strcmp(part_type,"classical") == 0) { //int *k_c = (int*) calloc(sizeof(int), num_procs); k[my_rank] = npro + remain; (num_elems_pro) = npro + remain; int p = 0; for ( p = 0; p < num_procs; p++ ) { MPI_Bcast(&k[p],1,MPI_INT,p,MPI_COMM_WORLD); } //int *k_c_sum = (int*) calloc(sizeof(int), num_procs); if ( my_rank == 0 ) { for (i = 1; i < num_procs; i++ ) { k_sum[i]=k_sum[i-1]+k[i-1]; } } //ditribute all B* array MPI_Scatterv(bs_a, k, k_sum, MPI_DOUBLE, *bs, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD); MPI_Scatterv(bn_a, k, k_sum, MPI_DOUBLE, *bn, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD); MPI_Scatterv(bw_a, k, k_sum, MPI_DOUBLE, *bw, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD); MPI_Scatterv(be_a, k, k_sum, MPI_DOUBLE, *be, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD); MPI_Scatterv(bl_a, k, k_sum, MPI_DOUBLE, *bl, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD); MPI_Scatterv(bh_a, k, k_sum, MPI_DOUBLE, *bh, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD); MPI_Scatterv(bp_a, k, k_sum, MPI_DOUBLE, *bp, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD); MPI_Scatterv(su_a, k, k_sum, MPI_DOUBLE, *su, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD); /* create a datatype to describe the subarrays of the global array */ /*int sizes[2] = {num_elems, 6}; int subsizes[2] = {npro, 6}; int starts[2] = {0,0}; MPI_Datatype type, subarrtype; MPI_Type_create_subarray(2, sizes, subsizes, starts, MPI_ORDER_C, MPI_INT, &type); MPI_Type_create_resized(type, 0, npro*sizeof(int), &subarrtype); MPI_Type_commit(&subarrtype);*/ //*lcc = (int**) calloc(sizeof(int*),(npro)); //for ( i = 0; i < npro; i++ ) { // (*lcc)[i] = &(data[6*i]); //(int *) calloc(sizeof(int),(6)); //} /*int *globalptr; if (my_rank == 0) globalptr = &(lcc_a[0][0]); // scatter the array to all processors int sendcounts[num_procs]; int displs[num_procs]; if (my_rank == 0) { for ( i = 0; i < num_procs; i++) sendcounts[i] = 1; // int disp = 0; // for (int i=0; i<; i++) { // for (int j=0; j<; j++) { // displs[i*procgridsize+j] = disp; // disp += 1; // } // disp += ((gridsize/procgridsize)-1)*procgridsize; // } } MPI_Scatterv(&(lcc_a[0][0]),sendcounts, k_c_sum, subarrtype, &((*lcc)[0][0]), npro*6, MPI_INT, 0, MPI_COMM_WORLD);*/ //initialization of computational array for ( i = 0; i < num_elems_pro; i++ ) { (*local_global_index)[i] = my_rank * npro + i; for (j = 0;j < 6;j++){ (*lcc)[i][j]=lcc_b[my_rank*npro+i][j]; } } for ( i = 0; i <= 10; i++ ) { (*oc)[i] = 0.0; (*cnorm)[i] = 1.0; } for ( i = 0; i < num_elems_pro; i++ ) { (*cgup)[i] = 0.0; (*var)[i] = 0.0; } for ( i = num_elems_pro; i < local_array_size; i++ ) { (*var)[i] = 0.0; (*cgup)[i] = 0.0; (*bs)[i] = 0.0; (*be)[i] = 0.0; (*bn)[i] = 0.0; (*bw)[i] = 0.0; (*bl)[i] = 0.0; (*bh)[i] = 0.0; } for ( i = 0; i < num_elems_pro; i++ ) { (*cgup)[i] = 1.0 / ((*bp)[i]); } //part type is not classics but metis }else{ *epart = (int*) calloc(sizeof(int), num_elems); *npart = (int*) calloc(sizeof(int), num_elems*8); if ( my_rank == 0 ) { //parametes and array for metis partition libary idx_t ne = (idx_t) num_elems; idx_t nn = (idx_t) points_num; idx_t ncommon = 4; idx_t nparts = num_procs; int node_num = ne*8; idx_t *eptr = (idx_t*) calloc(sizeof(idx_t), num_elems + 1); idx_t *eind = (idx_t*) calloc(sizeof(idx_t), node_num); idx_t objval_METIS; idx_t *epart_METIS = (idx_t*) calloc(sizeof(idx_t), num_elems); idx_t *npart_METIS = (idx_t*) calloc(sizeof(idx_t), node_num); int metis_final; for ( i = (*nintci); i <= (*nintcf + 1) ; i++ ) { eptr[i] = (idx_t) i*8; } for( i = 0; i < node_num; i++ ) { eind[i] = (idx_t) (*elems)[i]; } if ( strcmp(part_type,"dual") == 0 ) { metis_final = METIS_PartMeshDual(&ne,&nn,eptr, eind, NULL, NULL, &ncommon, &nparts, NULL,NULL, &objval_METIS, epart_METIS, npart_METIS); } else if ( strcmp(part_type,"noda") == 0 ) { metis_final = METIS_PartMeshNodal(&ne,&nn,eptr, eind, NULL, NULL, &nparts, NULL,NULL, &objval_METIS, epart_METIS, npart_METIS); } if ( metis_final != METIS_OK ) { printf("Metis part fails\n"); } (*objval)=(int*) calloc(sizeof(int), 1); (*objval)[0]=(int) objval_METIS; for ( i = 0; i < num_elems; i++ ) { (*epart)[i] = (int) epart_METIS[i]; } for ( i = 0; i < node_num; i++ ) { (*npart)[i] = (int) npart_METIS[i]; } }//single processor //Full METIS arrary should be avaible for every processor MPI_Bcast(*epart,num_elems,MPI_INT,0,MPI_COMM_WORLD); MPI_Bcast(*npart,num_elems*8,MPI_INT,0,MPI_COMM_WORLD); //ditribute data according to METIS Partition MPI_Barrier(MPI_COMM_WORLD); int p = 0; //int *k = (int*) calloc(sizeof(int), num_procs); //store local to global mapping for ( p = 0; p < num_procs; p++ ) { if (my_rank == p ) { for (j = 0; j < num_elems; j++ ) { if ( (*epart)[j] == my_rank ) { (*local_global_index)[k[my_rank]] = j ; for (i=0;i<6;i++){ (*lcc)[k[my_rank]][i]=lcc_b[j][i]; } k[my_rank] = k[my_rank] + 1; } } } MPI_Bcast(&k[p],1,MPI_INT,p,MPI_COMM_WORLD);/// send k[p] to other processors }///finish storing local to global mapping (num_elems_pro) = k[my_rank]; //int *k_sum = (int*) calloc(sizeof(int), num_procs); int *local_global_index_sum = (int*) calloc(sizeof(int), num_elems); if ( my_rank == 0 ) { for (i = 1; i < num_procs; i++ ) { k_sum[i]=k_sum[i-1]+k[i-1]; } } MPI_Gatherv( *local_global_index, k[my_rank], MPI_INT, local_global_index_sum, k, k_sum, MPI_INT, 0, MPI_COMM_WORLD); //copy B* array into new array accoring to order from metis partition double *bs_b = (double*) calloc(sizeof(double), (num_elems)); double *bn_b = (double*) calloc(sizeof(double), (num_elems)); double *bw_b = (double*) calloc(sizeof(double), (num_elems)); double *be_b = (double*) calloc(sizeof(double), (num_elems)); double *bl_b = (double*) calloc(sizeof(double), (num_elems)); double *bh_b = (double*) calloc(sizeof(double), (num_elems)); double *bp_b = (double*) calloc(sizeof(double), (num_elems)); double *su_b = (double*) calloc(sizeof(double), (num_elems)); if (my_rank==0){ for (i= 0; i<num_elems; i++){ j=local_global_index_sum[i]; bs_b[i]=bs_a[j]; bn_b[i]=bn_a[j]; bw_b[i]=bw_a[j]; be_b[i]=be_a[j]; bl_b[i]=bl_a[j]; bh_b[i]=bh_a[j]; bp_b[i]=bp_a[j]; su_b[i]=su_a[j]; } } MPI_Scatterv( bs_b, k, k_sum , MPI_DOUBLE, *bs, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD); MPI_Scatterv( bn_b, k, k_sum , MPI_DOUBLE, *bn, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD); MPI_Scatterv( bw_b, k, k_sum , MPI_DOUBLE, *bw, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD); MPI_Scatterv( be_b, k, k_sum , MPI_DOUBLE, *be, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD); MPI_Scatterv( bl_b, k, k_sum , MPI_DOUBLE, *bl, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD); MPI_Scatterv( bh_b, k, k_sum , MPI_DOUBLE, *bh, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD); MPI_Scatterv( bp_b, k, k_sum , MPI_DOUBLE, *bp, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD); MPI_Scatterv( su_b, k, k_sum , MPI_DOUBLE, *su, k[my_rank],MPI_DOUBLE,0, MPI_COMM_WORLD); //initialization computational array for ( i = 0; i <= 10; i++ ) { (*oc)[i] = 0.0; (*cnorm)[i] = 1.0; } for ( i = 0; i < num_elems_pro; i++ ) { (*cgup)[i] = 0.0; (*var)[i] = 0.0; } for ( i = num_elems_pro; i < local_array_size; i++ ) { (*var)[i] = 0.0; (*cgup)[i] = 0.0; (*bs)[i] = 0.0; (*be)[i] = 0.0; (*bn)[i] = 0.0; (*bw)[i] = 0.0; (*bl)[i] = 0.0; (*bh)[i] = 0.0; } for ( i = 0; i < num_elems_pro; i++ ) { (*cgup)[i] = 1.0 / ((*bp)[i]); } MPI_Barrier(MPI_COMM_WORLD); free(bp_b); free(bh_b); free(bl_b); free(bw_b); free(bn_b); free(be_b); free(bs_b); free(su_b); free(local_global_index_sum); }//finish choose part type section and all local array are stored MPI_Barrier(MPI_COMM_WORLD); //************Comunication List*********// *num_elems_local = num_elems_pro; *neighbors_count = num_procs-1; *send_count = (int*) calloc(sizeof(int), (num_procs)); *recv_count = (int*) calloc(sizeof(int), (num_procs)); *send_list = (int **) calloc(*neighbors_count+1, sizeof(int*)); for ( i = 0; i < *neighbors_count+1; i++ ) { (*send_list)[i] = (int *) calloc(6*num_elems_pro, sizeof(int)); } *recv_list = (int **) calloc(*neighbors_count+1, sizeof(int*)); for ( i = 0; i < *neighbors_count+1; i++ ) { (*recv_list)[i] = (int *) calloc(6*num_elems_pro, sizeof(int)); } //MPI_Barrier(MPI_COMM_WORLD); int num_elems_global=0; int* rank = (int*) calloc(sizeof(int), (num_procs)); int m = 0; for (i = 0; i < num_elems_pro; i++) { for (j = 0; j < 6; j++ ) { num_elems_global=(*lcc)[i][j]; // choose only ghost cell if (num_elems_global < num_elems){ // choose part type if (strcmp(part_type,"classical") == 0) { if (num_elems_global >= npro * num_procs){ rank[my_rank]= num_elems_global/npro-1; }else{ rank[my_rank]= num_elems_global/npro; } } else { rank[my_rank]=(*epart)[num_elems_global]; }///end choosing part type if (rank[my_rank] != my_rank ) { (*send_list)[rank[my_rank]][(*send_count)[rank[my_rank]]] = (*local_global_index)[i]; (*send_count)[rank[my_rank]]=(*send_count)[rank[my_rank]]+1; (*recv_list)[rank[my_rank]][(*recv_count)[rank[my_rank]]] = num_elems_global; (*recv_count)[rank[my_rank]]=(*recv_count)[rank[my_rank]]+1; } }///choose ghost cell }///end j for loop }///end i for loop free(lcc_b); if (my_rank == 0) { printf("Initializition finished successfully!\n"); } return 0; }
int test_distribution(char *file_in, char *file_vtk_out, int *local_global_index, int local_num_elems, double *scalars) { // global sized variables, for reading the input file int nintci_m, nintcf_m; int nextci_m, nextcf_m; int **lcc_m; double *bs_m, *be_m, *bn_m, *bw_m, *bh_m, *bl_m; double *bp_m; double *su_m; int points_count_m; int** points_m; int* elems_m; int i; // read the entire file int f_status = read_binary_geo( file_in, &nintci_m, &nintcf_m, &nextci_m, &nextcf_m, &lcc_m, &bs_m, &be_m, &bn_m, &bw_m, &bl_m, &bh_m, &bp_m, &su_m, &points_count_m, &points_m, &elems_m ); if ( f_status != 0 ) { printf( "Error in reading input file \n" ); return -1; } // allocate distribution vector double *distr; if ( ( distr = (double *) malloc( ( nintcf_m + 1 ) * sizeof(double) ) ) == NULL ) { printf( "malloc failed to allocate distr array" ); return -1; } for ( i = nintci_m; i < ( nintcf_m + 1 ); i++ ) { distr[i] = 0.0; } // copy the local values using the generated map for ( i = 0; i < local_num_elems; i++ ) { distr[local_global_index[i]] = scalars[i]; } // write vtk file vtk_write_unstr_grid_header( file_in, file_vtk_out, nintci_m, nintcf_m, points_count_m, points_m, elems_m ); vtk_append_double( file_vtk_out, "SCALARS", nintci_m, nintcf_m, distr ); printf( "Distribution VTK file succesfully generated! \n" ); // free the allocated memory free( su_m ); free( bp_m ); free( bh_m ); free( bl_m ); free( bw_m ); free( bn_m ); free( be_m ); free( bs_m ); free( elems_m ); for ( i = 0; i < nintcf_m + 1; i++ ) { free( lcc_m[i] ); } free( lcc_m ); for ( i = 0; i < points_count_m; i++ ) { free( points_m[i] ); } free( points_m ); free( distr ); return 0; }
int initialization(char* file_in, char* part_type, char* read_type, int nprocs, int myrank, int* nintci, int* nintcf, int* nextci, int* nextcf, int*** lcc, double** bs, double** be, double** bn, double** bw, double** bl, double** bh, double** bp, double** su, int* points_count, int*** points, int** elems, double** var, double** cgup, double** oc, double** cnorm, int** local_global_index) { /********** START INITIALIZATION **********/ printf("INIT!\n"); int i = 0; // read-in the input file int f_status = read_binary_geo(file_in, &*nintci, &*nintcf, &*nextci, &*nextcf, &*lcc, &*bs, &*be, &*bn, &*bw, &*bl, &*bh, &*bp, &*su, &*points_count, &*points, &*elems); // ====================== For testing purposes ====================== // For allread // int *loc_global_index; // int *rank = (int*) malloc(sizeof(int)*(*nintcf + 1)); // int nintci_loc, nintcf_loc, nextci_loc, nextcf_loc; // int r; // // int numproc = 3; // // for (r=0; r<numproc; r++) // { // // allread_calc_global_idx(&loc_global_index, &nintci_loc, &nintcf_loc, &nextci_loc, // &nextcf_loc, part_type, read_type, numproc, r, // *nintci, *nintcf, *nextci, // *nextcf, *lcc, *elems, *points_count); // // for (i=nintci_loc; i <= nintcf_loc; i++) { // rank[loc_global_index[i - nintci_loc]] = r; // } // // free(loc_global_index); // } // // For oneread int **loc_global_index; int *rank = (int*) malloc(sizeof(int)*(*nintcf + 1)); int *nintci_loc, *nintcf_loc, *nextci_loc, *nextcf_loc; int r; int numproc = 4; oneread_calc_global_idx(&loc_global_index, &nintci_loc, &nintcf_loc, &nextci_loc, &nextcf_loc, part_type, read_type, numproc, *nintci, *nintcf, *nextci, *nextcf, *lcc, *elems, *points_count); for (r=0;r<numproc;r++) { printf("%d\t%d\n", r, nintcf_loc[r]); for (i=nintci_loc[r]; i <= nintcf_loc[r]; i++) { rank[(loc_global_index[r][i - nintci_loc[r]])] = r; } } // for (r=0; r<numproc; r++) // { // free(loc_global_index[r]); // } // free(loc_global_index); // free(nintcf_loc); // free(nintci_loc); // free(nextci_loc); // free(nextcf_loc); // vtk_write_unstr_grid_header("a", "b.vtk", *nintci, *nintcf, *points_count, *points, *elems); // vtk_append_integer("b.vtk", "rank", *nintci, *nintcf, rank); double *scalars = (double*) calloc(nextci_loc[2], sizeof(double)); for (i=0; i<nextci_loc[2]; i++) { scalars[i] = 1.0; } test_distribution(file_in, "bb.vtk", loc_global_index[2], nextci_loc[2], scalars); free(scalars); free(rank); for (r=0; r<numproc; r++) { free(loc_global_index[r]); } free(loc_global_index); free(nintcf_loc); free(nintci_loc); free(nextci_loc); free(nextcf_loc); // ====================== For testing purposes ====================== if ( f_status != 0 ) return f_status; *var = (double*) calloc(sizeof(double), (*nextcf + 1)); *cgup = (double*) calloc(sizeof(double), (*nextcf + 1)); *cnorm = (double*) calloc(sizeof(double), (*nintcf + 1)); // initialize the arrays for ( i = 0; i <= 10; i++ ) { (*cnorm)[i] = 1.0; } for ( i = (*nintci); i <= (*nintcf); i++ ) { (*var)[i] = 0.0; } for ( i = (*nintci); i <= (*nintcf); i++ ) { (*cgup)[i] = 1.0 / ((*bp)[i]); } for ( i = (*nextci); i <= (*nextcf); i++ ) { (*var)[i] = 0.0; (*cgup)[i] = 0.0; (*bs)[i] = 0.0; (*be)[i] = 0.0; (*bn)[i] = 0.0; (*bw)[i] = 0.0; (*bh)[i] = 0.0; (*bl)[i] = 0.0; } return 0; }
int test_communication(char *file_in, char *file_vtk_out, int *local_global_index, int num_elems, int neighbors_count, int* send_count, int** send_list, int* recv_count, int** recv_list) { int i, j, id; int my_rank, num_procs; MPI_Comm_rank(MPI_COMM_WORLD, &my_rank); // Get current process id MPI_Comm_size(MPI_COMM_WORLD, &num_procs); // get number of processes // Read the geometry from the input file /** Simulation parameters parsed from the input datasets */ int nintci, nintcf; /// internal cells start and end index /// external cells start and end index. The external cells are only ghost cells. /// They are accessed only through internal cells int nextci, nextcf; int **lcc; /// link cell-to-cell array - stores neighboring information /// Boundary coefficients for each volume cell (South, East, North, West, High, Low) double *bs, *be, *bn, *bw, *bl, *bh; double *bp; /// Pole coefficient double *su; /// Source values /** Geometry data */ int points_count; /// total number of points that define the geometry int** points; /// coordinates of the points that define the cells - size [points_cnt][3] int* elems; /// definition of the cells using their nodes (points) - each cell has 8 points double* commlist; int ne_g; int read_input = read_binary_geo(file_in, &nintci, &nintcf, &nextci, &nextcf, &lcc, &bs, &be, &bn, &bw, &bl, &bh, &bp, &su, &points_count, &points, &elems); if (read_input != 0) printf("Could not read input file in test distribution"); ne_g = nintcf - nintci + 1; // allocate the commlist vector and initialize it with zeros if ((commlist = (double*) calloc(sizeof(double), ne_g)) == NULL ) { fprintf(stderr, "calloc failed to allocate distr"); return -1; } // set all internal cells for (i = 0; i < num_elems; i++) { id = local_global_index[i]; // get global id of the element commlist[id] = 15.0; } // set the elements which are to be sent for (i = 0; i < num_procs; i++) { // for all neighbors in of this process for (j = 0; j < send_count[i]; j++) { // for all elements in the list id = send_list[i][j]; // get global id of the element to be sent commlist[id] = 10.0; } } // set the elements which are to be received for (i = 0; i < num_procs; i++) { // for all neighbors in of this process for (j = 0; j < recv_count[i]; j++) { // for all elements in the list id = recv_list[i][j]; // get global id of the element to be received commlist[id] = 5.0; } } // write the vtk header const char experiment_name[] = "test for communication"; vtk_write_unstr_grid_header(experiment_name, file_vtk_out, nintci, nintcf, points_count, points, elems); // write the values to the vtk file vtk_append_double(file_vtk_out, "commlist", nintci, nintcf, commlist); return 0; }