int main(int argc, char **argv) { int ret; int p, ts; int var; int slice = 0; int nprocs = 1, rank = 0; char output_file_name[512]; // MPI initialization #if PIDX_HAVE_MPI MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); MPI_Comm_rank(MPI_COMM_WORLD, &rank); #endif if (rank == 0) { ret = parse_args(argc, argv); if (ret < 0) { usage(); #if PIDX_HAVE_MPI MPI_Abort(MPI_COMM_WORLD, -1); #else exit(-1); #endif } // check if the num procs is appropriate int num_bricks = (global_box_size[0] / local_box_size[0]) * (global_box_size[1] / local_box_size[1]) * (global_box_size[2] / local_box_size[2]); if(num_bricks != nprocs) { fprintf(stderr, "Error: number of sub-blocks (%d) doesn't match number of procs (%d)\n", num_bricks, nprocs); fprintf(stderr, "Incorrect distribution of data across processes i.e.\n(global_x / local_x) X (global_x / local_x) X (global_x / local_x) != nprocs\n(%d/%d) X (%d/%d) X (%d/%d) != %d\n", global_box_size[0], local_box_size[0], global_box_size[1], local_box_size[1], global_box_size[2], local_box_size[2], nprocs); #if PIDX_HAVE_MPI MPI_Abort(MPI_COMM_WORLD, -1); #else exit(-1); #endif } } #if 1 // The command line arguments are shared by all processes #if PIDX_HAVE_MPI MPI_Bcast(global_box_size, 3, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(local_box_size, 3, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(&time_step_count, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(&variable_count, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(&patch_count, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(&output_file_template, 512, MPI_CHAR, 0, MPI_COMM_WORLD); MPI_Bcast(&blocks_per_file, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(&bits_per_block, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(restructured_box_size, 5, MPI_LONG_LONG, 0, MPI_COMM_WORLD); MPI_Bcast(&compression_type, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(&compression_bit_rate, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(perform_phases, 6, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(debug_rst_hz, 2, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(dump_agg_io, 2, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(idx_count, 3, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(&aggregation_factor, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(&is_rank_z_ordering, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(hz_from_to, 2, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(rst_agg, 2, MPI_INT, 0, MPI_COMM_WORLD); #endif values_per_sample = malloc(sizeof(*values_per_sample) * variable_count); memset(values_per_sample, 0, sizeof(*values_per_sample) * variable_count); // Creating the filename sprintf(output_file_name, "%s%s", output_file_template,".idx"); // Calculating every process data's offset and size int sub_div[3]; sub_div[0] = (global_box_size[0] / local_box_size[0]); sub_div[1] = (global_box_size[1] / local_box_size[1]); sub_div[2] = (global_box_size[2] / local_box_size[2]); local_box_offset[2] = (rank / (sub_div[0] * sub_div[1])) * local_box_size[2]; slice = rank % (sub_div[0] * sub_div[1]); local_box_offset[1] = (slice / sub_div[0]) * local_box_size[1]; local_box_offset[0] = (slice % sub_div[0]) * local_box_size[0]; unsigned int rank_x = 0, rank_y = 0, rank_z = 0, rank_slice; if (is_rank_z_ordering == 0) { rank_z = rank / (sub_div[0] * sub_div[1]); rank_slice = rank % (sub_div[0] * sub_div[1]); rank_y = (rank_slice / sub_div[0]); rank_x = (rank_slice % sub_div[0]); } create_patches(); PIDX_file file; // IDX file descriptor PIDX_variable* variable; // variable descriptor variable = malloc(sizeof(*variable) * variable_count); memset(variable, 0, sizeof(*variable) * variable_count); PIDX_point global_bounding_box, **local_offset_point, **local_box_count_point; local_offset_point = malloc(sizeof(PIDX_point*) * variable_count); local_box_count_point = malloc(sizeof(PIDX_point*) * variable_count); for(var = 0; var < variable_count; var++) { local_offset_point[var] = malloc(sizeof(PIDX_point) * patch_count); local_box_count_point[var] = malloc(sizeof(PIDX_point) * patch_count); for(p = 0 ; p < patch_count ; p++) { PIDX_set_point_5D(local_offset_point[var][p], (int64_t)local_patch_offset[var][p][0], (int64_t)local_patch_offset[var][p][1], (int64_t)local_patch_offset[var][p][2], 0, 0); PIDX_set_point_5D(local_box_count_point[var][p], (int64_t)local_patch_size[var][p][0], (int64_t)local_patch_size[var][p][1], (int64_t)local_patch_size[var][p][2], 1, 1); } } PIDX_set_point_5D(global_bounding_box, (int64_t)global_box_size[0], (int64_t)global_box_size[1], (int64_t)global_box_size[2], 1, 1); PIDX_access access; PIDX_create_access(&access); #if PIDX_HAVE_MPI PIDX_set_mpi_access(access, MPI_COMM_WORLD); PIDX_set_idx_count(access, idx_count[0], idx_count[1], idx_count[2]); PIDX_set_process_extent(access, sub_div[0], sub_div[1], sub_div[2]); PIDX_set_process_rank_decomposition(access, rank_x, rank_y, rank_z); #endif for (ts = 0; ts < time_step_count; ts++) { PIDX_file_create(output_file_name, PIDX_MODE_CREATE, access, &file); PIDX_set_dims(file, global_bounding_box); PIDX_set_current_time_step(file, ts); PIDX_set_variable_count(file, variable_count); PIDX_set_resolution(file, hz_from_to[0], hz_from_to[1]); PIDX_set_block_size(file, bits_per_block); PIDX_set_block_count(file, blocks_per_file); // PIDX set restructuring box size PIDX_set_restructuring_box(file, restructured_box_size); // PIDX compression related calls if (compression_type == 0) PIDX_set_compression_type(file, PIDX_NO_COMPRESSION); if (compression_type == 1) PIDX_set_compression_type(file, PIDX_CHUNKING_ONLY); if (compression_type == 2) { PIDX_set_compression_type(file, PIDX_CHUNKING_ZFP); PIDX_set_lossy_compression_bit_rate(file, compression_bit_rate); } PIDX_debug_rst(file, debug_rst_hz[0]); PIDX_debug_hz(file, debug_rst_hz[1]); PIDX_dump_agg_info(file, dump_agg_io[0]); PIDX_dump_io_info(file, dump_agg_io[1]); if (perform_phases[0] == 0) PIDX_debug_disable_restructuring(file); if (perform_phases[1] == 0) PIDX_debug_disable_chunking(file); if (perform_phases[2] == 0) PIDX_debug_disable_hz(file); if (perform_phases[3] == 0) PIDX_debug_disable_compression(file); if (perform_phases[4] == 0) PIDX_debug_disable_agg(file); if (perform_phases[5] == 0) PIDX_debug_disable_io(file); if (rst_agg[0] == 0) PIDX_disable_rst(file); if (rst_agg[1] == 0) PIDX_disable_agg(file); for (var = 0; var < variable_count; var++) { char variable_name[512]; char data_type[512]; sprintf(variable_name, "variable_%d", var); sprintf(data_type, "%d*float64", values_per_sample[var]); ret = PIDX_variable_create(variable_name, sizeof(double) * 8, data_type, &variable[var]); if (ret != PIDX_success) report_error("PIDX_variable_create", __FILE__, __LINE__); for (p = 0 ; p < patch_count ; p++) { ret = PIDX_variable_write_data_layout(variable[var], local_offset_point[var][p], local_box_count_point[var][p], NULL, PIDX_row_major); if (ret != PIDX_success) report_error("PIDX_variable_data_layout", __FILE__, __LINE__); } ret = PIDX_append_and_write_variable(file, variable[var]); if (ret != PIDX_success) report_error("PIDX_append_and_write_variable", __FILE__, __LINE__); ret = PIDX_flush(file); } ret = PIDX_close(file); if (ret != PIDX_success) report_error("PIDX_close", __FILE__, __LINE__); } PIDX_close_access(access); for(var = 0; var < variable_count; var++) { free(local_offset_point[var]); free(local_box_count_point[var]); } free(local_offset_point); free(local_box_count_point); free(variable); variable = 0; free(values_per_sample); values_per_sample = 0; destroy_patches(); #endif #if PIDX_HAVE_MPI MPI_Finalize(); #endif return 0; }
int test_multi_patch_writer(struct Args args, int rank, int nprocs) { #if PIDX_HAVE_MPI int i = 0, j = 0, k = 0, u = 0, v = 0, p = 0, var, d; int spv = 0; int ts; int slice; int sub_div[5], offset_local[5]; PIDX_file file; // IDX file descriptor const char *output_file; // IDX File Name const int bits_per_block = 15; // Total number of samples in each block = 2 ^ bits_per_block const int blocks_per_file = 32; // Total number of blocks per file PIDX_variable *variable; // variable descriptor int *values_per_sample; // values per variable (example, scalar=1, vector=3) int *var_patch_count; // Number of patches a variable has (data blocks per variable that needs to be written to). int ***var_count; // Local extents of the variables in each process int ***var_offset; // Local counts of the variables in each process double ***var_double_scalar_data; //The command line arguments are shared by all processes MPI_Bcast(args.extents, 5, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(args.count_local, 5, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(&args.time_step, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(&args.variable_count, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(&args.output_file_template, 512, MPI_CHAR, 0, MPI_COMM_WORLD); variable = malloc(sizeof (PIDX_variable) * args.variable_count); // variable pointers memset(variable, 0, sizeof (PIDX_variable) * args.variable_count); values_per_sample = (int*) malloc(sizeof (int) * args.variable_count); var_patch_count = (int*) malloc(sizeof (int) * args.variable_count); for (var = 0; var < args.variable_count; var++) { values_per_sample[var] = 1; /* if(var % 4 == 3) var_patch_count[var] = 1; else if(var % 4 == 2) var_patch_count[var] = 2; else if(var % 4 == 1) var_patch_count[var] = 4; else if(var % 4 == 0) */ var_patch_count[var] = 8; } // Creating the filename args.output_file_name = (char*) malloc(sizeof (char) * 512); sprintf(args.output_file_name, "%s%s", args.output_file_template, ".idx"); // Calculating every process's offset and count sub_div[0] = (args.extents[0] / args.count_local[0]); sub_div[1] = (args.extents[1] / args.count_local[1]); sub_div[2] = (args.extents[2] / args.count_local[2]); offset_local[2] = (rank / (sub_div[0] * sub_div[1])) * args.count_local[2]; slice = rank % (sub_div[0] * sub_div[1]); offset_local[1] = (slice / sub_div[0]) * args.count_local[1]; offset_local[0] = (slice % sub_div[0]) * args.count_local[0]; offset_local[3] = 0; offset_local[4] = 0; args.count_local[3] = 1; args.count_local[4] = 1; var_count = malloc(sizeof(int**) * args.variable_count); var_offset = malloc(sizeof(int**) * args.variable_count); for(var = 0; var < args.variable_count; var++) { var_count[var] = malloc(sizeof(int*) * var_patch_count[var]); var_offset[var] = malloc(sizeof(int*) * var_patch_count[var]); for(i = 0; i < var_patch_count[var] ; i++) { var_count[var][i] = malloc(sizeof(int) * PIDX_MAX_DIMENSIONS); var_offset[var][i] = malloc(sizeof(int) * PIDX_MAX_DIMENSIONS); } } for(var = 0; var < args.variable_count; var++) { /* if(var % 4 == 3) // One patch for this variable { for(d = 0; d < PIDX_MAX_DIMENSIONS; d++) { var_count[var][0][d] = args.count_local[d]; var_offset[var][0][d] = offset_local[d]; } } else if(var % 4 == 2) // two patches for this variable { for(d = 0; d < PIDX_MAX_DIMENSIONS; d++) { var_count[var][0][d] = args.count_local[d]; var_offset[var][0][d] = offset_local[d]; var_count[var][1][d] = args.count_local[d]; var_offset[var][1][d] = offset_local[d]; } var_count[var][0][0] = args.count_local[0]/2; if(args.count_local[0] % 2 == 0) var_count[var][1][0] = args.count_local[0]/2; else var_count[var][1][0] = args.count_local[0]/2 + 1; var_offset[var][1][0] = offset_local[0] + args.count_local[0]/2; } else if(var % 4 == 1) { for(i = 0; i < var_patch_count[var]; i++) { for(d = 0; d < PIDX_MAX_DIMENSIONS; d++) { var_count[var][i][d] = args.count_local[d]; var_offset[var][i][d] = offset_local[d]; } } var_count[var][0][0] = args.count_local[0]/2; var_count[var][0][1] = args.count_local[1]/2; var_count[var][1][1] = args.count_local[1]/2; if(args.count_local[0] % 2 == 0) { var_count[var][1][0] = args.count_local[0]/2; var_count[var][3][0] = args.count_local[0]/2; var_offset[var][1][0] = var_offset[var][0][0] + args.count_local[0]/2; var_offset[var][3][0] = var_offset[var][0][0] + args.count_local[0]/2; } else { var_count[var][1][0] = args.count_local[0]/2 + 1; var_count[var][3][0] = args.count_local[0]/2 + 1; var_offset[var][1][0] = var_offset[var][0][0] + args.count_local[0]/2; var_offset[var][3][0] = var_offset[var][0][0] + args.count_local[0]/2; } var_count[var][2][0] = args.count_local[0]/2; if(args.count_local[1] % 2 == 0) { var_count[var][2][1] = args.count_local[1]/2; var_count[var][3][1] = args.count_local[1]/2; var_offset[var][2][1] = var_offset[var][0][1] + args.count_local[1]/2; var_offset[var][3][1] = var_offset[var][0][1] + args.count_local[1]/2; } else { var_count[var][2][1] = args.count_local[1]/2 + 1; var_count[var][3][1] = args.count_local[1]/2 + 1; var_offset[var][2][1] = var_offset[var][0][1] + args.count_local[1]/2; var_offset[var][3][1] = var_offset[var][0][1] + args.count_local[1]/2; } } */ //else if(var % 4 == 0) //{ for(i = 0; i < var_patch_count[var]; i++) { for(d = 0; d < PIDX_MAX_DIMENSIONS; d++) { var_count[var][i][d] = args.count_local[d]; var_offset[var][i][d] = offset_local[d]; } } var_count[var][0][0] = args.count_local[0]/2; var_count[var][0][1] = args.count_local[1]/2; var_count[var][4][0] = args.count_local[0]/2; var_count[var][4][1] = args.count_local[1]/2; var_count[var][1][1] = args.count_local[1]/2; var_count[var][5][1] = args.count_local[1]/2; if(args.count_local[0] % 2 == 0) { var_count[var][1][0] = args.count_local[0]/2; var_count[var][3][0] = args.count_local[0]/2; var_offset[var][1][0] = var_offset[var][0][0] + args.count_local[0]/2; var_offset[var][3][0] = var_offset[var][0][0] + args.count_local[0]/2; var_count[var][5][0] = args.count_local[0]/2; var_count[var][7][0] = args.count_local[0]/2; var_offset[var][5][0] = var_offset[var][0][0] + args.count_local[0]/2; var_offset[var][7][0] = var_offset[var][0][0] + args.count_local[0]/2; } else { var_count[var][1][0] = args.count_local[0]/2 + 1; var_count[var][3][0] = args.count_local[0]/2 + 1; var_offset[var][1][0] = var_offset[var][0][0] + args.count_local[0]/2; var_offset[var][3][0] = var_offset[var][0][0] + args.count_local[0]/2; var_count[var][5][0] = args.count_local[0]/2 + 1; var_count[var][7][0] = args.count_local[0]/2 + 1; var_offset[var][5][0] = var_offset[var][0][0] + args.count_local[0]/2; var_offset[var][7][0] = var_offset[var][0][0] + args.count_local[0]/2; } var_count[var][2][0] = args.count_local[0]/2; var_count[var][6][0] = args.count_local[0]/2; if(args.count_local[1] % 2 == 0) { var_count[var][2][1] = args.count_local[1]/2; var_count[var][3][1] = args.count_local[1]/2; var_offset[var][2][1] = var_offset[var][0][1] + args.count_local[1]/2; var_offset[var][3][1] = var_offset[var][0][1] + args.count_local[1]/2; var_count[var][6][1] = args.count_local[1]/2; var_count[var][7][1] = args.count_local[1]/2; var_offset[var][6][1] = var_offset[var][0][1] + args.count_local[1]/2; var_offset[var][7][1] = var_offset[var][0][1] + args.count_local[1]/2; } else { var_count[var][2][1] = args.count_local[1]/2 + 1; var_count[var][3][1] = args.count_local[1]/2 + 1; var_offset[var][2][1] = var_offset[var][0][1] + args.count_local[1]/2; var_offset[var][3][1] = var_offset[var][0][1] + args.count_local[1]/2; var_count[var][6][1] = args.count_local[1]/2 + 1; var_count[var][7][1] = args.count_local[1]/2 + 1; var_offset[var][6][1] = var_offset[var][0][1] + args.count_local[1]/2; var_offset[var][7][1] = var_offset[var][0][1] + args.count_local[1]/2; } var_count[var][0][2] = args.count_local[2]/2; var_count[var][1][2] = args.count_local[2]/2; var_count[var][2][2] = args.count_local[2]/2; var_count[var][3][2] = args.count_local[2]/2; if(args.count_local[1] % 2 == 0) { var_count[var][4][2] = args.count_local[2]/2; var_count[var][5][2] = args.count_local[2]/2; var_count[var][6][2] = args.count_local[2]/2; var_count[var][7][2] = args.count_local[2]/2; var_offset[var][4][2] = var_offset[var][0][2] + args.count_local[2]/2; var_offset[var][5][2] = var_offset[var][1][2] + args.count_local[2]/2; var_offset[var][6][2] = var_offset[var][2][2] + args.count_local[2]/2; var_offset[var][7][2] = var_offset[var][3][2] + args.count_local[2]/2; } else { var_count[var][4][2] = args.count_local[2]/2 + 1; var_count[var][5][2] = args.count_local[2]/2 + 1; var_count[var][6][2] = args.count_local[2]/2 + 1; var_count[var][7][2] = args.count_local[2]/2 + 1; var_offset[var][4][2] = var_offset[var][0][2] + args.count_local[2]/2; var_offset[var][5][2] = var_offset[var][1][2] + args.count_local[2]/2; var_offset[var][6][2] = var_offset[var][2][2] + args.count_local[2]/2; var_offset[var][7][2] = var_offset[var][3][2] + args.count_local[2]/2; } //} } output_file = args.output_file_name; PIDX_point global_bounding_box, **local_offset_point, **local_box_count_point; local_offset_point = malloc(sizeof(PIDX_point*) * args.variable_count); local_box_count_point = malloc(sizeof(PIDX_point*) * args.variable_count); for(var = 0; var < args.variable_count; var++) { local_offset_point[var] = malloc(sizeof(PIDX_point) * var_patch_count[var]); local_box_count_point[var] = malloc(sizeof(PIDX_point) * var_patch_count[var]); for(p = 0 ; p < var_patch_count[var] ; p++) { PIDX_set_point_5D(local_offset_point[var][p], (int64_t)var_offset[var][p][0], (int64_t)var_offset[var][p][1], (int64_t)var_offset[var][p][2], 0, 0); PIDX_set_point_5D(local_box_count_point[var][p], (int64_t)var_count[var][p][0], (int64_t)var_count[var][p][1], (int64_t)var_count[var][p][2], 1, 1); } } PIDX_set_point_5D(global_bounding_box, (int64_t)args.extents[0], (int64_t)args.extents[1], (int64_t)args.extents[2], 1, 1); for (ts = 0; ts < args.time_step; ts++) { PIDX_access access; PIDX_create_access(&access); #if PIDX_HAVE_MPI PIDX_set_mpi_access(access, 1, 1, 1, MPI_COMM_WORLD); #endif PIDX_file_create(output_file, PIDX_file_trunc, access, &file); PIDX_set_dims(file, global_bounding_box); PIDX_set_current_time_step(file, ts); PIDX_set_block_size(file, bits_per_block); PIDX_set_block_count(file, blocks_per_file); PIDX_set_variable_count(file, args.variable_count); #if 0 var_double_scalar_data = malloc(sizeof(double**) * args.variable_count); for (var = 0; var < args.variable_count; var++) { var_double_scalar_data[var] = malloc(sizeof(double*) * var_patch_count[var]); for(p = 0 ; p < var_patch_count[var] ; p++) { var_double_scalar_data[var][p] = malloc(sizeof (double) * var_count[var][p][0] * var_count[var][p][1] * var_count[var][p][2] * var_count[var][p][3] * var_count[var][p][4] * values_per_sample[var]); for (v = 0; v < var_count[var][p][4]; v++) for (u = 0; u < var_count[var][p][3]; u++) for (k = 0; k < var_count[var][p][2]; k++) for (j = 0; j < var_count[var][p][1]; j++) for (i = 0; i < var_count[var][p][0]; i++) { int64_t index = (int64_t) (var_count[var][p][0] * var_count[var][p][1] * var_count[var][p][2] * var_count[var][p][3] * v) + (var_count[var][p][0] * var_count[var][p][1] * var_count[var][p][2] * u) + (var_count[var][p][0] * var_count[var][p][1] * k) + (var_count[var][p][0] * j) + i; for (spv = 0; spv < values_per_sample[var]; spv++) var_double_scalar_data[var][p][index * values_per_sample[var] + spv] = (100 + ((args.extents[0] * args.extents[1] * args.extents[2] * args.extents[3] * (var_offset[var][p][4] + v)) + (args.extents[0] * args.extents[1] * args.extents[2] * (var_offset[var][p][3] + u)) + (args.extents[0] * args.extents[1]*(var_offset[var][p][2] + k))+(args.extents[0]*(var_offset[var][p][1] + j)) + (var_offset[var][p][0] + i))); } } } char variable_name[512]; char data_type[512]; for(var = 0; var < args.variable_count; var++) { sprintf(variable_name, "variable_%d", var); sprintf(data_type, "%d*float64", values_per_sample[var]); PIDX_variable_create(file, variable_name, values_per_sample[var] * sizeof(double) * 8, data_type, &variable[var]); for(p = 0 ; p < var_patch_count[var] ; p++) { if (rank == 0) printf("Writing patch %d\n", p); PIDX_append_and_write_variable(variable[var], local_offset_point[var][p], local_box_count_point[var][p], var_double_scalar_data[var][p], PIDX_row_major); } } PIDX_close(file); PIDX_close_access(access); for(var = 0; var < args.variable_count; var++) { for(p = 0; p < var_patch_count[0]; p++) { free(var_double_scalar_data[var][p]); var_double_scalar_data[var][p] = 0; } free(var_double_scalar_data[var]); var_double_scalar_data[var] = 0; } #endif #if 0 var_double_scalar_data = malloc(sizeof(double**) * args.variable_count); for (var = 0; var < args.variable_count; var++) { var_double_scalar_data[var] = malloc(sizeof(double*) * var_patch_count[var]); for(p = 0 ; p < var_patch_count[var] ; p++) { var_double_scalar_data[var][p] = malloc(sizeof (double) * var_count[var][p][0] * var_count[var][p][1] * var_count[var][p][2] * var_count[var][p][3] * var_count[var][p][4] * values_per_sample[var]); for (v = 0; v < var_count[var][p][4]; v++) for (u = 0; u < var_count[var][p][3]; u++) for (k = 0; k < var_count[var][p][2]; k++) for (j = 0; j < var_count[var][p][1]; j++) for (i = 0; i < var_count[var][p][0]; i++) { int64_t index = (int64_t) (var_count[var][p][0] * var_count[var][p][1] * var_count[var][p][2] * var_count[var][p][3] * v) + (var_count[var][p][0] * var_count[var][p][1] * var_count[var][p][2] * u) + (var_count[var][p][0] * var_count[var][p][1] * k) + (var_count[var][p][0] * j) + i; for (spv = 0; spv < values_per_sample[var]; spv++) var_double_scalar_data[var][p][index * values_per_sample[var] + spv] = (100 + ((args.extents[0] * args.extents[1] * args.extents[2] * args.extents[3] * (var_offset[var][p][4] + v)) + (args.extents[0] * args.extents[1] * args.extents[2] * (var_offset[var][p][3] + u)) + (args.extents[0] * args.extents[1]*(var_offset[var][p][2] + k))+(args.extents[0]*(var_offset[var][p][1] + j)) + (var_offset[var][p][0] + i))); } } char variable_name[512]; char data_type[512]; sprintf(variable_name, "variable_%d", var); sprintf(data_type, "%d*float64", values_per_sample[var]); PIDX_variable_create(file, variable_name, values_per_sample[var] * sizeof(double) * 8, data_type, &variable[var]); for(p = 0 ; p < var_patch_count[var] ; p++) { if (rank == 0) printf("Writing patch %d\n", p); PIDX_append_and_write_variable(variable[var], local_offset_point[var][p], local_box_count_point[var][p], var_double_scalar_data[var][p], PIDX_row_major); } PIDX_flush(file); for(p = 0; p < var_patch_count[var]; p++) { free(var_double_scalar_data[var][p]); var_double_scalar_data[var][p] = 0; } free(var_double_scalar_data[var]); var_double_scalar_data[var] = 0; } PIDX_close(file); PIDX_close_access(access); #endif #if 1 var_double_scalar_data = malloc(sizeof(double**) * args.variable_count); for (var = 0; var < args.variable_count; var++) { var_double_scalar_data[var] = malloc(sizeof(double*) * var_patch_count[var]); for(p = 0 ; p < var_patch_count[var] ; p++) { var_double_scalar_data[var][p] = malloc(sizeof (double) * var_count[var][p][0] * var_count[var][p][1] * var_count[var][p][2] * var_count[var][p][3] * var_count[var][p][4] * values_per_sample[var]); for (v = 0; v < var_count[var][p][4]; v++) for (u = 0; u < var_count[var][p][3]; u++) for (k = 0; k < var_count[var][p][2]; k++) for (j = 0; j < var_count[var][p][1]; j++) for (i = 0; i < var_count[var][p][0]; i++) { int64_t index = (int64_t) (var_count[var][p][0] * var_count[var][p][1] * var_count[var][p][2] * var_count[var][p][3] * v) + (var_count[var][p][0] * var_count[var][p][1] * var_count[var][p][2] * u) + (var_count[var][p][0] * var_count[var][p][1] * k) + (var_count[var][p][0] * j) + i; for (spv = 0; spv < values_per_sample[var]; spv++) var_double_scalar_data[var][p][index * values_per_sample[var] + spv] = (100 + ((args.extents[0] * args.extents[1] * args.extents[2] * args.extents[3] * (var_offset[var][p][4] + v)) + (args.extents[0] * args.extents[1] * args.extents[2] * (var_offset[var][p][3] + u)) + (args.extents[0] * args.extents[1]*(var_offset[var][p][2] + k))+(args.extents[0]*(var_offset[var][p][1] + j)) + (var_offset[var][p][0] + i))); } } } for (var = 0; var < args.variable_count; var++) { char variable_name[512]; char data_type[512]; sprintf(variable_name, "variable_%d", var); sprintf(data_type, "%d*float64", values_per_sample[var]); PIDX_variable_create(file, variable_name, values_per_sample[var] * sizeof(double) * 8, data_type, &variable[var]); for(p = 0 ; p < var_patch_count[var] ; p++) { //if (rank == 0) //printf("Writing patch %d\n", p); PIDX_append_and_write_variable(variable[var], local_offset_point[var][p], local_box_count_point[var][p], var_double_scalar_data[var][p], PIDX_row_major); } if(var % 2 == 0) PIDX_flush(file); //for(p = 0; p < var_patch_count[var]; p++) //{ //free(var_double_scalar_data[var][p]); //var_double_scalar_data[var][p] = 0; //} //free(var_double_scalar_data[var]); //var_double_scalar_data[var] = 0; } PIDX_close(file); PIDX_close_access(access); #endif free(var_double_scalar_data); var_double_scalar_data = 0; } for(var = 0; var < args.variable_count; var++) { free(local_offset_point[var]); free(local_box_count_point[var]); } free(local_offset_point); free(local_box_count_point); free(args.output_file_name); free(variable); variable = 0; free(values_per_sample); values_per_sample = 0; #endif return 0; }
int main(int argc, char **argv) { int sub_div[3], local_offset[3], count_local[3]; int t = 0, time_count = 0; sub_div[0] = 0; sub_div[1] = 0; sub_div[2] = 0; local_offset[0] = 0; local_offset[1] = 0; local_offset[2] = 0; count_local[0] = 0; count_local[1] = 0; count_local[2] = 0; #if HDF_IO hid_t file_id; hid_t plist_id; hid_t group_id; hid_t dataset_id; hid_t file_dataspace; hid_t mem_dataspace; #endif #if PIDX_IO PIDX_file file; PIDX_access access; PIDX_variable variable; #endif const int bits_per_block = 15; const int blocks_per_file = 512; int nprocs=1, rank=0, slice; MPI_Comm comm = MPI_COMM_WORLD; #if HDF_IO MPI_Info info = MPI_INFO_NULL; #endif MPI_Init(&argc, &argv); MPI_Comm_size(comm, &nprocs); MPI_Comm_rank(comm, &rank); output_file_name = (char*) malloc(sizeof (char) * 1024); sprintf(output_file_name, "%s%s", "/scratch/project/visus/datasets/flame", ".idx"); if (nprocs == 320) /// 4 x 5 x 16 { count_local[0] = 256; count_local[1] = 188; count_local[2] = 192; } else if (nprocs == 640) /// 4 x 5 x 32 { count_local[0] = 256; count_local[1] = 188; count_local[2] = 96; } else if (nprocs == 1280) /// 4 x 10 x 32 { count_local[0] = 256; count_local[1] = 94; count_local[2] = 96; } else if (nprocs == 2560) /// 8 x 10 x 32 { count_local[0] = 128; count_local[1] = 94; count_local[2] = 96; } sub_div[0] = (1024 / count_local[0]); sub_div[1] = (940 / count_local[1]); sub_div[2] = (3072 / count_local[2]); local_offset[2] = (rank / (sub_div[0] * sub_div[1])) * count_local[2]; slice = rank % (sub_div[0] * sub_div[1]); local_offset[1] = (slice / sub_div[0]) * count_local[1]; local_offset[0] = (slice % sub_div[0]) * count_local[0]; #if PIDX_IO PIDX_point global_bounding_box, local_offset_point, local_box_count_point; PIDX_set_point_5D(global_bounding_box, (int64_t)1024, (int64_t)940, (int64_t)3072, 1, 1); PIDX_set_point_5D(local_offset_point, (int64_t)local_offset[0], (int64_t)local_offset[1], (int64_t)local_offset[2], 0, 0); PIDX_set_point_5D(local_box_count_point, (int64_t)count_local[0], (int64_t)count_local[1], (int64_t)count_local[2], 1, 1); #endif #if HDF_IO hsize_t count[3]; count[0] = count_local[0]; count[1] = count_local[1]; count[2] = count_local[2]; hsize_t offset[3]; offset[0] = local_offset[0]; offset[1] = local_offset[1]; offset[2] = local_offset[2]; #endif /// P U V W ZMIX #if HDF_IO plist_id = H5Pcreate(H5P_FILE_ACCESS); H5Pset_fapl_mpio(plist_id, comm, info); #endif FILE *fp; fp = fopen("list", "r"); char file_name[1][1024]; while (!feof(fp)) { if (fscanf(fp, "%s", file_name[time_count]) != 1) break; if(rank == 0) printf("%s\n", file_name[time_count]); time_count++; } fclose(fp); #if PIDX_IO PIDX_create_access(&access); PIDX_set_mpi_access(access, 1, 1, 1, comm); PIDX_set_process_extent(access, sub_div[0], sub_div[1], sub_div[2]); #endif if (rank == 0) printf("Number of timesteps = %d\n", time_count); for (t = 0; t < time_count; t++) { #if HDF_IO file_id = H5Fopen(file_name[t], H5F_ACC_RDONLY, plist_id); #endif #if PIDX_IO PIDX_file_create(output_file_name, PIDX_file_trunc, access, &file); PIDX_set_dims(file, global_bounding_box); PIDX_set_current_time_step(file, t); PIDX_set_block_size(file, bits_per_block); PIDX_set_aggregation_factor(file, 1); PIDX_set_block_count(file, blocks_per_file); PIDX_set_variable_count(file, 1); PIDX_debug_rst(file, 0); PIDX_debug_hz(file, 0); PIDX_dump_agg_info(file, 0); PIDX_enable_hz(file, 1); PIDX_enable_agg(file, 1); PIDX_enable_io(file, 1); #endif #if HDF_IO group_id = H5Gopen(file_id, "/data", H5P_DEFAULT); dataset_id = H5Dopen2(group_id, "P", H5P_DEFAULT); mem_dataspace = H5Screate_simple (3, count, NULL); file_dataspace = H5Dget_space (dataset_id); H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, offset, NULL, count, NULL); #endif buffer = malloc(sizeof(double) * count_local[0] * count_local[1] * count_local[2]); memset(buffer, 0, sizeof(double) * count_local[0] * count_local[1] * count_local[2]); #if HDF_IO H5Dread(dataset_id, H5T_NATIVE_DOUBLE, mem_dataspace, file_dataspace, H5P_DEFAULT, buffer); H5Sclose(mem_dataspace); H5Sclose(file_dataspace); H5Dclose(dataset_id); H5Gclose(group_id); H5Fclose(file_id); if (rank == 0) printf("Finished reading HDF files\n"); #else int64_t i1, j1, k1; for (k1 = 0; k1 < count_local[2]; k1++) for (j1 = 0; j1 < count_local[1]; j1++) for (i1 = 0; i1 < count_local[0]; i1++) { int64_t index = (int64_t) (count_local[0] * count_local[1] * k1) + (count_local[0] * j1) + i1; buffer[index] = (256 * rank) / nprocs; } #endif #if PIDX_IO PIDX_variable_create(file, "P", sizeof(double) * 8, "1*float64", &variable); PIDX_append_and_write_variable(variable, local_offset_point, local_box_count_point, buffer, PIDX_column_major); if (rank == 0) printf("Starting to Write IDX %s Time Step %d\n", output_file_name, t); PIDX_close(file); #endif free(buffer); buffer = 0; } ////////// #if HDF_IO H5Pclose(plist_id); #endif #if PIDX_IO PIDX_close_access(access); #endif free(output_file_name); output_file_name = 0; MPI_Finalize(); return 0; }