int main(int argc, char **argv) { int ret; int p, ts; int var; int slice = 0; int nprocs = 1, rank = 0; char output_file_name[512]; // MPI initialization #if PIDX_HAVE_MPI MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &nprocs); MPI_Comm_rank(MPI_COMM_WORLD, &rank); #endif if (rank == 0) { ret = parse_args(argc, argv); if (ret < 0) { usage(); #if PIDX_HAVE_MPI MPI_Abort(MPI_COMM_WORLD, -1); #else exit(-1); #endif } // check if the num procs is appropriate int num_bricks = (global_box_size[0] / local_box_size[0]) * (global_box_size[1] / local_box_size[1]) * (global_box_size[2] / local_box_size[2]); if(num_bricks != nprocs) { fprintf(stderr, "Error: number of sub-blocks (%d) doesn't match number of procs (%d)\n", num_bricks, nprocs); fprintf(stderr, "Incorrect distribution of data across processes i.e.\n(global_x / local_x) X (global_x / local_x) X (global_x / local_x) != nprocs\n(%d/%d) X (%d/%d) X (%d/%d) != %d\n", global_box_size[0], local_box_size[0], global_box_size[1], local_box_size[1], global_box_size[2], local_box_size[2], nprocs); #if PIDX_HAVE_MPI MPI_Abort(MPI_COMM_WORLD, -1); #else exit(-1); #endif } } #if 1 // The command line arguments are shared by all processes #if PIDX_HAVE_MPI MPI_Bcast(global_box_size, 3, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(local_box_size, 3, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(&time_step_count, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(&variable_count, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(&patch_count, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(&output_file_template, 512, MPI_CHAR, 0, MPI_COMM_WORLD); MPI_Bcast(&blocks_per_file, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(&bits_per_block, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(restructured_box_size, 5, MPI_LONG_LONG, 0, MPI_COMM_WORLD); MPI_Bcast(&compression_type, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(&compression_bit_rate, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(perform_phases, 6, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(debug_rst_hz, 2, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(dump_agg_io, 2, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(idx_count, 3, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(&aggregation_factor, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(&is_rank_z_ordering, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(hz_from_to, 2, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(rst_agg, 2, MPI_INT, 0, MPI_COMM_WORLD); #endif values_per_sample = malloc(sizeof(*values_per_sample) * variable_count); memset(values_per_sample, 0, sizeof(*values_per_sample) * variable_count); // Creating the filename sprintf(output_file_name, "%s%s", output_file_template,".idx"); // Calculating every process data's offset and size int sub_div[3]; sub_div[0] = (global_box_size[0] / local_box_size[0]); sub_div[1] = (global_box_size[1] / local_box_size[1]); sub_div[2] = (global_box_size[2] / local_box_size[2]); local_box_offset[2] = (rank / (sub_div[0] * sub_div[1])) * local_box_size[2]; slice = rank % (sub_div[0] * sub_div[1]); local_box_offset[1] = (slice / sub_div[0]) * local_box_size[1]; local_box_offset[0] = (slice % sub_div[0]) * local_box_size[0]; unsigned int rank_x = 0, rank_y = 0, rank_z = 0, rank_slice; if (is_rank_z_ordering == 0) { rank_z = rank / (sub_div[0] * sub_div[1]); rank_slice = rank % (sub_div[0] * sub_div[1]); rank_y = (rank_slice / sub_div[0]); rank_x = (rank_slice % sub_div[0]); } create_patches(); PIDX_file file; // IDX file descriptor PIDX_variable* variable; // variable descriptor variable = malloc(sizeof(*variable) * variable_count); memset(variable, 0, sizeof(*variable) * variable_count); PIDX_point global_bounding_box, **local_offset_point, **local_box_count_point; local_offset_point = malloc(sizeof(PIDX_point*) * variable_count); local_box_count_point = malloc(sizeof(PIDX_point*) * variable_count); for(var = 0; var < variable_count; var++) { local_offset_point[var] = malloc(sizeof(PIDX_point) * patch_count); local_box_count_point[var] = malloc(sizeof(PIDX_point) * patch_count); for(p = 0 ; p < patch_count ; p++) { PIDX_set_point_5D(local_offset_point[var][p], (int64_t)local_patch_offset[var][p][0], (int64_t)local_patch_offset[var][p][1], (int64_t)local_patch_offset[var][p][2], 0, 0); PIDX_set_point_5D(local_box_count_point[var][p], (int64_t)local_patch_size[var][p][0], (int64_t)local_patch_size[var][p][1], (int64_t)local_patch_size[var][p][2], 1, 1); } } PIDX_set_point_5D(global_bounding_box, (int64_t)global_box_size[0], (int64_t)global_box_size[1], (int64_t)global_box_size[2], 1, 1); PIDX_access access; PIDX_create_access(&access); #if PIDX_HAVE_MPI PIDX_set_mpi_access(access, MPI_COMM_WORLD); PIDX_set_idx_count(access, idx_count[0], idx_count[1], idx_count[2]); PIDX_set_process_extent(access, sub_div[0], sub_div[1], sub_div[2]); PIDX_set_process_rank_decomposition(access, rank_x, rank_y, rank_z); #endif for (ts = 0; ts < time_step_count; ts++) { PIDX_file_create(output_file_name, PIDX_MODE_CREATE, access, &file); PIDX_set_dims(file, global_bounding_box); PIDX_set_current_time_step(file, ts); PIDX_set_variable_count(file, variable_count); PIDX_set_resolution(file, hz_from_to[0], hz_from_to[1]); PIDX_set_block_size(file, bits_per_block); PIDX_set_block_count(file, blocks_per_file); // PIDX set restructuring box size PIDX_set_restructuring_box(file, restructured_box_size); // PIDX compression related calls if (compression_type == 0) PIDX_set_compression_type(file, PIDX_NO_COMPRESSION); if (compression_type == 1) PIDX_set_compression_type(file, PIDX_CHUNKING_ONLY); if (compression_type == 2) { PIDX_set_compression_type(file, PIDX_CHUNKING_ZFP); PIDX_set_lossy_compression_bit_rate(file, compression_bit_rate); } PIDX_debug_rst(file, debug_rst_hz[0]); PIDX_debug_hz(file, debug_rst_hz[1]); PIDX_dump_agg_info(file, dump_agg_io[0]); PIDX_dump_io_info(file, dump_agg_io[1]); if (perform_phases[0] == 0) PIDX_debug_disable_restructuring(file); if (perform_phases[1] == 0) PIDX_debug_disable_chunking(file); if (perform_phases[2] == 0) PIDX_debug_disable_hz(file); if (perform_phases[3] == 0) PIDX_debug_disable_compression(file); if (perform_phases[4] == 0) PIDX_debug_disable_agg(file); if (perform_phases[5] == 0) PIDX_debug_disable_io(file); if (rst_agg[0] == 0) PIDX_disable_rst(file); if (rst_agg[1] == 0) PIDX_disable_agg(file); for (var = 0; var < variable_count; var++) { char variable_name[512]; char data_type[512]; sprintf(variable_name, "variable_%d", var); sprintf(data_type, "%d*float64", values_per_sample[var]); ret = PIDX_variable_create(variable_name, sizeof(double) * 8, data_type, &variable[var]); if (ret != PIDX_success) report_error("PIDX_variable_create", __FILE__, __LINE__); for (p = 0 ; p < patch_count ; p++) { ret = PIDX_variable_write_data_layout(variable[var], local_offset_point[var][p], local_box_count_point[var][p], NULL, PIDX_row_major); if (ret != PIDX_success) report_error("PIDX_variable_data_layout", __FILE__, __LINE__); } ret = PIDX_append_and_write_variable(file, variable[var]); if (ret != PIDX_success) report_error("PIDX_append_and_write_variable", __FILE__, __LINE__); ret = PIDX_flush(file); } ret = PIDX_close(file); if (ret != PIDX_success) report_error("PIDX_close", __FILE__, __LINE__); } PIDX_close_access(access); for(var = 0; var < variable_count; var++) { free(local_offset_point[var]); free(local_box_count_point[var]); } free(local_offset_point); free(local_box_count_point); free(variable); variable = 0; free(values_per_sample); values_per_sample = 0; destroy_patches(); #endif #if PIDX_HAVE_MPI MPI_Finalize(); #endif return 0; }
int main(int argc, char **argv) { init_mpi(argc, argv); parse_args(argc, argv); check_args(); calculate_per_process_offsets(); create_synthetic_simulation_data(); rank_0_print("Simulation Data Created\n"); int ret, var, ts; PIDX_file file; // IDX file descriptor PIDX_variable* variable; // variable descriptor variable = (PIDX_variable*)malloc(sizeof(*variable) * variable_count); memset(variable, 0, sizeof(*variable) * variable_count); PIDX_point global_size, local_offset, local_size; PIDX_set_point_5D(global_size, global_box_size[0], global_box_size[1], global_box_size[2], 1, 1); PIDX_set_point_5D(local_offset, local_box_offset[0], local_box_offset[1], local_box_offset[2], 0, 0); PIDX_set_point_5D(local_size, local_box_size[0], local_box_size[1], local_box_size[2], 1, 1); // Creating access PIDX_access access; PIDX_create_access(&access); #if PIDX_HAVE_MPI PIDX_set_mpi_access(access, MPI_COMM_WORLD); #endif for (ts = 0; ts < time_step_count; ts++) { // PIDX mandatory calls ret = PIDX_file_create(output_file_name, PIDX_MODE_CREATE, access, global_size, &file); if (ret != PIDX_success) terminate_with_error_msg("PIDX_file_create"); ret = PIDX_set_current_time_step(file, ts); if (ret != PIDX_success) terminate_with_error_msg("PIDX_set_current_time_step"); ret = PIDX_set_variable_count(file, variable_count); if (ret != PIDX_success) terminate_with_error_msg("PIDX_set_variable_count"); PIDX_disable_agg(file); PIDX_save_big_endian(file); //PIDX_dump_rst_info(file, 1); //PIDX_debug_rst(file, 1); //PIDX_debug_hz(file, 1); PIDX_point reg_patch_size; PIDX_set_point_5D(reg_patch_size, 128, 128, 128, 1, 1); PIDX_set_restructuring_box(file, reg_patch_size); //PIDX_GLOBAL_PARTITION_IDX_IO //PIDX_IDX_IO ret = PIDX_set_io_mode(file, PIDX_RAW_IO); if (ret != PIDX_success) terminate_with_error_msg("PIDX_set_variable_count"); ret = PIDX_set_partition_size(file, partition_size[0], partition_size[1], partition_size[2]); if (ret != PIDX_success) terminate_with_error_msg("PIDX_set_partition_size"); PIDX_set_block_count(file, 128); //ret = PIDX_set_aggregator_multiplier(file, aggregator_multiplier); //if (ret != PIDX_success) terminate_with_error_msg("PIDX_set_partition_size"); /* int io_type = PIDX_IDX_IO; switch (io_type) { case PIDX_GLOBAL_PARTITION_IDX_IO: PIDX_set_block_count(file,blocks_per_file); PIDX_set_block_size(file, 13); break; case PIDX_IDX_IO: PIDX_set_block_count(file,blocks_per_file); break; case PIDX_RAW_IO: PIDX_raw_io_pipe_length(file, 2); PIDX_point reg_patch_size; PIDX_set_point_5D(reg_patch_size, 128, 128, 128, 1, 1); PIDX_set_restructuring_box(file, reg_patch_size); break; } */ //ret = PIDX_debug_disable_agg(file); //if (ret != PIDX_success) terminate_with_error_msg("PIDX_debug_output"); //ret = PIDX_debug_disable_io(file); //if (ret != PIDX_success) terminate_with_error_msg("PIDX_debug_output"); //ret = PIDX_debug_disable_hz(file); //if (ret != PIDX_success) terminate_with_error_msg("PIDX_debug_output"); for (var = 0; var < variable_count; var++) { if (bpv[var] == 32) { ret = PIDX_variable_create(var_name[var], bpv[var], FLOAT32 , &variable[var]); if (ret != PIDX_success) terminate_with_error_msg("PIDX_variable_create"); } else if (bpv[var] == 192) { ret = PIDX_variable_create(var_name[var], bpv[var], FLOAT64_RGB , &variable[var]); if (ret != PIDX_success) terminate_with_error_msg("PIDX_variable_create"); } else if (bpv[var] == 64) { ret = PIDX_variable_create(var_name[var], bpv[var], FLOAT64 , &variable[var]); if (ret != PIDX_success) terminate_with_error_msg("PIDX_variable_create"); } ret = PIDX_variable_write_data_layout(variable[var], local_offset, local_size, data[var], PIDX_row_major); if (ret != PIDX_success) terminate_with_error_msg("PIDX_variable_data_layout"); ret = PIDX_append_and_write_variable(file, variable[var]); if (ret != PIDX_success) terminate_with_error_msg("PIDX_append_and_write_variable"); //PIDX_flush(file); } ret = PIDX_close(file); if (ret != PIDX_success) terminate_with_error_msg("PIDX_close"); } ret = PIDX_close_access(access); if (ret != PIDX_success) terminate_with_error_msg("PIDX_close_access"); free(variable); variable = 0; destroy_synthetic_simulation_data(); shutdown_mpi(); return 0; }