int main(int argc, char **argv)
{
int ret;
int p, ts;
int var;
int slice = 0;
int nprocs = 1, rank = 0;
char output_file_name[512];
// MPI initialization
#if PIDX_HAVE_MPI
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
#endif
if (rank == 0)
{
ret = parse_args(argc, argv);
if (ret < 0)
{
usage();
#if PIDX_HAVE_MPI
MPI_Abort(MPI_COMM_WORLD, -1);
#else
exit(-1);
#endif
}
// check if the num procs is appropriate
int num_bricks = (global_box_size[0] / local_box_size[0]) * (global_box_size[1] / local_box_size[1]) * (global_box_size[2] / local_box_size[2]);
if(num_bricks != nprocs)
{
fprintf(stderr, "Error: number of sub-blocks (%d) doesn't match number of procs (%d)\n", num_bricks, nprocs);
fprintf(stderr, "Incorrect distribution of data across processes i.e.\n(global_x / local_x) X (global_x / local_x) X (global_x / local_x) != nprocs\n(%d/%d) X (%d/%d) X (%d/%d) != %d\n", global_box_size[0], local_box_size[0], global_box_size[1], local_box_size[1], global_box_size[2], local_box_size[2], nprocs);
#if PIDX_HAVE_MPI
MPI_Abort(MPI_COMM_WORLD, -1);
#else
exit(-1);
#endif
}
}
#if 1
// The command line arguments are shared by all processes
#if PIDX_HAVE_MPI
MPI_Bcast(global_box_size, 3, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(local_box_size, 3, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(&time_step_count, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(&variable_count, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(&patch_count, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(&output_file_template, 512, MPI_CHAR, 0, MPI_COMM_WORLD);
MPI_Bcast(&blocks_per_file, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(&bits_per_block, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(restructured_box_size, 5, MPI_LONG_LONG, 0, MPI_COMM_WORLD);
MPI_Bcast(&compression_type, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(&compression_bit_rate, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(perform_phases, 6, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(debug_rst_hz, 2, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(dump_agg_io, 2, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(idx_count, 3, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(&aggregation_factor, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(&is_rank_z_ordering, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(hz_from_to, 2, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(rst_agg, 2, MPI_INT, 0, MPI_COMM_WORLD);
#endif
values_per_sample = malloc(sizeof(*values_per_sample) * variable_count);
memset(values_per_sample, 0, sizeof(*values_per_sample) * variable_count);
// Creating the filename
sprintf(output_file_name, "%s%s", output_file_template,".idx");
// Calculating every process data's offset and size
int sub_div[3];
sub_div[0] = (global_box_size[0] / local_box_size[0]);
sub_div[1] = (global_box_size[1] / local_box_size[1]);
sub_div[2] = (global_box_size[2] / local_box_size[2]);
local_box_offset[2] = (rank / (sub_div[0] * sub_div[1])) * local_box_size[2];
slice = rank % (sub_div[0] * sub_div[1]);
local_box_offset[1] = (slice / sub_div[0]) * local_box_size[1];
local_box_offset[0] = (slice % sub_div[0]) * local_box_size[0];
unsigned int rank_x = 0, rank_y = 0, rank_z = 0, rank_slice;
if (is_rank_z_ordering == 0)
{
rank_z = rank / (sub_div[0] * sub_div[1]);
rank_slice = rank % (sub_div[0] * sub_div[1]);
rank_y = (rank_slice / sub_div[0]);
rank_x = (rank_slice % sub_div[0]);
}
create_patches();
PIDX_file file; // IDX file descriptor
PIDX_variable* variable; // variable descriptor
variable = malloc(sizeof(*variable) * variable_count);
memset(variable, 0, sizeof(*variable) * variable_count);
PIDX_point global_bounding_box, **local_offset_point, **local_box_count_point;
local_offset_point = malloc(sizeof(PIDX_point*) * variable_count);
local_box_count_point = malloc(sizeof(PIDX_point*) * variable_count);
for(var = 0; var < variable_count; var++)
{
local_offset_point[var] = malloc(sizeof(PIDX_point) * patch_count);
local_box_count_point[var] = malloc(sizeof(PIDX_point) * patch_count);
for(p = 0 ; p < patch_count ; p++)
{
PIDX_set_point_5D(local_offset_point[var][p], (int64_t)local_patch_offset[var][p][0], (int64_t)local_patch_offset[var][p][1], (int64_t)local_patch_offset[var][p][2], 0, 0);
PIDX_set_point_5D(local_box_count_point[var][p], (int64_t)local_patch_size[var][p][0], (int64_t)local_patch_size[var][p][1], (int64_t)local_patch_size[var][p][2], 1, 1);
}
}
PIDX_set_point_5D(global_bounding_box, (int64_t)global_box_size[0], (int64_t)global_box_size[1], (int64_t)global_box_size[2], 1, 1);
PIDX_access access;
PIDX_create_access(&access);
#if PIDX_HAVE_MPI
PIDX_set_mpi_access(access, MPI_COMM_WORLD);
PIDX_set_idx_count(access, idx_count[0], idx_count[1], idx_count[2]);
PIDX_set_process_extent(access, sub_div[0], sub_div[1], sub_div[2]);
PIDX_set_process_rank_decomposition(access, rank_x, rank_y, rank_z);
#endif
for (ts = 0; ts < time_step_count; ts++)
{
PIDX_file_create(output_file_name, PIDX_MODE_CREATE, access, &file);
PIDX_set_dims(file, global_bounding_box);
PIDX_set_current_time_step(file, ts);
PIDX_set_variable_count(file, variable_count);
PIDX_set_resolution(file, hz_from_to[0], hz_from_to[1]);
PIDX_set_block_size(file, bits_per_block);
PIDX_set_block_count(file, blocks_per_file);
// PIDX set restructuring box size
PIDX_set_restructuring_box(file, restructured_box_size);
// PIDX compression related calls
if (compression_type == 0)
PIDX_set_compression_type(file, PIDX_NO_COMPRESSION);
if (compression_type == 1)
PIDX_set_compression_type(file, PIDX_CHUNKING_ONLY);
if (compression_type == 2)
{
PIDX_set_compression_type(file, PIDX_CHUNKING_ZFP);
PIDX_set_lossy_compression_bit_rate(file, compression_bit_rate);
}
PIDX_debug_rst(file, debug_rst_hz[0]);
PIDX_debug_hz(file, debug_rst_hz[1]);
PIDX_dump_agg_info(file, dump_agg_io[0]);
PIDX_dump_io_info(file, dump_agg_io[1]);
if (perform_phases[0] == 0)
PIDX_debug_disable_restructuring(file);
if (perform_phases[1] == 0)
PIDX_debug_disable_chunking(file);
if (perform_phases[2] == 0)
PIDX_debug_disable_hz(file);
if (perform_phases[3] == 0)
PIDX_debug_disable_compression(file);
if (perform_phases[4] == 0)
PIDX_debug_disable_agg(file);
if (perform_phases[5] == 0)
PIDX_debug_disable_io(file);
if (rst_agg[0] == 0)
PIDX_disable_rst(file);
if (rst_agg[1] == 0)
PIDX_disable_agg(file);
for (var = 0; var < variable_count; var++)
{
char variable_name[512];
char data_type[512];
sprintf(variable_name, "variable_%d", var);
sprintf(data_type, "%d*float64", values_per_sample[var]);
ret = PIDX_variable_create(variable_name, sizeof(double) * 8, data_type, &variable[var]);
if (ret != PIDX_success) report_error("PIDX_variable_create", __FILE__, __LINE__);
for (p = 0 ; p < patch_count ; p++)
{
ret = PIDX_variable_write_data_layout(variable[var], local_offset_point[var][p], local_box_count_point[var][p], NULL, PIDX_row_major);
if (ret != PIDX_success) report_error("PIDX_variable_data_layout", __FILE__, __LINE__);
}
ret = PIDX_append_and_write_variable(file, variable[var]);
if (ret != PIDX_success) report_error("PIDX_append_and_write_variable", __FILE__, __LINE__);
ret = PIDX_flush(file);
}
ret = PIDX_close(file);
if (ret != PIDX_success) report_error("PIDX_close", __FILE__, __LINE__);
}
PIDX_close_access(access);
for(var = 0; var < variable_count; var++)
{
free(local_offset_point[var]);
free(local_box_count_point[var]);
}
free(local_offset_point);
free(local_box_count_point);
free(variable);
variable = 0;
free(values_per_sample);
values_per_sample = 0;
destroy_patches();
#endif
#if PIDX_HAVE_MPI
MPI_Finalize();
#endif
return 0;
}
/// main
int main(int argc, char **argv)
{
init_mpi(argc, argv);
parse_args(argc, argv);
check_args();
calculate_per_process_offsets();
create_synthetic_simulation_data();
int var, p;
PIDX_point global_bounding_box, **local_offset_point, **local_box_count_point;
local_offset_point = malloc(sizeof(PIDX_point*) * variable_count);
local_box_count_point = malloc(sizeof(PIDX_point*) * variable_count);
for(var = 0; var < variable_count; var++)
{
local_offset_point[var] = malloc(sizeof(PIDX_point) * patch_count);
local_box_count_point[var] = malloc(sizeof(PIDX_point) * patch_count);
for(p = 0 ; p < patch_count ; p++)
{
PIDX_set_point_5D(local_offset_point[var][p], (int64_t)var_offset[var][p][0], (int64_t)var_offset[var][p][1], (int64_t)var_offset[var][p][2], 0, 0);
PIDX_set_point_5D(local_box_count_point[var][p], (int64_t)var_count[var][p][0], (int64_t)var_count[var][p][1], (int64_t)var_count[var][p][2], 1, 1);
}
}
PIDX_file file; // IDX file descriptor
PIDX_variable* variable; // variable descriptor
variable = malloc(sizeof(*variable) * variable_count);
memset(variable, 0, sizeof(*variable) * variable_count);
PIDX_set_point_5D(global_bounding_box, (int64_t)global_box_size[0], (int64_t)global_box_size[1], (int64_t)global_box_size[2], 1, 1);
PIDX_access access;
PIDX_create_access(&access);
#if PIDX_HAVE_MPI
PIDX_set_mpi_access(access, MPI_COMM_WORLD);
#endif
int ts;
for (ts = 0; ts < time_step_count; ts++)
{
PIDX_file_create(output_file_name, PIDX_MODE_CREATE, access, &file);
PIDX_set_dims(file, global_bounding_box);
PIDX_set_current_time_step(file, ts);
PIDX_set_variable_count(file, variable_count);
PIDX_debug_rst(file, 1);
PIDX_debug_hz(file, 1);
for (var = 0; var < variable_count; var++)
{
char variable_name[512];
sprintf(variable_name, "variable_%d", var);
PIDX_variable_create(variable_name, sizeof(double) * 8, FLOAT64, &variable[var]);
for (p = 0 ; p < patch_count ; p++)
PIDX_variable_write_data_layout(variable[var], local_offset_point[var][p], local_box_count_point[var][p], double_data[var][p], PIDX_row_major);
PIDX_append_and_write_variable(file, variable[var]/*, local_offset_point[var][p], local_box_count_point[var][p], double_data[var][p], PIDX_row_major*/);
}
PIDX_close(file);
}
PIDX_close_access(access);
destroy_synthetic_simulation_data();
for(var = 0; var < variable_count; var++)
{
free(local_offset_point[var]);
free(local_box_count_point[var]);
}
free(local_offset_point);
free(local_box_count_point);
free(variable);
variable = 0;
shutdown_mpi();
return 0;
}
int main(int argc, char **argv)
{
int sub_div[3], local_offset[3], count_local[3];
int t = 0, time_count = 0;
sub_div[0] = 0;
sub_div[1] = 0;
sub_div[2] = 0;
local_offset[0] = 0;
local_offset[1] = 0;
local_offset[2] = 0;
count_local[0] = 0;
count_local[1] = 0;
count_local[2] = 0;
#if HDF_IO
hid_t file_id;
hid_t plist_id;
hid_t group_id;
hid_t dataset_id;
hid_t file_dataspace;
hid_t mem_dataspace;
#endif
#if PIDX_IO
PIDX_file file;
PIDX_access access;
PIDX_variable variable;
#endif
const int bits_per_block = 15;
const int blocks_per_file = 512;
int nprocs=1, rank=0, slice;
MPI_Comm comm = MPI_COMM_WORLD;
#if HDF_IO
MPI_Info info = MPI_INFO_NULL;
#endif
MPI_Init(&argc, &argv);
MPI_Comm_size(comm, &nprocs);
MPI_Comm_rank(comm, &rank);
output_file_name = (char*) malloc(sizeof (char) * 1024);
sprintf(output_file_name, "%s%s", "/scratch/project/visus/datasets/flame", ".idx");
if (nprocs == 320) /// 4 x 5 x 16
{
count_local[0] = 256;
count_local[1] = 188;
count_local[2] = 192;
}
else if (nprocs == 640) /// 4 x 5 x 32
{
count_local[0] = 256;
count_local[1] = 188;
count_local[2] = 96;
}
else if (nprocs == 1280) /// 4 x 10 x 32
{
count_local[0] = 256;
count_local[1] = 94;
count_local[2] = 96;
}
else if (nprocs == 2560) /// 8 x 10 x 32
{
count_local[0] = 128;
count_local[1] = 94;
count_local[2] = 96;
}
sub_div[0] = (1024 / count_local[0]);
sub_div[1] = (940 / count_local[1]);
sub_div[2] = (3072 / count_local[2]);
local_offset[2] = (rank / (sub_div[0] * sub_div[1])) * count_local[2];
slice = rank % (sub_div[0] * sub_div[1]);
local_offset[1] = (slice / sub_div[0]) * count_local[1];
local_offset[0] = (slice % sub_div[0]) * count_local[0];
#if PIDX_IO
PIDX_point global_bounding_box, local_offset_point, local_box_count_point;
PIDX_set_point_5D(global_bounding_box, (int64_t)1024, (int64_t)940, (int64_t)3072, 1, 1);
PIDX_set_point_5D(local_offset_point, (int64_t)local_offset[0], (int64_t)local_offset[1], (int64_t)local_offset[2], 0, 0);
PIDX_set_point_5D(local_box_count_point, (int64_t)count_local[0], (int64_t)count_local[1], (int64_t)count_local[2], 1, 1);
#endif
#if HDF_IO
hsize_t count[3];
count[0] = count_local[0];
count[1] = count_local[1];
count[2] = count_local[2];
hsize_t offset[3];
offset[0] = local_offset[0];
offset[1] = local_offset[1];
offset[2] = local_offset[2];
#endif
/// P U V W ZMIX
#if HDF_IO
plist_id = H5Pcreate(H5P_FILE_ACCESS);
H5Pset_fapl_mpio(plist_id, comm, info);
#endif
FILE *fp;
fp = fopen("list", "r");
char file_name[1][1024];
while (!feof(fp))
{
if (fscanf(fp, "%s", file_name[time_count]) != 1)
break;
if(rank == 0)
printf("%s\n", file_name[time_count]);
time_count++;
}
fclose(fp);
#if PIDX_IO
PIDX_create_access(&access);
PIDX_set_mpi_access(access, 1, 1, 1, comm);
PIDX_set_process_extent(access, sub_div[0], sub_div[1], sub_div[2]);
#endif
if (rank == 0)
printf("Number of timesteps = %d\n", time_count);
for (t = 0; t < time_count; t++)
{
#if HDF_IO
file_id = H5Fopen(file_name[t], H5F_ACC_RDONLY, plist_id);
#endif
#if PIDX_IO
PIDX_file_create(output_file_name, PIDX_file_trunc, access, &file);
PIDX_set_dims(file, global_bounding_box);
PIDX_set_current_time_step(file, t);
PIDX_set_block_size(file, bits_per_block);
PIDX_set_aggregation_factor(file, 1);
PIDX_set_block_count(file, blocks_per_file);
PIDX_set_variable_count(file, 1);
PIDX_debug_rst(file, 0);
PIDX_debug_hz(file, 0);
PIDX_dump_agg_info(file, 0);
PIDX_enable_hz(file, 1);
PIDX_enable_agg(file, 1);
PIDX_enable_io(file, 1);
#endif
#if HDF_IO
group_id = H5Gopen(file_id, "/data", H5P_DEFAULT);
dataset_id = H5Dopen2(group_id, "P", H5P_DEFAULT);
mem_dataspace = H5Screate_simple (3, count, NULL);
file_dataspace = H5Dget_space (dataset_id);
H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, offset, NULL, count, NULL);
#endif
buffer = malloc(sizeof(double) * count_local[0] * count_local[1] * count_local[2]);
memset(buffer, 0, sizeof(double) * count_local[0] * count_local[1] * count_local[2]);
#if HDF_IO
H5Dread(dataset_id, H5T_NATIVE_DOUBLE, mem_dataspace, file_dataspace, H5P_DEFAULT, buffer);
H5Sclose(mem_dataspace);
H5Sclose(file_dataspace);
H5Dclose(dataset_id);
H5Gclose(group_id);
H5Fclose(file_id);
if (rank == 0)
printf("Finished reading HDF files\n");
#else
int64_t i1, j1, k1;
for (k1 = 0; k1 < count_local[2]; k1++)
for (j1 = 0; j1 < count_local[1]; j1++)
for (i1 = 0; i1 < count_local[0]; i1++)
{
int64_t index = (int64_t) (count_local[0] * count_local[1] * k1) + (count_local[0] * j1) + i1;
buffer[index] = (256 * rank) / nprocs;
}
#endif
#if PIDX_IO
PIDX_variable_create(file, "P", sizeof(double) * 8, "1*float64", &variable);
PIDX_append_and_write_variable(variable, local_offset_point, local_box_count_point, buffer, PIDX_column_major);
if (rank == 0)
printf("Starting to Write IDX %s Time Step %d\n", output_file_name, t);
PIDX_close(file);
#endif
free(buffer);
buffer = 0;
}
//////////
#if HDF_IO
H5Pclose(plist_id);
#endif
#if PIDX_IO
PIDX_close_access(access);
#endif
free(output_file_name);
output_file_name = 0;
MPI_Finalize();
return 0;
}