int main(int argc, char **argv)
{
int ret;
int p, ts;
int var;
int slice = 0;
int nprocs = 1, rank = 0;
char output_file_name[512];
// MPI initialization
#if PIDX_HAVE_MPI
MPI_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
#endif
if (rank == 0)
{
ret = parse_args(argc, argv);
if (ret < 0)
{
usage();
#if PIDX_HAVE_MPI
MPI_Abort(MPI_COMM_WORLD, -1);
#else
exit(-1);
#endif
}
// check if the num procs is appropriate
int num_bricks = (global_box_size[0] / local_box_size[0]) * (global_box_size[1] / local_box_size[1]) * (global_box_size[2] / local_box_size[2]);
if(num_bricks != nprocs)
{
fprintf(stderr, "Error: number of sub-blocks (%d) doesn't match number of procs (%d)\n", num_bricks, nprocs);
fprintf(stderr, "Incorrect distribution of data across processes i.e.\n(global_x / local_x) X (global_x / local_x) X (global_x / local_x) != nprocs\n(%d/%d) X (%d/%d) X (%d/%d) != %d\n", global_box_size[0], local_box_size[0], global_box_size[1], local_box_size[1], global_box_size[2], local_box_size[2], nprocs);
#if PIDX_HAVE_MPI
MPI_Abort(MPI_COMM_WORLD, -1);
#else
exit(-1);
#endif
}
}
#if 1
// The command line arguments are shared by all processes
#if PIDX_HAVE_MPI
MPI_Bcast(global_box_size, 3, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(local_box_size, 3, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(&time_step_count, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(&variable_count, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(&patch_count, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(&output_file_template, 512, MPI_CHAR, 0, MPI_COMM_WORLD);
MPI_Bcast(&blocks_per_file, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(&bits_per_block, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(restructured_box_size, 5, MPI_LONG_LONG, 0, MPI_COMM_WORLD);
MPI_Bcast(&compression_type, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(&compression_bit_rate, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(perform_phases, 6, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(debug_rst_hz, 2, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(dump_agg_io, 2, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(idx_count, 3, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(&aggregation_factor, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(&is_rank_z_ordering, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(hz_from_to, 2, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(rst_agg, 2, MPI_INT, 0, MPI_COMM_WORLD);
#endif
values_per_sample = malloc(sizeof(*values_per_sample) * variable_count);
memset(values_per_sample, 0, sizeof(*values_per_sample) * variable_count);
// Creating the filename
sprintf(output_file_name, "%s%s", output_file_template,".idx");
// Calculating every process data's offset and size
int sub_div[3];
sub_div[0] = (global_box_size[0] / local_box_size[0]);
sub_div[1] = (global_box_size[1] / local_box_size[1]);
sub_div[2] = (global_box_size[2] / local_box_size[2]);
local_box_offset[2] = (rank / (sub_div[0] * sub_div[1])) * local_box_size[2];
slice = rank % (sub_div[0] * sub_div[1]);
local_box_offset[1] = (slice / sub_div[0]) * local_box_size[1];
local_box_offset[0] = (slice % sub_div[0]) * local_box_size[0];
unsigned int rank_x = 0, rank_y = 0, rank_z = 0, rank_slice;
if (is_rank_z_ordering == 0)
{
rank_z = rank / (sub_div[0] * sub_div[1]);
rank_slice = rank % (sub_div[0] * sub_div[1]);
rank_y = (rank_slice / sub_div[0]);
rank_x = (rank_slice % sub_div[0]);
}
create_patches();
PIDX_file file; // IDX file descriptor
PIDX_variable* variable; // variable descriptor
variable = malloc(sizeof(*variable) * variable_count);
memset(variable, 0, sizeof(*variable) * variable_count);
PIDX_point global_bounding_box, **local_offset_point, **local_box_count_point;
local_offset_point = malloc(sizeof(PIDX_point*) * variable_count);
local_box_count_point = malloc(sizeof(PIDX_point*) * variable_count);
for(var = 0; var < variable_count; var++)
{
local_offset_point[var] = malloc(sizeof(PIDX_point) * patch_count);
local_box_count_point[var] = malloc(sizeof(PIDX_point) * patch_count);
for(p = 0 ; p < patch_count ; p++)
{
PIDX_set_point_5D(local_offset_point[var][p], (int64_t)local_patch_offset[var][p][0], (int64_t)local_patch_offset[var][p][1], (int64_t)local_patch_offset[var][p][2], 0, 0);
PIDX_set_point_5D(local_box_count_point[var][p], (int64_t)local_patch_size[var][p][0], (int64_t)local_patch_size[var][p][1], (int64_t)local_patch_size[var][p][2], 1, 1);
}
}
PIDX_set_point_5D(global_bounding_box, (int64_t)global_box_size[0], (int64_t)global_box_size[1], (int64_t)global_box_size[2], 1, 1);
PIDX_access access;
PIDX_create_access(&access);
#if PIDX_HAVE_MPI
PIDX_set_mpi_access(access, MPI_COMM_WORLD);
PIDX_set_idx_count(access, idx_count[0], idx_count[1], idx_count[2]);
PIDX_set_process_extent(access, sub_div[0], sub_div[1], sub_div[2]);
PIDX_set_process_rank_decomposition(access, rank_x, rank_y, rank_z);
#endif
for (ts = 0; ts < time_step_count; ts++)
{
PIDX_file_create(output_file_name, PIDX_MODE_CREATE, access, &file);
PIDX_set_dims(file, global_bounding_box);
PIDX_set_current_time_step(file, ts);
PIDX_set_variable_count(file, variable_count);
PIDX_set_resolution(file, hz_from_to[0], hz_from_to[1]);
PIDX_set_block_size(file, bits_per_block);
PIDX_set_block_count(file, blocks_per_file);
// PIDX set restructuring box size
PIDX_set_restructuring_box(file, restructured_box_size);
// PIDX compression related calls
if (compression_type == 0)
PIDX_set_compression_type(file, PIDX_NO_COMPRESSION);
if (compression_type == 1)
PIDX_set_compression_type(file, PIDX_CHUNKING_ONLY);
if (compression_type == 2)
{
PIDX_set_compression_type(file, PIDX_CHUNKING_ZFP);
PIDX_set_lossy_compression_bit_rate(file, compression_bit_rate);
}
PIDX_debug_rst(file, debug_rst_hz[0]);
PIDX_debug_hz(file, debug_rst_hz[1]);
PIDX_dump_agg_info(file, dump_agg_io[0]);
PIDX_dump_io_info(file, dump_agg_io[1]);
if (perform_phases[0] == 0)
PIDX_debug_disable_restructuring(file);
if (perform_phases[1] == 0)
PIDX_debug_disable_chunking(file);
if (perform_phases[2] == 0)
PIDX_debug_disable_hz(file);
if (perform_phases[3] == 0)
PIDX_debug_disable_compression(file);
if (perform_phases[4] == 0)
PIDX_debug_disable_agg(file);
if (perform_phases[5] == 0)
PIDX_debug_disable_io(file);
if (rst_agg[0] == 0)
PIDX_disable_rst(file);
if (rst_agg[1] == 0)
PIDX_disable_agg(file);
for (var = 0; var < variable_count; var++)
{
char variable_name[512];
char data_type[512];
sprintf(variable_name, "variable_%d", var);
sprintf(data_type, "%d*float64", values_per_sample[var]);
ret = PIDX_variable_create(variable_name, sizeof(double) * 8, data_type, &variable[var]);
if (ret != PIDX_success) report_error("PIDX_variable_create", __FILE__, __LINE__);
for (p = 0 ; p < patch_count ; p++)
{
ret = PIDX_variable_write_data_layout(variable[var], local_offset_point[var][p], local_box_count_point[var][p], NULL, PIDX_row_major);
if (ret != PIDX_success) report_error("PIDX_variable_data_layout", __FILE__, __LINE__);
}
ret = PIDX_append_and_write_variable(file, variable[var]);
if (ret != PIDX_success) report_error("PIDX_append_and_write_variable", __FILE__, __LINE__);
ret = PIDX_flush(file);
}
ret = PIDX_close(file);
if (ret != PIDX_success) report_error("PIDX_close", __FILE__, __LINE__);
}
PIDX_close_access(access);
for(var = 0; var < variable_count; var++)
{
free(local_offset_point[var]);
free(local_box_count_point[var]);
}
free(local_offset_point);
free(local_box_count_point);
free(variable);
variable = 0;
free(values_per_sample);
values_per_sample = 0;
destroy_patches();
#endif
#if PIDX_HAVE_MPI
MPI_Finalize();
#endif
return 0;
}
int test_multi_patch_writer(struct Args args, int rank, int nprocs)
{
#if PIDX_HAVE_MPI
int i = 0, j = 0, k = 0, u = 0, v = 0, p = 0, var, d;
int spv = 0;
int ts;
int slice;
int sub_div[5], offset_local[5];
PIDX_file file; // IDX file descriptor
const char *output_file; // IDX File Name
const int bits_per_block = 15; // Total number of samples in each block = 2 ^ bits_per_block
const int blocks_per_file = 32; // Total number of blocks per file
PIDX_variable *variable; // variable descriptor
int *values_per_sample; // values per variable (example, scalar=1, vector=3)
int *var_patch_count; // Number of patches a variable has (data blocks per variable that needs to be written to).
int ***var_count; // Local extents of the variables in each process
int ***var_offset; // Local counts of the variables in each process
double ***var_double_scalar_data;
//The command line arguments are shared by all processes
MPI_Bcast(args.extents, 5, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(args.count_local, 5, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(&args.time_step, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(&args.variable_count, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(&args.output_file_template, 512, MPI_CHAR, 0, MPI_COMM_WORLD);
variable = malloc(sizeof (PIDX_variable) * args.variable_count); // variable pointers
memset(variable, 0, sizeof (PIDX_variable) * args.variable_count);
values_per_sample = (int*) malloc(sizeof (int) * args.variable_count);
var_patch_count = (int*) malloc(sizeof (int) * args.variable_count);
for (var = 0; var < args.variable_count; var++)
{
values_per_sample[var] = 1;
/*
if(var % 4 == 3)
var_patch_count[var] = 1;
else if(var % 4 == 2)
var_patch_count[var] = 2;
else if(var % 4 == 1)
var_patch_count[var] = 4;
else if(var % 4 == 0)
*/
var_patch_count[var] = 8;
}
// Creating the filename
args.output_file_name = (char*) malloc(sizeof (char) * 512);
sprintf(args.output_file_name, "%s%s", args.output_file_template, ".idx");
// Calculating every process's offset and count
sub_div[0] = (args.extents[0] / args.count_local[0]);
sub_div[1] = (args.extents[1] / args.count_local[1]);
sub_div[2] = (args.extents[2] / args.count_local[2]);
offset_local[2] = (rank / (sub_div[0] * sub_div[1])) * args.count_local[2];
slice = rank % (sub_div[0] * sub_div[1]);
offset_local[1] = (slice / sub_div[0]) * args.count_local[1];
offset_local[0] = (slice % sub_div[0]) * args.count_local[0];
offset_local[3] = 0;
offset_local[4] = 0;
args.count_local[3] = 1;
args.count_local[4] = 1;
var_count = malloc(sizeof(int**) * args.variable_count);
var_offset = malloc(sizeof(int**) * args.variable_count);
for(var = 0; var < args.variable_count; var++)
{
var_count[var] = malloc(sizeof(int*) * var_patch_count[var]);
var_offset[var] = malloc(sizeof(int*) * var_patch_count[var]);
for(i = 0; i < var_patch_count[var] ; i++)
{
var_count[var][i] = malloc(sizeof(int) * PIDX_MAX_DIMENSIONS);
var_offset[var][i] = malloc(sizeof(int) * PIDX_MAX_DIMENSIONS);
}
}
for(var = 0; var < args.variable_count; var++)
{
/*
if(var % 4 == 3) // One patch for this variable
{
for(d = 0; d < PIDX_MAX_DIMENSIONS; d++)
{
var_count[var][0][d] = args.count_local[d];
var_offset[var][0][d] = offset_local[d];
}
}
else if(var % 4 == 2) // two patches for this variable
{
for(d = 0; d < PIDX_MAX_DIMENSIONS; d++)
{
var_count[var][0][d] = args.count_local[d];
var_offset[var][0][d] = offset_local[d];
var_count[var][1][d] = args.count_local[d];
var_offset[var][1][d] = offset_local[d];
}
var_count[var][0][0] = args.count_local[0]/2;
if(args.count_local[0] % 2 == 0)
var_count[var][1][0] = args.count_local[0]/2;
else
var_count[var][1][0] = args.count_local[0]/2 + 1;
var_offset[var][1][0] = offset_local[0] + args.count_local[0]/2;
}
else if(var % 4 == 1)
{
for(i = 0; i < var_patch_count[var]; i++)
{
for(d = 0; d < PIDX_MAX_DIMENSIONS; d++)
{
var_count[var][i][d] = args.count_local[d];
var_offset[var][i][d] = offset_local[d];
}
}
var_count[var][0][0] = args.count_local[0]/2;
var_count[var][0][1] = args.count_local[1]/2;
var_count[var][1][1] = args.count_local[1]/2;
if(args.count_local[0] % 2 == 0)
{
var_count[var][1][0] = args.count_local[0]/2;
var_count[var][3][0] = args.count_local[0]/2;
var_offset[var][1][0] = var_offset[var][0][0] + args.count_local[0]/2;
var_offset[var][3][0] = var_offset[var][0][0] + args.count_local[0]/2;
}
else
{
var_count[var][1][0] = args.count_local[0]/2 + 1;
var_count[var][3][0] = args.count_local[0]/2 + 1;
var_offset[var][1][0] = var_offset[var][0][0] + args.count_local[0]/2;
var_offset[var][3][0] = var_offset[var][0][0] + args.count_local[0]/2;
}
var_count[var][2][0] = args.count_local[0]/2;
if(args.count_local[1] % 2 == 0)
{
var_count[var][2][1] = args.count_local[1]/2;
var_count[var][3][1] = args.count_local[1]/2;
var_offset[var][2][1] = var_offset[var][0][1] + args.count_local[1]/2;
var_offset[var][3][1] = var_offset[var][0][1] + args.count_local[1]/2;
}
else
{
var_count[var][2][1] = args.count_local[1]/2 + 1;
var_count[var][3][1] = args.count_local[1]/2 + 1;
var_offset[var][2][1] = var_offset[var][0][1] + args.count_local[1]/2;
var_offset[var][3][1] = var_offset[var][0][1] + args.count_local[1]/2;
}
}
*/
//else if(var % 4 == 0)
//{
for(i = 0; i < var_patch_count[var]; i++)
{
for(d = 0; d < PIDX_MAX_DIMENSIONS; d++)
{
var_count[var][i][d] = args.count_local[d];
var_offset[var][i][d] = offset_local[d];
}
}
var_count[var][0][0] = args.count_local[0]/2;
var_count[var][0][1] = args.count_local[1]/2;
var_count[var][4][0] = args.count_local[0]/2;
var_count[var][4][1] = args.count_local[1]/2;
var_count[var][1][1] = args.count_local[1]/2;
var_count[var][5][1] = args.count_local[1]/2;
if(args.count_local[0] % 2 == 0)
{
var_count[var][1][0] = args.count_local[0]/2;
var_count[var][3][0] = args.count_local[0]/2;
var_offset[var][1][0] = var_offset[var][0][0] + args.count_local[0]/2;
var_offset[var][3][0] = var_offset[var][0][0] + args.count_local[0]/2;
var_count[var][5][0] = args.count_local[0]/2;
var_count[var][7][0] = args.count_local[0]/2;
var_offset[var][5][0] = var_offset[var][0][0] + args.count_local[0]/2;
var_offset[var][7][0] = var_offset[var][0][0] + args.count_local[0]/2;
}
else
{
var_count[var][1][0] = args.count_local[0]/2 + 1;
var_count[var][3][0] = args.count_local[0]/2 + 1;
var_offset[var][1][0] = var_offset[var][0][0] + args.count_local[0]/2;
var_offset[var][3][0] = var_offset[var][0][0] + args.count_local[0]/2;
var_count[var][5][0] = args.count_local[0]/2 + 1;
var_count[var][7][0] = args.count_local[0]/2 + 1;
var_offset[var][5][0] = var_offset[var][0][0] + args.count_local[0]/2;
var_offset[var][7][0] = var_offset[var][0][0] + args.count_local[0]/2;
}
var_count[var][2][0] = args.count_local[0]/2;
var_count[var][6][0] = args.count_local[0]/2;
if(args.count_local[1] % 2 == 0)
{
var_count[var][2][1] = args.count_local[1]/2;
var_count[var][3][1] = args.count_local[1]/2;
var_offset[var][2][1] = var_offset[var][0][1] + args.count_local[1]/2;
var_offset[var][3][1] = var_offset[var][0][1] + args.count_local[1]/2;
var_count[var][6][1] = args.count_local[1]/2;
var_count[var][7][1] = args.count_local[1]/2;
var_offset[var][6][1] = var_offset[var][0][1] + args.count_local[1]/2;
var_offset[var][7][1] = var_offset[var][0][1] + args.count_local[1]/2;
}
else
{
var_count[var][2][1] = args.count_local[1]/2 + 1;
var_count[var][3][1] = args.count_local[1]/2 + 1;
var_offset[var][2][1] = var_offset[var][0][1] + args.count_local[1]/2;
var_offset[var][3][1] = var_offset[var][0][1] + args.count_local[1]/2;
var_count[var][6][1] = args.count_local[1]/2 + 1;
var_count[var][7][1] = args.count_local[1]/2 + 1;
var_offset[var][6][1] = var_offset[var][0][1] + args.count_local[1]/2;
var_offset[var][7][1] = var_offset[var][0][1] + args.count_local[1]/2;
}
var_count[var][0][2] = args.count_local[2]/2;
var_count[var][1][2] = args.count_local[2]/2;
var_count[var][2][2] = args.count_local[2]/2;
var_count[var][3][2] = args.count_local[2]/2;
if(args.count_local[1] % 2 == 0)
{
var_count[var][4][2] = args.count_local[2]/2;
var_count[var][5][2] = args.count_local[2]/2;
var_count[var][6][2] = args.count_local[2]/2;
var_count[var][7][2] = args.count_local[2]/2;
var_offset[var][4][2] = var_offset[var][0][2] + args.count_local[2]/2;
var_offset[var][5][2] = var_offset[var][1][2] + args.count_local[2]/2;
var_offset[var][6][2] = var_offset[var][2][2] + args.count_local[2]/2;
var_offset[var][7][2] = var_offset[var][3][2] + args.count_local[2]/2;
}
else
{
var_count[var][4][2] = args.count_local[2]/2 + 1;
var_count[var][5][2] = args.count_local[2]/2 + 1;
var_count[var][6][2] = args.count_local[2]/2 + 1;
var_count[var][7][2] = args.count_local[2]/2 + 1;
var_offset[var][4][2] = var_offset[var][0][2] + args.count_local[2]/2;
var_offset[var][5][2] = var_offset[var][1][2] + args.count_local[2]/2;
var_offset[var][6][2] = var_offset[var][2][2] + args.count_local[2]/2;
var_offset[var][7][2] = var_offset[var][3][2] + args.count_local[2]/2;
}
//}
}
output_file = args.output_file_name;
PIDX_point global_bounding_box, **local_offset_point, **local_box_count_point;
local_offset_point = malloc(sizeof(PIDX_point*) * args.variable_count);
local_box_count_point = malloc(sizeof(PIDX_point*) * args.variable_count);
for(var = 0; var < args.variable_count; var++)
{
local_offset_point[var] = malloc(sizeof(PIDX_point) * var_patch_count[var]);
local_box_count_point[var] = malloc(sizeof(PIDX_point) * var_patch_count[var]);
for(p = 0 ; p < var_patch_count[var] ; p++)
{
PIDX_set_point_5D(local_offset_point[var][p], (int64_t)var_offset[var][p][0], (int64_t)var_offset[var][p][1], (int64_t)var_offset[var][p][2], 0, 0);
PIDX_set_point_5D(local_box_count_point[var][p], (int64_t)var_count[var][p][0], (int64_t)var_count[var][p][1], (int64_t)var_count[var][p][2], 1, 1);
}
}
PIDX_set_point_5D(global_bounding_box, (int64_t)args.extents[0], (int64_t)args.extents[1], (int64_t)args.extents[2], 1, 1);
for (ts = 0; ts < args.time_step; ts++)
{
PIDX_access access;
PIDX_create_access(&access);
#if PIDX_HAVE_MPI
PIDX_set_mpi_access(access, 1, 1, 1, MPI_COMM_WORLD);
#endif
PIDX_file_create(output_file, PIDX_file_trunc, access, &file);
PIDX_set_dims(file, global_bounding_box);
PIDX_set_current_time_step(file, ts);
PIDX_set_block_size(file, bits_per_block);
PIDX_set_block_count(file, blocks_per_file);
PIDX_set_variable_count(file, args.variable_count);
#if 0
var_double_scalar_data = malloc(sizeof(double**) * args.variable_count);
for (var = 0; var < args.variable_count; var++)
{
var_double_scalar_data[var] = malloc(sizeof(double*) * var_patch_count[var]);
for(p = 0 ; p < var_patch_count[var] ; p++)
{
var_double_scalar_data[var][p] = malloc(sizeof (double) * var_count[var][p][0] * var_count[var][p][1] * var_count[var][p][2] * var_count[var][p][3] * var_count[var][p][4] * values_per_sample[var]);
for (v = 0; v < var_count[var][p][4]; v++)
for (u = 0; u < var_count[var][p][3]; u++)
for (k = 0; k < var_count[var][p][2]; k++)
for (j = 0; j < var_count[var][p][1]; j++)
for (i = 0; i < var_count[var][p][0]; i++)
{
int64_t index = (int64_t) (var_count[var][p][0] * var_count[var][p][1] * var_count[var][p][2] * var_count[var][p][3] * v) +
(var_count[var][p][0] * var_count[var][p][1] * var_count[var][p][2] * u) + (var_count[var][p][0] * var_count[var][p][1] * k) +
(var_count[var][p][0] * j) + i;
for (spv = 0; spv < values_per_sample[var]; spv++)
var_double_scalar_data[var][p][index * values_per_sample[var] + spv] = (100 +
((args.extents[0] * args.extents[1] * args.extents[2] * args.extents[3] * (var_offset[var][p][4] + v)) +
(args.extents[0] * args.extents[1] * args.extents[2] * (var_offset[var][p][3] + u)) +
(args.extents[0] * args.extents[1]*(var_offset[var][p][2] + k))+(args.extents[0]*(var_offset[var][p][1] + j)) + (var_offset[var][p][0] + i)));
}
}
}
char variable_name[512];
char data_type[512];
for(var = 0; var < args.variable_count; var++)
{
sprintf(variable_name, "variable_%d", var);
sprintf(data_type, "%d*float64", values_per_sample[var]);
PIDX_variable_create(file, variable_name, values_per_sample[var] * sizeof(double) * 8, data_type, &variable[var]);
for(p = 0 ; p < var_patch_count[var] ; p++)
{
if (rank == 0)
printf("Writing patch %d\n", p);
PIDX_append_and_write_variable(variable[var], local_offset_point[var][p], local_box_count_point[var][p], var_double_scalar_data[var][p], PIDX_row_major);
}
}
PIDX_close(file);
PIDX_close_access(access);
for(var = 0; var < args.variable_count; var++)
{
for(p = 0; p < var_patch_count[0]; p++)
{
free(var_double_scalar_data[var][p]);
var_double_scalar_data[var][p] = 0;
}
free(var_double_scalar_data[var]);
var_double_scalar_data[var] = 0;
}
#endif
#if 0
var_double_scalar_data = malloc(sizeof(double**) * args.variable_count);
for (var = 0; var < args.variable_count; var++)
{
var_double_scalar_data[var] = malloc(sizeof(double*) * var_patch_count[var]);
for(p = 0 ; p < var_patch_count[var] ; p++)
{
var_double_scalar_data[var][p] = malloc(sizeof (double) * var_count[var][p][0] * var_count[var][p][1] * var_count[var][p][2] * var_count[var][p][3] * var_count[var][p][4] * values_per_sample[var]);
for (v = 0; v < var_count[var][p][4]; v++)
for (u = 0; u < var_count[var][p][3]; u++)
for (k = 0; k < var_count[var][p][2]; k++)
for (j = 0; j < var_count[var][p][1]; j++)
for (i = 0; i < var_count[var][p][0]; i++)
{
int64_t index = (int64_t) (var_count[var][p][0] * var_count[var][p][1] * var_count[var][p][2] * var_count[var][p][3] * v) +
(var_count[var][p][0] * var_count[var][p][1] * var_count[var][p][2] * u) + (var_count[var][p][0] * var_count[var][p][1] * k) +
(var_count[var][p][0] * j) + i;
for (spv = 0; spv < values_per_sample[var]; spv++)
var_double_scalar_data[var][p][index * values_per_sample[var] + spv] = (100 +
((args.extents[0] * args.extents[1] * args.extents[2] * args.extents[3] * (var_offset[var][p][4] + v)) +
(args.extents[0] * args.extents[1] * args.extents[2] * (var_offset[var][p][3] + u)) +
(args.extents[0] * args.extents[1]*(var_offset[var][p][2] + k))+(args.extents[0]*(var_offset[var][p][1] + j)) + (var_offset[var][p][0] + i)));
}
}
char variable_name[512];
char data_type[512];
sprintf(variable_name, "variable_%d", var);
sprintf(data_type, "%d*float64", values_per_sample[var]);
PIDX_variable_create(file, variable_name, values_per_sample[var] * sizeof(double) * 8, data_type, &variable[var]);
for(p = 0 ; p < var_patch_count[var] ; p++)
{
if (rank == 0)
printf("Writing patch %d\n", p);
PIDX_append_and_write_variable(variable[var], local_offset_point[var][p], local_box_count_point[var][p], var_double_scalar_data[var][p], PIDX_row_major);
}
PIDX_flush(file);
for(p = 0; p < var_patch_count[var]; p++)
{
free(var_double_scalar_data[var][p]);
var_double_scalar_data[var][p] = 0;
}
free(var_double_scalar_data[var]);
var_double_scalar_data[var] = 0;
}
PIDX_close(file);
PIDX_close_access(access);
#endif
#if 1
var_double_scalar_data = malloc(sizeof(double**) * args.variable_count);
for (var = 0; var < args.variable_count; var++)
{
var_double_scalar_data[var] = malloc(sizeof(double*) * var_patch_count[var]);
for(p = 0 ; p < var_patch_count[var] ; p++)
{
var_double_scalar_data[var][p] = malloc(sizeof (double) * var_count[var][p][0] * var_count[var][p][1] * var_count[var][p][2] * var_count[var][p][3] * var_count[var][p][4] * values_per_sample[var]);
for (v = 0; v < var_count[var][p][4]; v++)
for (u = 0; u < var_count[var][p][3]; u++)
for (k = 0; k < var_count[var][p][2]; k++)
for (j = 0; j < var_count[var][p][1]; j++)
for (i = 0; i < var_count[var][p][0]; i++)
{
int64_t index = (int64_t) (var_count[var][p][0] * var_count[var][p][1] * var_count[var][p][2] * var_count[var][p][3] * v) +
(var_count[var][p][0] * var_count[var][p][1] * var_count[var][p][2] * u) + (var_count[var][p][0] * var_count[var][p][1] * k) +
(var_count[var][p][0] * j) + i;
for (spv = 0; spv < values_per_sample[var]; spv++)
var_double_scalar_data[var][p][index * values_per_sample[var] + spv] = (100 +
((args.extents[0] * args.extents[1] * args.extents[2] * args.extents[3] * (var_offset[var][p][4] + v)) +
(args.extents[0] * args.extents[1] * args.extents[2] * (var_offset[var][p][3] + u)) +
(args.extents[0] * args.extents[1]*(var_offset[var][p][2] + k))+(args.extents[0]*(var_offset[var][p][1] + j)) + (var_offset[var][p][0] + i)));
}
}
}
for (var = 0; var < args.variable_count; var++)
{
char variable_name[512];
char data_type[512];
sprintf(variable_name, "variable_%d", var);
sprintf(data_type, "%d*float64", values_per_sample[var]);
PIDX_variable_create(file, variable_name, values_per_sample[var] * sizeof(double) * 8, data_type, &variable[var]);
for(p = 0 ; p < var_patch_count[var] ; p++)
{
//if (rank == 0)
//printf("Writing patch %d\n", p);
PIDX_append_and_write_variable(variable[var], local_offset_point[var][p], local_box_count_point[var][p], var_double_scalar_data[var][p], PIDX_row_major);
}
if(var % 2 == 0)
PIDX_flush(file);
//for(p = 0; p < var_patch_count[var]; p++)
//{
//free(var_double_scalar_data[var][p]);
//var_double_scalar_data[var][p] = 0;
//}
//free(var_double_scalar_data[var]);
//var_double_scalar_data[var] = 0;
}
PIDX_close(file);
PIDX_close_access(access);
#endif
free(var_double_scalar_data); var_double_scalar_data = 0;
}
for(var = 0; var < args.variable_count; var++)
{
free(local_offset_point[var]);
free(local_box_count_point[var]);
}
free(local_offset_point);
free(local_box_count_point);
free(args.output_file_name);
free(variable);
variable = 0;
free(values_per_sample);
values_per_sample = 0;
#endif
return 0;
}
//----------------------------------------------------------------
int main(int argc, char **argv)
{
init_mpi(argc, argv);
printf("start %d %d\n",rank,process_count);
int i;
if(rank==0) {
parse_args(argc, argv);
// var_count = read_list_in_file(var_file, &netcdf_var_names);
// rank_0_print("Number of variables = %d\n", var_count);
// time_step_count = read_list_in_file(netcdf_file_list, &netcdf_file_names);
// rank_0_print("Number of timesteps = %d\n", time_step_count);
check_args();
}
// The command line arguments are shared by all processes
#if PIDX_HAVE_MPI
MPI_Bcast(global_box_size, 3, MPI_LONG_LONG, 0, MPI_COMM_WORLD);
MPI_Bcast(local_box_size, 3, MPI_LONG_LONG, 0, MPI_COMM_WORLD);
// MPI_Bcast(&time_step_count, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(&var_file, 512, MPI_CHAR, 0, MPI_COMM_WORLD);
MPI_Bcast(&netcdf_file_list, 512, MPI_CHAR, 0, MPI_COMM_WORLD);
// MPI_Bcast(&var_count, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(&output_file_name, 512, MPI_CHAR, 0, MPI_COMM_WORLD);
#endif
//TODO: Only the rank 0 should read the input files and broadcast the data
var_count = read_list_in_file(var_file, &netcdf_var_names);
rank_0_print("Number of variables = %d\n", var_count);
time_step_count = read_list_in_file(netcdf_file_list, &netcdf_file_names);
rank_0_print("Number of timesteps = %d\n", time_step_count);
calculate_per_process_offsets();
create_pidx_var_names();
PIDX_access pidx_access;
create_pidx_access(&pidx_access);
PIDX_time_step_caching_ON();
determine_var_types();
create_pidx_vars();
int t = 0,plist_id;
for (t = 0; t < time_step_count; ++t)
{
rank_0_print("Processing time step %d (file %s)\n", t, netcdf_file_names[t]);
PIDX_file pidx_file;
int ret = PIDX_file_create(output_file_name, PIDX_MODE_CREATE, pidx_access, &pidx_file);
if (ret != PIDX_success)
terminate_with_error_msg("ERROR: Failed to create PIDX file\n");
set_pidx_params(pidx_file);
PIDX_set_current_time_step(pidx_file, t);
int file_id = ncmpi_open(MPI_COMM_WORLD,netcdf_file_names[t], NC_NOWRITE, MPI_INFO_NULL, &plist_id);
if (file_id !=0)
terminate_with_error_msg("ERROR: Failed to open file %s\n", netcdf_file_names[t]);
int v = 0;
for(v = 0; v < var_count; ++v)
{
rank_0_print("Processing variable %s\n", netcdf_var_names[v]);
var_data = malloc(var_types[v].atomic_type * var_types[v].num_values * local_box_size[0] * local_box_size[1] * local_box_size[2]);
read_var_from_netcdf(plist_id, netcdf_var_names[v], var_types[v]);
write_var_to_idx(pidx_file, pidx_var_names[v], pidx_vars[v]);
if (PIDX_flush(pidx_file) != PIDX_success)
terminate_with_error_msg("ERROR: Failed to flush variable %s, time step %d\n", pidx_var_names[v], t);
free(var_data);
}
ncmpi_close(plist_id);
PIDX_close(pidx_file);
}
PIDX_time_step_caching_OFF();
PIDX_close_access(pidx_access);
free_memory();
shutdown_mpi();
return 0;
}
