///< Parse the input arguments
static void parse_args(int argc, char **argv)
{
char flags[] = "f:";
int one_opt = 0;
while ((one_opt = getopt(argc, argv, flags)) != EOF)
{
/* postpone error checking for after while loop */
switch (one_opt)
{
case('f'): // output file name
if (sprintf(output_file_template, "%s", optarg) < 0)
terminate_with_error_msg("Invalid output file name template\n%s", usage);
sprintf(output_file_name, "%s", output_file_template);
break;
//case('c'): // number of timesteps
// if (sscanf(optarg, "%d", &cores) < 0)
// terminate_with_error_msg("Invalid variable file\n%s", usage);
// break;
default:
terminate_with_error_msg("Wrong arguments\n%s", usage);
}
}
}
//----------------------------------------------------------------
static void init_mpi(int argc, char **argv)
{
if (MPI_Init(&argc, &argv) != MPI_SUCCESS)
terminate_with_error_msg("ERROR: MPI_Init error\n");
if (MPI_Comm_size(MPI_COMM_WORLD, &process_count) != MPI_SUCCESS)
terminate_with_error_msg("ERROR: MPI_Comm_size error\n");
if (MPI_Comm_rank(MPI_COMM_WORLD, &rank) != MPI_SUCCESS)
terminate_with_error_msg("ERROR: MPI_Comm_rank error\n");
}
//----------------------------------------------------------------
static void create_pidx_access(PIDX_access *pidx_access)
{
int ret = PIDX_create_access(pidx_access);
if (ret != PIDX_success)
terminate_with_error_msg("ERROR: Failed to create PIDX access\n");
#if PIDX_HAVE_MPI
ret = PIDX_set_mpi_access(*pidx_access, MPI_COMM_WORLD);
if (ret != PIDX_success)
terminate_with_error_msg("ERROR: Failed to create PIDX MPI access\n");
#endif
}
//----------------------------------------------------------------
// Return a negative value when failed, otherwise return 0
int read_var_from_netcdf(int file_id, const char *var_name, struct Type type)
{
int TIMES=1;
int LATS=local_box_size[1];
int LONS=local_box_size[0];
assert(var_name != 0);
assert(var_data != 0);
int varidp,ndims,nvars,ngatts,unlimited;
nc_type xtypep;
//int dataset_id = H5Dopen2(file_id, var_name, H5P_DEFAULT);
int dataset_id = ncmpi_inq_varid(file_id, var_name, &varidp);
MPI_Offset start[]={local_box_offset[2],local_box_offset[1],local_box_offset[0]};
MPI_Offset count[]={TIMES,LATS,LONS};
dataset_id = ncmpi_inq_vartype(file_id,varidp, &xtypep);
// if (dataset_id !=0)
// terminate_with_error_msg("ERROR: Failed to open NetCDF dataset for variable %s\n", var_name);
int read_error = 0;
if (type.atomic_type == DOUBLE)
read_error = ncmpi_get_vara_double(file_id, varidp, start, count, var_data);
else if (type.atomic_type == FLOAT){
ncmpi_begin_indep_data(file_id);
read_error = ncmpi_get_vara_float(file_id, varidp, start, count, (float *)var_data);
ncmpi_end_indep_data(file_id);
if (read_error != NC_NOERR)
terminate_with_error_msg("ERROR: Can not read the data for the variable %s \n", var_name);
}
else if (type.atomic_type == INT)
read_error = ncmpi_get_vara_int(file_id, varidp, start, count, var_data);
// else if (type.atomic_type == UINT)
// read_error = ncmpi_get_vara_uint(file_id, varidp, start, count, var_data);
// else if (type.atomic_type == CHAR)
// read_error = ncmpi_get_vara_char(file_id, varidp, start, count, var_data);
else if (type.atomic_type == UCHAR)
read_error = ncmpi_get_vara_uchar(file_id, varidp, start, count, var_data);
else
terminate_with_error_msg("ERROR: Unsupported type. Type = %d\n", type.atomic_type);
if (read_error < 0)
return -1;
return 0;
}
//----------------------------------------------------------------
static void check_args()
{
if (global_box_size[X] < local_box_size[X] || global_box_size[Y] < local_box_size[Y] || global_box_size[Z] < local_box_size[Z])
terminate_with_error_msg("ERROR: Global box is smaller than local box in one of the dimensions\n");
// check if the number of processes given by the user is consistent with the actual number of processes needed
int brick_count = (int)((global_box_size[X] + local_box_size[X] - 1) / local_box_size[X]) *
(int)((global_box_size[Y] + local_box_size[Y] - 1) / local_box_size[Y]) *
(int)((global_box_size[Z] + local_box_size[Z] - 1) / local_box_size[Z]);
if(brick_count != process_count)
terminate_with_error_msg("ERROR: Number of sub-blocks (%d) doesn't match number of processes (%d)\n", brick_count, process_count);
}
//----------------------------------------------------------------
static void set_pidx_params(PIDX_file pidx_file)
{
int ret = PIDX_set_point_5D(pidx_global_box_size, global_box_size[X], global_box_size[Y], global_box_size[Z], 1, 1);
ret = PIDX_set_dims(pidx_file, pidx_global_box_size);
if (ret != PIDX_success)
terminate_with_error_msg("ERROR: Failed to set PIDX global dimensions\n");
ret = PIDX_set_variable_count(pidx_file, var_count);
if (ret != 0)
terminate_with_error_msg("ERROR: Failed to set PIDX variable count\n");
// PIDX_set_block_size(pidx_file,16);
// PIDX_set_block_count(pidx_file,16);
}
//----------------------------------------------------------------
// Open the first NETCDF file and query all the types for all the variables
static void determine_var_types()
{
assert(netcdf_file_names != 0);
assert(netcdf_var_names != 0);
struct Type type;
int plist_id,i = 0,varidp,ndimsp;
int file_id = ncmpi_open(MPI_COMM_WORLD,netcdf_file_names[0], NC_NOWRITE, MPI_INFO_NULL, &plist_id);
if (file_id != NC_NOERR)
terminate_with_error_msg("ERROR: Cannot open file %s\n", netcdf_file_names[0]);
var_types = (struct Type *)calloc(var_count, sizeof(*var_types));
nc_type xtypep;
for (i = 0; i < var_count; ++i)
{
int dataset_id = ncmpi_inq_varid (plist_id, netcdf_var_names[i], &varidp);
dataset_id = ncmpi_inq_vartype(plist_id,varidp, &xtypep);
if (dataset_id != NC_NOERR)
terminate_with_error_msg("ERROR: Cannot read the datatype of the variable %s\n", netcdf_var_names[i]);
int num_dims = ncmpi_inq_varndims(plist_id,varidp,&ndimsp);
if (ndimsp > 3 )
{
//TODO:probably we can make it more clever to handle more dimensions as they are not related to the variable itself on netcdf
type.atomic_type = INVALID; // we don't support arrays of more than 3 dimension
return;
}
else if (xtypep == NC_FLOAT || xtypep == NC_INT)
{
type.num_values = 1;
}
else // we don't support HD5_COMPOUND datatype for example
{
type.atomic_type = INVALID;
}
type.atomic_type = from_netcdf_atomic_type(xtypep);
var_types[i] = type;
if (var_types[i].atomic_type == INVALID)
terminate_with_error_msg("ERROR: The datatype of the %s variable is not supported\n", netcdf_var_names[i]);
}
ncmpi_close(plist_id);
}
//----------------------------------------------------------------
static void write_var_to_idx(PIDX_file pidx_file, const char *var_name, PIDX_variable pidx_var)
{
assert(var_name != 0);
assert(var_data != 0);
PIDX_set_point_5D(pidx_local_box_offset, local_box_offset[X], local_box_offset[Y], local_box_offset[Z], 0, 0);
PIDX_set_point_5D(pidx_local_box_size, local_box_size[X], local_box_size[Y], local_box_size[Z], 1, 1);
int ret = PIDX_success,i;
ret = PIDX_variable_write_data_layout(pidx_var, pidx_local_box_offset, pidx_local_box_size, var_data, PIDX_ROW_OR_COLUMN_MAJOR);
if (ret != PIDX_success)
terminate_with_error_msg("ERROR: PIDX failed to specify variable data layout for %s\n", var_name);
ret = PIDX_append_and_write_variable(pidx_file, pidx_var);
if (ret != PIDX_success)
terminate_with_error_msg("ERROR: PIDX failed to append and write variable %s\n", var_name);
}
//----------------------------------------------------------------
static void parse_args(int argc, char **argv)
{
char flags[] = "g:l:f:t:v:p:";
int one_opt = 0;
while ((one_opt = getopt(argc, argv, flags)) != EOF)
{
/* postpone error checking for after while loop */
switch (one_opt)
{
case('g'): // global dimension
if ((sscanf(optarg, "%lldx%lldx%lld", &global_box_size[X], &global_box_size[Y], &global_box_size[Z]) == EOF) || (global_box_size[X] < 1 || global_box_size[Y] < 1 || global_box_size[Z] < 1))
terminate_with_error_msg("Invalid global dimensions\n%s", usage);
break;
case('l'): // local dimension
if ((sscanf(optarg, "%lldx%lldx%lld", &local_box_size[X], &local_box_size[Y], &local_box_size[Z]) == EOF) ||(local_box_size[X] < 1 || local_box_size[Y] < 1 || local_box_size[Z] < 1))
terminate_with_error_msg("Invalid local dimension\n%s", usage);
break;
case('f'): // output file name
if (sprintf(output_file_template, "%s", optarg) < 0)
terminate_with_error_msg("Invalid output file name template\n%s", usage);
sprintf(output_file_name, "%s%s", output_file_template, ".idx");
break;
case('t'): // number of timesteps
if (sscanf(optarg, "%d", &time_step_count) < 0)
terminate_with_error_msg("Invalid variable file\n%s", usage);
break;
case('v'): // number of variables
if(!isNumber(optarg)){ // the param is a file with the list of variables
if (sprintf(var_list, "%s", optarg) > 0)
parse_var_list();
else
terminate_with_error_msg("Invalid variable list file\n%s", usage);
}else { // the param is a number of variables (default: 1*float32)
if(sscanf(optarg, "%d", &variable_count) > 0)
generate_vars();
else
terminate_with_error_msg("Invalid number of variables\n%s", usage);
}
break;
case('p'): // number of patches
if (sscanf(optarg, "%d", &patch_count) < 0)
terminate_with_error_msg("Invalid patch count\n%s", usage);
break;
default:
terminate_with_error_msg("Wrong arguments\n%s", usage);
}
}
}
//----------------------------------------------------------------
static void create_pidx_var_names()
{
assert(netcdf_var_names != 0);
pidx_var_names = (char **)calloc(var_count, sizeof(*pidx_var_names));
if (pidx_var_names == 0)
terminate_with_error_msg("ERROR: Failed to allocate memory to store PIDX var names\n. Bytes requested = %d (values) * %u (bytes)\n", var_count, sizeof(*pidx_var_names));
int i = 0;
for (i = 0; i < var_count; ++i)
{
pidx_var_names[i] = (char *)calloc(strlen(netcdf_var_names[i]) + 1, sizeof(*pidx_var_names[i]));
if (pidx_var_names == 0)
terminate_with_error_msg("ERROR: Failed to allocate memory to store the PIDX var name for %s\n. Bytes requested = %d (values) * %u (bytes)\n", netcdf_var_names[i], strlen(netcdf_var_names[i]) + 1, sizeof(*pidx_var_names[i]));
netcdf_var_name_to_pidx_var_name(netcdf_var_names[i], pidx_var_names[i]);
rank_0_print("NetCDF variable %s becomes\n PIDX variable %s\n", netcdf_var_names[i], pidx_var_names[i]);
}
}
//----------------------------------------------------------------
static void set_pidx_variable(int var)
{
int p = 0;
PIDX_return_code ret = 0;
PIDX_variable_create(var_name[var], bpv[var], type_name[var], &variable[var]);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_variable_create");
for (p = 0 ; p < patch_count ; p++)
{
ret = PIDX_variable_write_data_layout(variable[var], local_offset_point[var][p], local_box_count_point[var][p], data[var][p], PIDX_row_major);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_variable_write_data_layout");
}
ret = PIDX_append_and_write_variable(file, variable[var]);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_append_and_write_variable");
return;
}
//----------------------------------------------------------------
static void destroy_pidx_var_point_and_access()
{
if (PIDX_close_access(p_access) != PIDX_success)
terminate_with_error_msg("PIDX_close_access");
free(variable);
variable = 0;
return;
}
///< Parse the input arguments
static void parse_args(int argc, char **argv)
{
char flags[] = "g:l:p:f:t:v:b:";
int one_opt = 0;
while ((one_opt = getopt(argc, argv, flags)) != EOF)
{
/* postpone error checking for after while loop */
switch (one_opt)
{
case('g'): // global dimension
if ((sscanf(optarg, "%lldx%lldx%lld", &global_box_size[0], &global_box_size[1], &global_box_size[2]) == EOF) || (global_box_size[0] < 1 || global_box_size[1] < 1 || global_box_size[2] < 1))
terminate_with_error_msg("Invalid global dimensions\n%s", usage);
break;
case('l'): // local dimension
if ((sscanf(optarg, "%lldx%lldx%lld", &local_box_size[0], &local_box_size[1], &local_box_size[2]) == EOF) ||(local_box_size[0] < 1 || local_box_size[1] < 1 || local_box_size[2] < 1))
terminate_with_error_msg("Invalid local dimension\n%s", usage);
break;
case('p'): // local dimension
if ((sscanf(optarg, "%dx%dx%d", &partition_size[0], &partition_size[1], &partition_size[2]) == EOF) ||(partition_size[0] < 1 || partition_size[1] < 1 || partition_size[2] < 1))
terminate_with_error_msg("Invalid partition dimension\n%s", usage);
break;
case('f'): // output file name
if (sprintf(output_file_template, "%s", optarg) < 0)
terminate_with_error_msg("Invalid output file name template\n%s", usage);
sprintf(output_file_name, "%s%s", output_file_template, ".idx");
break;
case('t'): // number of timesteps
if (sscanf(optarg, "%d", &time_step_count) < 0)
terminate_with_error_msg("Invalid variable file\n%s", usage);
break;
case('v'): // number of variables
if (sprintf(var_list, "%s", optarg) < 0)
terminate_with_error_msg("Invalid output file name template\n%s", usage);
parse_var_list();
break;
case('b'): // blocks per file
if (sscanf(optarg, "%d", &blocks_per_file) < 0)
terminate_with_error_msg("Invalid variable file\n%s", usage);
break;
default:
terminate_with_error_msg("Wrong arguments\n%s", usage);
}
}
}
//----------------------------------------------------------------
// Read all the lines in a file into a list.
// Return the number of lines read.
// A line starting with // is treated as a comment and does not increase the count.
static int read_list_in_file(const char *file_name, char ***list_ptr)
{
assert(file_name != 0);
assert(list_ptr != 0);
rank_0_print("Opening file %s\n", file_name);
FILE *fp = 0;
if ((fp = fopen(file_name, "r")) == 0) {
terminate_with_error_msg("ERROR: Cannot open file here %s\n", file_name);
}
// first pass, count the number of non-comment lines
int line_count = 0;
char line[512]; // a line cannot be longer than 512 characters
//char *line = malloc(sizeof(char) * 512);
while (fgets(line, 512, fp) != 0)
{
if (line[0] != '/' || line[1] != '/')
++line_count;
}
// second pass, actually read the data
*list_ptr = (char **)calloc(line_count, sizeof(*list_ptr));
char **list = *list_ptr;
if (list == 0)
terminate_with_error_msg("ERROR: Failed to allocate memory for a list of names. Bytes requested = %d (items) * %u (bytes)\n", line_count, sizeof(*list_ptr));
rewind(fp);
int i = 0;
while (fgets(line, sizeof(line), fp) != 0)
{
if (line[0] != '/' || line[1] != '/')
{
line[strcspn(line, "\r\n")] = 0; // trim the newline character at the end if any
list[i] = strdup(line);
++i;
}
}
fclose(fp);
return line_count;
}
//----------------------------------------------------------------
static void create_pidx_vars()
{
assert(netcdf_var_names != 0);
assert(var_types != 0);
pidx_vars = (PIDX_variable *)calloc(var_count, sizeof(*pidx_vars));
int i = 0;
for (i = 0; i < var_count; ++i)
{
char type_string[32] = { 0 };
to_idx_type_string(var_types[i], type_string);
int ret = PIDX_variable_create(pidx_var_names[i], var_types[i].atomic_type * 8, type_string, &pidx_vars[i]);
if (ret != PIDX_success)
terminate_with_error_msg("ERROR: PIDX failed to create PIDX variable %s\n", pidx_var_names[i]);
}
}
//----------------------------------------------------------------
static void set_pidx_file(int ts)
{
PIDX_return_code ret;
ret = PIDX_file_create(output_file_name, PIDX_MODE_CREATE, p_access, global_size, &file);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_file_create");
PIDX_set_current_time_step(file, ts);
PIDX_set_variable_count(file, variable_count);
//PIDX_debug_rst(file, 1);
//PIDX_debug_hz(file, 1);
// Selecting raw I/O mode
PIDX_set_io_mode(file, PIDX_IDX_IO);
return;
}
//----------------------------------------------------------------
static void to_idx_type_string(struct Type type, char *type_string)
{
assert(type_string != 0);
if (type.atomic_type == DOUBLE)
sprintf(type_string, "%lld*float64", type.num_values);
else if (type.atomic_type == FLOAT)
sprintf(type_string, "%lld*float32", type.num_values);
else if (type.atomic_type == INT)
sprintf(type_string, "%lld*int32", type.num_values);
else if (type.atomic_type == UINT)
sprintf(type_string, "%lld*uint32", type.num_values);
else if (type.atomic_type == CHAR)
sprintf(type_string, "%lld*int8", type.num_values);
else if (type.atomic_type == UCHAR)
sprintf(type_string, "%lld*uint8", type.num_values);
else
terminate_with_error_msg("ERROR: Unsupported type. Type = %d\n", type.atomic_type);
}
//----------------------------------------------------------------
///< Parse the input arguments
static void parse_args(int argc, char **argv)
{
if (argc != 11)
terminate_with_error_msg("ERROR: Wrong number of arguments.\n%s", usage);
char flags[] = "g:l:f:i:v:";
int opt = 0;
while ((opt = getopt(argc, argv, flags)) != -1)
{
switch (opt)
{
case('g'): // global dimension
if ((sscanf(optarg, "%lldx%lldx%lld", &global_box_size[0], &global_box_size[1], &global_box_size[2]) == EOF) ||
(global_box_size[0] < 1 || global_box_size[1] < 1 || global_box_size[2] < 1))
terminate_with_error_msg("Invalid global dimensions\n%s", usage);
break;
case('l'): // local dimension
if ((sscanf(optarg, "%lldx%lldx%lld", &local_box_size[0], &local_box_size[1], &local_box_size[2]) == EOF) ||
(local_box_size[0] < 1 || local_box_size[1] < 1 || local_box_size[2] < 1))
terminate_with_error_msg("Invalid local dimension\n%s", usage);
break;
case('f'): // output file name
if (sprintf(output_file_template, "%s", optarg) < 0)
terminate_with_error_msg("Invalid output file name template\n%s", usage);
sprintf(output_file_name, "%s%s", output_file_template, ".idx");
break;
case('i'): // a file with a list of NetCDF files
if (sprintf(netcdf_file_list, "%s", optarg) < 0)
terminate_with_error_msg("Invalid input file\n%s", usage);
break;
case('v'): // a file with a list of variables
if (sprintf(var_file, "%s", optarg) < 0)
terminate_with_error_msg("Invalid variable file\n%s", usage);
break;
default:
terminate_with_error_msg("Wrong arguments\n%s", usage);
}
}
}
int main(int argc, char **argv)
{
init_mpi(argc, argv);
parse_args(argc, argv);
check_args();
calculate_per_process_offsets();
create_synthetic_simulation_data();
rank_0_print("Simulation Data Created\n");
int ret;
int var;
int ts;
PIDX_file file; // IDX file descriptor
PIDX_variable* variable; // variable descriptor
variable = malloc(sizeof(*variable) * variable_count);
memset(variable, 0, sizeof(*variable) * variable_count);
PIDX_point global_size, local_offset, local_size;
PIDX_set_point_5D(global_size, global_box_size[0], global_box_size[1], global_box_size[2], 1, 1);
PIDX_set_point_5D(local_offset, local_box_offset[0], local_box_offset[1], local_box_offset[2], 0, 0);
PIDX_set_point_5D(local_size, local_box_size[0], local_box_size[1], local_box_size[2], 1, 1);
// Creating access
PIDX_access access;
PIDX_create_access(&access);
#if PIDX_HAVE_MPI
PIDX_set_mpi_access(access, MPI_COMM_WORLD);
#endif
for (ts = 0; ts < time_step_count; ts++)
{
// PIDX mandatory calls
ret = PIDX_file_create(output_file_name, PIDX_MODE_CREATE, access, global_size, &file);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_file_create");
ret = PIDX_set_current_time_step(file, ts);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_set_current_time_step");
ret = PIDX_set_variable_count(file, variable_count);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_set_variable_count");
ret = PIDX_set_resolution(file, 0, reduced_resolution);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_set_resolution");
char var_name[512];
for (var = 0; var < variable_count; var++)
{
sprintf(var_name, "variable_%d", var);
ret = PIDX_variable_create(var_name, sizeof(unsigned long long) * 8, FLOAT64, &variable[var]);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_variable_create");
ret = PIDX_variable_write_data_layout(variable[var], local_offset, local_size, data[var], PIDX_row_major);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_variable_data_layout");
ret = PIDX_append_and_write_variable(file, variable[var]);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_append_and_write_variable");
}
ret = PIDX_close(file);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_close");
}
ret = PIDX_close_access(access);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_close_access");
free(variable);
variable = 0;
destroy_synthetic_simulation_data();
shutdown_mpi();
return 0;
}
int main(int argc, char **argv)
{
double start_time, end_time;
start_time = get_time();
int ret = 0;
init_mpi(argc, argv);
parse_args(argc, argv);
rank_0_print("Merge Program\n");
#if 0
comm = MPI_COMM_WORLD;
#endif
ret = IDX_file_open(output_file_name);
if (ret != 0) terminate_with_error_msg("PIDX_file_create");
maxh = strlen(bitSequence);
fprintf(stderr, "Partition size :: and count %d %d %d :: %d %d %d\n", idx_count[0], idx_count[1], idx_count[2], idx_size[0], idx_size[1], idx_size[2]);
fprintf(stderr, "bitstring %s maxh = %d\n", bitSequence, maxh);
// shared_block_level is the level upto which the idx blocks are shared
int shared_block_level = (int)log2(idx_count[0] * idx_count[1] * idx_count[2]) + bits_per_block + 1;
if (shared_block_level >= maxh)
shared_block_level = maxh;
int shared_block_count = pow(2, shared_block_level - 1) / samples_per_block;
fprintf(stderr, "Shared block level = %d Shared block count = %d\n", shared_block_level, shared_block_count);
int level = 0;
int ts = 0;
// Iteration through all the timesteps
for (ts = start_time_step; ts <= end_time_step; ts++)
{
// Iteration through all the shared blocks
//for (level = 0; level < shared_block_level; level = level + 1)
{
int hz_index = (int)pow(2, level - 1);
int file_no = hz_index / (blocks_per_file * samples_per_block);
int file_count;
char existing_file_name[PIDX_FILE_PATH_LENGTH];
char new_file_name[PIDX_FILE_PATH_LENGTH];
int ic = 0;
if (level <= bits_per_block + log2(blocks_per_file) + 1)
file_count = 1;
else
file_count = (int)pow(2, level - (bits_per_block + log2(blocks_per_file) + 1));
// file_no is the index of the file that needs to be opened to read from all the partitions
// they contain the shared blocks
fprintf(stderr, "Opening file %d\n", file_no);
#if 1
// iterate throuh all the files that contains the shared blocks
// most likely this will be only the first file of all the partitions
// so fc = 1
int fc = 0;
for (fc = file_no; fc < file_no + file_count; fc++)
{
// malloc for the header for the outpur blocks, i.e. the merged blocks
uint32_t* write_binheader;
int write_binheader_count;
write_binheader_count = 10 + 10 * blocks_per_file * variable_count;
int write_binheader_length = write_binheader_count * sizeof (*write_binheader);
write_binheader = malloc(write_binheader_length);
memset(write_binheader, 0, write_binheader_length);
//iterate through all the variables/fields
int var = 0;
off_t var_offset = 0;
for (var = 0; var < 1; var++)
{
unsigned char *write_data_buffer = malloc(samples_per_block * shared_block_count * bpv[var]/8);
memset(write_data_buffer, 0, samples_per_block * shared_block_count * bpv[var]/8);
//fprintf(stderr, "Write bufer size = %d [%d x %d x %d]\n", samples_per_block * shared_block_count * bpv[var]/8, (int)pow(2, bits_per_block), shared_block_count, bpv[var]/8);
// shared block data
// doube pointer (number o fpartitions x number of shared blocks)
unsigned char **read_data_buffer = malloc(idx_count[0] * idx_count[1] * idx_count[2] * sizeof(*read_data_buffer));
memset(read_data_buffer, 0, idx_count[0] * idx_count[1] * idx_count[2] * sizeof(*read_data_buffer));
// shared block header info
uint32_t** read_binheader = malloc(idx_count[0] * idx_count[1] * idx_count[2] * sizeof(*read_binheader));
memset(read_binheader, 0, idx_count[0] * idx_count[1] * idx_count[2] * sizeof(*read_binheader));
file_initialize_time_step(ts, output_file_name, output_file_template);
generate_file_name(blocks_per_file, output_file_template, fc, new_file_name, PATH_MAX);
//fprintf(stderr, "Merged blocks to be written in %s\n", new_file_name);
// iterate through all the parttions
for (ic = 0; ic < idx_count[0] * idx_count[1] * idx_count[2]; ic++)
{
char file_name_skeleton[PIDX_FILE_PATH_LENGTH];
strncpy(file_name_skeleton, output_file_name, strlen(output_file_name) - 4);
file_name_skeleton[strlen(output_file_name) - 4] = '\0';
if (idx_count[0] != 1 || idx_count[1] != 1 || idx_count[2] != 1)
sprintf(partition_file_name, "%s_%d.idx", file_name_skeleton, ic);
else
strcpy(partition_file_name, output_file_name);
file_initialize_time_step(ts, partition_file_name, partition_file_template);
generate_file_name(blocks_per_file, partition_file_template, fc, existing_file_name, PATH_MAX);
int read_binheader_count;
read_binheader_count = 10 + 10 * blocks_per_file * variable_count;
read_binheader[ic] = (uint32_t*) malloc(sizeof (*read_binheader[ic])*(read_binheader_count));
memset(read_binheader[ic], 0, sizeof (*(read_binheader[ic]))*(read_binheader_count));
fprintf(stderr, "[%d] Partition File name %s\n", ic, existing_file_name);
// file exists
if ( access( partition_file_name, F_OK ) != -1 )
{
// contins data from the shared blocks
read_data_buffer[ic] = malloc(samples_per_block * shared_block_count * bpv[var]/8);
memset(read_data_buffer[ic], 0, samples_per_block * shared_block_count * bpv[var]/8);
int fd;
fd = open(existing_file_name, O_RDONLY | O_BINARY);
if (fd < 0)
{
fprintf(stderr, "[File : %s] [Line : %d] open\n", __FILE__, __LINE__);
continue;
return 0;
}
// reads the header infor from binary file of the partitions
ret = read(fd, read_binheader[ic], (sizeof (*(read_binheader[ic])) * read_binheader_count));
if (ret < 0)
{
fprintf(stderr, "[File : %s] [Line : %d] read\n", __FILE__, __LINE__);
return 0;
}
//assert(ret == (sizeof (*(read_binheader[ic])) * read_binheader_count));
// copy the header from the partition file to the merged output file
// do it only for first partition (this gets all info other than block offset nd count)
if (ic == 0)
memcpy(write_binheader, read_binheader[ic], 10 * sizeof (*write_binheader));
int bpf = 0;
size_t data_size = 0;
off_t data_offset = 0;
for (bpf = 0; bpf < shared_block_count; bpf++)
{
data_offset = ntohl(read_binheader[ic][(bpf + var * blocks_per_file)*10 + 12]);
data_size = ntohl(read_binheader[ic][(bpf + var * blocks_per_file)*10 + 14]);
fprintf(stderr, "[%s] [Partition %d Block %d Variable %d] --> Offset %d Count %d\n", partition_file_name, ic, bpf, var, (int)data_offset, (int)data_size);
if (data_offset != 0 && data_size != 0)
{
pread(fd, read_data_buffer[ic] + (bpf * samples_per_block * (bpv[var] / 8)), data_size, data_offset);
write_binheader[((bpf + var * blocks_per_file)*10 + 12)] = htonl(write_binheader_length + (bpf * data_size) + var * shared_block_count);
write_binheader[((bpf + var * blocks_per_file)*10 + 14)] = htonl(data_size);
// Merge happening while the shared block is being read
// Hardcoded stupid merge
// checks if value is not zero then copies to the write block
int m = 0;
for (m = 0; m < data_size / (bpv[var] / 8) ; m++)
{
double temp;
memcpy(&temp, read_data_buffer[ic] + (bpf * samples_per_block + m) * sizeof(double), sizeof(double));
if (temp != 0)
memcpy(write_data_buffer + ((bpf * samples_per_block) + m) * sizeof(double), &temp, sizeof(double));
}
}
}
close(fd);
}
else
continue;
}
//Merge after all the reads
for (ic = 0; ic < idx_count[0] * idx_count[1] * idx_count[2]; ic++)
{
//input is read_data_buffer**
//output is write_data_buffer*
}
if ( access( new_file_name, F_OK ) != -1 )
{
// file exists
int fd;
fd = open(new_file_name, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR);
{
}
close(fd);
}
else
{
// file doesn't exist
/*
int r;
for (r = 0; r < (shared_block_count * samples_per_block * bpv[var]/8) / sizeof(double); r++)
{
double dval;
memcpy(&dval, write_data_buffer + r * sizeof(double), sizeof(double));
fprintf(stderr, "value at %d = %f\n", r, dval);
}
*/
int fd;
fd = open(new_file_name, O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR);
pwrite(fd, write_binheader, sizeof (*write_binheader)*(write_binheader_count), 0);
pwrite(fd, write_data_buffer, shared_block_count * samples_per_block * bpv[var]/8, sizeof (*write_binheader)*(write_binheader_count));
close(fd);
}
}
}
#endif
}
}
shutdown_mpi();
end_time = get_time();
fprintf(stderr, "Total time taken = %f %f\n", end_time, start_time);
return 0;
}
//----------------------------------------------------------------
int main(int argc, char **argv)
{
init_mpi(argc, argv);
printf("start %d %d\n",rank,process_count);
int i;
if(rank==0) {
parse_args(argc, argv);
// var_count = read_list_in_file(var_file, &netcdf_var_names);
// rank_0_print("Number of variables = %d\n", var_count);
// time_step_count = read_list_in_file(netcdf_file_list, &netcdf_file_names);
// rank_0_print("Number of timesteps = %d\n", time_step_count);
check_args();
}
// The command line arguments are shared by all processes
#if PIDX_HAVE_MPI
MPI_Bcast(global_box_size, 3, MPI_LONG_LONG, 0, MPI_COMM_WORLD);
MPI_Bcast(local_box_size, 3, MPI_LONG_LONG, 0, MPI_COMM_WORLD);
// MPI_Bcast(&time_step_count, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(&var_file, 512, MPI_CHAR, 0, MPI_COMM_WORLD);
MPI_Bcast(&netcdf_file_list, 512, MPI_CHAR, 0, MPI_COMM_WORLD);
// MPI_Bcast(&var_count, 1, MPI_INT, 0, MPI_COMM_WORLD);
MPI_Bcast(&output_file_name, 512, MPI_CHAR, 0, MPI_COMM_WORLD);
#endif
//TODO: Only the rank 0 should read the input files and broadcast the data
var_count = read_list_in_file(var_file, &netcdf_var_names);
rank_0_print("Number of variables = %d\n", var_count);
time_step_count = read_list_in_file(netcdf_file_list, &netcdf_file_names);
rank_0_print("Number of timesteps = %d\n", time_step_count);
calculate_per_process_offsets();
create_pidx_var_names();
PIDX_access pidx_access;
create_pidx_access(&pidx_access);
PIDX_time_step_caching_ON();
determine_var_types();
create_pidx_vars();
int t = 0,plist_id;
for (t = 0; t < time_step_count; ++t)
{
rank_0_print("Processing time step %d (file %s)\n", t, netcdf_file_names[t]);
PIDX_file pidx_file;
int ret = PIDX_file_create(output_file_name, PIDX_MODE_CREATE, pidx_access, &pidx_file);
if (ret != PIDX_success)
terminate_with_error_msg("ERROR: Failed to create PIDX file\n");
set_pidx_params(pidx_file);
PIDX_set_current_time_step(pidx_file, t);
int file_id = ncmpi_open(MPI_COMM_WORLD,netcdf_file_names[t], NC_NOWRITE, MPI_INFO_NULL, &plist_id);
if (file_id !=0)
terminate_with_error_msg("ERROR: Failed to open file %s\n", netcdf_file_names[t]);
int v = 0;
for(v = 0; v < var_count; ++v)
{
rank_0_print("Processing variable %s\n", netcdf_var_names[v]);
var_data = malloc(var_types[v].atomic_type * var_types[v].num_values * local_box_size[0] * local_box_size[1] * local_box_size[2]);
read_var_from_netcdf(plist_id, netcdf_var_names[v], var_types[v]);
write_var_to_idx(pidx_file, pidx_var_names[v], pidx_vars[v]);
if (PIDX_flush(pidx_file) != PIDX_success)
terminate_with_error_msg("ERROR: Failed to flush variable %s, time step %d\n", pidx_var_names[v], t);
free(var_data);
}
ncmpi_close(plist_id);
PIDX_close(pidx_file);
}
PIDX_time_step_caching_OFF();
PIDX_close_access(pidx_access);
free_memory();
shutdown_mpi();
return 0;
}
int main(int argc, char **argv)
{
init_mpi(argc, argv);
parse_args(argc, argv);
check_args();
calculate_per_process_offsets();
create_synthetic_simulation_data();
rank_0_print("Simulation Data Created\n");
int ret, var, ts;
PIDX_file file; // IDX file descriptor
PIDX_variable* variable; // variable descriptor
variable = (PIDX_variable*)malloc(sizeof(*variable) * variable_count);
memset(variable, 0, sizeof(*variable) * variable_count);
PIDX_point global_size, local_offset, local_size;
PIDX_set_point_5D(global_size, global_box_size[0], global_box_size[1], global_box_size[2], 1, 1);
PIDX_set_point_5D(local_offset, local_box_offset[0], local_box_offset[1], local_box_offset[2], 0, 0);
PIDX_set_point_5D(local_size, local_box_size[0], local_box_size[1], local_box_size[2], 1, 1);
// Creating access
PIDX_access access;
PIDX_create_access(&access);
#if PIDX_HAVE_MPI
PIDX_set_mpi_access(access, MPI_COMM_WORLD);
#endif
for (ts = 0; ts < time_step_count; ts++)
{
// PIDX mandatory calls
ret = PIDX_file_create(output_file_name, PIDX_MODE_CREATE, access, global_size, &file);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_file_create");
ret = PIDX_set_current_time_step(file, ts);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_set_current_time_step");
ret = PIDX_set_variable_count(file, variable_count);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_set_variable_count");
PIDX_disable_agg(file);
PIDX_save_big_endian(file);
//PIDX_dump_rst_info(file, 1);
//PIDX_debug_rst(file, 1);
//PIDX_debug_hz(file, 1);
PIDX_point reg_patch_size;
PIDX_set_point_5D(reg_patch_size, 128, 128, 128, 1, 1);
PIDX_set_restructuring_box(file, reg_patch_size);
//PIDX_GLOBAL_PARTITION_IDX_IO
//PIDX_IDX_IO
ret = PIDX_set_io_mode(file, PIDX_RAW_IO);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_set_variable_count");
ret = PIDX_set_partition_size(file, partition_size[0], partition_size[1], partition_size[2]);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_set_partition_size");
PIDX_set_block_count(file, 128);
//ret = PIDX_set_aggregator_multiplier(file, aggregator_multiplier);
//if (ret != PIDX_success) terminate_with_error_msg("PIDX_set_partition_size");
/*
int io_type = PIDX_IDX_IO;
switch (io_type)
{
case PIDX_GLOBAL_PARTITION_IDX_IO:
PIDX_set_block_count(file,blocks_per_file);
PIDX_set_block_size(file, 13);
break;
case PIDX_IDX_IO:
PIDX_set_block_count(file,blocks_per_file);
break;
case PIDX_RAW_IO:
PIDX_raw_io_pipe_length(file, 2);
PIDX_point reg_patch_size;
PIDX_set_point_5D(reg_patch_size, 128, 128, 128, 1, 1);
PIDX_set_restructuring_box(file, reg_patch_size);
break;
}
*/
//ret = PIDX_debug_disable_agg(file);
//if (ret != PIDX_success) terminate_with_error_msg("PIDX_debug_output");
//ret = PIDX_debug_disable_io(file);
//if (ret != PIDX_success) terminate_with_error_msg("PIDX_debug_output");
//ret = PIDX_debug_disable_hz(file);
//if (ret != PIDX_success) terminate_with_error_msg("PIDX_debug_output");
for (var = 0; var < variable_count; var++)
{
if (bpv[var] == 32)
{
ret = PIDX_variable_create(var_name[var], bpv[var], FLOAT32 , &variable[var]);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_variable_create");
}
else if (bpv[var] == 192)
{
ret = PIDX_variable_create(var_name[var], bpv[var], FLOAT64_RGB , &variable[var]);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_variable_create");
}
else if (bpv[var] == 64)
{
ret = PIDX_variable_create(var_name[var], bpv[var], FLOAT64 , &variable[var]);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_variable_create");
}
ret = PIDX_variable_write_data_layout(variable[var], local_offset, local_size, data[var], PIDX_row_major);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_variable_data_layout");
ret = PIDX_append_and_write_variable(file, variable[var]);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_append_and_write_variable");
//PIDX_flush(file);
}
ret = PIDX_close(file);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_close");
}
ret = PIDX_close_access(access);
if (ret != PIDX_success) terminate_with_error_msg("PIDX_close_access");
free(variable);
variable = 0;
destroy_synthetic_simulation_data();
shutdown_mpi();
return 0;
}
