Example #1
0
int main(int argc, char **argv)
{
  int ts = 0, var = 0;
  init_mpi(argc, argv);
  parse_args(argc, argv);
  check_args();
  calculate_per_process_offsets();
  create_synthetic_simulation_data();

  rank_0_print("Simulation Data Created\n");

  create_pidx_var_point_and_access();
  for (ts = 0; ts < time_step_count; ts++)
  {
    set_pidx_file(ts);
    for (var = 0; var < variable_count; var++)
      set_pidx_variable(var);
    PIDX_close(file);
  }
  destroy_pidx_var_point_and_access();

  destroy_synthetic_simulation_data();
  shutdown_mpi();

  return 0;
}
Example #2
0
/// main
int main(int argc, char **argv)
{
  init_mpi(argc, argv);
  parse_args(argc, argv);
  check_args();
  calculate_per_process_offsets();
  create_synthetic_simulation_data();

  int var, p;
  PIDX_point global_bounding_box, **local_offset_point, **local_box_count_point;

  local_offset_point = malloc(sizeof(PIDX_point*) * variable_count);
  local_box_count_point = malloc(sizeof(PIDX_point*) * variable_count);

  for(var = 0; var < variable_count; var++)
  {
    local_offset_point[var] = malloc(sizeof(PIDX_point) * patch_count);
    local_box_count_point[var] = malloc(sizeof(PIDX_point) * patch_count);
    for(p = 0 ; p < patch_count ; p++)
    {
      PIDX_set_point_5D(local_offset_point[var][p], (int64_t)var_offset[var][p][0], (int64_t)var_offset[var][p][1], (int64_t)var_offset[var][p][2], 0, 0);
      PIDX_set_point_5D(local_box_count_point[var][p], (int64_t)var_count[var][p][0], (int64_t)var_count[var][p][1], (int64_t)var_count[var][p][2], 1, 1);
    }
  }
  PIDX_file file;            // IDX file descriptor
  PIDX_variable* variable;   // variable descriptor

  variable = malloc(sizeof(*variable) * variable_count);
  memset(variable, 0, sizeof(*variable) * variable_count);

  PIDX_set_point_5D(global_bounding_box, (int64_t)global_box_size[0], (int64_t)global_box_size[1], (int64_t)global_box_size[2], 1, 1);

  PIDX_access access;
  PIDX_create_access(&access);

#if PIDX_HAVE_MPI
  PIDX_set_mpi_access(access, MPI_COMM_WORLD);
#endif

  int ts;
  for (ts = 0; ts < time_step_count; ts++)
  {
    PIDX_file_create(output_file_name, PIDX_MODE_CREATE, access, &file);
    PIDX_set_dims(file, global_bounding_box);
    PIDX_set_current_time_step(file, ts);
    PIDX_set_variable_count(file, variable_count);

    PIDX_debug_rst(file, 1);
    PIDX_debug_hz(file, 1);

    for (var = 0; var < variable_count; var++)
    {
      char variable_name[512];
      sprintf(variable_name, "variable_%d", var);
      PIDX_variable_create(variable_name, sizeof(double) * 8, FLOAT64, &variable[var]);

      for (p = 0 ; p < patch_count ; p++)
        PIDX_variable_write_data_layout(variable[var], local_offset_point[var][p], local_box_count_point[var][p], double_data[var][p], PIDX_row_major);

      PIDX_append_and_write_variable(file, variable[var]/*, local_offset_point[var][p], local_box_count_point[var][p], double_data[var][p], PIDX_row_major*/);
    }

    PIDX_close(file);
  }
  PIDX_close_access(access);

  destroy_synthetic_simulation_data();

  for(var = 0; var < variable_count; var++)
  {
    free(local_offset_point[var]);
    free(local_box_count_point[var]);
  }
  free(local_offset_point);
  free(local_box_count_point);

  free(variable);
  variable = 0;

  shutdown_mpi();

  return 0;
}
Example #3
0
int main(int argc, char **argv)
{
  int ret;
  int i, j, k;
  int var, vps;
  int slice = 0;
  int nprocs = 1, rank = 0;
  char output_file_name[512];

  // MPI initialization
#if PIDX_HAVE_MPI
  MPI_Init(&argc, &argv);
  MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
#endif

  double **data;             // variable data buffer
  int *values_per_sample;    // Example: 1 for scalar 3 for vector

  int local_box_offset[3];

  if (rank == 0)
  {
    ret = parse_args(argc, argv);
    if (ret < 0)
    {
      usage();
#if PIDX_HAVE_MPI
      MPI_Abort(MPI_COMM_WORLD, -1);
#else
      exit(-1);
#endif
    }

    // check if the num procs is appropriate
    int num_bricks = (global_box_size[0] / local_box_size[0]) * (global_box_size[1] / local_box_size[1]) * (global_box_size[2] / local_box_size[2]);
    if(num_bricks != nprocs)
    {
      fprintf(stderr, "Error: number of sub-blocks (%d) doesn't match number of procs (%d)\n", num_bricks, nprocs);
      fprintf(stderr, "Incorrect distribution of data across processes i.e.\n(global_x / local_x) X (global_x / local_x) X (global_x / local_x) != nprocs\n(%d/%d) X (%d/%d) X (%d/%d) != %d\n", global_box_size[0], local_box_size[0], global_box_size[1], local_box_size[1], global_box_size[2], local_box_size[2], nprocs);

#if PIDX_HAVE_MPI
      MPI_Abort(MPI_COMM_WORLD, -1);
#else
      exit(-1);
#endif
    }
  }

  //  The command line arguments are shared by all processes
#if PIDX_HAVE_MPI
  MPI_Bcast(global_box_size, 3, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(local_box_size, 3, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(&time_step_count, 1, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(&output_file_template, 512, MPI_CHAR, 0, MPI_COMM_WORLD);
#endif

  // Creating the filename
  sprintf(output_file_name, "%s%s", output_file_template,".idx");

  // Calculating every process data's offset and size
  int sub_div[3];
  sub_div[0] = (global_box_size[0] / local_box_size[0]);
  sub_div[1] = (global_box_size[1] / local_box_size[1]);
  sub_div[2] = (global_box_size[2] / local_box_size[2]);
  local_box_offset[2] = (rank / (sub_div[0] * sub_div[1])) * local_box_size[2];
  slice = rank % (sub_div[0] * sub_div[1]);
  local_box_offset[1] = (slice / sub_div[0]) * local_box_size[1];
  local_box_offset[0] = (slice % sub_div[0]) * local_box_size[0];

  PIDX_file file;            // IDX file descriptor
  PIDX_variable* variable;   // variable descriptor

  PIDX_point global_size, local_offset, local_size;
  PIDX_set_point_5D(global_size, global_box_size[0], global_box_size[1], global_box_size[2], 1, 1);
  PIDX_set_point_5D(local_offset, local_box_offset[0], local_box_offset[1], local_box_offset[2], 0, 0);
  PIDX_set_point_5D(local_size, local_box_size[0], local_box_size[1], local_box_size[2], 1, 1);

  //  Creating access
  PIDX_access access;
  PIDX_create_access(&access);
#if PIDX_HAVE_MPI
  PIDX_set_mpi_access(access, MPI_COMM_WORLD);
#endif

  //  PIDX mandatory calls
  ret = PIDX_file_open(output_file_name, PIDX_MODE_RDONLY, access, &file);
  if (ret != PIDX_success)  report_error("PIDX_file_create", __FILE__, __LINE__);

  ret = PIDX_get_dims(file, global_size);
  if (ret != PIDX_success)  report_error("PIDX_set_dims", __FILE__, __LINE__);

  ret = PIDX_get_variable_count(file, &variable_count);
  if (ret != PIDX_success)  report_error("PIDX_set_variable_count", __FILE__, __LINE__);

  ret = PIDX_set_current_time_step(file, 0);
  if (ret != PIDX_success)  report_error("PIDX_set_current_time_step", __FILE__, __LINE__);

  values_per_sample = malloc(sizeof(*values_per_sample) * variable_count);
  memset(values_per_sample, 0, sizeof(*values_per_sample) * variable_count);

  variable = malloc(sizeof(*variable) * variable_count);
  memset(variable, 0, sizeof(*variable) * variable_count);

  data = (double**)malloc(sizeof(*data) * variable_count);
  memset(data, 0, sizeof(*data) * variable_count);

  //printf("Variable Count %d\n", variable_count);
  for (var = 0; var < variable_count; var++)
  {
    ret = PIDX_get_next_variable(file, &variable[var]);
    if (ret != PIDX_success)  report_error("PIDX_get_next_variable", __FILE__, __LINE__);

    values_per_sample[var] = variable[var]->values_per_sample;

    int bits_per_sample = 0;
    ret = PIDX_default_bits_per_datatype(variable[var]->type_name, &bits_per_sample);
    if (ret != PIDX_success)  report_error("PIDX_default_bytes_per_datatype", __FILE__, __LINE__);

    data[var] = malloc((bits_per_sample/8) * local_box_size[0] * local_box_size[1] * local_box_size[2]  * variable[var]->values_per_sample);
    memset(data[var], 0, (bits_per_sample/8) * local_box_size[0] * local_box_size[1] * local_box_size[2]  * variable[var]->values_per_sample);

    ret = PIDX_read_next_variable(file, variable[var]);
    if (ret != PIDX_success)  report_error("PIDX_read_next_variable", __FILE__, __LINE__);
  }

  ret = PIDX_reset_variable_counter(file);
  if (ret != PIDX_success)  report_error("PIDX_reset_variable_counter", __FILE__, __LINE__);

  for (var = 0; var < variable_count; var++)
  {
    ret = PIDX_get_next_variable(file, &variable[var]);
    if (ret != PIDX_success)  report_error("PIDX_get_next_variable", __FILE__, __LINE__);

    ret = PIDX_variable_read_data_layout(variable[var], local_offset, local_size, data[var], PIDX_row_major);
    if (ret != PIDX_success)  report_error("PIDX_variable_read_data_layout", __FILE__, __LINE__);

    ret = PIDX_read_next_variable(file, variable[var]);
    if (ret != PIDX_success)  report_error("PIDX_read_next_variable", __FILE__, __LINE__);
  }

  ret = PIDX_close(file);
  if (ret != PIDX_success)  report_error("PIDX_close", __FILE__, __LINE__);

   ret = PIDX_close_access(access);
  if (ret != PIDX_success)  report_error("PIDX_close_access", __FILE__, __LINE__);

  int read_error_count = 0, read_count = 0;
  for(var = 0; var < variable_count; var++)
  {
    for (k = 0; k < local_box_size[2]; k++)
      for (j = 0; j < local_box_size[1]; j++)
        for (i = 0; i < local_box_size[0]; i++)
        {
          int64_t index = (int64_t) (local_box_size[0] * local_box_size[1] * k) + (local_box_size[0] * j) + i;
          for (vps = 0; vps < values_per_sample[var]; vps++)
            if (data[var][index * values_per_sample[var] + vps] != 100 + var + vps + ((global_box_size[0] * global_box_size[1]*(local_box_offset[2] + k))+(global_box_size[0]*(local_box_offset[1] + j)) + (local_box_offset[0] + i)))
            {
              read_error_count++;
              if (rank == 0)
                printf("[%d %d %d] Read error %f %d\n", i,j ,k, data[var][index * values_per_sample[var] + vps], 100 + var + vps + ((global_box_size[0] * global_box_size[1]*(local_box_offset[2] + k))+(global_box_size[0]*(local_box_offset[1] + j)) + (local_box_offset[0] + i)));
            }
            else
              read_count++;
        }
  }
  int var_sample_count = 0;
  for(var = 0; var < variable_count; var++)
    var_sample_count = var_sample_count + values_per_sample[var];

  printf("Read Error Count + Right Count : [%d + %d] Total Count = %d\n", read_error_count, read_count, global_box_size[0]*global_box_size[1]*global_box_size[2] * var_sample_count);

  for(var = 0; var < variable_count; var++)
  {
    free(data[var]);
    data[var] = 0;
  }
  free(data);
  data = 0;

  free(values_per_sample);

  //  MPI finalize
#if PIDX_HAVE_MPI
  MPI_Finalize();
#endif

  return 0;
}
Example #4
0
int main(int argc, char **argv)
{
  int ret;
  int p, ts;
  int var;
  int slice = 0;
  int nprocs = 1, rank = 0;
  char output_file_name[512];

  // MPI initialization
#if PIDX_HAVE_MPI
  MPI_Init(&argc, &argv);
  MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
#endif

  if (rank == 0)
  {
    ret = parse_args(argc, argv);
    if (ret < 0)
    {
      usage();
#if PIDX_HAVE_MPI
      MPI_Abort(MPI_COMM_WORLD, -1);
#else
      exit(-1);
#endif
    }

    // check if the num procs is appropriate
    int num_bricks = (global_box_size[0] / local_box_size[0]) * (global_box_size[1] / local_box_size[1]) * (global_box_size[2] / local_box_size[2]);
    if(num_bricks != nprocs)
    {
      fprintf(stderr, "Error: number of sub-blocks (%d) doesn't match number of procs (%d)\n", num_bricks, nprocs);
      fprintf(stderr, "Incorrect distribution of data across processes i.e.\n(global_x / local_x) X (global_x / local_x) X (global_x / local_x) != nprocs\n(%d/%d) X (%d/%d) X (%d/%d) != %d\n", global_box_size[0], local_box_size[0], global_box_size[1], local_box_size[1], global_box_size[2], local_box_size[2], nprocs);

#if PIDX_HAVE_MPI
      MPI_Abort(MPI_COMM_WORLD, -1);
#else
      exit(-1);
#endif
    }
  }

#if 1
  //  The command line arguments are shared by all processes
#if PIDX_HAVE_MPI
  MPI_Bcast(global_box_size, 3, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(local_box_size, 3, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(&time_step_count, 1, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(&variable_count, 1, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(&patch_count, 1, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(&output_file_template, 512, MPI_CHAR, 0, MPI_COMM_WORLD);
  MPI_Bcast(&blocks_per_file, 1, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(&bits_per_block, 1, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(restructured_box_size, 5, MPI_LONG_LONG, 0, MPI_COMM_WORLD);
  MPI_Bcast(&compression_type, 1, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(&compression_bit_rate, 1, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(perform_phases, 6, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(debug_rst_hz, 2, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(dump_agg_io, 2, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(idx_count, 3, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(&aggregation_factor, 1, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(&is_rank_z_ordering, 1, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(hz_from_to, 2, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(rst_agg, 2, MPI_INT, 0, MPI_COMM_WORLD);
#endif

  values_per_sample = malloc(sizeof(*values_per_sample) * variable_count);
  memset(values_per_sample, 0, sizeof(*values_per_sample) * variable_count);

  // Creating the filename
  sprintf(output_file_name, "%s%s", output_file_template,".idx");

  // Calculating every process data's offset and size
  int sub_div[3];
  sub_div[0] = (global_box_size[0] / local_box_size[0]);
  sub_div[1] = (global_box_size[1] / local_box_size[1]);
  sub_div[2] = (global_box_size[2] / local_box_size[2]);
  local_box_offset[2] = (rank / (sub_div[0] * sub_div[1])) * local_box_size[2];
  slice = rank % (sub_div[0] * sub_div[1]);
  local_box_offset[1] = (slice / sub_div[0]) * local_box_size[1];
  local_box_offset[0] = (slice % sub_div[0]) * local_box_size[0];

  unsigned int rank_x = 0, rank_y = 0, rank_z = 0, rank_slice;
  if (is_rank_z_ordering == 0)
  {
    rank_z = rank / (sub_div[0] * sub_div[1]);
    rank_slice = rank % (sub_div[0] * sub_div[1]);
    rank_y = (rank_slice / sub_div[0]);
    rank_x = (rank_slice % sub_div[0]);
  }

  create_patches();

  PIDX_file file;            // IDX file descriptor
  PIDX_variable* variable;   // variable descriptor

  variable = malloc(sizeof(*variable) * variable_count);
  memset(variable, 0, sizeof(*variable) * variable_count);

  PIDX_point global_bounding_box, **local_offset_point, **local_box_count_point;

  local_offset_point = malloc(sizeof(PIDX_point*) * variable_count);
  local_box_count_point = malloc(sizeof(PIDX_point*) * variable_count);
  for(var = 0; var < variable_count; var++)
  {
    local_offset_point[var] = malloc(sizeof(PIDX_point) * patch_count);
    local_box_count_point[var] = malloc(sizeof(PIDX_point) * patch_count);
    for(p = 0 ; p < patch_count ; p++)
    {
      PIDX_set_point_5D(local_offset_point[var][p], (int64_t)local_patch_offset[var][p][0], (int64_t)local_patch_offset[var][p][1], (int64_t)local_patch_offset[var][p][2], 0, 0);
      PIDX_set_point_5D(local_box_count_point[var][p], (int64_t)local_patch_size[var][p][0], (int64_t)local_patch_size[var][p][1], (int64_t)local_patch_size[var][p][2], 1, 1);
    }
  }
  PIDX_set_point_5D(global_bounding_box, (int64_t)global_box_size[0], (int64_t)global_box_size[1], (int64_t)global_box_size[2], 1, 1);

  PIDX_access access;
  PIDX_create_access(&access);

#if PIDX_HAVE_MPI
  PIDX_set_mpi_access(access, MPI_COMM_WORLD);
  PIDX_set_idx_count(access, idx_count[0], idx_count[1], idx_count[2]);
  PIDX_set_process_extent(access, sub_div[0], sub_div[1], sub_div[2]);
  PIDX_set_process_rank_decomposition(access, rank_x, rank_y, rank_z);
#endif

  for (ts = 0; ts < time_step_count; ts++)
  {
    PIDX_file_create(output_file_name, PIDX_MODE_CREATE, access, &file);
    PIDX_set_dims(file, global_bounding_box);
    PIDX_set_current_time_step(file, ts);
    PIDX_set_variable_count(file, variable_count);

    PIDX_set_resolution(file, hz_from_to[0], hz_from_to[1]);
    PIDX_set_block_size(file, bits_per_block);
    PIDX_set_block_count(file, blocks_per_file);

    // PIDX set restructuring box size
    PIDX_set_restructuring_box(file, restructured_box_size);

    // PIDX compression related calls
    if (compression_type == 0)
      PIDX_set_compression_type(file, PIDX_NO_COMPRESSION);
    if (compression_type == 1)
      PIDX_set_compression_type(file, PIDX_CHUNKING_ONLY);
    if (compression_type == 2)
    {
      PIDX_set_compression_type(file, PIDX_CHUNKING_ZFP);
      PIDX_set_lossy_compression_bit_rate(file, compression_bit_rate);
    }

    PIDX_debug_rst(file, debug_rst_hz[0]);
    PIDX_debug_hz(file, debug_rst_hz[1]);
    PIDX_dump_agg_info(file, dump_agg_io[0]);
    PIDX_dump_io_info(file, dump_agg_io[1]);

    if (perform_phases[0] == 0)
      PIDX_debug_disable_restructuring(file);
    if (perform_phases[1] == 0)
      PIDX_debug_disable_chunking(file);
    if (perform_phases[2] == 0)
      PIDX_debug_disable_hz(file);
    if (perform_phases[3] == 0)
      PIDX_debug_disable_compression(file);
    if (perform_phases[4] == 0)
      PIDX_debug_disable_agg(file);
    if (perform_phases[5] == 0)
      PIDX_debug_disable_io(file);

    if (rst_agg[0] == 0)
      PIDX_disable_rst(file);
    if (rst_agg[1] == 0)
      PIDX_disable_agg(file);

    for (var = 0; var < variable_count; var++)
    {
      char variable_name[512];
      char data_type[512];
      sprintf(variable_name, "variable_%d", var);
      sprintf(data_type, "%d*float64", values_per_sample[var]);

      ret = PIDX_variable_create(variable_name, sizeof(double) * 8, data_type, &variable[var]);
      if (ret != PIDX_success)  report_error("PIDX_variable_create", __FILE__, __LINE__);

      for (p = 0 ; p < patch_count ; p++)
      {
        ret = PIDX_variable_write_data_layout(variable[var], local_offset_point[var][p], local_box_count_point[var][p], NULL, PIDX_row_major);
        if (ret != PIDX_success)  report_error("PIDX_variable_data_layout", __FILE__, __LINE__);
      }

      ret = PIDX_append_and_write_variable(file, variable[var]);
      if (ret != PIDX_success)  report_error("PIDX_append_and_write_variable", __FILE__, __LINE__);

      ret = PIDX_flush(file);
    }
    ret = PIDX_close(file);
    if (ret != PIDX_success)  report_error("PIDX_close", __FILE__, __LINE__);
  }
  PIDX_close_access(access);

  for(var = 0; var < variable_count; var++)
  {
    free(local_offset_point[var]);
    free(local_box_count_point[var]);
  }
  free(local_offset_point);
  free(local_box_count_point);

  free(variable);
  variable = 0;
  free(values_per_sample);
  values_per_sample = 0;

  destroy_patches();
#endif

#if PIDX_HAVE_MPI
  MPI_Finalize();
#endif

  return 0;
}
int test_multi_patch_writer(struct Args args, int rank, int nprocs) 
{
#if PIDX_HAVE_MPI
  int i = 0, j = 0, k = 0, u = 0, v = 0, p = 0, var, d;
  int spv = 0;
  int ts;
  int slice;
  int sub_div[5], offset_local[5];

  PIDX_file file;                                                // IDX file descriptor
  const char *output_file;                                                      // IDX File Name
  const int bits_per_block = 15;                                                // Total number of samples in each block = 2 ^ bits_per_block
  const int blocks_per_file = 32;                                               // Total number of blocks per file
  
  PIDX_variable *variable;                                       // variable descriptor
  int *values_per_sample;                                                       // values per variable (example, scalar=1, vector=3)
  int *var_patch_count;                                                         // Number of patches a variable has (data blocks per variable that needs to be written to).
  
  int      ***var_count;                                                              // Local extents of the variables in each process
  int      ***var_offset;                                                              // Local counts of the variables in each process
  
  double   ***var_double_scalar_data;
  
  //The command line arguments are shared by all processes
  MPI_Bcast(args.extents, 5, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(args.count_local, 5, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(&args.time_step, 1, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(&args.variable_count, 1, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(&args.output_file_template, 512, MPI_CHAR, 0, MPI_COMM_WORLD);
  
  variable = malloc(sizeof (PIDX_variable) * args.variable_count);        // variable pointers
  memset(variable, 0, sizeof (PIDX_variable) * args.variable_count);
  
  values_per_sample = (int*) malloc(sizeof (int) * args.variable_count);                  
  var_patch_count = (int*) malloc(sizeof (int) * args.variable_count);                  
  for (var = 0; var < args.variable_count; var++)
  {
    values_per_sample[var] = 1;
    /*
    if(var % 4 == 3)
      var_patch_count[var] = 1;
    else if(var % 4 == 2)
      var_patch_count[var] = 2;
    else if(var % 4 == 1)
      var_patch_count[var] = 4;
    else if(var % 4 == 0)
    */
    var_patch_count[var] = 8;
  }
  
  
  //   Creating the filename 
  args.output_file_name = (char*) malloc(sizeof (char) * 512);
  sprintf(args.output_file_name, "%s%s", args.output_file_template, ".idx");

  //   Calculating every process's offset and count  
  sub_div[0] = (args.extents[0] / args.count_local[0]);
  sub_div[1] = (args.extents[1] / args.count_local[1]);
  sub_div[2] = (args.extents[2] / args.count_local[2]);
  offset_local[2] = (rank / (sub_div[0] * sub_div[1])) * args.count_local[2];
  slice = rank % (sub_div[0] * sub_div[1]);
  offset_local[1] = (slice / sub_div[0]) * args.count_local[1];
  offset_local[0] = (slice % sub_div[0]) * args.count_local[0];

  offset_local[3] = 0;
  offset_local[4] = 0;
  args.count_local[3] = 1;
  args.count_local[4] = 1;

  var_count = malloc(sizeof(int**) * args.variable_count);
  var_offset = malloc(sizeof(int**) * args.variable_count);
  
  for(var = 0; var < args.variable_count; var++)
  {
    var_count[var] = malloc(sizeof(int*) * var_patch_count[var]);
    var_offset[var] = malloc(sizeof(int*) * var_patch_count[var]);
    for(i = 0; i < var_patch_count[var] ; i++)
    {
      var_count[var][i] = malloc(sizeof(int) * PIDX_MAX_DIMENSIONS);
      var_offset[var][i] = malloc(sizeof(int) * PIDX_MAX_DIMENSIONS);    
    }
  }
  
  for(var = 0; var < args.variable_count; var++)
  {
    /*
    if(var % 4 == 3)                                                       // One patch for this variable
    {
      for(d = 0; d < PIDX_MAX_DIMENSIONS; d++)
      {
	var_count[var][0][d] = args.count_local[d];
	var_offset[var][0][d] = offset_local[d];
      }
    }
    else if(var % 4 == 2)                                                  // two patches for this variable
    {
      for(d = 0; d < PIDX_MAX_DIMENSIONS; d++)
      {
	var_count[var][0][d] = args.count_local[d];
	var_offset[var][0][d] = offset_local[d];
	var_count[var][1][d] = args.count_local[d];
	var_offset[var][1][d] = offset_local[d];
      }
      
      var_count[var][0][0] = args.count_local[0]/2;
      if(args.count_local[0] % 2 == 0)
	var_count[var][1][0] = args.count_local[0]/2;
      else
	var_count[var][1][0] = args.count_local[0]/2 + 1;
      
      var_offset[var][1][0] = offset_local[0] + args.count_local[0]/2;
    }
    else if(var % 4 == 1)
    {
      for(i = 0; i < var_patch_count[var]; i++)
      {
	for(d = 0; d < PIDX_MAX_DIMENSIONS; d++)
	{
	  var_count[var][i][d] = args.count_local[d];
	  var_offset[var][i][d] = offset_local[d];
	}
      }
      var_count[var][0][0] = args.count_local[0]/2;
      var_count[var][0][1] = args.count_local[1]/2;
      
      var_count[var][1][1] = args.count_local[1]/2;
      if(args.count_local[0] % 2 == 0)
      {
	var_count[var][1][0] = args.count_local[0]/2;
	var_count[var][3][0] = args.count_local[0]/2;
	var_offset[var][1][0] = var_offset[var][0][0] + args.count_local[0]/2;
	var_offset[var][3][0] = var_offset[var][0][0] + args.count_local[0]/2;
      }
      else
      {
	var_count[var][1][0] = args.count_local[0]/2 + 1;
	var_count[var][3][0] = args.count_local[0]/2 + 1;
	var_offset[var][1][0] = var_offset[var][0][0] + args.count_local[0]/2;
	var_offset[var][3][0] = var_offset[var][0][0] + args.count_local[0]/2;
      }
      
      var_count[var][2][0] = args.count_local[0]/2;
      if(args.count_local[1] % 2 == 0)
      {
	var_count[var][2][1] = args.count_local[1]/2;
	var_count[var][3][1] = args.count_local[1]/2;
	var_offset[var][2][1] = var_offset[var][0][1] + args.count_local[1]/2;
	var_offset[var][3][1] = var_offset[var][0][1] + args.count_local[1]/2;
      }
      else
      {
	var_count[var][2][1] = args.count_local[1]/2 + 1;
	var_count[var][3][1] = args.count_local[1]/2 + 1;
	var_offset[var][2][1] = var_offset[var][0][1] + args.count_local[1]/2;
	var_offset[var][3][1] = var_offset[var][0][1] + args.count_local[1]/2;
      }
    }
    */
    //else if(var % 4 == 0)
    //{
      for(i = 0; i < var_patch_count[var]; i++)
      {
	for(d = 0; d < PIDX_MAX_DIMENSIONS; d++)
	{
	  var_count[var][i][d] = args.count_local[d];
	  var_offset[var][i][d] = offset_local[d];
	}
      }
      var_count[var][0][0] = args.count_local[0]/2;
      var_count[var][0][1] = args.count_local[1]/2;
      
      var_count[var][4][0] = args.count_local[0]/2;
      var_count[var][4][1] = args.count_local[1]/2;
      
      var_count[var][1][1] = args.count_local[1]/2;
      var_count[var][5][1] = args.count_local[1]/2;
      if(args.count_local[0] % 2 == 0)
      {
	var_count[var][1][0] = args.count_local[0]/2;
	var_count[var][3][0] = args.count_local[0]/2;
	var_offset[var][1][0] = var_offset[var][0][0] + args.count_local[0]/2;
	var_offset[var][3][0] = var_offset[var][0][0] + args.count_local[0]/2;
	
	var_count[var][5][0] = args.count_local[0]/2;
	var_count[var][7][0] = args.count_local[0]/2;
	var_offset[var][5][0] = var_offset[var][0][0] + args.count_local[0]/2;
	var_offset[var][7][0] = var_offset[var][0][0] + args.count_local[0]/2;
      }
      else
      {
	var_count[var][1][0] = args.count_local[0]/2 + 1;
	var_count[var][3][0] = args.count_local[0]/2 + 1;
	var_offset[var][1][0] = var_offset[var][0][0] + args.count_local[0]/2;
	var_offset[var][3][0] = var_offset[var][0][0] + args.count_local[0]/2;
	
	var_count[var][5][0] = args.count_local[0]/2 + 1;
	var_count[var][7][0] = args.count_local[0]/2 + 1;
	var_offset[var][5][0] = var_offset[var][0][0] + args.count_local[0]/2;
	var_offset[var][7][0] = var_offset[var][0][0] + args.count_local[0]/2;
      }
      
      var_count[var][2][0] = args.count_local[0]/2;
      var_count[var][6][0] = args.count_local[0]/2;
      
      if(args.count_local[1] % 2 == 0)
      {
	var_count[var][2][1] = args.count_local[1]/2;
	var_count[var][3][1] = args.count_local[1]/2;
	var_offset[var][2][1] = var_offset[var][0][1] + args.count_local[1]/2;
	var_offset[var][3][1] = var_offset[var][0][1] + args.count_local[1]/2;
	
	var_count[var][6][1] = args.count_local[1]/2;
	var_count[var][7][1] = args.count_local[1]/2;
	var_offset[var][6][1] = var_offset[var][0][1] + args.count_local[1]/2;
	var_offset[var][7][1] = var_offset[var][0][1] + args.count_local[1]/2;
      }
      else
      {
	var_count[var][2][1] = args.count_local[1]/2 + 1;
	var_count[var][3][1] = args.count_local[1]/2 + 1;
	var_offset[var][2][1] = var_offset[var][0][1] + args.count_local[1]/2;
	var_offset[var][3][1] = var_offset[var][0][1] + args.count_local[1]/2;
	
	var_count[var][6][1] = args.count_local[1]/2 + 1;
	var_count[var][7][1] = args.count_local[1]/2 + 1;
	var_offset[var][6][1] = var_offset[var][0][1] + args.count_local[1]/2;
	var_offset[var][7][1] = var_offset[var][0][1] + args.count_local[1]/2;
      }
      
      var_count[var][0][2] = args.count_local[2]/2;
      var_count[var][1][2] = args.count_local[2]/2;
      var_count[var][2][2] = args.count_local[2]/2;
      var_count[var][3][2] = args.count_local[2]/2;
      if(args.count_local[1] % 2 == 0)
      {
	var_count[var][4][2] = args.count_local[2]/2;
	var_count[var][5][2] = args.count_local[2]/2;
	var_count[var][6][2] = args.count_local[2]/2;
	var_count[var][7][2] = args.count_local[2]/2;
	
	var_offset[var][4][2] = var_offset[var][0][2] + args.count_local[2]/2;
	var_offset[var][5][2] = var_offset[var][1][2] + args.count_local[2]/2;
	var_offset[var][6][2] = var_offset[var][2][2] + args.count_local[2]/2;
	var_offset[var][7][2] = var_offset[var][3][2] + args.count_local[2]/2;
      }
      else
      {
	var_count[var][4][2] = args.count_local[2]/2 + 1;
	var_count[var][5][2] = args.count_local[2]/2 + 1;
	var_count[var][6][2] = args.count_local[2]/2 + 1;
	var_count[var][7][2] = args.count_local[2]/2 + 1;
	
	var_offset[var][4][2] = var_offset[var][0][2] + args.count_local[2]/2;
	var_offset[var][5][2] = var_offset[var][1][2] + args.count_local[2]/2;
	var_offset[var][6][2] = var_offset[var][2][2] + args.count_local[2]/2;
	var_offset[var][7][2] = var_offset[var][3][2] + args.count_local[2]/2;
      }
    //}
  }
  
  output_file = args.output_file_name;
  
  PIDX_point global_bounding_box, **local_offset_point, **local_box_count_point;
  
  local_offset_point = malloc(sizeof(PIDX_point*) * args.variable_count);
  local_box_count_point = malloc(sizeof(PIDX_point*) * args.variable_count);
  for(var = 0; var < args.variable_count; var++)
  {
    local_offset_point[var] = malloc(sizeof(PIDX_point) * var_patch_count[var]);
    local_box_count_point[var] = malloc(sizeof(PIDX_point) * var_patch_count[var]);
    for(p = 0 ; p < var_patch_count[var] ; p++)
    {
      PIDX_set_point_5D(local_offset_point[var][p], (int64_t)var_offset[var][p][0], (int64_t)var_offset[var][p][1], (int64_t)var_offset[var][p][2], 0, 0);
      PIDX_set_point_5D(local_box_count_point[var][p], (int64_t)var_count[var][p][0], (int64_t)var_count[var][p][1], (int64_t)var_count[var][p][2], 1, 1);       
    }
  }
  
  PIDX_set_point_5D(global_bounding_box, (int64_t)args.extents[0], (int64_t)args.extents[1], (int64_t)args.extents[2], 1, 1);
  
  for (ts = 0; ts < args.time_step; ts++) 
  {
    PIDX_access access;
    PIDX_create_access(&access);

#if PIDX_HAVE_MPI
    PIDX_set_mpi_access(access, 1, 1, 1, MPI_COMM_WORLD);
#endif
    
    PIDX_file_create(output_file, PIDX_file_trunc, access, &file);
    PIDX_set_dims(file, global_bounding_box);
    PIDX_set_current_time_step(file, ts);
    PIDX_set_block_size(file, bits_per_block);
    PIDX_set_block_count(file, blocks_per_file);
    PIDX_set_variable_count(file, args.variable_count);
    
#if 0
    var_double_scalar_data = malloc(sizeof(double**) * args.variable_count);
    for (var = 0; var < args.variable_count; var++)
    {
      var_double_scalar_data[var] = malloc(sizeof(double*) * var_patch_count[var]);
      for(p = 0 ; p < var_patch_count[var] ; p++)
      {
	var_double_scalar_data[var][p] = malloc(sizeof (double) * var_count[var][p][0] * var_count[var][p][1] * var_count[var][p][2] * var_count[var][p][3] * var_count[var][p][4] * values_per_sample[var]);
	for (v = 0; v < var_count[var][p][4]; v++)
	  for (u = 0; u < var_count[var][p][3]; u++)
	    for (k = 0; k < var_count[var][p][2]; k++)
	      for (j = 0; j < var_count[var][p][1]; j++)
		for (i = 0; i < var_count[var][p][0]; i++) 
		{
		  int64_t index = (int64_t) (var_count[var][p][0] * var_count[var][p][1] * var_count[var][p][2] * var_count[var][p][3] * v) + 
				    (var_count[var][p][0] * var_count[var][p][1] * var_count[var][p][2] * u) + (var_count[var][p][0] * var_count[var][p][1] * k) + 
				    (var_count[var][p][0] * j) + i;
		  for (spv = 0; spv < values_per_sample[var]; spv++)
		    var_double_scalar_data[var][p][index * values_per_sample[var] + spv] = (100 + 
		      ((args.extents[0] * args.extents[1] * args.extents[2] * args.extents[3] * (var_offset[var][p][4] + v)) + 
		      (args.extents[0] * args.extents[1] * args.extents[2] * (var_offset[var][p][3] + u)) + 
		      (args.extents[0] * args.extents[1]*(var_offset[var][p][2] + k))+(args.extents[0]*(var_offset[var][p][1] + j)) + (var_offset[var][p][0] + i)));
		}
      }
    }
    
    char variable_name[512];
    char data_type[512];
    for(var = 0; var < args.variable_count; var++)
    {
      sprintf(variable_name, "variable_%d", var);
      sprintf(data_type, "%d*float64", values_per_sample[var]);
      PIDX_variable_create(file, variable_name, values_per_sample[var] * sizeof(double) * 8, data_type, &variable[var]);
      
      for(p = 0 ; p < var_patch_count[var] ; p++)
      {
	if (rank == 0)
	  printf("Writing patch %d\n", p);
	PIDX_append_and_write_variable(variable[var], local_offset_point[var][p], local_box_count_point[var][p], var_double_scalar_data[var][p], PIDX_row_major);
      }
    }
    
    PIDX_close(file);
    PIDX_close_access(access);
    
    for(var = 0; var < args.variable_count; var++)
    {
      for(p = 0; p < var_patch_count[0]; p++)
      {
	free(var_double_scalar_data[var][p]);  
	var_double_scalar_data[var][p] = 0;
      }
      free(var_double_scalar_data[var]);  
      var_double_scalar_data[var] = 0;
    }
#endif

#if 0
    var_double_scalar_data = malloc(sizeof(double**) * args.variable_count);
    for (var = 0; var < args.variable_count; var++)
    {
      var_double_scalar_data[var] = malloc(sizeof(double*) * var_patch_count[var]);
      for(p = 0 ; p < var_patch_count[var] ; p++)
      {
	var_double_scalar_data[var][p] = malloc(sizeof (double) * var_count[var][p][0] * var_count[var][p][1] * var_count[var][p][2] * var_count[var][p][3] * var_count[var][p][4] * values_per_sample[var]);
	for (v = 0; v < var_count[var][p][4]; v++)
	  for (u = 0; u < var_count[var][p][3]; u++)
	    for (k = 0; k < var_count[var][p][2]; k++)
	      for (j = 0; j < var_count[var][p][1]; j++)
		for (i = 0; i < var_count[var][p][0]; i++) 
		{
		  int64_t index = (int64_t) (var_count[var][p][0] * var_count[var][p][1] * var_count[var][p][2] * var_count[var][p][3] * v) + 
				    (var_count[var][p][0] * var_count[var][p][1] * var_count[var][p][2] * u) + (var_count[var][p][0] * var_count[var][p][1] * k) + 
				    (var_count[var][p][0] * j) + i;
		  for (spv = 0; spv < values_per_sample[var]; spv++)
		    var_double_scalar_data[var][p][index * values_per_sample[var] + spv] = (100 + 
		      ((args.extents[0] * args.extents[1] * args.extents[2] * args.extents[3] * (var_offset[var][p][4] + v)) + 
		      (args.extents[0] * args.extents[1] * args.extents[2] * (var_offset[var][p][3] + u)) + 
		      (args.extents[0] * args.extents[1]*(var_offset[var][p][2] + k))+(args.extents[0]*(var_offset[var][p][1] + j)) + (var_offset[var][p][0] + i)));
		}
      }
    
    
      char variable_name[512];
      char data_type[512];
      sprintf(variable_name, "variable_%d", var);
      sprintf(data_type, "%d*float64", values_per_sample[var]);
      PIDX_variable_create(file, variable_name, values_per_sample[var] * sizeof(double) * 8, data_type, &variable[var]);
      
      for(p = 0 ; p < var_patch_count[var] ; p++)
      {
	if (rank == 0)
	  printf("Writing patch %d\n", p);
	PIDX_append_and_write_variable(variable[var], local_offset_point[var][p], local_box_count_point[var][p], var_double_scalar_data[var][p], PIDX_row_major);
      }
      
      PIDX_flush(file);
      for(p = 0; p < var_patch_count[var]; p++)
      {
	free(var_double_scalar_data[var][p]);  
	var_double_scalar_data[var][p] = 0;
      }
      
      free(var_double_scalar_data[var]);  
      var_double_scalar_data[var] = 0;
    }
    
    PIDX_close(file);
    PIDX_close_access(access);
#endif
    
    
#if 1
  
  var_double_scalar_data = malloc(sizeof(double**) * args.variable_count);
    for (var = 0; var < args.variable_count; var++)
    {
      var_double_scalar_data[var] = malloc(sizeof(double*) * var_patch_count[var]);
      for(p = 0 ; p < var_patch_count[var] ; p++)
      {
	var_double_scalar_data[var][p] = malloc(sizeof (double) * var_count[var][p][0] * var_count[var][p][1] * var_count[var][p][2] * var_count[var][p][3] * var_count[var][p][4] * values_per_sample[var]);
	for (v = 0; v < var_count[var][p][4]; v++)
	  for (u = 0; u < var_count[var][p][3]; u++)
	    for (k = 0; k < var_count[var][p][2]; k++)
	      for (j = 0; j < var_count[var][p][1]; j++)
		for (i = 0; i < var_count[var][p][0]; i++) 
		{
		  int64_t index = (int64_t) (var_count[var][p][0] * var_count[var][p][1] * var_count[var][p][2] * var_count[var][p][3] * v) + 
				    (var_count[var][p][0] * var_count[var][p][1] * var_count[var][p][2] * u) + (var_count[var][p][0] * var_count[var][p][1] * k) + 
				    (var_count[var][p][0] * j) + i;
		  for (spv = 0; spv < values_per_sample[var]; spv++)
		    var_double_scalar_data[var][p][index * values_per_sample[var] + spv] = (100 + 
		      ((args.extents[0] * args.extents[1] * args.extents[2] * args.extents[3] * (var_offset[var][p][4] + v)) + 
		      (args.extents[0] * args.extents[1] * args.extents[2] * (var_offset[var][p][3] + u)) + 
		      (args.extents[0] * args.extents[1]*(var_offset[var][p][2] + k))+(args.extents[0]*(var_offset[var][p][1] + j)) + (var_offset[var][p][0] + i)));
		}
      }
    }
    
    for (var = 0; var < args.variable_count; var++)
    {
      char variable_name[512];
      char data_type[512];
      sprintf(variable_name, "variable_%d", var);
      sprintf(data_type, "%d*float64", values_per_sample[var]);
      PIDX_variable_create(file, variable_name, values_per_sample[var] * sizeof(double) * 8, data_type, &variable[var]);
      
      for(p = 0 ; p < var_patch_count[var] ; p++)
      {
	//if (rank == 0)
	  //printf("Writing patch %d\n", p);
	PIDX_append_and_write_variable(variable[var], local_offset_point[var][p], local_box_count_point[var][p], var_double_scalar_data[var][p], PIDX_row_major);
      }
      
      if(var % 2 == 0)
	PIDX_flush(file);
      //for(p = 0; p < var_patch_count[var]; p++)
      //{
	//free(var_double_scalar_data[var][p]);  
	//var_double_scalar_data[var][p] = 0;
      //}
      
      //free(var_double_scalar_data[var]);  
      //var_double_scalar_data[var] = 0;
    }
    
    PIDX_close(file);
    PIDX_close_access(access);
  
#endif
    
    free(var_double_scalar_data);  var_double_scalar_data = 0;
  }
  
  for(var = 0; var < args.variable_count; var++)
  {
    free(local_offset_point[var]);
    free(local_box_count_point[var]);
  }
  free(local_offset_point);
  free(local_box_count_point);
  
  free(args.output_file_name);
  free(variable);
  variable = 0;
  free(values_per_sample);
  values_per_sample = 0;
#endif
  return 0;
}
Example #6
0
//----------------------------------------------------------------
int main(int argc, char **argv)
{
  init_mpi(argc, argv);
  printf("start %d %d\n",rank,process_count);
   int i;
  if(rank==0) {
    parse_args(argc, argv);
//    var_count = read_list_in_file(var_file, &netcdf_var_names);
//    rank_0_print("Number of variables = %d\n", var_count);
//    time_step_count = read_list_in_file(netcdf_file_list, &netcdf_file_names);
//    rank_0_print("Number of timesteps = %d\n", time_step_count);
    check_args();
  }

   
  //  The command line arguments are shared by all processes
#if PIDX_HAVE_MPI
  MPI_Bcast(global_box_size, 3, MPI_LONG_LONG, 0, MPI_COMM_WORLD);
  MPI_Bcast(local_box_size, 3, MPI_LONG_LONG, 0, MPI_COMM_WORLD);
//  MPI_Bcast(&time_step_count, 1, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(&var_file, 512, MPI_CHAR, 0, MPI_COMM_WORLD);
  MPI_Bcast(&netcdf_file_list, 512, MPI_CHAR, 0, MPI_COMM_WORLD);

//  MPI_Bcast(&var_count, 1, MPI_INT, 0, MPI_COMM_WORLD);
  
  MPI_Bcast(&output_file_name, 512, MPI_CHAR, 0, MPI_COMM_WORLD);
#endif

//TODO: Only the rank 0 should read the input files and broadcast the data
    var_count = read_list_in_file(var_file, &netcdf_var_names);
    rank_0_print("Number of variables = %d\n", var_count);


    time_step_count = read_list_in_file(netcdf_file_list, &netcdf_file_names);
    rank_0_print("Number of timesteps = %d\n", time_step_count);

  calculate_per_process_offsets();
  
  create_pidx_var_names();

  PIDX_access pidx_access;
  create_pidx_access(&pidx_access);
  PIDX_time_step_caching_ON();

  determine_var_types();
  create_pidx_vars();

  int t = 0,plist_id;
  for (t = 0; t < time_step_count; ++t)
  {
    rank_0_print("Processing time step %d (file %s)\n", t, netcdf_file_names[t]);

    PIDX_file pidx_file;
    int ret = PIDX_file_create(output_file_name, PIDX_MODE_CREATE, pidx_access, &pidx_file);
  

    if (ret != PIDX_success)
      terminate_with_error_msg("ERROR: Failed to create PIDX file\n");
    set_pidx_params(pidx_file);
    PIDX_set_current_time_step(pidx_file, t);

    int file_id = ncmpi_open(MPI_COMM_WORLD,netcdf_file_names[t], NC_NOWRITE, MPI_INFO_NULL, &plist_id);

    if (file_id !=0)
      terminate_with_error_msg("ERROR: Failed to open file %s\n", netcdf_file_names[t]);

    int v = 0;
    for(v = 0; v < var_count; ++v)
    {
      rank_0_print("Processing variable %s\n", netcdf_var_names[v]);
      var_data = malloc(var_types[v].atomic_type * var_types[v].num_values * local_box_size[0] * local_box_size[1] * local_box_size[2]);

      read_var_from_netcdf(plist_id, netcdf_var_names[v], var_types[v]);
     
      write_var_to_idx(pidx_file, pidx_var_names[v], pidx_vars[v]);
      if (PIDX_flush(pidx_file) != PIDX_success)
        terminate_with_error_msg("ERROR: Failed to flush variable %s, time step %d\n", pidx_var_names[v], t);
      free(var_data);
    }

    ncmpi_close(plist_id);
    PIDX_close(pidx_file);
  }

  PIDX_time_step_caching_OFF();
  PIDX_close_access(pidx_access);

  free_memory();
  shutdown_mpi();

  return 0;
}
Example #7
0
int main(int argc, char **argv) 
{
  int sub_div[3], local_offset[3], count_local[3];
  int t = 0, time_count = 0;
  
  sub_div[0] = 0;
  sub_div[1] = 0;
  sub_div[2] = 0;
  local_offset[0] = 0;
  local_offset[1] = 0;
  local_offset[2] = 0;
  count_local[0] = 0;
  count_local[1] = 0;
  count_local[2] = 0;
  
#if HDF_IO
  hid_t file_id;
  hid_t plist_id;     
  hid_t group_id;
  hid_t dataset_id;
  hid_t file_dataspace;
  hid_t mem_dataspace;
#endif
  
#if PIDX_IO
  PIDX_file file;
  PIDX_access access;
  PIDX_variable variable;
#endif
  
  const int bits_per_block = 15;
  const int blocks_per_file = 512;
  int nprocs=1, rank=0, slice;
  
  MPI_Comm comm  = MPI_COMM_WORLD;
  
#if HDF_IO
  MPI_Info info  = MPI_INFO_NULL;
#endif
  
  MPI_Init(&argc, &argv);
  MPI_Comm_size(comm, &nprocs);
  MPI_Comm_rank(comm, &rank); 
  
  output_file_name = (char*) malloc(sizeof (char) * 1024);
  sprintf(output_file_name, "%s%s", "/scratch/project/visus/datasets/flame", ".idx");
  
  if (nprocs == 320)            /// 4 x 5 x 16
  {
    count_local[0] = 256;
    count_local[1] = 188;
    count_local[2] = 192;
  }
  else if (nprocs == 640)       /// 4 x 5 x 32
  {
    count_local[0] = 256;
    count_local[1] = 188;
    count_local[2] = 96;
  }
  else if (nprocs == 1280)      /// 4 x 10 x 32
  {
    count_local[0] = 256;
    count_local[1] = 94;
    count_local[2] = 96;
  }
  else if (nprocs == 2560)      /// 8 x 10 x 32
  {
    count_local[0] = 128;
    count_local[1] = 94;
    count_local[2] = 96;
  }
  
  sub_div[0] = (1024 / count_local[0]);
  sub_div[1] = (940 / count_local[1]);
  sub_div[2] = (3072 / count_local[2]);
  local_offset[2] = (rank / (sub_div[0] * sub_div[1])) * count_local[2];
  slice = rank % (sub_div[0] * sub_div[1]);
  local_offset[1] = (slice / sub_div[0]) * count_local[1];
  local_offset[0] = (slice % sub_div[0]) * count_local[0];
 
#if PIDX_IO
  PIDX_point global_bounding_box, local_offset_point, local_box_count_point;
  PIDX_set_point_5D(global_bounding_box, (int64_t)1024, (int64_t)940, (int64_t)3072, 1, 1);
  PIDX_set_point_5D(local_offset_point, (int64_t)local_offset[0], (int64_t)local_offset[1], (int64_t)local_offset[2], 0, 0);
  PIDX_set_point_5D(local_box_count_point, (int64_t)count_local[0], (int64_t)count_local[1], (int64_t)count_local[2], 1, 1);
#endif
  
#if HDF_IO
  hsize_t count[3];
  count[0] = count_local[0];
  count[1] = count_local[1];
  count[2] = count_local[2];
  
  hsize_t offset[3];
  offset[0] = local_offset[0];
  offset[1] = local_offset[1];
  offset[2] = local_offset[2];
#endif
  
  /// P   U   V   W   ZMIX
  
#if HDF_IO
  plist_id = H5Pcreate(H5P_FILE_ACCESS);
  H5Pset_fapl_mpio(plist_id, comm, info);
#endif
 
  FILE *fp;
  fp = fopen("list", "r");
  char file_name[1][1024];
  while (!feof(fp)) 
  {
    if (fscanf(fp, "%s", file_name[time_count]) != 1)
      break;
    if(rank == 0)
      printf("%s\n", file_name[time_count]);
    time_count++;
  }
  fclose(fp);
  
#if PIDX_IO
  PIDX_create_access(&access);
  PIDX_set_mpi_access(access, 1, 1, 1, comm);
  PIDX_set_process_extent(access, sub_div[0], sub_div[1], sub_div[2]);
#endif
  
  if (rank == 0)
    printf("Number of timesteps = %d\n", time_count);
  for (t = 0; t < time_count; t++)
  {
#if HDF_IO
    file_id = H5Fopen(file_name[t], H5F_ACC_RDONLY, plist_id);
#endif
     
#if PIDX_IO
    PIDX_file_create(output_file_name, PIDX_file_trunc, access, &file);
    PIDX_set_dims(file, global_bounding_box);
    PIDX_set_current_time_step(file, t);
    PIDX_set_block_size(file, bits_per_block);
    PIDX_set_aggregation_factor(file, 1);
    PIDX_set_block_count(file, blocks_per_file);
    PIDX_set_variable_count(file, 1);
    
    PIDX_debug_rst(file, 0);
    PIDX_debug_hz(file, 0);
    PIDX_dump_agg_info(file, 0);
    
    PIDX_enable_hz(file, 1);
    PIDX_enable_agg(file, 1);
    PIDX_enable_io(file, 1);
#endif
    
#if HDF_IO
    group_id = H5Gopen(file_id, "/data", H5P_DEFAULT);
    dataset_id = H5Dopen2(group_id, "P", H5P_DEFAULT);
    
    mem_dataspace = H5Screate_simple (3, count, NULL);
    file_dataspace = H5Dget_space (dataset_id);
    H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, offset, NULL, count, NULL);
#endif
    
    buffer = malloc(sizeof(double) * count_local[0] * count_local[1] * count_local[2]);
    memset(buffer, 0, sizeof(double) * count_local[0] * count_local[1] * count_local[2]);
    
#if HDF_IO
    H5Dread(dataset_id, H5T_NATIVE_DOUBLE, mem_dataspace, file_dataspace, H5P_DEFAULT, buffer);
    H5Sclose(mem_dataspace);
    H5Sclose(file_dataspace);
    H5Dclose(dataset_id);
    H5Gclose(group_id);
    H5Fclose(file_id);
    
    if (rank == 0)
      printf("Finished reading HDF files\n");
#else
    int64_t i1, j1, k1;
    for (k1 = 0; k1 < count_local[2]; k1++)
      for (j1 = 0; j1 < count_local[1]; j1++)
        for (i1 = 0; i1 < count_local[0]; i1++)
        {
          int64_t index = (int64_t) (count_local[0] * count_local[1] * k1) + (count_local[0] * j1) + i1;
          buffer[index] = (256 * rank) / nprocs;
        }

#endif
    
#if PIDX_IO
    PIDX_variable_create(file, "P", sizeof(double) * 8, "1*float64", &variable);
    PIDX_append_and_write_variable(variable, local_offset_point, local_box_count_point, buffer, PIDX_column_major);
    
    if (rank == 0)
      printf("Starting to Write IDX %s Time Step %d\n", output_file_name, t);
    
    PIDX_close(file);
#endif
    
    free(buffer);
    buffer = 0;
  }
  
  //////////
#if HDF_IO
  H5Pclose(plist_id);
#endif
  
#if PIDX_IO
  PIDX_close_access(access);
#endif
  
  free(output_file_name);
  output_file_name = 0;
  
  MPI_Finalize();
  
  return 0;
}
Example #8
0
int main(int argc, char **argv)
{
  int ret;
  int i;
  int slice = 0;
  int nprocs = 1, rank = 0;
  char output_file_name[512];

  // MPI initialization
#if PIDX_HAVE_MPI
  MPI_Init(&argc, &argv);
  MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
#endif

  double *data;             // variable data buffer
  int local_box_offset[3];

  if (rank == 0)
  {
    ret = parse_args(argc, argv);
    if (ret < 0)
    {
      usage();
#if PIDX_HAVE_MPI
      MPI_Abort(MPI_COMM_WORLD, -1);
#else
      exit(-1);
#endif
    }

    // check if the num procs is appropriate
    int num_bricks = (global_box_size[0] / local_box_size[0]) * (global_box_size[1] / local_box_size[1]) * (global_box_size[2] / local_box_size[2]);
    if(num_bricks != nprocs)
    {
      fprintf(stderr, "Error: number of sub-blocks (%d) doesn't match number of procs (%d)\n", num_bricks, nprocs);
      fprintf(stderr, "Incorrect distribution of data across processes i.e.\n(global_x / local_x) X (global_x / local_x) X (global_x / local_x) != nprocs\n(%d/%d) X (%d/%d) X (%d/%d) != %d\n", global_box_size[0], local_box_size[0], global_box_size[1], local_box_size[1], global_box_size[2], local_box_size[2], nprocs);

#if PIDX_HAVE_MPI
      MPI_Abort(MPI_COMM_WORLD, -1);
#else
      exit(-1);
#endif
    }
  }

  //  The command line arguments are shared by all processes
#if PIDX_HAVE_MPI
  MPI_Bcast(global_box_size, 3, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(local_box_size, 3, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(&time_step_count, 1, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(&roi_type, 1, MPI_INT, 0, MPI_COMM_WORLD);
  MPI_Bcast(&output_file_template, 512, MPI_CHAR, 0, MPI_COMM_WORLD);
#endif


  // Creating the filename
  sprintf(output_file_name, "%s%s", output_file_template,".idx");

  // Calculating every process data's offset and size
  int sub_div[3];
  sub_div[0] = (global_box_size[0] / local_box_size[0]);
  sub_div[1] = (global_box_size[1] / local_box_size[1]);
  sub_div[2] = (global_box_size[2] / local_box_size[2]);
  local_box_offset[2] = (rank / (sub_div[0] * sub_div[1])) * local_box_size[2];
  slice = rank % (sub_div[0] * sub_div[1]);
  local_box_offset[1] = (slice / sub_div[0]) * local_box_size[1];
  local_box_offset[0] = (slice % sub_div[0]) * local_box_size[0];

  int ts;
  PIDX_file file;            // IDX file descriptor
  PIDX_variable variable;   // variable descriptor
  PIDX_point global_size, local_offset, local_size;

  switch (roi_type)
  {
    case 0:
      data = malloc(sizeof (uint64_t) * local_box_size[0] * local_box_size[1] * local_box_size[2]);
      for (i = 0; i < local_box_size[0] * local_box_size[1] * local_box_size[2]; i++)
        data[i] = rank;

      PIDX_set_point_5D(local_size, local_box_size[0], local_box_size[1], local_box_size[2], 1, 1);
      break;

    case 1:
      if (rank % 2 == 0)
      {
        data = malloc(sizeof (uint64_t) * local_box_size[0] * local_box_size[1] * local_box_size[2]);
        for (i = 0; i < local_box_size[0] * local_box_size[1] * local_box_size[2]; i++)
          data[i] = 100;

        PIDX_set_point_5D(local_size, local_box_size[0], local_box_size[1], local_box_size[2], 1, 1);
      }
      else
      {
        data = NULL;
        PIDX_set_point_5D(local_size, 0, 0, 0, 1, 1);
      }
      break;

    case 2:
      if ((rank / (global_box_size[1]/local_box_size[1])) % 2 == 0)
      {
        if (rank % 2 == 0)
        {
          data = malloc(sizeof (uint64_t) * local_box_size[0] * local_box_size[1] * local_box_size[2]);
          for (i = 0; i < local_box_size[0] * local_box_size[1] * local_box_size[2]; i++)
            data[i] = 100;

          PIDX_set_point_5D(local_size, local_box_size[0], local_box_size[1], local_box_size[2], 1, 1);
        }
        else
        {
          data = NULL;
          PIDX_set_point_5D(local_size, 0, 0, 0, 1, 1);
        }
      }
      else
      {
          if (rank % 2 == 1)
          {
            data = malloc(sizeof (uint64_t) * local_box_size[0] * local_box_size[1] * local_box_size[2]);
            for (i = 0; i < local_box_size[0] * local_box_size[1] * local_box_size[2]; i++)
              data[i] = 100;

            PIDX_set_point_5D(local_size, local_box_size[0], local_box_size[1], local_box_size[2], 1, 1);
          }
          else
          {
            data = NULL;
            PIDX_set_point_5D(local_size, 0, 0, 0, 1, 1);
          }

      }
      break;

    default:
      break;
  }

  PIDX_set_point_5D(global_size, global_box_size[0], global_box_size[1], global_box_size[2], 1, 1);
  PIDX_set_point_5D(local_offset, local_box_offset[0], local_box_offset[1], local_box_offset[2], 0, 0);

  //  Creating access
  PIDX_access access;
  PIDX_create_access(&access);
#if PIDX_HAVE_MPI
  PIDX_set_mpi_access(access, MPI_COMM_WORLD);
#endif

  for (ts = 0; ts < time_step_count; ts++)
  {
    //  PIDX mandatory calls
    ret = PIDX_file_create(output_file_name, PIDX_MODE_CREATE, access, &file);
    if (ret != PIDX_success)  report_error("PIDX_file_create", __FILE__, __LINE__);

    ret = PIDX_set_dims(file, global_size);
    if (ret != PIDX_success)  report_error("PIDX_set_dims", __FILE__, __LINE__);
    ret = PIDX_set_current_time_step(file, ts);
    if (ret != PIDX_success)  report_error("PIDX_set_current_time_step", __FILE__, __LINE__);
    ret = PIDX_set_variable_count(file, 1);
    if (ret != PIDX_success)  report_error("PIDX_set_variable_count", __FILE__, __LINE__);

    ret = PIDX_set_ROI_writes(file);
    if (ret != PIDX_success)  report_error("PIDX_set_ROI_writes", __FILE__, __LINE__);

    ret = PIDX_variable_create("var", sizeof(uint64_t) * 8, FLOAT64, &variable);
    if (ret != PIDX_success)  report_error("PIDX_variable_create", __FILE__, __LINE__);

    ret = PIDX_variable_write_data_layout(variable, local_offset, local_size, data, PIDX_row_major);
    if (ret != PIDX_success)  report_error("PIDX_variable_data_layout", __FILE__, __LINE__);

    ret = PIDX_append_and_write_variable(file, variable);
    if (ret != PIDX_success)  report_error("PIDX_append_and_write_variable", __FILE__, __LINE__);

    ret = PIDX_close(file);
    if (ret != PIDX_success)  report_error("PIDX_close", __FILE__, __LINE__);
  }

  ret = PIDX_close_access(access);
  if (ret != PIDX_success)  report_error("PIDX_close_access", __FILE__, __LINE__);

  switch (roi_type)
  {
    case 0:
      free(data);
      break;

    case 1:
      if (rank % 2 == 0)
        free(data);
      break;

    case 2:
      if ((rank / (global_box_size[1]/local_box_size[1])) % 2 == 0)
      {
        if (rank % 2 == 0)
          free(data);
      }
      else
      {
        if (rank % 2 == 1)
          free(data);
      }
      break;

    default:
      break;
  }

  //  MPI finalize
#if PIDX_HAVE_MPI
  MPI_Finalize();
#endif

  return 0;
}
int main(int argc, char **argv)
{
  init_mpi(argc, argv);
  parse_args(argc, argv);
  check_args();
  calculate_per_process_offsets();
  create_synthetic_simulation_data();

  rank_0_print("Simulation Data Created\n");

  int ret;
  int var;
  int ts;
  PIDX_file file;            // IDX file descriptor
  PIDX_variable* variable;   // variable descriptor

  variable = malloc(sizeof(*variable) * variable_count);
  memset(variable, 0, sizeof(*variable) * variable_count);

  PIDX_point global_size, local_offset, local_size;
  PIDX_set_point_5D(global_size, global_box_size[0], global_box_size[1], global_box_size[2], 1, 1);
  PIDX_set_point_5D(local_offset, local_box_offset[0], local_box_offset[1], local_box_offset[2], 0, 0);
  PIDX_set_point_5D(local_size, local_box_size[0], local_box_size[1], local_box_size[2], 1, 1);

  //  Creating access
  PIDX_access access;
  PIDX_create_access(&access);
#if PIDX_HAVE_MPI
  PIDX_set_mpi_access(access, MPI_COMM_WORLD);
#endif

  for (ts = 0; ts < time_step_count; ts++)
  {
    //  PIDX mandatory calls
    ret = PIDX_file_create(output_file_name, PIDX_MODE_CREATE, access, global_size, &file);
    if (ret != PIDX_success)  terminate_with_error_msg("PIDX_file_create");

    ret = PIDX_set_current_time_step(file, ts);
    if (ret != PIDX_success)  terminate_with_error_msg("PIDX_set_current_time_step");
    ret = PIDX_set_variable_count(file, variable_count);
    if (ret != PIDX_success)  terminate_with_error_msg("PIDX_set_variable_count");

    ret = PIDX_set_resolution(file, 0, reduced_resolution);
    if (ret != PIDX_success)  terminate_with_error_msg("PIDX_set_resolution");

    char var_name[512];
    for (var = 0; var < variable_count; var++)
    {
      sprintf(var_name, "variable_%d", var);

      ret = PIDX_variable_create(var_name, sizeof(unsigned long long) * 8, FLOAT64, &variable[var]);
      if (ret != PIDX_success)  terminate_with_error_msg("PIDX_variable_create");

      ret = PIDX_variable_write_data_layout(variable[var], local_offset, local_size, data[var], PIDX_row_major);
      if (ret != PIDX_success)  terminate_with_error_msg("PIDX_variable_data_layout");

      ret = PIDX_append_and_write_variable(file, variable[var]);
      if (ret != PIDX_success)  terminate_with_error_msg("PIDX_append_and_write_variable");
    }

    ret = PIDX_close(file);
    if (ret != PIDX_success)  terminate_with_error_msg("PIDX_close");
  }

  ret = PIDX_close_access(access);
  if (ret != PIDX_success)  terminate_with_error_msg("PIDX_close_access");

  free(variable);
  variable = 0;

  destroy_synthetic_simulation_data();
  shutdown_mpi();

  return 0;
}
Example #10
0
int main(int argc, char **argv)
{
  init_mpi(argc, argv);
  parse_args(argc, argv);
  check_args();
  calculate_per_process_offsets();
  create_synthetic_simulation_data();

  rank_0_print("Simulation Data Created\n");

  int ret, var, ts;
  PIDX_file file;            // IDX file descriptor
  PIDX_variable* variable;   // variable descriptor

  variable = (PIDX_variable*)malloc(sizeof(*variable) * variable_count);
  memset(variable, 0, sizeof(*variable) * variable_count);

  PIDX_point global_size, local_offset, local_size;
  PIDX_set_point_5D(global_size, global_box_size[0], global_box_size[1], global_box_size[2], 1, 1);
  PIDX_set_point_5D(local_offset, local_box_offset[0], local_box_offset[1], local_box_offset[2], 0, 0);
  PIDX_set_point_5D(local_size, local_box_size[0], local_box_size[1], local_box_size[2], 1, 1);

  //  Creating access
  PIDX_access access;
  PIDX_create_access(&access);
#if PIDX_HAVE_MPI
  PIDX_set_mpi_access(access, MPI_COMM_WORLD);
#endif

  for (ts = 0; ts < time_step_count; ts++)
  {
    //  PIDX mandatory calls
    ret = PIDX_file_create(output_file_name, PIDX_MODE_CREATE, access, global_size, &file);
    if (ret != PIDX_success)  terminate_with_error_msg("PIDX_file_create");

    ret = PIDX_set_current_time_step(file, ts);
    if (ret != PIDX_success)  terminate_with_error_msg("PIDX_set_current_time_step");
    ret = PIDX_set_variable_count(file, variable_count);
    if (ret != PIDX_success)  terminate_with_error_msg("PIDX_set_variable_count");

    PIDX_disable_agg(file);

    PIDX_save_big_endian(file);

    //PIDX_dump_rst_info(file, 1);

    //PIDX_debug_rst(file, 1);
    //PIDX_debug_hz(file, 1);

    PIDX_point reg_patch_size;
    PIDX_set_point_5D(reg_patch_size, 128, 128, 128, 1, 1);
    PIDX_set_restructuring_box(file, reg_patch_size);
    //PIDX_GLOBAL_PARTITION_IDX_IO
    //PIDX_IDX_IO
    ret = PIDX_set_io_mode(file, PIDX_RAW_IO);
    if (ret != PIDX_success)  terminate_with_error_msg("PIDX_set_variable_count");

    ret = PIDX_set_partition_size(file, partition_size[0], partition_size[1], partition_size[2]);
    if (ret != PIDX_success)  terminate_with_error_msg("PIDX_set_partition_size");

    PIDX_set_block_count(file, 128);

    //ret = PIDX_set_aggregator_multiplier(file, aggregator_multiplier);
    //if (ret != PIDX_success)  terminate_with_error_msg("PIDX_set_partition_size");

    /*
    int io_type = PIDX_IDX_IO;
    switch (io_type)
    {
      case PIDX_GLOBAL_PARTITION_IDX_IO:
        PIDX_set_block_count(file,blocks_per_file);
        PIDX_set_block_size(file, 13);
        break;

      case PIDX_IDX_IO:
        PIDX_set_block_count(file,blocks_per_file);
        break;

      case PIDX_RAW_IO:
        PIDX_raw_io_pipe_length(file, 2);
        PIDX_point reg_patch_size;
        PIDX_set_point_5D(reg_patch_size, 128, 128, 128, 1, 1);
        PIDX_set_restructuring_box(file, reg_patch_size);
        break;
    }
    */

    //ret = PIDX_debug_disable_agg(file);
    //if (ret != PIDX_success)  terminate_with_error_msg("PIDX_debug_output");

    //ret = PIDX_debug_disable_io(file);
    //if (ret != PIDX_success)  terminate_with_error_msg("PIDX_debug_output");

    //ret = PIDX_debug_disable_hz(file);
    //if (ret != PIDX_success)  terminate_with_error_msg("PIDX_debug_output");

    for (var = 0; var < variable_count; var++)
    {
      if (bpv[var] == 32)
      {
        ret = PIDX_variable_create(var_name[var],  bpv[var], FLOAT32 , &variable[var]);
        if (ret != PIDX_success)  terminate_with_error_msg("PIDX_variable_create");
      }
      else if (bpv[var] == 192)
      {
        ret = PIDX_variable_create(var_name[var],  bpv[var], FLOAT64_RGB , &variable[var]);
        if (ret != PIDX_success)  terminate_with_error_msg("PIDX_variable_create");
      }
      else if (bpv[var] == 64)
      {
        ret = PIDX_variable_create(var_name[var],  bpv[var], FLOAT64 , &variable[var]);
        if (ret != PIDX_success)  terminate_with_error_msg("PIDX_variable_create");
      }

      ret = PIDX_variable_write_data_layout(variable[var], local_offset, local_size, data[var], PIDX_row_major);
      if (ret != PIDX_success)  terminate_with_error_msg("PIDX_variable_data_layout");

      ret = PIDX_append_and_write_variable(file, variable[var]);
      if (ret != PIDX_success)  terminate_with_error_msg("PIDX_append_and_write_variable");

      //PIDX_flush(file);
    }

    ret = PIDX_close(file);
    if (ret != PIDX_success)  terminate_with_error_msg("PIDX_close");
  }

  ret = PIDX_close_access(access);
  if (ret != PIDX_success)  terminate_with_error_msg("PIDX_close_access");

  free(variable);
  variable = 0;

  destroy_synthetic_simulation_data();
  shutdown_mpi();

  return 0;
}