PIDX_return_code destroy_block_layout_and_buffers(PIDX_io file, int svi, int evi)
{
int ret;
PIDX_time time = file->time;
time->group_cleanup_start = PIDX_get_time();
ret = destroy_agg_io_buffer(file, svi, evi);
if (ret != PIDX_success)
{
fprintf(stderr,"File %s Line %d\n", __FILE__, __LINE__);
return PIDX_err_file;
}
ret = delete_rst_block_layout(file);
if (ret != PIDX_success)
{
fprintf(stderr,"File %s Line %d\n", __FILE__, __LINE__);
return PIDX_err_file;
}
time->group_cleanup_end = PIDX_get_time();
return PIDX_success;
}
PIDX_return_code populate_block_layout_and_buffers(PIDX_io file, int svi, int evi, int mode, int partitioning_mode)
{
// Three tasks
// 1. Create the bitstring
// 2. populate the block layout and aggregation related buffers
// 3. Poplate the idx binary file hierarchy
PIDX_time time = file->time;
time->bit_string_start = PIDX_get_time();
// calculates maxh and bitstring
if (partitioning_mode == PIDX_IDX_IO)
{
if (populate_global_bit_string(file, mode) != PIDX_success)
{
fprintf(stderr,"File %s Line %d\n", __FILE__, __LINE__);
return PIDX_err_file;
}
}
else if (partitioning_mode == PIDX_LOCAL_PARTITION_IDX_IO)
{
if (populate_local_bit_string(file, mode) != PIDX_success)
{
fprintf(stderr,"File %s Line %d\n", __FILE__, __LINE__);
return PIDX_err_file;
}
}
// selects layout levels based on maxh
select_io_mode(file);
time->bit_string_end = PIDX_get_time();
time->layout_start = PIDX_get_time();
// calculates the block layoutven this is pure IDX only non-share block layout is populated
if (populate_rst_block_layouts(file, svi, file->idx_b->hz_file0_from, file->idx_b->hz_n_file0_to) != PIDX_success)
{
fprintf(stderr,"File %s Line %d\n", __FILE__, __LINE__);
return PIDX_err_file;
}
// Calculate the hz level upto which aggregation is possible
if (find_agg_level(file, svi, evi) != PIDX_success)
{
fprintf(stderr,"File %s Line %d\n", __FILE__, __LINE__);
return PIDX_err_file;
}
// Creates the agg and io ids
if (create_agg_io_buffer(file) != PIDX_success)
{
fprintf(stderr,"File %s Line %d\n", __FILE__, __LINE__);
return PIDX_err_file;
}
time->layout_end = PIDX_get_time();
time->header_io_start = PIDX_get_time();
// Creates the file heirarchy and writes the header info for all binary files
if (write_headers(file, svi, evi, mode) != PIDX_success)
{
fprintf(stderr,"File %s Line %d\n", __FILE__, __LINE__);
return PIDX_err_file;
}
time->header_io_end = PIDX_get_time();
return PIDX_success;
}
int PIDX_aggregated_io(PIDX_file_io_id io_id, Agg_buffer agg_buf, PIDX_block_layout block_layout, int MODE)
{
int64_t data_offset = 0;
char file_name[PATH_MAX];
int i = 0, k = 0;
uint32_t *headers;
int total_header_size = 0;
#ifdef PIDX_RECORD_TIME
double t1, t2, t3, t4, t5;
#endif
#if PIDX_HAVE_MPI
int mpi_ret;
MPI_File fh;
MPI_Status status;
#else
int fh;
#endif
int total_chunk_size = (io_id->idx->chunk_size[0] * io_id->idx->chunk_size[1] * io_id->idx->chunk_size[2] * io_id->idx->chunk_size[3] * io_id->idx->chunk_size[4]);
if (enable_caching == 1 && agg_buf->var_number == io_id->init_index && agg_buf->sample_number == 0)
{
#ifdef PIDX_RECORD_TIME
t1 = PIDX_get_time();
#endif
int adjusted_file_index = 0;
int l = pow(2, ((int)log2((unsigned int) agg_buf->file_number * io_id->idx->blocks_per_file)));
adjusted_file_index = (l * (io_id->idx_d->idx_count[0] * io_id->idx_d->idx_count[1] * io_id->idx_d->idx_count[2]) + (((unsigned int) agg_buf->file_number * io_id->idx->blocks_per_file) - l) + (io_id->idx_d->color * l)) / io_id->idx->blocks_per_file;
generate_file_name(io_id->idx->blocks_per_file, io_id->idx->filename_template, (unsigned int) /*agg_buf->file_number*/adjusted_file_index, file_name, PATH_MAX);
#if !SIMULATE_IO
#if PIDX_HAVE_MPI
mpi_ret = MPI_File_open(MPI_COMM_SELF, file_name, MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() failed filename %s.\n", __FILE__, __LINE__, file_name);
return PIDX_err_io;
}
#else
fh = open(file_name, O_WRONLY);
#endif
#endif
#ifdef PIDX_RECORD_TIME
t2 = PIDX_get_time();
#endif
data_offset = 0;
total_header_size = (10 + (10 * io_id->idx->blocks_per_file)) * sizeof (uint32_t) * io_id->idx->variable_count;
headers = (uint32_t*)malloc(total_header_size);
memset(headers, 0, total_header_size);
#if !SIMULATE_IO
if (enable_caching == 1)
memcpy (headers, cached_header_copy, total_header_size);
else
{
//TODO
}
#endif
#ifdef PIDX_RECORD_TIME
t3 = PIDX_get_time();
#endif
uint64_t header_size = (io_id->idx_d->start_fs_block * io_id->idx_d->fs_block_size);
#if !SIMULATE_IO
unsigned char* temp_buffer = (unsigned char*)realloc(agg_buf->buffer, agg_buf->buffer_size + header_size);
if (temp_buffer == NULL)
{
fprintf(stderr, "[%s] [%d] realloc() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
else
{
agg_buf->buffer = temp_buffer;
memmove(agg_buf->buffer + header_size, agg_buf->buffer, agg_buf->buffer_size);
memcpy(agg_buf->buffer, headers, total_header_size);
memset(agg_buf->buffer + total_header_size, 0, (header_size - total_header_size));
}
#endif
free(headers);
#if !SIMULATE_IO
#if PIDX_HAVE_MPI
mpi_ret = MPI_File_write_at(fh, 0, agg_buf->buffer, agg_buf->buffer_size + header_size, MPI_BYTE, &status);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_write_at() failed for filename %s.\n", __FILE__, __LINE__, file_name);
return PIDX_err_io;
}
int write_count;
MPI_Get_count(&status, MPI_BYTE, &write_count);
if (write_count != agg_buf->buffer_size + header_size)
{
fprintf(stderr, "[%s] [%d] MPI_File_write_at() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
#else
ssize_t write_count = pwrite(fh, agg_buf->buffer, agg_buf->buffer_size + header_size, 0);
if (write_count != agg_buf->buffer_size + header_size)
{
fprintf(stderr, "[%s] [%d] pwrite() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
#endif
#endif
#ifdef PIDX_RECORD_TIME
t4 = PIDX_get_time();
#endif
#if !SIMULATE_IO
#if PIDX_HAVE_MPI
mpi_ret = MPI_File_close(&fh);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
#else
close(fh);
#endif
#endif
#ifdef PIDX_RECORD_TIME
t5 = PIDX_get_time();
#endif
#ifdef PIDX_RECORD_TIME
printf("V0. [R %d] [O 0 C %lld] [FVS %d %d %d] Time: O %f H %f W %f C %f\n", rank, (long long)agg_buf->buffer_size + header_size, agg_buf->file_number, agg_buf->var_number, agg_buf->sample_number, (t2-t1), (t3-t2), (t4-t3), (t5-t4));
#else
#endif
}
else if (agg_buf->var_number != -1 && agg_buf->sample_number != -1 && agg_buf->file_number != -1)
{
#ifdef PIDX_RECORD_TIME
t1 = PIDX_get_time();
#endif
int adjusted_file_index = 0;
int l = pow(2, ((int)log2((unsigned int) agg_buf->file_number * io_id->idx->blocks_per_file)));
adjusted_file_index = (l * (io_id->idx_d->idx_count[0] * io_id->idx_d->idx_count[1] * io_id->idx_d->idx_count[2]) + (((unsigned int) agg_buf->file_number * io_id->idx->blocks_per_file) - l) + (io_id->idx_d->color * l)) / io_id->idx->blocks_per_file;
generate_file_name(io_id->idx->blocks_per_file, io_id->idx->filename_template, (unsigned int) adjusted_file_index/*agg_buf->file_number*/, file_name, PATH_MAX);
#if !SIMULATE_IO
#if PIDX_HAVE_MPI
if (MODE == PIDX_WRITE)
{
mpi_ret = MPI_File_open(MPI_COMM_SELF, file_name, MPI_MODE_WRONLY, MPI_INFO_NULL, &fh);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() filename %s failed.\n", __FILE__, __LINE__, file_name);
return PIDX_err_io;
}
}
else
{
mpi_ret = MPI_File_open(MPI_COMM_SELF, file_name, MPI_MODE_RDONLY, MPI_INFO_NULL, &fh);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() filename %s failed.\n", __FILE__, __LINE__, file_name);
return PIDX_err_io;
}
}
#else
if (MODE == PIDX_WRITE)
fh = open(file_name, O_WRONLY);
else
fh = open(file_name, O_RDONLY);
#endif
#endif
#ifdef PIDX_RECORD_TIME
t2 = PIDX_get_time();
#endif
data_offset = 0;
data_offset += io_id->idx_d->start_fs_block * io_id->idx_d->fs_block_size;
if (MODE == PIDX_WRITE)
{
for (k = 0; k < agg_buf->var_number; k++)
{
PIDX_variable vark = io_id->idx->variable[k];
int bytes_per_datatype = ((vark->bits_per_value/8) * total_chunk_size) / (io_id->idx->compression_factor);
int64_t prev_var_sample = (int64_t) block_layout->block_count_per_file[agg_buf->file_number] * io_id->idx_d->samples_per_block * bytes_per_datatype * io_id->idx->variable[k]->values_per_sample;
data_offset = (int64_t) data_offset + prev_var_sample;
}
for (i = 0; i < agg_buf->sample_number; i++)
data_offset = (int64_t) data_offset + agg_buf->buffer_size;
}
else
{
int total_header_size = (10 + (10 * io_id->idx->blocks_per_file)) * sizeof (uint32_t) * io_id->idx->variable_count;
headers = malloc(total_header_size);
memset(headers, 0, total_header_size);
#if PIDX_HAVE_MPI
mpi_ret = MPI_File_read_at(fh, 0, headers, total_header_size , MPI_BYTE, &status);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "Data offset = %lld [%s] [%d] MPI_File_write_at() failed for filename %s.\n", (long long) data_offset, __FILE__, __LINE__, file_name);
return PIDX_err_io;
}
#endif
}
#if !SIMULATE_IO
#if PIDX_HAVE_MPI
if (MODE == PIDX_WRITE)
{
//int rank;
//MPI_Comm_rank(io_id->comm, &rank);
//printf("W [%d] [%d %d %d] size = %d and offset = %d\n", rank, agg_buf->file_number, agg_buf->var_number, agg_buf->sample_number, agg_buf->buffer_size, data_offset);
mpi_ret = MPI_File_write_at(fh, data_offset, agg_buf->buffer, agg_buf->buffer_size , MPI_BYTE, &status);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "Data offset = %lld [%s] [%d] MPI_File_write_at() failed for filename %s.\n", (long long) data_offset, __FILE__, __LINE__, file_name);
return PIDX_err_io;
}
int write_count = 0;
MPI_Get_count(&status, MPI_BYTE, &write_count);
if (write_count != agg_buf->buffer_size)
{
fprintf(stderr, "[%s] [%d] MPI_File_write_at() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
}
else
{
int data_size = 0;
int block_count = 0;
for (i = 0; i < io_id->idx->blocks_per_file; i++)
{
if (PIDX_blocks_is_block_present(agg_buf->file_number * io_id->idx->blocks_per_file + i, block_layout))
{
data_offset = htonl(headers[12 + ((i + (io_id->idx->blocks_per_file * agg_buf->var_number))*10 )]);
data_size = htonl(headers[14 + ((i + (io_id->idx->blocks_per_file * agg_buf->var_number))*10 )]);
mpi_ret = MPI_File_read_at(fh, data_offset, agg_buf->buffer + (block_count * io_id->idx_d->samples_per_block * (io_id->idx->variable[agg_buf->var_number]->bits_per_value/8) * io_id->idx->variable[agg_buf->var_number]->values_per_sample * io_id->idx->chunk_size[0] * io_id->idx->chunk_size[1] * io_id->idx->chunk_size[2]) / io_id->idx->compression_factor, /*agg_buf->buffer_size*/data_size , MPI_BYTE, &status);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "Data offset = %lld [%s] [%d] MPI_File_write_at() failed for filename %s.\n", (long long) data_offset, __FILE__, __LINE__, file_name);
return PIDX_err_io;
}
int read_count = 0;
MPI_Get_count(&status, MPI_BYTE, &read_count);
if (read_count != /*agg_buf->buffer_size*/data_size)
{
fprintf(stderr, "[%s] [%d] MPI_File_write_at() failed. %d != %lldd\n", __FILE__, __LINE__, read_count, (long long)agg_buf->buffer_size);
return PIDX_err_io;
}
block_count++;
}
}
free(headers);
}
#else
if (MODE == PIDX_WRITE)
{
ssize_t write_count = pwrite(fh, agg_buf->buffer, agg_buf->buffer_size, data_offset);
if (write_count != agg_buf->buffer_size)
{
fprintf(stderr, "[%s] [%d] pwrite() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
}
else
{
ssize_t read_count = pread(fh, agg_buf->buffer, agg_buf->buffer_size, data_offset);
if (read_count != agg_buf->buffer_size)
{
fprintf(stderr, "[%s] [%d] pread() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
}
#endif
#endif
#ifdef PIDX_RECORD_TIME
t3 = PIDX_get_time();
#endif
#if !SIMULATE_IO
#if PIDX_HAVE_MPI
mpi_ret = MPI_File_close(&fh);
if (mpi_ret != MPI_SUCCESS)
{
fprintf(stderr, "[%s] [%d] MPI_File_open() failed.\n", __FILE__, __LINE__);
return PIDX_err_io;
}
#else
close(fh);
#endif
#endif
#ifdef PIDX_RECORD_TIME
t4 = PIDX_get_time();
#endif
#ifdef PIDX_RECORD_TIME
printf("V. [R %d] [O %lld C %lld] [FVS %d %d %d] Time: O %f H %f W %f C %f\n", rank, (long long) data_offset, (long long)agg_buf->buffer_size, agg_buf->file_number, agg_buf->var_number, agg_buf->sample_number, (t2-t1), (t2-t2), (t3-t2), (t4-t3));
#endif
}
return PIDX_success;
}