void write_index (int fd, uint64_t file_offset, struct adios_index_struct_v1 * index)
{
char * buffer = NULL;
uint64_t buffer_size = 0L;
uint64_t buffer_offset = 0L;
adios_write_index_v1 (&buffer, &buffer_size, &buffer_offset, file_offset, index);
adios_write_version_v1 (&buffer, &buffer_size, &buffer_offset);
printf ("Index metadata size is %llu bytes\n", buffer_offset);
if (do_write_index)
{
printf ("Ready to write index to file offset %llu. Size is %llu bytes\n",
file_offset, buffer_offset);
lseek (fd, file_offset, SEEK_SET);
ssize_t s = write (fd, buffer, buffer_offset);
if (s != buffer_offset) {
printf ("ERROR: Only wrote %llu bytes of index data instead of the attempted %llu bytes\n",
(unsigned long long)s, buffer_offset);
} else {
printf ("Wrote index to file offset %llu. Size is %llu bytes\n", file_offset, buffer_offset);
}
printf ("Truncate file to size %llu\n", file_offset+s);
ftruncate (fd, file_offset+s);
}
else
{
printf ("To actually recover the file by rewriting the index, use -f option.\n"
"Final file will be truncted to %llu bytes\n", file_offset+buffer_offset);
}
}
void adios_posix_close (struct adios_file_struct * fd
,struct adios_method_struct * method
)
{
struct adios_POSIX_data_struct * p = (struct adios_POSIX_data_struct *)
method->method_data;
struct adios_attribute_struct * a = fd->group->attributes;
struct adios_index_process_group_struct_v1 * new_pg_root = 0;
struct adios_index_var_struct_v1 * new_vars_root = 0;
struct adios_index_attribute_struct_v1 * new_attrs_root = 0;
START_TIMER (ADIOS_TIMER_POSIX_AD_CLOSE);
switch (fd->mode)
{
case adios_mode_write:
{
if (fd->shared_buffer == adios_flag_no)
{
off_t new_off;
// set it up so that it will start at 0, but have correct sizes
new_off = lseek (p->b.f, 0, SEEK_CUR);
fd->offset = fd->base_offset - p->vars_start;
fd->vars_start = 0;
fd->buffer_size = 0;
adios_write_close_vars_v1 (fd);
// fd->vars_start gets updated with the size written
fd->offset = lseek (p->b.f, p->vars_start, SEEK_SET);
START_TIMER (ADIOS_TIMER_POSIX_IO);
ssize_t s = write (p->b.f, fd->buffer, p->vars_header_size);
STOP_TIMER (ADIOS_TIMER_POSIX_IO);
if (s != fd->vars_start)
{
fprintf (stderr, "POSIX method tried to write %llu, "
"only wrote %lld\n"
,fd->vars_start
,(int64_t)s
);
}
fd->offset = 0;
fd->bytes_written = 0;
adios_shared_buffer_free (&p->b);
new_off = lseek (p->b.f, new_off, SEEK_SET); // go back to end
adios_write_open_attributes_v1 (fd);
p->vars_start = lseek (p->b.f, fd->offset, SEEK_CUR); // save loc
p->vars_header_size = p->vars_start - fd->base_offset;
p->vars_start -= fd->offset; // adjust to start of header
fd->base_offset += fd->offset; // add size of header
fd->offset = 0;
fd->bytes_written = 0;
if (!fd->group->process_id) { // from ADIOS 1.4, only rank 0 writes attributes
while (a)
{
adios_write_attribute_v1 (fd, a);
if (fd->base_offset + fd->bytes_written > fd->pg_start_in_file + fd->write_size_bytes)
fprintf (stderr, "adios_posix_write exceeds pg bound. File is corrupted. "
"Need to enlarge group size. \n");
START_TIMER (ADIOS_TIMER_POSIX_MD);
ssize_t s = write (p->b.f, fd->buffer, fd->bytes_written);
STOP_TIMER (ADIOS_TIMER_POSIX_MD);
if (s != fd->bytes_written)
{
fprintf (stderr, "POSIX method tried to write %llu, "
"only wrote %lld\n"
,fd->bytes_written
,(int64_t)s
);
}
fd->base_offset += s;
fd->offset = 0;
fd->bytes_written = 0;
adios_shared_buffer_free (&p->b);
a = a->next;
}
}
// set it up so that it will start at 0, but have correct sizes
fd->offset = fd->base_offset - p->vars_start;
fd->vars_start = 0;
fd->buffer_size = 0;
adios_write_close_attributes_v1 (fd);
fd->offset = lseek (p->b.f, p->vars_start, SEEK_SET);
// fd->vars_start gets updated with the size written
START_TIMER (ADIOS_TIMER_POSIX_MD);
s = write (p->b.f, fd->buffer, p->vars_header_size);
STOP_TIMER (ADIOS_TIMER_POSIX_MD);
if (s != p->vars_header_size)
{
fprintf (stderr, "POSIX method tried to write %llu, "
"only wrote %lld\n"
,p->vars_header_size
,(int64_t)s
);
}
fd->offset = 0;
fd->bytes_written = 0;
}
// buffering or not, write the index
char * buffer = 0;
uint64_t buffer_size = 0;
uint64_t buffer_offset = 0;
uint64_t index_start = fd->base_offset + fd->offset;
// build index
adios_build_index_v1 (fd, p->index);
// if collective, gather the indexes from the rest and call
// adios_merge_index_v1 (&new_pg_root, &new_vars_root, pg, vars);
adios_write_index_v1 (&buffer, &buffer_size, &buffer_offset
,index_start, p->index);
adios_write_version_v1 (&buffer, &buffer_size, &buffer_offset);
START_TIMER (ADIOS_TIMER_POSIX_IO);
adios_posix_do_write (fd, method, buffer, buffer_offset); // Buffered vars written here
STOP_TIMER (ADIOS_TIMER_POSIX_IO);
#ifdef HAVE_MPI
if (p->group_comm != MPI_COMM_SELF)
{
if (p->rank == 0)
{
int * index_sizes = malloc (4 * p->size);
int * index_offsets = malloc (4 * p->size);
char * recv_buffer = 0;
int i;
uint32_t size = 0, total_size = 0;
START_TIMER (ADIOS_TIMER_POSIX_COMM);
MPI_Gather (&size, 1, MPI_INT
,index_sizes, 1, MPI_INT
,0, p->group_comm
);
STOP_TIMER (ADIOS_TIMER_POSIX_COMM);
for (i = 0; i < p->size; i++)
{
index_offsets [i] = total_size;
total_size += index_sizes [i];
}
recv_buffer = malloc (total_size);
START_TIMER (ADIOS_TIMER_POSIX_COMM);
MPI_Gatherv (&size, 0, MPI_BYTE
,recv_buffer, index_sizes, index_offsets
,MPI_BYTE, 0, p->group_comm
);
STOP_TIMER (ADIOS_TIMER_POSIX_COMM);
char * buffer_save = p->b.buff;
uint64_t buffer_size_save = p->b.length;
uint64_t offset_save = p->b.offset;
for (i = 1; i < p->size; i++)
{
p->b.buff = recv_buffer + index_offsets [i];
p->b.length = index_sizes [i];
p->b.offset = 0;
adios_parse_process_group_index_v1 (&p->b
,&new_pg_root
);
adios_parse_vars_index_v1 (&p->b, &new_vars_root, NULL, NULL);
// do not merge attributes from other processes from 1.4
/*
adios_parse_attributes_index_v1 (&p->b
,&new_attrs_root
);
*/
adios_merge_index_v1 (p->index, new_pg_root,
new_vars_root, new_attrs_root);
new_pg_root = 0;
new_vars_root = 0;
new_attrs_root = 0;
}
p->b.buff = buffer_save;
p->b.length = buffer_size_save;
p->b.offset = offset_save;
free (recv_buffer);
free (index_sizes);
free (index_offsets);
char * global_index_buffer = 0;
uint64_t global_index_buffer_size = 0;
uint64_t global_index_buffer_offset = 0;
uint64_t global_index_start = 0;
uint16_t flag = 0;
adios_write_index_v1 (&global_index_buffer, &global_index_buffer_size
,&global_index_buffer_offset, global_index_start
,p->index);
flag |= ADIOS_VERSION_HAVE_SUBFILE;
adios_write_version_flag_v1 (&global_index_buffer
,&global_index_buffer_size
,&global_index_buffer_offset
,flag
);
START_TIMER (ADIOS_TIMER_POSIX_MD);
ssize_t s = write (p->mf, global_index_buffer, global_index_buffer_offset);
STOP_TIMER (ADIOS_TIMER_POSIX_MD);
if (s != global_index_buffer_offset)
{
fprintf (stderr, "POSIX method tried to write %llu, "
"only wrote %lld\n"
,fd->bytes_written
,(int64_t)s
);
}
close (p->mf);
}
else
{
START_TIMER (ADIOS_TIMER_POSIX_COMM);
// Added this explicit cast to avoid truncation of low-order bytes on BGP
int i_buffer_size = (int) buffer_size;
MPI_Gather (&i_buffer_size, 1, MPI_INT
,0, 0, MPI_INT
,0, p->group_comm
);
MPI_Gatherv (buffer, buffer_size, MPI_BYTE
,0, 0, 0, MPI_BYTE
,0, p->group_comm
);
STOP_TIMER (ADIOS_TIMER_POSIX_COMM);
}
}
#endif
free (buffer);
break;
}
case adios_mode_append:
case adios_mode_update:
{
if (fd->shared_buffer == adios_flag_no)
{
off_t new_off;
// set it up so that it will start at 0, but have correct sizes
new_off = lseek (p->b.f, 0, SEEK_CUR);
fd->offset = fd->base_offset - p->vars_start;
fd->vars_start = 0;
fd->buffer_size = 0;
adios_write_close_vars_v1 (fd);
// fd->vars_start gets updated with the size written
fd->offset = lseek (p->b.f, p->vars_start, SEEK_SET);
START_TIMER (ADIOS_TIMER_POSIX_IO);
ssize_t s = write (p->b.f, fd->buffer, p->vars_header_size);
STOP_TIMER (ADIOS_TIMER_POSIX_IO);
if (s != fd->vars_start)
{
fprintf (stderr, "POSIX method tried to write %llu, "
"only wrote %lld\n"
,fd->vars_start
,(int64_t)s
);
}
fd->offset = 0;
fd->bytes_written = 0;
adios_shared_buffer_free (&p->b);
new_off = lseek (p->b.f, new_off, SEEK_SET); // go back to end
adios_write_open_attributes_v1 (fd);
p->vars_start = lseek (p->b.f, fd->offset, SEEK_CUR); // save loc
p->vars_header_size = p->vars_start - fd->base_offset;
p->vars_start -= fd->offset; // adjust to start of header
fd->base_offset += fd->offset; // add size of header
fd->offset = 0;
fd->bytes_written = 0;
if (!fd->group->process_id) { // from ADIOS 1.4, only rank 0 writes attributes
while (a)
{
adios_write_attribute_v1 (fd, a);
START_TIMER (ADIOS_TIMER_POSIX_MD);
ssize_t s = write (p->b.f, fd->buffer, fd->bytes_written);
STOP_TIMER (ADIOS_TIMER_POSIX_MD);
if (s != fd->bytes_written)
{
fprintf (stderr, "POSIX method tried to write %llu, "
"only wrote %lld\n"
,fd->bytes_written
,(int64_t)s
);
}
fd->base_offset += s;
fd->offset = 0;
fd->bytes_written = 0;
adios_shared_buffer_free (&p->b);
a = a->next;
}
}
// set it up so that it will start at 0, but have correct sizes
fd->offset = fd->base_offset - p->vars_start;
fd->vars_start = 0;
fd->buffer_size = 0;
adios_write_close_attributes_v1 (fd);
fd->offset = lseek (p->b.f, p->vars_start, SEEK_SET);
// fd->vars_start gets updated with the size written
START_TIMER (ADIOS_TIMER_POSIX_MD);
s = write (p->b.f, fd->buffer, p->vars_header_size);
STOP_TIMER (ADIOS_TIMER_POSIX_MD);
if (s != p->vars_header_size)
{
fprintf (stderr, "POSIX method tried to write %llu, "
"only wrote %lld\n"
,p->vars_header_size
,(int64_t)s
);
}
fd->offset = 0;
fd->bytes_written = 0;
}
char * buffer = 0;
uint64_t buffer_size = 0;
uint64_t buffer_offset = 0;
uint64_t index_start = fd->base_offset + fd->offset;
// build index
adios_build_index_v1 (fd, p->index);
adios_write_index_v1 (&buffer, &buffer_size, &buffer_offset
,index_start, p->index);
#ifdef HAVE_MPI
if (p->group_comm != MPI_COMM_SELF)
{
if (p->rank == 0)
{
int * index_sizes = malloc (4 * p->size);
int * index_offsets = malloc (4 * p->size);
char * recv_buffer = 0;
int i;
uint32_t size = 0, total_size = 0;
START_TIMER (ADIOS_TIMER_POSIX_COMM);
MPI_Gather (&size, 1, MPI_INT
,index_sizes, 1, MPI_INT
,0, p->group_comm
);
STOP_TIMER (ADIOS_TIMER_POSIX_COMM);
for (i = 0; i < p->size; i++)
{
index_offsets [i] = total_size;
total_size += index_sizes [i];
}
recv_buffer = malloc (total_size);
START_TIMER (ADIOS_TIMER_POSIX_COMM);
MPI_Gatherv (&size, 0, MPI_BYTE
,recv_buffer, index_sizes, index_offsets
,MPI_BYTE, 0, p->group_comm
);
STOP_TIMER (ADIOS_TIMER_POSIX_COMM);
char * buffer_save = p->b.buff;
uint64_t buffer_size_save = p->b.length;
uint64_t offset_save = p->b.offset;
for (i = 1; i < p->size; i++)
{
p->b.buff = recv_buffer + index_offsets [i];
p->b.length = index_sizes [i];
p->b.offset = 0;
adios_parse_process_group_index_v1 (&p->b
,&new_pg_root
);
adios_parse_vars_index_v1 (&p->b, &new_vars_root, NULL, NULL);
// do not merge attributes from other processes from 1.4
/*
adios_parse_attributes_index_v1 (&p->b
,&new_attrs_root
);
*/
adios_merge_index_v1 (p->index,new_pg_root,
new_vars_root, new_attrs_root);
new_pg_root = 0;
new_vars_root = 0;
new_attrs_root = 0;
}
adios_sort_index_v1 (&p->index->pg_root
,&p->index->vars_root
,&p->index->attrs_root
);
p->b.buff = buffer_save;
p->b.length = buffer_size_save;
p->b.offset = offset_save;
free (recv_buffer);
free (index_sizes);
free (index_offsets);
char * global_index_buffer = 0;
uint64_t global_index_buffer_size = 0;
uint64_t global_index_buffer_offset = 0;
uint64_t global_index_start = 0;
uint16_t flag = 0;
adios_write_index_v1 (&global_index_buffer, &global_index_buffer_size
,&global_index_buffer_offset, global_index_start
,p->index);
flag |= ADIOS_VERSION_HAVE_SUBFILE;
adios_write_version_flag_v1 (&global_index_buffer
,&global_index_buffer_size
,&global_index_buffer_offset
,flag
);
START_TIMER (ADIOS_TIMER_POSIX_MD);
ssize_t s = write (p->mf, global_index_buffer, global_index_buffer_offset);
STOP_TIMER (ADIOS_TIMER_POSIX_MD);
if (s != global_index_buffer_offset)
{
fprintf (stderr, "POSIX method tried to write %llu, "
"only wrote %lld, Mode: a\n"
,global_index_buffer_offset
,(int64_t)s
);
}
close (p->mf);
free (global_index_buffer);
}
else
{
START_TIMER (ADIOS_TIMER_POSIX_COMM);
MPI_Gather (&buffer_size, 1, MPI_INT
,0, 0, MPI_INT
,0, p->group_comm
);
MPI_Gatherv (buffer, buffer_size, MPI_BYTE
,0, 0, 0, MPI_BYTE
,0, p->group_comm
);
STOP_TIMER (ADIOS_TIMER_POSIX_COMM);
}
}
#endif
adios_write_version_v1 (&buffer, &buffer_size, &buffer_offset);
START_TIMER (ADIOS_TIMER_POSIX_MD);
adios_posix_do_write (fd, method, buffer, buffer_offset);
STOP_TIMER (ADIOS_TIMER_POSIX_MD);
free (buffer);
break;
}
case adios_mode_read:
{
// read the index to find the place to start reading
adios_posix_do_read (fd, method);
struct adios_var_struct * v = fd->group->vars;
while (v)
{
v->data = 0;
v = v->next;
}
break;
}
default:
{
fprintf (stderr, "Unknown file mode: %d\n", fd->mode);
return;
}
}
adios_posix_close_internal (&p->b);
adios_clear_index_v1 (p->index);
STOP_TIMER (ADIOS_TIMER_POSIX_AD_CLOSE);
#if defined ADIOS_TIMERS || defined ADIOS_TIMER_EVENTS
//Finished timing this cycle, swap the timing buffers
adios_timing_destroy(fd->group->prev_timing_obj);
fd->group->prev_timing_obj = fd->group->timing_obj;
fd->group->timing_obj = 0;
// prev_timing_obj points to unwritten timing info, timing_obj is
// ready to allocate at the next open
#endif
}
void adios_posix1_close (struct adios_file_struct * fd
,struct adios_method_struct * method
)
{
struct adios_POSIX1_data_struct * p = (struct adios_POSIX1_data_struct *)
method->method_data;
struct adios_attribute_struct * a = fd->group->attributes;
switch (fd->mode)
{
case adios_mode_write:
{
if (fd->shared_buffer == adios_flag_no)
{
off_t new_off;
// set it up so that it will start at 0, but have correct sizes
new_off = lseek (p->b.f, 0, SEEK_CUR);
fd->offset = fd->base_offset - p->vars_start;
fd->vars_start = 0;
fd->buffer_size = 0;
adios_write_close_vars_v1 (fd);
// fd->vars_start gets updated with the size written
fd->offset = lseek (p->b.f, p->vars_start, SEEK_SET);
ssize_t s = write (p->b.f, fd->buffer, p->vars_header_size);
if (s != fd->vars_start)
{
fprintf (stderr, "POSIX1 method tried to write %llu, "
"only wrote %lld\n"
,fd->vars_start
,(int64_t)s
);
}
fd->offset = 0;
fd->bytes_written = 0;
adios_shared_buffer_free (&p->b);
new_off = lseek (p->b.f, new_off, SEEK_SET); // go back to end
adios_write_open_attributes_v1 (fd);
p->vars_start = lseek (p->b.f, fd->offset, SEEK_CUR); // save loc
p->vars_header_size = p->vars_start - fd->base_offset;
p->vars_start -= fd->offset; // adjust to start of header
fd->base_offset += fd->offset; // add size of header
fd->offset = 0;
fd->bytes_written = 0;
while (a)
{
adios_write_attribute_v1 (fd, a);
if (fd->base_offset + fd->bytes_written > fd->pg_start_in_file + fd->write_size_bytes)
fprintf (stderr, "adios_posix1_write exceeds pg bound. File is corrupted. "
"Need to enlarge group size. \n");
ssize_t s = write (p->b.f, fd->buffer, fd->bytes_written);
if (s != fd->bytes_written)
{
fprintf (stderr, "POSIX1 method tried to write %llu, "
"only wrote %lld\n"
,fd->bytes_written
,(int64_t)s
);
}
fd->base_offset += s;
fd->offset = 0;
fd->bytes_written = 0;
adios_shared_buffer_free (&p->b);
a = a->next;
}
// set it up so that it will start at 0, but have correct sizes
fd->offset = fd->base_offset - p->vars_start;
fd->vars_start = 0;
fd->buffer_size = 0;
adios_write_close_attributes_v1 (fd);
fd->offset = lseek (p->b.f, p->vars_start, SEEK_SET);
// fd->vars_start gets updated with the size written
s = write (p->b.f, fd->buffer, p->vars_header_size);
if (s != p->vars_header_size)
{
fprintf (stderr, "POSIX1 method tried to write %llu, "
"only wrote %lld\n"
,p->vars_header_size
,(int64_t)s
);
}
fd->offset = 0;
fd->bytes_written = 0;
}
// buffering or not, write the index
char * buffer = 0;
uint64_t buffer_size = 0;
uint64_t buffer_offset = 0;
uint64_t index_start = fd->base_offset + fd->offset;
// build index
adios_build_index_v1 (fd, p->index);
adios_write_index_v1 (&buffer, &buffer_size, &buffer_offset
,index_start, p->index);
adios_write_version_v1 (&buffer, &buffer_size, &buffer_offset);
adios_posix1_do_write (fd, method, buffer, buffer_offset);
free (buffer);
break;
}
case adios_mode_append:
{
if (fd->shared_buffer == adios_flag_no)
{
off_t new_off;
// set it up so that it will start at 0, but have correct sizes
new_off = lseek (p->b.f, 0, SEEK_CUR);
fd->offset = fd->base_offset - p->vars_start;
fd->vars_start = 0;
fd->buffer_size = 0;
adios_write_close_vars_v1 (fd);
// fd->vars_start gets updated with the size written
fd->offset = lseek (p->b.f, p->vars_start, SEEK_SET);
ssize_t s = write (p->b.f, fd->buffer, p->vars_header_size);
if (s != fd->vars_start)
{
fprintf (stderr, "POSIX1 method tried to write %llu, "
"only wrote %lld\n"
,fd->vars_start
,(int64_t)s
);
}
fd->offset = 0;
fd->bytes_written = 0;
adios_shared_buffer_free (&p->b);
new_off = lseek (p->b.f, new_off, SEEK_SET); // go back to end
adios_write_open_attributes_v1 (fd);
p->vars_start = lseek (p->b.f, fd->offset, SEEK_CUR); // save loc
p->vars_header_size = p->vars_start - fd->base_offset;
p->vars_start -= fd->offset; // adjust to start of header
fd->base_offset += fd->offset; // add size of header
fd->offset = 0;
fd->bytes_written = 0;
while (a)
{
adios_write_attribute_v1 (fd, a);
ssize_t s = write (p->b.f, fd->buffer, fd->bytes_written);
if (s != fd->bytes_written)
{
fprintf (stderr, "POSIX1 method tried to write %llu, "
"only wrote %lld\n"
,fd->bytes_written
,(int64_t)s
);
}
fd->base_offset += s;
fd->offset = 0;
fd->bytes_written = 0;
adios_shared_buffer_free (&p->b);
a = a->next;
}
// set it up so that it will start at 0, but have correct sizes
fd->offset = fd->base_offset - p->vars_start;
fd->vars_start = 0;
fd->buffer_size = 0;
adios_write_close_attributes_v1 (fd);
fd->offset = lseek (p->b.f, p->vars_start, SEEK_SET);
// fd->vars_start gets updated with the size written
s = write (p->b.f, fd->buffer, p->vars_header_size);
if (s != p->vars_header_size)
{
fprintf (stderr, "POSIX1 method tried to write %llu, "
"only wrote %lld\n"
,p->vars_header_size
,(int64_t)s
);
}
fd->offset = 0;
fd->bytes_written = 0;
}
char * buffer = 0;
uint64_t buffer_size = 0;
uint64_t buffer_offset = 0;
uint64_t index_start = fd->base_offset + fd->offset;
// build index. Note: It merges with old indices already stored
// in p->index in adios_posix_open's append case
adios_build_index_v1 (fd, p->index);
// merge in old indicies
//adios_merge_index_v1 (p->index, new_pg_root,
// new_vars_root, new_attrs_root
// );
adios_write_index_v1 (&buffer, &buffer_size, &buffer_offset
,index_start, p->index);
adios_write_version_v1 (&buffer, &buffer_size, &buffer_offset);
adios_posix1_do_write (fd, method, buffer, buffer_offset);
free (buffer);
break;
}
case adios_mode_read:
{
// read the index to find the place to start reading
adios_posix1_do_read (fd, method);
struct adios_var_struct * v = fd->group->vars;
while (v)
{
v->data = 0;
v = v->next;
}
break;
}
default:
{
fprintf (stderr, "Unknown file mode: %d\n", fd->mode);
return;
}
}
adios_posix_close_internal (&p->b);
adios_clear_index_v1 (p->index);
}