int main (int argc, char ** argv)
{
long int tmp;
int c;
while ((c = getopt_long(argc, argv, optstring, options, NULL)) != -1) {
switch (c) {
case 'n':
case 't':
errno = 0;
tmp = strtol(optarg, (char **)NULL, 0);
if (errno) {
fprintf(stderr, "Error: could not convert -%c value: %s\n",
c, optarg);
return 1;
}
if (c == 'n')
nsubfiles=tmp;
else
nthreads=tmp;
break;
case 'h':
display_help();
return 0;
break;
case 'v':
verbose++;
break;
default:
printf("Unrecognized argument: %s\n", optarg);
break;
}
}
/* Check if we have a file defined */
if (optind >= argc) {
printf ("Missing file name\n");
display_help();
return 1;
}
filename = strdup(argv[optind++]);
if (nsubfiles < 1)
nsubfiles = get_nsubfiles (filename);
if (nsubfiles < 1) {
printf ("Cannot determine the number of subfiles. To avoid this problem, "
"provide the number of subfiles manually with the -n <N> option.\n");
return -1;
}
if (nthreads < 1)
nthreads = 1;
if (nthreads > nsubfiles) {
printf ("Warning: asked for processing %d subfiles using %d threads. "
"We will utilize only %d threads.\n",
nsubfiles, nthreads, nsubfiles);
nthreads = nsubfiles;
}
if (verbose>1)
printf ("Create metadata file %s from %d subfiles using %d threads\n",
filename, nsubfiles, nthreads);
/* Initialize global variables */
b = malloc (nsubfiles * sizeof (struct adios_bp_buffer_struct_v1*));
subindex = malloc (nthreads * sizeof (struct adios_index_struct_v1*));
/* Split the processing work among T threads */
int tid;
#if HAVE_PTHREAD
pthread_t *thread = (pthread_t *) malloc (nthreads * sizeof(pthread_t));
struct thread_args *targs = (struct thread_args*)
malloc (nthreads * sizeof(struct thread_args));
int K = nsubfiles/nthreads; // base number of files to be processed by one thread
int L = nsubfiles%nthreads; // this many threads processes one more subfile
int startidx, endidx;
int rc;
//printf ("K=%d L=%d\n", K, L);
endidx = -1;
for (tid=0; tid<nthreads; tid++)
{
startidx = endidx + 1;
endidx = startidx + K - 1;
targs[tid].tid = tid;
targs[tid].startidx = startidx;
targs[tid].endidx = endidx;
if (tid < L) {
endidx++;
}
if (verbose)
printf ("Process subfiles from %d to %d with thread %d\n",
targs[tid].startidx, targs[tid].endidx, targs[tid]);
if (tid < nthreads-1) {
/* Start worker thread. */
pthread_attr_t attr;
pthread_attr_init (&attr);
pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_JOINABLE);
rc = pthread_create (&thread[tid], &attr, thread_main, &targs[tid]);
if (rc) {
printf ("ERROR: Thread %d: Cannot create thread, err code = %d\n",
tid, rc);
}
pthread_attr_destroy(&attr);
}
else
{
// last "thread" is the main thread
process_subfiles (tid, startidx, endidx);
}
}
// wait here for everyone to finish
for (tid=0; tid<nthreads-1; tid++)
{
void *status;
rc = pthread_join (thread[tid], &status);
if (rc) {
printf ("ERROR: Thread %d: Cannot join thread, err code = %d\n", tid, rc);
} else {
if (verbose>1)
printf ("Thread %d: Joined thread.\n", tid);
}
}
free (targs);
free (thread);
#else /* non-threaded version */
nthreads = 1;
process_subfiles (0, 0, nsubfiles-1);
#endif
/* Merge the T indexes into the global output index */
struct adios_index_struct_v1 * globalindex;
globalindex = adios_alloc_index_v1(1);
for (tid=0; tid<nthreads; tid++)
{
adios_merge_index_v1 (globalindex,
subindex[tid]->pg_root,
subindex[tid]->vars_root,
subindex[tid]->attrs_root);
}
write_index (globalindex, filename);
/* Clean-up */
adios_clear_index_v1 (globalindex);
/*... already cleaned-up by globalindex clearing
for (tid=0; tid<nthreads; tid++) {
adios_clear_index_v1 (subindex[tid]);
free (subindex[tid]);
}
*/
free (subindex);
free (globalindex);
free (b);
return 0;
}
static void adios_posix_do_read (struct adios_file_struct * fd
,struct adios_method_struct * method
)
{
struct adios_POSIX_data_struct * p = (struct adios_POSIX_data_struct *)
method->method_data;
struct adios_var_struct * v = fd->group->vars;
uint32_t version = 0;
adios_posix_read_version (&p->b);
adios_parse_version (&p->b, &version);
version &= ADIOS_VERSION_NUM_MASK;
switch (version)
{
case 1:
case 2:
{
struct adios_index_struct_v1 * index = adios_alloc_index_v1(0); // no hashtables
struct adios_index_process_group_struct_v1 * pg_root = index->pg_root;
struct adios_index_process_group_struct_v1 * pg_root_temp = 0;
adios_posix_read_index_offsets (&p->b);
adios_parse_index_offsets_v1 (&p->b);
adios_posix_read_process_group_index (&p->b);
adios_parse_process_group_index_v1 (&p->b, &pg_root);
#if 1
adios_posix_read_vars_index (&p->b);
adios_parse_vars_index_v1 (&p->b, &index->vars_root, NULL, NULL);
adios_posix_read_attributes_index (&p->b);
adios_parse_attributes_index_v1 (&p->b, &index->attrs_root);
#endif
// the three section headers
struct adios_process_group_header_struct_v1 pg_header;
struct adios_vars_header_struct_v1 vars_header;
struct adios_attributes_header_struct_v1 attrs_header;
struct adios_var_header_struct_v1 var_header;
struct adios_var_payload_struct_v1 var_payload;
struct adios_attribute_struct_v1 attribute;
int i;
pg_root_temp = pg_root;
while (pg_root_temp && pg_root_temp->next)
pg_root_temp = pg_root_temp->next;
p->b.read_pg_offset = pg_root_temp->offset_in_file;
if (pg_root_temp->next)
{
p->b.read_pg_size = pg_root_temp->next->offset_in_file
- pg_root_temp->offset_in_file;
}
else
{
p->b.read_pg_size = p->b.pg_index_offset
- pg_root_temp->offset_in_file;
}
adios_posix_read_process_group (&p->b);
adios_parse_process_group_header_v1 (&p->b, &pg_header);
adios_parse_vars_header_v1 (&p->b, &vars_header);
for (i = 0; i < vars_header.count; i++)
{
memset (&var_payload, 0
,sizeof (struct adios_var_payload_struct_v1)
);
adios_parse_var_data_header_v1 (&p->b, &var_header);
struct adios_var_struct * v1 = v;
while (v1)
{
if ( strcasecmp (var_header.name, v1->name)
|| strcasecmp (var_header.path, v1->path)
)
{
v1 = v1->next;
}
else
break;
}
if (v1)
{
var_payload.payload = v1->data;
adios_parse_var_data_payload_v1 (&p->b, &var_header
,&var_payload
,v1->data_size
);
}
else
{
adios_parse_var_data_payload_v1 (&p->b, &var_header
,NULL, 0
);
}
adios_clear_var_header_v1 (&var_header);
}
#if 1
adios_parse_attributes_header_v1 (&p->b, &attrs_header);
for (i = 0; i < attrs_header.count; i++)
{
adios_parse_attribute_v1 (&p->b, &attribute);
adios_clear_attribute_v1 (&attribute);
}
#endif
adios_clear_process_group_header_v1 (&pg_header);
adios_clear_index_v1 (index);
break;
}
default:
fprintf (stderr, "POSIX read: file version unknown: %u\n", version);
return;
}
adios_buffer_struct_clear (&p->b);
}
void adios_posix_close (struct adios_file_struct * fd
,struct adios_method_struct * method
)
{
struct adios_POSIX_data_struct * p = (struct adios_POSIX_data_struct *)
method->method_data;
struct adios_attribute_struct * a = fd->group->attributes;
struct adios_index_process_group_struct_v1 * new_pg_root = 0;
struct adios_index_var_struct_v1 * new_vars_root = 0;
struct adios_index_attribute_struct_v1 * new_attrs_root = 0;
START_TIMER (ADIOS_TIMER_POSIX_AD_CLOSE);
switch (fd->mode)
{
case adios_mode_write:
{
if (fd->shared_buffer == adios_flag_no)
{
off_t new_off;
// set it up so that it will start at 0, but have correct sizes
new_off = lseek (p->b.f, 0, SEEK_CUR);
fd->offset = fd->base_offset - p->vars_start;
fd->vars_start = 0;
fd->buffer_size = 0;
adios_write_close_vars_v1 (fd);
// fd->vars_start gets updated with the size written
fd->offset = lseek (p->b.f, p->vars_start, SEEK_SET);
START_TIMER (ADIOS_TIMER_POSIX_IO);
ssize_t s = write (p->b.f, fd->buffer, p->vars_header_size);
STOP_TIMER (ADIOS_TIMER_POSIX_IO);
if (s != fd->vars_start)
{
fprintf (stderr, "POSIX method tried to write %llu, "
"only wrote %lld\n"
,fd->vars_start
,(int64_t)s
);
}
fd->offset = 0;
fd->bytes_written = 0;
adios_shared_buffer_free (&p->b);
new_off = lseek (p->b.f, new_off, SEEK_SET); // go back to end
adios_write_open_attributes_v1 (fd);
p->vars_start = lseek (p->b.f, fd->offset, SEEK_CUR); // save loc
p->vars_header_size = p->vars_start - fd->base_offset;
p->vars_start -= fd->offset; // adjust to start of header
fd->base_offset += fd->offset; // add size of header
fd->offset = 0;
fd->bytes_written = 0;
if (!fd->group->process_id) { // from ADIOS 1.4, only rank 0 writes attributes
while (a)
{
adios_write_attribute_v1 (fd, a);
if (fd->base_offset + fd->bytes_written > fd->pg_start_in_file + fd->write_size_bytes)
fprintf (stderr, "adios_posix_write exceeds pg bound. File is corrupted. "
"Need to enlarge group size. \n");
START_TIMER (ADIOS_TIMER_POSIX_MD);
ssize_t s = write (p->b.f, fd->buffer, fd->bytes_written);
STOP_TIMER (ADIOS_TIMER_POSIX_MD);
if (s != fd->bytes_written)
{
fprintf (stderr, "POSIX method tried to write %llu, "
"only wrote %lld\n"
,fd->bytes_written
,(int64_t)s
);
}
fd->base_offset += s;
fd->offset = 0;
fd->bytes_written = 0;
adios_shared_buffer_free (&p->b);
a = a->next;
}
}
// set it up so that it will start at 0, but have correct sizes
fd->offset = fd->base_offset - p->vars_start;
fd->vars_start = 0;
fd->buffer_size = 0;
adios_write_close_attributes_v1 (fd);
fd->offset = lseek (p->b.f, p->vars_start, SEEK_SET);
// fd->vars_start gets updated with the size written
START_TIMER (ADIOS_TIMER_POSIX_MD);
s = write (p->b.f, fd->buffer, p->vars_header_size);
STOP_TIMER (ADIOS_TIMER_POSIX_MD);
if (s != p->vars_header_size)
{
fprintf (stderr, "POSIX method tried to write %llu, "
"only wrote %lld\n"
,p->vars_header_size
,(int64_t)s
);
}
fd->offset = 0;
fd->bytes_written = 0;
}
// buffering or not, write the index
char * buffer = 0;
uint64_t buffer_size = 0;
uint64_t buffer_offset = 0;
uint64_t index_start = fd->base_offset + fd->offset;
// build index
adios_build_index_v1 (fd, p->index);
// if collective, gather the indexes from the rest and call
// adios_merge_index_v1 (&new_pg_root, &new_vars_root, pg, vars);
adios_write_index_v1 (&buffer, &buffer_size, &buffer_offset
,index_start, p->index);
adios_write_version_v1 (&buffer, &buffer_size, &buffer_offset);
START_TIMER (ADIOS_TIMER_POSIX_IO);
adios_posix_do_write (fd, method, buffer, buffer_offset); // Buffered vars written here
STOP_TIMER (ADIOS_TIMER_POSIX_IO);
#ifdef HAVE_MPI
if (p->group_comm != MPI_COMM_SELF)
{
if (p->rank == 0)
{
int * index_sizes = malloc (4 * p->size);
int * index_offsets = malloc (4 * p->size);
char * recv_buffer = 0;
int i;
uint32_t size = 0, total_size = 0;
START_TIMER (ADIOS_TIMER_POSIX_COMM);
MPI_Gather (&size, 1, MPI_INT
,index_sizes, 1, MPI_INT
,0, p->group_comm
);
STOP_TIMER (ADIOS_TIMER_POSIX_COMM);
for (i = 0; i < p->size; i++)
{
index_offsets [i] = total_size;
total_size += index_sizes [i];
}
recv_buffer = malloc (total_size);
START_TIMER (ADIOS_TIMER_POSIX_COMM);
MPI_Gatherv (&size, 0, MPI_BYTE
,recv_buffer, index_sizes, index_offsets
,MPI_BYTE, 0, p->group_comm
);
STOP_TIMER (ADIOS_TIMER_POSIX_COMM);
char * buffer_save = p->b.buff;
uint64_t buffer_size_save = p->b.length;
uint64_t offset_save = p->b.offset;
for (i = 1; i < p->size; i++)
{
p->b.buff = recv_buffer + index_offsets [i];
p->b.length = index_sizes [i];
p->b.offset = 0;
adios_parse_process_group_index_v1 (&p->b
,&new_pg_root
);
adios_parse_vars_index_v1 (&p->b, &new_vars_root, NULL, NULL);
// do not merge attributes from other processes from 1.4
/*
adios_parse_attributes_index_v1 (&p->b
,&new_attrs_root
);
*/
adios_merge_index_v1 (p->index, new_pg_root,
new_vars_root, new_attrs_root);
new_pg_root = 0;
new_vars_root = 0;
new_attrs_root = 0;
}
p->b.buff = buffer_save;
p->b.length = buffer_size_save;
p->b.offset = offset_save;
free (recv_buffer);
free (index_sizes);
free (index_offsets);
char * global_index_buffer = 0;
uint64_t global_index_buffer_size = 0;
uint64_t global_index_buffer_offset = 0;
uint64_t global_index_start = 0;
uint16_t flag = 0;
adios_write_index_v1 (&global_index_buffer, &global_index_buffer_size
,&global_index_buffer_offset, global_index_start
,p->index);
flag |= ADIOS_VERSION_HAVE_SUBFILE;
adios_write_version_flag_v1 (&global_index_buffer
,&global_index_buffer_size
,&global_index_buffer_offset
,flag
);
START_TIMER (ADIOS_TIMER_POSIX_MD);
ssize_t s = write (p->mf, global_index_buffer, global_index_buffer_offset);
STOP_TIMER (ADIOS_TIMER_POSIX_MD);
if (s != global_index_buffer_offset)
{
fprintf (stderr, "POSIX method tried to write %llu, "
"only wrote %lld\n"
,fd->bytes_written
,(int64_t)s
);
}
close (p->mf);
}
else
{
START_TIMER (ADIOS_TIMER_POSIX_COMM);
// Added this explicit cast to avoid truncation of low-order bytes on BGP
int i_buffer_size = (int) buffer_size;
MPI_Gather (&i_buffer_size, 1, MPI_INT
,0, 0, MPI_INT
,0, p->group_comm
);
MPI_Gatherv (buffer, buffer_size, MPI_BYTE
,0, 0, 0, MPI_BYTE
,0, p->group_comm
);
STOP_TIMER (ADIOS_TIMER_POSIX_COMM);
}
}
#endif
free (buffer);
break;
}
case adios_mode_append:
case adios_mode_update:
{
if (fd->shared_buffer == adios_flag_no)
{
off_t new_off;
// set it up so that it will start at 0, but have correct sizes
new_off = lseek (p->b.f, 0, SEEK_CUR);
fd->offset = fd->base_offset - p->vars_start;
fd->vars_start = 0;
fd->buffer_size = 0;
adios_write_close_vars_v1 (fd);
// fd->vars_start gets updated with the size written
fd->offset = lseek (p->b.f, p->vars_start, SEEK_SET);
START_TIMER (ADIOS_TIMER_POSIX_IO);
ssize_t s = write (p->b.f, fd->buffer, p->vars_header_size);
STOP_TIMER (ADIOS_TIMER_POSIX_IO);
if (s != fd->vars_start)
{
fprintf (stderr, "POSIX method tried to write %llu, "
"only wrote %lld\n"
,fd->vars_start
,(int64_t)s
);
}
fd->offset = 0;
fd->bytes_written = 0;
adios_shared_buffer_free (&p->b);
new_off = lseek (p->b.f, new_off, SEEK_SET); // go back to end
adios_write_open_attributes_v1 (fd);
p->vars_start = lseek (p->b.f, fd->offset, SEEK_CUR); // save loc
p->vars_header_size = p->vars_start - fd->base_offset;
p->vars_start -= fd->offset; // adjust to start of header
fd->base_offset += fd->offset; // add size of header
fd->offset = 0;
fd->bytes_written = 0;
if (!fd->group->process_id) { // from ADIOS 1.4, only rank 0 writes attributes
while (a)
{
adios_write_attribute_v1 (fd, a);
START_TIMER (ADIOS_TIMER_POSIX_MD);
ssize_t s = write (p->b.f, fd->buffer, fd->bytes_written);
STOP_TIMER (ADIOS_TIMER_POSIX_MD);
if (s != fd->bytes_written)
{
fprintf (stderr, "POSIX method tried to write %llu, "
"only wrote %lld\n"
,fd->bytes_written
,(int64_t)s
);
}
fd->base_offset += s;
fd->offset = 0;
fd->bytes_written = 0;
adios_shared_buffer_free (&p->b);
a = a->next;
}
}
// set it up so that it will start at 0, but have correct sizes
fd->offset = fd->base_offset - p->vars_start;
fd->vars_start = 0;
fd->buffer_size = 0;
adios_write_close_attributes_v1 (fd);
fd->offset = lseek (p->b.f, p->vars_start, SEEK_SET);
// fd->vars_start gets updated with the size written
START_TIMER (ADIOS_TIMER_POSIX_MD);
s = write (p->b.f, fd->buffer, p->vars_header_size);
STOP_TIMER (ADIOS_TIMER_POSIX_MD);
if (s != p->vars_header_size)
{
fprintf (stderr, "POSIX method tried to write %llu, "
"only wrote %lld\n"
,p->vars_header_size
,(int64_t)s
);
}
fd->offset = 0;
fd->bytes_written = 0;
}
char * buffer = 0;
uint64_t buffer_size = 0;
uint64_t buffer_offset = 0;
uint64_t index_start = fd->base_offset + fd->offset;
// build index
adios_build_index_v1 (fd, p->index);
adios_write_index_v1 (&buffer, &buffer_size, &buffer_offset
,index_start, p->index);
#ifdef HAVE_MPI
if (p->group_comm != MPI_COMM_SELF)
{
if (p->rank == 0)
{
int * index_sizes = malloc (4 * p->size);
int * index_offsets = malloc (4 * p->size);
char * recv_buffer = 0;
int i;
uint32_t size = 0, total_size = 0;
START_TIMER (ADIOS_TIMER_POSIX_COMM);
MPI_Gather (&size, 1, MPI_INT
,index_sizes, 1, MPI_INT
,0, p->group_comm
);
STOP_TIMER (ADIOS_TIMER_POSIX_COMM);
for (i = 0; i < p->size; i++)
{
index_offsets [i] = total_size;
total_size += index_sizes [i];
}
recv_buffer = malloc (total_size);
START_TIMER (ADIOS_TIMER_POSIX_COMM);
MPI_Gatherv (&size, 0, MPI_BYTE
,recv_buffer, index_sizes, index_offsets
,MPI_BYTE, 0, p->group_comm
);
STOP_TIMER (ADIOS_TIMER_POSIX_COMM);
char * buffer_save = p->b.buff;
uint64_t buffer_size_save = p->b.length;
uint64_t offset_save = p->b.offset;
for (i = 1; i < p->size; i++)
{
p->b.buff = recv_buffer + index_offsets [i];
p->b.length = index_sizes [i];
p->b.offset = 0;
adios_parse_process_group_index_v1 (&p->b
,&new_pg_root
);
adios_parse_vars_index_v1 (&p->b, &new_vars_root, NULL, NULL);
// do not merge attributes from other processes from 1.4
/*
adios_parse_attributes_index_v1 (&p->b
,&new_attrs_root
);
*/
adios_merge_index_v1 (p->index,new_pg_root,
new_vars_root, new_attrs_root);
new_pg_root = 0;
new_vars_root = 0;
new_attrs_root = 0;
}
adios_sort_index_v1 (&p->index->pg_root
,&p->index->vars_root
,&p->index->attrs_root
);
p->b.buff = buffer_save;
p->b.length = buffer_size_save;
p->b.offset = offset_save;
free (recv_buffer);
free (index_sizes);
free (index_offsets);
char * global_index_buffer = 0;
uint64_t global_index_buffer_size = 0;
uint64_t global_index_buffer_offset = 0;
uint64_t global_index_start = 0;
uint16_t flag = 0;
adios_write_index_v1 (&global_index_buffer, &global_index_buffer_size
,&global_index_buffer_offset, global_index_start
,p->index);
flag |= ADIOS_VERSION_HAVE_SUBFILE;
adios_write_version_flag_v1 (&global_index_buffer
,&global_index_buffer_size
,&global_index_buffer_offset
,flag
);
START_TIMER (ADIOS_TIMER_POSIX_MD);
ssize_t s = write (p->mf, global_index_buffer, global_index_buffer_offset);
STOP_TIMER (ADIOS_TIMER_POSIX_MD);
if (s != global_index_buffer_offset)
{
fprintf (stderr, "POSIX method tried to write %llu, "
"only wrote %lld, Mode: a\n"
,global_index_buffer_offset
,(int64_t)s
);
}
close (p->mf);
free (global_index_buffer);
}
else
{
START_TIMER (ADIOS_TIMER_POSIX_COMM);
MPI_Gather (&buffer_size, 1, MPI_INT
,0, 0, MPI_INT
,0, p->group_comm
);
MPI_Gatherv (buffer, buffer_size, MPI_BYTE
,0, 0, 0, MPI_BYTE
,0, p->group_comm
);
STOP_TIMER (ADIOS_TIMER_POSIX_COMM);
}
}
#endif
adios_write_version_v1 (&buffer, &buffer_size, &buffer_offset);
START_TIMER (ADIOS_TIMER_POSIX_MD);
adios_posix_do_write (fd, method, buffer, buffer_offset);
STOP_TIMER (ADIOS_TIMER_POSIX_MD);
free (buffer);
break;
}
case adios_mode_read:
{
// read the index to find the place to start reading
adios_posix_do_read (fd, method);
struct adios_var_struct * v = fd->group->vars;
while (v)
{
v->data = 0;
v = v->next;
}
break;
}
default:
{
fprintf (stderr, "Unknown file mode: %d\n", fd->mode);
return;
}
}
adios_posix_close_internal (&p->b);
adios_clear_index_v1 (p->index);
STOP_TIMER (ADIOS_TIMER_POSIX_AD_CLOSE);
#if defined ADIOS_TIMERS || defined ADIOS_TIMER_EVENTS
//Finished timing this cycle, swap the timing buffers
adios_timing_destroy(fd->group->prev_timing_obj);
fd->group->prev_timing_obj = fd->group->timing_obj;
fd->group->timing_obj = 0;
// prev_timing_obj points to unwritten timing info, timing_obj is
// ready to allocate at the next open
#endif
}
void adios_posix1_close (struct adios_file_struct * fd
,struct adios_method_struct * method
)
{
struct adios_POSIX1_data_struct * p = (struct adios_POSIX1_data_struct *)
method->method_data;
struct adios_attribute_struct * a = fd->group->attributes;
switch (fd->mode)
{
case adios_mode_write:
{
if (fd->shared_buffer == adios_flag_no)
{
off_t new_off;
// set it up so that it will start at 0, but have correct sizes
new_off = lseek (p->b.f, 0, SEEK_CUR);
fd->offset = fd->base_offset - p->vars_start;
fd->vars_start = 0;
fd->buffer_size = 0;
adios_write_close_vars_v1 (fd);
// fd->vars_start gets updated with the size written
fd->offset = lseek (p->b.f, p->vars_start, SEEK_SET);
ssize_t s = write (p->b.f, fd->buffer, p->vars_header_size);
if (s != fd->vars_start)
{
fprintf (stderr, "POSIX1 method tried to write %llu, "
"only wrote %lld\n"
,fd->vars_start
,(int64_t)s
);
}
fd->offset = 0;
fd->bytes_written = 0;
adios_shared_buffer_free (&p->b);
new_off = lseek (p->b.f, new_off, SEEK_SET); // go back to end
adios_write_open_attributes_v1 (fd);
p->vars_start = lseek (p->b.f, fd->offset, SEEK_CUR); // save loc
p->vars_header_size = p->vars_start - fd->base_offset;
p->vars_start -= fd->offset; // adjust to start of header
fd->base_offset += fd->offset; // add size of header
fd->offset = 0;
fd->bytes_written = 0;
while (a)
{
adios_write_attribute_v1 (fd, a);
if (fd->base_offset + fd->bytes_written > fd->pg_start_in_file + fd->write_size_bytes)
fprintf (stderr, "adios_posix1_write exceeds pg bound. File is corrupted. "
"Need to enlarge group size. \n");
ssize_t s = write (p->b.f, fd->buffer, fd->bytes_written);
if (s != fd->bytes_written)
{
fprintf (stderr, "POSIX1 method tried to write %llu, "
"only wrote %lld\n"
,fd->bytes_written
,(int64_t)s
);
}
fd->base_offset += s;
fd->offset = 0;
fd->bytes_written = 0;
adios_shared_buffer_free (&p->b);
a = a->next;
}
// set it up so that it will start at 0, but have correct sizes
fd->offset = fd->base_offset - p->vars_start;
fd->vars_start = 0;
fd->buffer_size = 0;
adios_write_close_attributes_v1 (fd);
fd->offset = lseek (p->b.f, p->vars_start, SEEK_SET);
// fd->vars_start gets updated with the size written
s = write (p->b.f, fd->buffer, p->vars_header_size);
if (s != p->vars_header_size)
{
fprintf (stderr, "POSIX1 method tried to write %llu, "
"only wrote %lld\n"
,p->vars_header_size
,(int64_t)s
);
}
fd->offset = 0;
fd->bytes_written = 0;
}
// buffering or not, write the index
char * buffer = 0;
uint64_t buffer_size = 0;
uint64_t buffer_offset = 0;
uint64_t index_start = fd->base_offset + fd->offset;
// build index
adios_build_index_v1 (fd, p->index);
adios_write_index_v1 (&buffer, &buffer_size, &buffer_offset
,index_start, p->index);
adios_write_version_v1 (&buffer, &buffer_size, &buffer_offset);
adios_posix1_do_write (fd, method, buffer, buffer_offset);
free (buffer);
break;
}
case adios_mode_append:
{
if (fd->shared_buffer == adios_flag_no)
{
off_t new_off;
// set it up so that it will start at 0, but have correct sizes
new_off = lseek (p->b.f, 0, SEEK_CUR);
fd->offset = fd->base_offset - p->vars_start;
fd->vars_start = 0;
fd->buffer_size = 0;
adios_write_close_vars_v1 (fd);
// fd->vars_start gets updated with the size written
fd->offset = lseek (p->b.f, p->vars_start, SEEK_SET);
ssize_t s = write (p->b.f, fd->buffer, p->vars_header_size);
if (s != fd->vars_start)
{
fprintf (stderr, "POSIX1 method tried to write %llu, "
"only wrote %lld\n"
,fd->vars_start
,(int64_t)s
);
}
fd->offset = 0;
fd->bytes_written = 0;
adios_shared_buffer_free (&p->b);
new_off = lseek (p->b.f, new_off, SEEK_SET); // go back to end
adios_write_open_attributes_v1 (fd);
p->vars_start = lseek (p->b.f, fd->offset, SEEK_CUR); // save loc
p->vars_header_size = p->vars_start - fd->base_offset;
p->vars_start -= fd->offset; // adjust to start of header
fd->base_offset += fd->offset; // add size of header
fd->offset = 0;
fd->bytes_written = 0;
while (a)
{
adios_write_attribute_v1 (fd, a);
ssize_t s = write (p->b.f, fd->buffer, fd->bytes_written);
if (s != fd->bytes_written)
{
fprintf (stderr, "POSIX1 method tried to write %llu, "
"only wrote %lld\n"
,fd->bytes_written
,(int64_t)s
);
}
fd->base_offset += s;
fd->offset = 0;
fd->bytes_written = 0;
adios_shared_buffer_free (&p->b);
a = a->next;
}
// set it up so that it will start at 0, but have correct sizes
fd->offset = fd->base_offset - p->vars_start;
fd->vars_start = 0;
fd->buffer_size = 0;
adios_write_close_attributes_v1 (fd);
fd->offset = lseek (p->b.f, p->vars_start, SEEK_SET);
// fd->vars_start gets updated with the size written
s = write (p->b.f, fd->buffer, p->vars_header_size);
if (s != p->vars_header_size)
{
fprintf (stderr, "POSIX1 method tried to write %llu, "
"only wrote %lld\n"
,p->vars_header_size
,(int64_t)s
);
}
fd->offset = 0;
fd->bytes_written = 0;
}
char * buffer = 0;
uint64_t buffer_size = 0;
uint64_t buffer_offset = 0;
uint64_t index_start = fd->base_offset + fd->offset;
// build index. Note: It merges with old indices already stored
// in p->index in adios_posix_open's append case
adios_build_index_v1 (fd, p->index);
// merge in old indicies
//adios_merge_index_v1 (p->index, new_pg_root,
// new_vars_root, new_attrs_root
// );
adios_write_index_v1 (&buffer, &buffer_size, &buffer_offset
,index_start, p->index);
adios_write_version_v1 (&buffer, &buffer_size, &buffer_offset);
adios_posix1_do_write (fd, method, buffer, buffer_offset);
free (buffer);
break;
}
case adios_mode_read:
{
// read the index to find the place to start reading
adios_posix1_do_read (fd, method);
struct adios_var_struct * v = fd->group->vars;
while (v)
{
v->data = 0;
v = v->next;
}
break;
}
default:
{
fprintf (stderr, "Unknown file mode: %d\n", fd->mode);
return;
}
}
adios_posix_close_internal (&p->b);
adios_clear_index_v1 (p->index);
}