void adios_posix1_init (const PairStruct * parameters
,struct adios_method_struct * method
)
{
struct adios_POSIX1_data_struct * p = 0;
if (!adios_posix1_initialized)
{
adios_posix1_initialized = 1;
}
method->method_data = malloc (sizeof (struct adios_POSIX1_data_struct));
p = (struct adios_POSIX1_data_struct *) method->method_data;
adios_buffer_struct_init (&p->b);
p->index = adios_alloc_index_v1(1); // with hashtables
p->vars_start = 0;
p->vars_header_size = 0;
}
void adios_posix_init (const PairStruct * parameters
,struct adios_method_struct * method
)
{
struct adios_POSIX_data_struct * p = 0;
if (!adios_posix_initialized)
{
adios_posix_initialized = 1;
}
method->method_data = malloc (sizeof (struct adios_POSIX_data_struct));
p = (struct adios_POSIX_data_struct *) method->method_data;
adios_buffer_struct_init (&p->b);
p->index = adios_alloc_index_v1(1); // with hashtables
p->vars_start = 0;
p->vars_header_size = 0;
#ifdef HAVE_MPI
p->mf = 0;
p->group_comm = MPI_COMM_NULL;
p->rank = 0;
p->size = 0;
#endif
}
int process_subfiles (int tid, int startidx, int endidx)
{
char fn[256];
uint32_t version = 0;
int idx;
int rc = 0;
subindex[tid] = adios_alloc_index_v1(1);
for (idx=startidx; idx<=endidx; idx++)
{
b[idx] = malloc (sizeof (struct adios_bp_buffer_struct_v1));
adios_buffer_struct_init (b[idx]);
snprintf (fn, 256, "%s.dir/%s.%d", filename, filename, idx);
rc = adios_posix_open_read_internal (fn, "", b[idx]);
if (!rc)
{
fprintf (stderr, "bpmeta: file not found: %s\n", fn);
return -1;
}
adios_posix_read_version (b[idx]);
adios_parse_version (b[idx], &version);
version = version & ADIOS_VERSION_NUM_MASK;
if (verbose) {
//printf (DIVIDER);
printf ("Thread %d: Metadata of %s:\n", tid, fn);
printf ("Thread %d: BP format version: %d\n", tid, version);
}
if (version < 2)
{
fprintf (stderr, "bpmeta: This version of bpmeta can only work with BP format version 2 and up. "
"Use an older bpmeta from adios 1.6 to work with this file.\n");
adios_posix_close_internal (b[idx]);
return -1;
}
struct adios_index_process_group_struct_v1 * new_pg_root = 0;
struct adios_index_var_struct_v1 * new_vars_root = 0;
struct adios_index_attribute_struct_v1 * new_attrs_root = 0;
adios_posix_read_index_offsets (b[idx]);
adios_parse_index_offsets_v1 (b[idx]);
/*
printf ("End of process groups = %llu\n", b->end_of_pgs);
printf ("Process Groups Index Offset = %llu\n", b->pg_index_offset);
printf ("Process Groups Index Size = %llu\n", b->pg_size);
printf ("Variable Index Offset = %llu\n", b->vars_index_offset);
printf ("Variable Index Size = %llu\n", b->vars_size);
printf ("Attribute Index Offset = %llu\n", b->attrs_index_offset);
printf ("Attribute Index Size = %llu\n", b->attrs_size);
*/
adios_posix_read_process_group_index (b[idx]);
adios_parse_process_group_index_v1 (b[idx], &new_pg_root);
print_pg_index (tid, new_pg_root);
adios_posix_read_vars_index (b[idx]);
adios_parse_vars_index_v1 (b[idx], &new_vars_root, NULL, NULL);
print_variable_index (tid, new_vars_root);
adios_posix_read_attributes_index (b[idx]);
adios_parse_attributes_index_v1 (b[idx], &new_attrs_root);
print_attribute_index (tid, new_attrs_root);
adios_merge_index_v1 (subindex[tid], new_pg_root, new_vars_root, new_attrs_root);
adios_posix_close_internal (b[idx]);
adios_shared_buffer_free (b[idx]);
}
if (verbose>1) {
//printf (DIVIDER);
printf ("Thread %d: End of reading all subfiles\n", tid);
}
return 0;
}
int main (int argc, char ** argv)
{
long int tmp;
int c;
while ((c = getopt_long(argc, argv, optstring, options, NULL)) != -1) {
switch (c) {
case 'n':
case 't':
errno = 0;
tmp = strtol(optarg, (char **)NULL, 0);
if (errno) {
fprintf(stderr, "Error: could not convert -%c value: %s\n",
c, optarg);
return 1;
}
if (c == 'n')
nsubfiles=tmp;
else
nthreads=tmp;
break;
case 'h':
display_help();
return 0;
break;
case 'v':
verbose++;
break;
default:
printf("Unrecognized argument: %s\n", optarg);
break;
}
}
/* Check if we have a file defined */
if (optind >= argc) {
printf ("Missing file name\n");
display_help();
return 1;
}
filename = strdup(argv[optind++]);
if (nsubfiles < 1)
nsubfiles = get_nsubfiles (filename);
if (nsubfiles < 1) {
printf ("Cannot determine the number of subfiles. To avoid this problem, "
"provide the number of subfiles manually with the -n <N> option.\n");
return -1;
}
if (nthreads < 1)
nthreads = 1;
if (nthreads > nsubfiles) {
printf ("Warning: asked for processing %d subfiles using %d threads. "
"We will utilize only %d threads.\n",
nsubfiles, nthreads, nsubfiles);
nthreads = nsubfiles;
}
if (verbose>1)
printf ("Create metadata file %s from %d subfiles using %d threads\n",
filename, nsubfiles, nthreads);
/* Initialize global variables */
b = malloc (nsubfiles * sizeof (struct adios_bp_buffer_struct_v1*));
subindex = malloc (nthreads * sizeof (struct adios_index_struct_v1*));
/* Split the processing work among T threads */
int tid;
#if HAVE_PTHREAD
pthread_t *thread = (pthread_t *) malloc (nthreads * sizeof(pthread_t));
struct thread_args *targs = (struct thread_args*)
malloc (nthreads * sizeof(struct thread_args));
int K = nsubfiles/nthreads; // base number of files to be processed by one thread
int L = nsubfiles%nthreads; // this many threads processes one more subfile
int startidx, endidx;
int rc;
//printf ("K=%d L=%d\n", K, L);
endidx = -1;
for (tid=0; tid<nthreads; tid++)
{
startidx = endidx + 1;
endidx = startidx + K - 1;
targs[tid].tid = tid;
targs[tid].startidx = startidx;
targs[tid].endidx = endidx;
if (tid < L) {
endidx++;
}
if (verbose)
printf ("Process subfiles from %d to %d with thread %d\n",
targs[tid].startidx, targs[tid].endidx, targs[tid]);
if (tid < nthreads-1) {
/* Start worker thread. */
pthread_attr_t attr;
pthread_attr_init (&attr);
pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_JOINABLE);
rc = pthread_create (&thread[tid], &attr, thread_main, &targs[tid]);
if (rc) {
printf ("ERROR: Thread %d: Cannot create thread, err code = %d\n",
tid, rc);
}
pthread_attr_destroy(&attr);
}
else
{
// last "thread" is the main thread
process_subfiles (tid, startidx, endidx);
}
}
// wait here for everyone to finish
for (tid=0; tid<nthreads-1; tid++)
{
void *status;
rc = pthread_join (thread[tid], &status);
if (rc) {
printf ("ERROR: Thread %d: Cannot join thread, err code = %d\n", tid, rc);
} else {
if (verbose>1)
printf ("Thread %d: Joined thread.\n", tid);
}
}
free (targs);
free (thread);
#else /* non-threaded version */
nthreads = 1;
process_subfiles (0, 0, nsubfiles-1);
#endif
/* Merge the T indexes into the global output index */
struct adios_index_struct_v1 * globalindex;
globalindex = adios_alloc_index_v1(1);
for (tid=0; tid<nthreads; tid++)
{
adios_merge_index_v1 (globalindex,
subindex[tid]->pg_root,
subindex[tid]->vars_root,
subindex[tid]->attrs_root);
}
write_index (globalindex, filename);
/* Clean-up */
adios_clear_index_v1 (globalindex);
/*... already cleaned-up by globalindex clearing
for (tid=0; tid<nthreads; tid++) {
adios_clear_index_v1 (subindex[tid]);
free (subindex[tid]);
}
*/
free (subindex);
free (globalindex);
free (b);
return 0;
}
static void adios_posix_do_read (struct adios_file_struct * fd
,struct adios_method_struct * method
)
{
struct adios_POSIX_data_struct * p = (struct adios_POSIX_data_struct *)
method->method_data;
struct adios_var_struct * v = fd->group->vars;
uint32_t version = 0;
adios_posix_read_version (&p->b);
adios_parse_version (&p->b, &version);
version &= ADIOS_VERSION_NUM_MASK;
switch (version)
{
case 1:
case 2:
{
struct adios_index_struct_v1 * index = adios_alloc_index_v1(0); // no hashtables
struct adios_index_process_group_struct_v1 * pg_root = index->pg_root;
struct adios_index_process_group_struct_v1 * pg_root_temp = 0;
adios_posix_read_index_offsets (&p->b);
adios_parse_index_offsets_v1 (&p->b);
adios_posix_read_process_group_index (&p->b);
adios_parse_process_group_index_v1 (&p->b, &pg_root);
#if 1
adios_posix_read_vars_index (&p->b);
adios_parse_vars_index_v1 (&p->b, &index->vars_root, NULL, NULL);
adios_posix_read_attributes_index (&p->b);
adios_parse_attributes_index_v1 (&p->b, &index->attrs_root);
#endif
// the three section headers
struct adios_process_group_header_struct_v1 pg_header;
struct adios_vars_header_struct_v1 vars_header;
struct adios_attributes_header_struct_v1 attrs_header;
struct adios_var_header_struct_v1 var_header;
struct adios_var_payload_struct_v1 var_payload;
struct adios_attribute_struct_v1 attribute;
int i;
pg_root_temp = pg_root;
while (pg_root_temp && pg_root_temp->next)
pg_root_temp = pg_root_temp->next;
p->b.read_pg_offset = pg_root_temp->offset_in_file;
if (pg_root_temp->next)
{
p->b.read_pg_size = pg_root_temp->next->offset_in_file
- pg_root_temp->offset_in_file;
}
else
{
p->b.read_pg_size = p->b.pg_index_offset
- pg_root_temp->offset_in_file;
}
adios_posix_read_process_group (&p->b);
adios_parse_process_group_header_v1 (&p->b, &pg_header);
adios_parse_vars_header_v1 (&p->b, &vars_header);
for (i = 0; i < vars_header.count; i++)
{
memset (&var_payload, 0
,sizeof (struct adios_var_payload_struct_v1)
);
adios_parse_var_data_header_v1 (&p->b, &var_header);
struct adios_var_struct * v1 = v;
while (v1)
{
if ( strcasecmp (var_header.name, v1->name)
|| strcasecmp (var_header.path, v1->path)
)
{
v1 = v1->next;
}
else
break;
}
if (v1)
{
var_payload.payload = v1->data;
adios_parse_var_data_payload_v1 (&p->b, &var_header
,&var_payload
,v1->data_size
);
}
else
{
adios_parse_var_data_payload_v1 (&p->b, &var_header
,NULL, 0
);
}
adios_clear_var_header_v1 (&var_header);
}
#if 1
adios_parse_attributes_header_v1 (&p->b, &attrs_header);
for (i = 0; i < attrs_header.count; i++)
{
adios_parse_attribute_v1 (&p->b, &attribute);
adios_clear_attribute_v1 (&attribute);
}
#endif
adios_clear_process_group_header_v1 (&pg_header);
adios_clear_index_v1 (index);
break;
}
default:
fprintf (stderr, "POSIX read: file version unknown: %u\n", version);
return;
}
adios_buffer_struct_clear (&p->b);
}
int main (int argc, char ** argv)
{
char * filename;
int rc = 0;
if (argc < 2 || argc > 3)
{
print_usage (argc, argv);
return -1;
}
if (argv [1][0] && argv [1][0] == '-')
{
if ( !strcmp (argv [1], "-f")
|| !strcmp (argv [1], "--force")
)
{
do_write_index = 1;
filename = argv [2];
}
else
{
print_usage (argc, argv);
return -1;
}
}
else
{
filename = argv [1];
do_write_index = 0;
}
have_subfiles = 0;
struct adios_bp_buffer_struct_v1 * b = 0;
uint32_t version = 0;
b = malloc (sizeof (struct adios_bp_buffer_struct_v1));
adios_buffer_struct_init (b);
int flags = O_RDONLY;
if (do_write_index)
flags = O_RDWR;
int fd = open (filename, flags);
if (fd < 0) {
fprintf (stderr, "recover: cannot open file %s\n", filename);
if (errno)
fprintf (stderr, "%s\n", strerror(errno));
return -1;
}
struct stat statbuf;
fstat (fd, &statbuf);
uint64_t file_size = statbuf.st_size;
b->f = fd;
printf ("File size in bytes: %llu\n", file_size);
/* Variables to build new index */
struct adios_index_struct_v1 * index = adios_alloc_index_v1(1);
//struct adios_index_process_group_struct_v1 * pg_root = 0;
struct adios_index_process_group_struct_v1 * pg = 0;
//struct adios_index_var_struct_v1 * vars_root = 0;
//struct adios_index_attribute_struct_v1 * attrs_root = 0;
uint64_t pg_num = 0L;
uint64_t curr_offset = 0L;
uint64_t new_offset = 0L;
uint64_t pgsize_reported; // size of current pg (as indicated in PG header (wrongly))
uint64_t pgsize_actual; // size of current pg based on processing (accurate)
int found_pg = 0;
printf (DIVIDER);
found_pg = find_pg (fd, new_offset, file_size, &pgsize_reported);
// pass over the PGs from beginning of file
while (found_pg)
{
curr_offset = new_offset;
pg_num++;
printf ("PG %llu found at offset %llu\n", pg_num, curr_offset);
/* Let's process the group */
/* Allocate PG index struct here to allow to be used below */
pg = (struct adios_index_process_group_struct_v1 *)
malloc (sizeof(struct adios_index_process_group_struct_v1));
// setup where to read the process group from (and size)
pg->offset_in_file = curr_offset;
//b->read_pg_offset = pg->offset_in_file;
b->read_pg_offset = curr_offset;
b->read_pg_size = pgsize_reported;
/* Temporary variables for parsing one PG */
struct adios_process_group_header_struct_v1 pg_header;
struct adios_vars_header_struct_v1 vars_header;
struct adios_attributes_header_struct_v1 attrs_header;
struct adios_var_header_struct_v1 var_header;
struct adios_var_payload_struct_v1 var_payload;
struct adios_attribute_struct_v1 attribute;
init_dimensions (); // store scalar values from this PG temporarily
/* Read the whole PG into a buffer and start parsing */
adios_posix_read_process_group (b);
adios_parse_process_group_header_v1 (b, &pg_header);
print_process_group_header (pg_num, &pg_header);
add_pg_to_index (index, &pg_header, curr_offset);
adios_parse_vars_header_v1 (b, &vars_header);
print_vars_header (&vars_header);
int i;
for (i = 0; i < vars_header.count; i++)
{
var_payload.payload = 0;
adios_parse_var_data_header_v1 (b, &var_header);
print_var_header (&var_header);
if ( var_header.dims == 0)
{
// Load scalars to save them for handling as dimension values
var_payload.payload = malloc (var_header.payload_size + 1);
adios_parse_var_data_payload_v1 (b, &var_header, &var_payload
,var_header.payload_size
);
}
else
{
// Just parse to move the offset in buffer, don't read data
adios_parse_var_data_payload_v1 (b, &var_header, NULL, 0);
}
store_scalar_dimensions (&var_header, &var_payload);
add_var_to_index (index, &pg_header, &var_header, &var_payload);
if (var_payload.payload)
{
free (var_payload.payload);
var_payload.payload = 0;
}
//printf ("\n");
}
adios_parse_attributes_header_v1 (b, &attrs_header);
print_attrs_header (&attrs_header);
for (i = 0; i < attrs_header.count; i++)
{
adios_parse_attribute_v1 (b, &attribute);
//print_attribute (&attribute);
//printf ("\n");
}
pgsize_actual = b->offset;
printf ("Actual size of group by processing: %llu bytes\n", pgsize_actual);
pg = pg->next;
printf (DIVIDER);
found_pg = 0;
if (curr_offset + pgsize_actual < file_size)
{
new_offset = curr_offset + pgsize_actual;
found_pg = find_pg (fd, curr_offset+pgsize_actual, file_size, &pgsize_reported);
}
if (!found_pg &&
pgsize_actual != pgsize_reported &&
curr_offset + pgsize_reported < file_size)
{
new_offset = curr_offset + pgsize_reported;
found_pg = find_pg (fd, curr_offset+pgsize_reported, file_size, &pgsize_reported);
}
}
// The end of the last successfully processed PG
// This will be the start of the index data
curr_offset += pgsize_actual;
printf (DIVIDER);
printf ("Found %llu PGs to be processable\n", pg_num);
write_index (fd, curr_offset, index);
adios_posix_close_internal (b); // will close fd
return 0;
}
