static void adios_posix_do_read (struct adios_file_struct * fd
,struct adios_method_struct * method
)
{
struct adios_POSIX_data_struct * p = (struct adios_POSIX_data_struct *)
method->method_data;
struct adios_var_struct * v = fd->group->vars;
uint32_t version = 0;
adios_posix_read_version (&p->b);
adios_parse_version (&p->b, &version);
version &= ADIOS_VERSION_NUM_MASK;
switch (version)
{
case 1:
case 2:
{
struct adios_index_struct_v1 * index = adios_alloc_index_v1(0); // no hashtables
struct adios_index_process_group_struct_v1 * pg_root = index->pg_root;
struct adios_index_process_group_struct_v1 * pg_root_temp = 0;
adios_posix_read_index_offsets (&p->b);
adios_parse_index_offsets_v1 (&p->b);
adios_posix_read_process_group_index (&p->b);
adios_parse_process_group_index_v1 (&p->b, &pg_root);
#if 1
adios_posix_read_vars_index (&p->b);
adios_parse_vars_index_v1 (&p->b, &index->vars_root, NULL, NULL);
adios_posix_read_attributes_index (&p->b);
adios_parse_attributes_index_v1 (&p->b, &index->attrs_root);
#endif
// the three section headers
struct adios_process_group_header_struct_v1 pg_header;
struct adios_vars_header_struct_v1 vars_header;
struct adios_attributes_header_struct_v1 attrs_header;
struct adios_var_header_struct_v1 var_header;
struct adios_var_payload_struct_v1 var_payload;
struct adios_attribute_struct_v1 attribute;
int i;
pg_root_temp = pg_root;
while (pg_root_temp && pg_root_temp->next)
pg_root_temp = pg_root_temp->next;
p->b.read_pg_offset = pg_root_temp->offset_in_file;
if (pg_root_temp->next)
{
p->b.read_pg_size = pg_root_temp->next->offset_in_file
- pg_root_temp->offset_in_file;
}
else
{
p->b.read_pg_size = p->b.pg_index_offset
- pg_root_temp->offset_in_file;
}
adios_posix_read_process_group (&p->b);
adios_parse_process_group_header_v1 (&p->b, &pg_header);
adios_parse_vars_header_v1 (&p->b, &vars_header);
for (i = 0; i < vars_header.count; i++)
{
memset (&var_payload, 0
,sizeof (struct adios_var_payload_struct_v1)
);
adios_parse_var_data_header_v1 (&p->b, &var_header);
struct adios_var_struct * v1 = v;
while (v1)
{
if ( strcasecmp (var_header.name, v1->name)
|| strcasecmp (var_header.path, v1->path)
)
{
v1 = v1->next;
}
else
break;
}
if (v1)
{
var_payload.payload = v1->data;
adios_parse_var_data_payload_v1 (&p->b, &var_header
,&var_payload
,v1->data_size
);
}
else
{
adios_parse_var_data_payload_v1 (&p->b, &var_header
,NULL, 0
);
}
adios_clear_var_header_v1 (&var_header);
}
#if 1
adios_parse_attributes_header_v1 (&p->b, &attrs_header);
for (i = 0; i < attrs_header.count; i++)
{
adios_parse_attribute_v1 (&p->b, &attribute);
adios_clear_attribute_v1 (&attribute);
}
#endif
adios_clear_process_group_header_v1 (&pg_header);
adios_clear_index_v1 (index);
break;
}
default:
fprintf (stderr, "POSIX read: file version unknown: %u\n", version);
return;
}
adios_buffer_struct_clear (&p->b);
}
int main (int argc, char ** argv)
{
char * filename;
int rc = 0;
struct dump_struct dump;
if (argc < 2 || argc > 4)
{
fprintf (stderr, "usage: %s [-d [var]|--dump [var]] <filename>\n"
,argv [0]
);
return -1;
}
if (argv [1][0] && argv [1][0] == '-')
{
if ( !strcmp (argv [1], "-d")
|| !strcmp (argv [1], "--dump")
)
{
dump.do_dump = 1;
if (argc > 3)
{
dump.dump_var = argv [2];
filename = argv [3];
printf("%s %s\n",dump.dump_var,filename);
}
else
{
dump.dump_var = 0;
filename = argv [2];
printf("ALLVARS %s\n",filename);
}
}
else
{
fprintf (stderr, "usage: %s [-d [var]|--dump [var]] <filename>\n"
,argv [0]
);
return -1;
}
}
else
{
filename = argv [1];
dump.do_dump = 0;
dump.dump_var = 0;
}
have_subfiles = 0;
struct adios_bp_buffer_struct_v1 * b = 0;
uint32_t version = 0;
b = malloc (sizeof (struct adios_bp_buffer_struct_v1));
adios_buffer_struct_init (b);
rc = adios_posix_open_read_internal (filename, "", b);
if (!rc)
{
fprintf (stderr, "bpdump: file not found: %s\n", filename);
return -1;
}
adios_posix_read_version (b);
adios_parse_version (b, &version);
version = version & ADIOS_VERSION_NUM_MASK;
printf ("BP format version: %d\n", version);
if (version < 2)
{
fprintf (stderr, "bpdump: This version of bpdump can only dump BP format version 2. "
"Use an older bpdump from adios 1.6 to dump this file.\n");
adios_posix_close_internal (b);
return -1;
}
struct adios_index_process_group_struct_v1 * pg_root = 0;
struct adios_index_process_group_struct_v1 * pg = 0;
struct adios_index_var_struct_v1 * vars_root = 0;
struct adios_index_attribute_struct_v1 * attrs_root = 0;
printf (DIVIDER);
printf ("Process Groups Index:\n");
adios_posix_read_index_offsets (b);
adios_parse_index_offsets_v1 (b);
/*
printf ("End of process groups = %" PRIu64 "\n", b->end_of_pgs);
printf ("Process Groups Index Offset = %" PRIu64 "\n", b->pg_index_offset);
printf ("Process Groups Index Size = %" PRIu64 "\n", b->pg_size);
printf ("Variable Index Offset = %" PRIu64 "\n", b->vars_index_offset);
printf ("Variable Index Size = %" PRIu64 "\n", b->vars_size);
printf ("Attribute Index Offset = %" PRIu64 "\n", b->attrs_index_offset);
printf ("Attribute Index Size = %" PRIu64 "\n", b->attrs_size);
*/
adios_posix_read_process_group_index (b);
adios_parse_process_group_index_v1 (b, &pg_root, NULL);
print_process_group_index (pg_root);
printf (DIVIDER);
printf ("Vars Index:\n");
adios_posix_read_vars_index (b);
adios_parse_vars_index_v1 (b, &vars_root, NULL, NULL);
print_vars_index (vars_root);
printf (DIVIDER);
printf ("Attributes Index:\n");
adios_posix_read_attributes_index (b);
adios_parse_attributes_index_v1 (b, &attrs_root);
print_attributes_index (attrs_root);
if (version & ADIOS_VERSION_HAVE_SUBFILE)
{
printf (DIVIDER);
return 0;
}
uint64_t pg_num = 0;
pg = pg_root;
while (pg)
{
pg_num++;
/* Avoid processing PG's whose offset is beyond the start of index (they are actually in subfiles) */
/* Note: only variables have subfile index, PG's don't so we need to be this indirect in the test */
if (pg->offset_in_file >= b->pg_index_offset)
{
printf ("Process Group %" PRIu64 " offset is beyond the footer.", pg_num);
if (have_subfiles) {
printf (" It is probably in a subfile but bpdump does not process subfiles.\n");
} else {
printf (" Since there are no subfiles, this probably means the BP file is corrupt.\n"
"offset=%" PRIu64 " footer starts at %" PRIu64 "\n", pg->offset_in_file, b->pg_index_offset);
}
pg = pg->next;
continue;
}
int var_dims_count = 0;
struct var_dim * var_dims = 0;
printf (DIVIDER);
struct adios_process_group_header_struct_v1 pg_header;
struct adios_vars_header_struct_v1 vars_header;
struct adios_attributes_header_struct_v1 attrs_header;
struct adios_var_header_struct_v1 var_header;
struct adios_var_payload_struct_v1 var_payload;
struct adios_attribute_struct_v1 attribute;
// setup where to read the process group from (and size)
b->read_pg_offset = pg->offset_in_file;
if (pg->next)
{
b->read_pg_size = pg->next->offset_in_file
- pg->offset_in_file;
}
else
{
b->read_pg_size = b->pg_index_offset
- pg->offset_in_file;
}
adios_posix_read_process_group (b);
adios_parse_process_group_header_v1 (b, &pg_header);
print_process_group_header (pg_num, &pg_header);
//printf ("\tSize of group in fil: %" PRIu64 " bytes\n", b->read_pg_size);
adios_parse_vars_header_v1 (b, &vars_header);
print_vars_header (&vars_header);
dump.host_language_fortran = pg_header.host_language_fortran;
int i;
for (i = 0; i < vars_header.count; i++)
{
var_payload.payload = 0;
adios_parse_var_data_header_v1 (b, &var_header);
print_var_header (&var_header);
if ( var_header.dims == 0 ||
( dump.do_dump &&
dump.dump_var &&
!strcasecmp (dump.dump_var, var_header.name)
)
)
{
// add one for string null terminators
var_payload.payload = malloc (var_header.payload_size + 1);
adios_parse_var_data_payload_v1 (b, &var_header, &var_payload
,var_header.payload_size
);
}
else
{
adios_parse_var_data_payload_v1 (b, &var_header, NULL, 0);
}
if (var_header.is_dim == adios_flag_yes)
{
var_dims = realloc (var_dims, (var_dims_count + 1)
* sizeof (struct var_dim)
);
var_dims [var_dims_count].id = var_header.id;
var_dims [var_dims_count].rank = *(unsigned int *)
var_payload.payload;
var_dims_count++;
}
if (dump.do_dump)
{
// make sure the buffer is big enough or send in null
print_var_payload (&var_header, &var_payload, &dump
,var_dims_count, var_dims
);
}
if (var_payload.payload)
{
free (var_payload.payload);
var_payload.payload = 0;
}
printf ("\n");
}
adios_parse_attributes_header_v1 (b, &attrs_header);
print_attrs_header (&attrs_header);
for (i = 0; i < attrs_header.count; i++)
{
adios_parse_attribute_v1 (b, &attribute);
print_attribute (&attribute);
printf ("\n");
}
var_dims_count = 0;
if (var_dims)
free (var_dims);
pg = pg->next;
}
printf (DIVIDER);
printf ("End of %s\n", filename);
adios_posix_close_internal (b);
return 0;
}
int main (int argc, char ** argv)
{
char * filename;
int rc = 0;
if (argc < 2 || argc > 3)
{
print_usage (argc, argv);
return -1;
}
if (argv [1][0] && argv [1][0] == '-')
{
if ( !strcmp (argv [1], "-f")
|| !strcmp (argv [1], "--force")
)
{
do_write_index = 1;
filename = argv [2];
}
else
{
print_usage (argc, argv);
return -1;
}
}
else
{
filename = argv [1];
do_write_index = 0;
}
have_subfiles = 0;
struct adios_bp_buffer_struct_v1 * b = 0;
uint32_t version = 0;
b = malloc (sizeof (struct adios_bp_buffer_struct_v1));
adios_buffer_struct_init (b);
int flags = O_RDONLY;
if (do_write_index)
flags = O_RDWR;
int fd = open (filename, flags);
if (fd < 0) {
fprintf (stderr, "recover: cannot open file %s\n", filename);
if (errno)
fprintf (stderr, "%s\n", strerror(errno));
return -1;
}
struct stat statbuf;
fstat (fd, &statbuf);
uint64_t file_size = statbuf.st_size;
b->f = fd;
printf ("File size in bytes: %llu\n", file_size);
/* Variables to build new index */
struct adios_index_struct_v1 * index = adios_alloc_index_v1(1);
//struct adios_index_process_group_struct_v1 * pg_root = 0;
struct adios_index_process_group_struct_v1 * pg = 0;
//struct adios_index_var_struct_v1 * vars_root = 0;
//struct adios_index_attribute_struct_v1 * attrs_root = 0;
uint64_t pg_num = 0L;
uint64_t curr_offset = 0L;
uint64_t new_offset = 0L;
uint64_t pgsize_reported; // size of current pg (as indicated in PG header (wrongly))
uint64_t pgsize_actual; // size of current pg based on processing (accurate)
int found_pg = 0;
printf (DIVIDER);
found_pg = find_pg (fd, new_offset, file_size, &pgsize_reported);
// pass over the PGs from beginning of file
while (found_pg)
{
curr_offset = new_offset;
pg_num++;
printf ("PG %llu found at offset %llu\n", pg_num, curr_offset);
/* Let's process the group */
/* Allocate PG index struct here to allow to be used below */
pg = (struct adios_index_process_group_struct_v1 *)
malloc (sizeof(struct adios_index_process_group_struct_v1));
// setup where to read the process group from (and size)
pg->offset_in_file = curr_offset;
//b->read_pg_offset = pg->offset_in_file;
b->read_pg_offset = curr_offset;
b->read_pg_size = pgsize_reported;
/* Temporary variables for parsing one PG */
struct adios_process_group_header_struct_v1 pg_header;
struct adios_vars_header_struct_v1 vars_header;
struct adios_attributes_header_struct_v1 attrs_header;
struct adios_var_header_struct_v1 var_header;
struct adios_var_payload_struct_v1 var_payload;
struct adios_attribute_struct_v1 attribute;
init_dimensions (); // store scalar values from this PG temporarily
/* Read the whole PG into a buffer and start parsing */
adios_posix_read_process_group (b);
adios_parse_process_group_header_v1 (b, &pg_header);
print_process_group_header (pg_num, &pg_header);
add_pg_to_index (index, &pg_header, curr_offset);
adios_parse_vars_header_v1 (b, &vars_header);
print_vars_header (&vars_header);
int i;
for (i = 0; i < vars_header.count; i++)
{
var_payload.payload = 0;
adios_parse_var_data_header_v1 (b, &var_header);
print_var_header (&var_header);
if ( var_header.dims == 0)
{
// Load scalars to save them for handling as dimension values
var_payload.payload = malloc (var_header.payload_size + 1);
adios_parse_var_data_payload_v1 (b, &var_header, &var_payload
,var_header.payload_size
);
}
else
{
// Just parse to move the offset in buffer, don't read data
adios_parse_var_data_payload_v1 (b, &var_header, NULL, 0);
}
store_scalar_dimensions (&var_header, &var_payload);
add_var_to_index (index, &pg_header, &var_header, &var_payload);
if (var_payload.payload)
{
free (var_payload.payload);
var_payload.payload = 0;
}
//printf ("\n");
}
adios_parse_attributes_header_v1 (b, &attrs_header);
print_attrs_header (&attrs_header);
for (i = 0; i < attrs_header.count; i++)
{
adios_parse_attribute_v1 (b, &attribute);
//print_attribute (&attribute);
//printf ("\n");
}
pgsize_actual = b->offset;
printf ("Actual size of group by processing: %llu bytes\n", pgsize_actual);
pg = pg->next;
printf (DIVIDER);
found_pg = 0;
if (curr_offset + pgsize_actual < file_size)
{
new_offset = curr_offset + pgsize_actual;
found_pg = find_pg (fd, curr_offset+pgsize_actual, file_size, &pgsize_reported);
}
if (!found_pg &&
pgsize_actual != pgsize_reported &&
curr_offset + pgsize_reported < file_size)
{
new_offset = curr_offset + pgsize_reported;
found_pg = find_pg (fd, curr_offset+pgsize_reported, file_size, &pgsize_reported);
}
}
// The end of the last successfully processed PG
// This will be the start of the index data
curr_offset += pgsize_actual;
printf (DIVIDER);
printf ("Found %llu PGs to be processable\n", pg_num);
write_index (fd, curr_offset, index);
adios_posix_close_internal (b); // will close fd
return 0;
}