static void adios_posix_do_read (struct adios_file_struct * fd ,struct adios_method_struct * method ) { struct adios_POSIX_data_struct * p = (struct adios_POSIX_data_struct *) method->method_data; struct adios_var_struct * v = fd->group->vars; uint32_t version = 0; adios_posix_read_version (&p->b); adios_parse_version (&p->b, &version); version &= ADIOS_VERSION_NUM_MASK; switch (version) { case 1: case 2: { struct adios_index_struct_v1 * index = adios_alloc_index_v1(0); // no hashtables struct adios_index_process_group_struct_v1 * pg_root = index->pg_root; struct adios_index_process_group_struct_v1 * pg_root_temp = 0; adios_posix_read_index_offsets (&p->b); adios_parse_index_offsets_v1 (&p->b); adios_posix_read_process_group_index (&p->b); adios_parse_process_group_index_v1 (&p->b, &pg_root); #if 1 adios_posix_read_vars_index (&p->b); adios_parse_vars_index_v1 (&p->b, &index->vars_root, NULL, NULL); adios_posix_read_attributes_index (&p->b); adios_parse_attributes_index_v1 (&p->b, &index->attrs_root); #endif // the three section headers struct adios_process_group_header_struct_v1 pg_header; struct adios_vars_header_struct_v1 vars_header; struct adios_attributes_header_struct_v1 attrs_header; struct adios_var_header_struct_v1 var_header; struct adios_var_payload_struct_v1 var_payload; struct adios_attribute_struct_v1 attribute; int i; pg_root_temp = pg_root; while (pg_root_temp && pg_root_temp->next) pg_root_temp = pg_root_temp->next; p->b.read_pg_offset = pg_root_temp->offset_in_file; if (pg_root_temp->next) { p->b.read_pg_size = pg_root_temp->next->offset_in_file - pg_root_temp->offset_in_file; } else { p->b.read_pg_size = p->b.pg_index_offset - pg_root_temp->offset_in_file; } adios_posix_read_process_group (&p->b); adios_parse_process_group_header_v1 (&p->b, &pg_header); adios_parse_vars_header_v1 (&p->b, &vars_header); for (i = 0; i < vars_header.count; i++) { memset (&var_payload, 0 ,sizeof (struct adios_var_payload_struct_v1) ); adios_parse_var_data_header_v1 (&p->b, &var_header); struct adios_var_struct * v1 = v; while (v1) { if ( strcasecmp (var_header.name, v1->name) || strcasecmp (var_header.path, v1->path) ) { v1 = v1->next; } else break; } if (v1) { var_payload.payload = v1->data; adios_parse_var_data_payload_v1 (&p->b, &var_header ,&var_payload ,v1->data_size ); } else { adios_parse_var_data_payload_v1 (&p->b, &var_header ,NULL, 0 ); } adios_clear_var_header_v1 (&var_header); } #if 1 adios_parse_attributes_header_v1 (&p->b, &attrs_header); for (i = 0; i < attrs_header.count; i++) { adios_parse_attribute_v1 (&p->b, &attribute); adios_clear_attribute_v1 (&attribute); } #endif adios_clear_process_group_header_v1 (&pg_header); adios_clear_index_v1 (index); break; } default: fprintf (stderr, "POSIX read: file version unknown: %u\n", version); return; } adios_buffer_struct_clear (&p->b); }
int main (int argc, char ** argv) { char * filename; int rc = 0; struct dump_struct dump; if (argc < 2 || argc > 4) { fprintf (stderr, "usage: %s [-d [var]|--dump [var]] <filename>\n" ,argv [0] ); return -1; } if (argv [1][0] && argv [1][0] == '-') { if ( !strcmp (argv [1], "-d") || !strcmp (argv [1], "--dump") ) { dump.do_dump = 1; if (argc > 3) { dump.dump_var = argv [2]; filename = argv [3]; printf("%s %s\n",dump.dump_var,filename); } else { dump.dump_var = 0; filename = argv [2]; printf("ALLVARS %s\n",filename); } } else { fprintf (stderr, "usage: %s [-d [var]|--dump [var]] <filename>\n" ,argv [0] ); return -1; } } else { filename = argv [1]; dump.do_dump = 0; dump.dump_var = 0; } have_subfiles = 0; struct adios_bp_buffer_struct_v1 * b = 0; uint32_t version = 0; b = malloc (sizeof (struct adios_bp_buffer_struct_v1)); adios_buffer_struct_init (b); rc = adios_posix_open_read_internal (filename, "", b); if (!rc) { fprintf (stderr, "bpdump: file not found: %s\n", filename); return -1; } adios_posix_read_version (b); adios_parse_version (b, &version); version = version & ADIOS_VERSION_NUM_MASK; printf ("BP format version: %d\n", version); if (version < 2) { fprintf (stderr, "bpdump: This version of bpdump can only dump BP format version 2. " "Use an older bpdump from adios 1.6 to dump this file.\n"); adios_posix_close_internal (b); return -1; } struct adios_index_process_group_struct_v1 * pg_root = 0; struct adios_index_process_group_struct_v1 * pg = 0; struct adios_index_var_struct_v1 * vars_root = 0; struct adios_index_attribute_struct_v1 * attrs_root = 0; printf (DIVIDER); printf ("Process Groups Index:\n"); adios_posix_read_index_offsets (b); adios_parse_index_offsets_v1 (b); /* printf ("End of process groups = %" PRIu64 "\n", b->end_of_pgs); printf ("Process Groups Index Offset = %" PRIu64 "\n", b->pg_index_offset); printf ("Process Groups Index Size = %" PRIu64 "\n", b->pg_size); printf ("Variable Index Offset = %" PRIu64 "\n", b->vars_index_offset); printf ("Variable Index Size = %" PRIu64 "\n", b->vars_size); printf ("Attribute Index Offset = %" PRIu64 "\n", b->attrs_index_offset); printf ("Attribute Index Size = %" PRIu64 "\n", b->attrs_size); */ adios_posix_read_process_group_index (b); adios_parse_process_group_index_v1 (b, &pg_root, NULL); print_process_group_index (pg_root); printf (DIVIDER); printf ("Vars Index:\n"); adios_posix_read_vars_index (b); adios_parse_vars_index_v1 (b, &vars_root, NULL, NULL); print_vars_index (vars_root); printf (DIVIDER); printf ("Attributes Index:\n"); adios_posix_read_attributes_index (b); adios_parse_attributes_index_v1 (b, &attrs_root); print_attributes_index (attrs_root); if (version & ADIOS_VERSION_HAVE_SUBFILE) { printf (DIVIDER); return 0; } uint64_t pg_num = 0; pg = pg_root; while (pg) { pg_num++; /* Avoid processing PG's whose offset is beyond the start of index (they are actually in subfiles) */ /* Note: only variables have subfile index, PG's don't so we need to be this indirect in the test */ if (pg->offset_in_file >= b->pg_index_offset) { printf ("Process Group %" PRIu64 " offset is beyond the footer.", pg_num); if (have_subfiles) { printf (" It is probably in a subfile but bpdump does not process subfiles.\n"); } else { printf (" Since there are no subfiles, this probably means the BP file is corrupt.\n" "offset=%" PRIu64 " footer starts at %" PRIu64 "\n", pg->offset_in_file, b->pg_index_offset); } pg = pg->next; continue; } int var_dims_count = 0; struct var_dim * var_dims = 0; printf (DIVIDER); struct adios_process_group_header_struct_v1 pg_header; struct adios_vars_header_struct_v1 vars_header; struct adios_attributes_header_struct_v1 attrs_header; struct adios_var_header_struct_v1 var_header; struct adios_var_payload_struct_v1 var_payload; struct adios_attribute_struct_v1 attribute; // setup where to read the process group from (and size) b->read_pg_offset = pg->offset_in_file; if (pg->next) { b->read_pg_size = pg->next->offset_in_file - pg->offset_in_file; } else { b->read_pg_size = b->pg_index_offset - pg->offset_in_file; } adios_posix_read_process_group (b); adios_parse_process_group_header_v1 (b, &pg_header); print_process_group_header (pg_num, &pg_header); //printf ("\tSize of group in fil: %" PRIu64 " bytes\n", b->read_pg_size); adios_parse_vars_header_v1 (b, &vars_header); print_vars_header (&vars_header); dump.host_language_fortran = pg_header.host_language_fortran; int i; for (i = 0; i < vars_header.count; i++) { var_payload.payload = 0; adios_parse_var_data_header_v1 (b, &var_header); print_var_header (&var_header); if ( var_header.dims == 0 || ( dump.do_dump && dump.dump_var && !strcasecmp (dump.dump_var, var_header.name) ) ) { // add one for string null terminators var_payload.payload = malloc (var_header.payload_size + 1); adios_parse_var_data_payload_v1 (b, &var_header, &var_payload ,var_header.payload_size ); } else { adios_parse_var_data_payload_v1 (b, &var_header, NULL, 0); } if (var_header.is_dim == adios_flag_yes) { var_dims = realloc (var_dims, (var_dims_count + 1) * sizeof (struct var_dim) ); var_dims [var_dims_count].id = var_header.id; var_dims [var_dims_count].rank = *(unsigned int *) var_payload.payload; var_dims_count++; } if (dump.do_dump) { // make sure the buffer is big enough or send in null print_var_payload (&var_header, &var_payload, &dump ,var_dims_count, var_dims ); } if (var_payload.payload) { free (var_payload.payload); var_payload.payload = 0; } printf ("\n"); } adios_parse_attributes_header_v1 (b, &attrs_header); print_attrs_header (&attrs_header); for (i = 0; i < attrs_header.count; i++) { adios_parse_attribute_v1 (b, &attribute); print_attribute (&attribute); printf ("\n"); } var_dims_count = 0; if (var_dims) free (var_dims); pg = pg->next; } printf (DIVIDER); printf ("End of %s\n", filename); adios_posix_close_internal (b); return 0; }
int main (int argc, char ** argv) { char * filename; int rc = 0; if (argc < 2 || argc > 3) { print_usage (argc, argv); return -1; } if (argv [1][0] && argv [1][0] == '-') { if ( !strcmp (argv [1], "-f") || !strcmp (argv [1], "--force") ) { do_write_index = 1; filename = argv [2]; } else { print_usage (argc, argv); return -1; } } else { filename = argv [1]; do_write_index = 0; } have_subfiles = 0; struct adios_bp_buffer_struct_v1 * b = 0; uint32_t version = 0; b = malloc (sizeof (struct adios_bp_buffer_struct_v1)); adios_buffer_struct_init (b); int flags = O_RDONLY; if (do_write_index) flags = O_RDWR; int fd = open (filename, flags); if (fd < 0) { fprintf (stderr, "recover: cannot open file %s\n", filename); if (errno) fprintf (stderr, "%s\n", strerror(errno)); return -1; } struct stat statbuf; fstat (fd, &statbuf); uint64_t file_size = statbuf.st_size; b->f = fd; printf ("File size in bytes: %llu\n", file_size); /* Variables to build new index */ struct adios_index_struct_v1 * index = adios_alloc_index_v1(1); //struct adios_index_process_group_struct_v1 * pg_root = 0; struct adios_index_process_group_struct_v1 * pg = 0; //struct adios_index_var_struct_v1 * vars_root = 0; //struct adios_index_attribute_struct_v1 * attrs_root = 0; uint64_t pg_num = 0L; uint64_t curr_offset = 0L; uint64_t new_offset = 0L; uint64_t pgsize_reported; // size of current pg (as indicated in PG header (wrongly)) uint64_t pgsize_actual; // size of current pg based on processing (accurate) int found_pg = 0; printf (DIVIDER); found_pg = find_pg (fd, new_offset, file_size, &pgsize_reported); // pass over the PGs from beginning of file while (found_pg) { curr_offset = new_offset; pg_num++; printf ("PG %llu found at offset %llu\n", pg_num, curr_offset); /* Let's process the group */ /* Allocate PG index struct here to allow to be used below */ pg = (struct adios_index_process_group_struct_v1 *) malloc (sizeof(struct adios_index_process_group_struct_v1)); // setup where to read the process group from (and size) pg->offset_in_file = curr_offset; //b->read_pg_offset = pg->offset_in_file; b->read_pg_offset = curr_offset; b->read_pg_size = pgsize_reported; /* Temporary variables for parsing one PG */ struct adios_process_group_header_struct_v1 pg_header; struct adios_vars_header_struct_v1 vars_header; struct adios_attributes_header_struct_v1 attrs_header; struct adios_var_header_struct_v1 var_header; struct adios_var_payload_struct_v1 var_payload; struct adios_attribute_struct_v1 attribute; init_dimensions (); // store scalar values from this PG temporarily /* Read the whole PG into a buffer and start parsing */ adios_posix_read_process_group (b); adios_parse_process_group_header_v1 (b, &pg_header); print_process_group_header (pg_num, &pg_header); add_pg_to_index (index, &pg_header, curr_offset); adios_parse_vars_header_v1 (b, &vars_header); print_vars_header (&vars_header); int i; for (i = 0; i < vars_header.count; i++) { var_payload.payload = 0; adios_parse_var_data_header_v1 (b, &var_header); print_var_header (&var_header); if ( var_header.dims == 0) { // Load scalars to save them for handling as dimension values var_payload.payload = malloc (var_header.payload_size + 1); adios_parse_var_data_payload_v1 (b, &var_header, &var_payload ,var_header.payload_size ); } else { // Just parse to move the offset in buffer, don't read data adios_parse_var_data_payload_v1 (b, &var_header, NULL, 0); } store_scalar_dimensions (&var_header, &var_payload); add_var_to_index (index, &pg_header, &var_header, &var_payload); if (var_payload.payload) { free (var_payload.payload); var_payload.payload = 0; } //printf ("\n"); } adios_parse_attributes_header_v1 (b, &attrs_header); print_attrs_header (&attrs_header); for (i = 0; i < attrs_header.count; i++) { adios_parse_attribute_v1 (b, &attribute); //print_attribute (&attribute); //printf ("\n"); } pgsize_actual = b->offset; printf ("Actual size of group by processing: %llu bytes\n", pgsize_actual); pg = pg->next; printf (DIVIDER); found_pg = 0; if (curr_offset + pgsize_actual < file_size) { new_offset = curr_offset + pgsize_actual; found_pg = find_pg (fd, curr_offset+pgsize_actual, file_size, &pgsize_reported); } if (!found_pg && pgsize_actual != pgsize_reported && curr_offset + pgsize_reported < file_size) { new_offset = curr_offset + pgsize_reported; found_pg = find_pg (fd, curr_offset+pgsize_reported, file_size, &pgsize_reported); } } // The end of the last successfully processed PG // This will be the start of the index data curr_offset += pgsize_actual; printf (DIVIDER); printf ("Found %llu PGs to be processable\n", pg_num); write_index (fd, curr_offset, index); adios_posix_close_internal (b); // will close fd return 0; }