Exemplo n.º 1
0
static void adios_posix_do_read (struct adios_file_struct * fd
                                ,struct adios_method_struct * method
                                )
{
    struct adios_POSIX_data_struct * p = (struct adios_POSIX_data_struct *)
                                                          method->method_data;
    struct adios_var_struct * v = fd->group->vars;

    uint32_t version = 0;

    adios_posix_read_version (&p->b);
    adios_parse_version (&p->b, &version);
    version &= ADIOS_VERSION_NUM_MASK;

    switch (version)
    {
        case 1:
        case 2:
        {
            struct adios_index_struct_v1 * index = adios_alloc_index_v1(0); // no hashtables
            struct adios_index_process_group_struct_v1 * pg_root = index->pg_root;
            struct adios_index_process_group_struct_v1 * pg_root_temp = 0;

            adios_posix_read_index_offsets (&p->b);
            adios_parse_index_offsets_v1 (&p->b);

            adios_posix_read_process_group_index (&p->b);
            adios_parse_process_group_index_v1 (&p->b, &pg_root);
#if 1
            adios_posix_read_vars_index (&p->b);
            adios_parse_vars_index_v1 (&p->b, &index->vars_root, NULL, NULL);

            adios_posix_read_attributes_index (&p->b);
            adios_parse_attributes_index_v1 (&p->b, &index->attrs_root);
#endif

            // the three section headers
            struct adios_process_group_header_struct_v1 pg_header;
            struct adios_vars_header_struct_v1 vars_header;
            struct adios_attributes_header_struct_v1 attrs_header;

            struct adios_var_header_struct_v1 var_header;
            struct adios_var_payload_struct_v1 var_payload;
            struct adios_attribute_struct_v1 attribute;

            int i;

            pg_root_temp = pg_root;
            while (pg_root_temp && pg_root_temp->next)
                pg_root_temp = pg_root_temp->next;

            p->b.read_pg_offset = pg_root_temp->offset_in_file;
            if (pg_root_temp->next)
            {
                p->b.read_pg_size =   pg_root_temp->next->offset_in_file
                                    - pg_root_temp->offset_in_file;
            }
            else
            {
                p->b.read_pg_size =   p->b.pg_index_offset
                                    - pg_root_temp->offset_in_file;
            }

            adios_posix_read_process_group (&p->b);
            adios_parse_process_group_header_v1 (&p->b, &pg_header);

            adios_parse_vars_header_v1 (&p->b, &vars_header);

            for (i = 0; i < vars_header.count; i++)
            {
                memset (&var_payload, 0
                       ,sizeof (struct adios_var_payload_struct_v1)
                       );
                adios_parse_var_data_header_v1 (&p->b, &var_header);

                struct adios_var_struct * v1 = v;
                while (v1)
                {
                    if (   strcasecmp (var_header.name, v1->name)
                        || strcasecmp (var_header.path, v1->path)
                       )
                    {
                        v1 = v1->next;
                    }
                    else
                        break;
                }

                if (v1)
                {
                    var_payload.payload = v1->data;
                    adios_parse_var_data_payload_v1 (&p->b, &var_header
                                                    ,&var_payload
                                                    ,v1->data_size
                                                    );
                }
                else
                {
                    adios_parse_var_data_payload_v1 (&p->b, &var_header
                                                    ,NULL, 0
                                                    );
                }

                adios_clear_var_header_v1 (&var_header);
            }

#if 1
            adios_parse_attributes_header_v1 (&p->b, &attrs_header);

            for (i = 0; i < attrs_header.count; i++)
            {
                adios_parse_attribute_v1 (&p->b, &attribute);
                adios_clear_attribute_v1 (&attribute);
            }
#endif
            adios_clear_process_group_header_v1 (&pg_header);
            adios_clear_index_v1 (index);
            break;
        }

        default:
            fprintf (stderr, "POSIX read: file version unknown: %u\n", version);
            return;
    }

    adios_buffer_struct_clear (&p->b);
}
Exemplo n.º 2
0
int main (int argc, char ** argv)
{
    char * filename;
    int rc = 0;
    struct dump_struct dump;

    if (argc < 2 || argc > 4)
    {
        fprintf (stderr, "usage: %s [-d [var]|--dump [var]] <filename>\n"
                ,argv [0]
                );

        return -1;
    }

    if (argv [1][0] && argv [1][0] == '-')
    {
        if (   !strcmp (argv [1], "-d")
            || !strcmp (argv [1], "--dump")
           )
        {
            dump.do_dump = 1;
            if (argc > 3)
            {
                dump.dump_var = argv [2];
                filename = argv [3];
                printf("%s %s\n",dump.dump_var,filename);
            }
            else
            {
                dump.dump_var = 0;
                filename = argv [2];
                printf("ALLVARS %s\n",filename);
            }
        }
        else
        {
            fprintf (stderr, "usage: %s [-d [var]|--dump [var]] <filename>\n"
                    ,argv [0]
                    );

            return -1;
        }
    }
    else
    {
        filename = argv [1];
        dump.do_dump = 0;
        dump.dump_var = 0;
    }

    have_subfiles = 0;
    struct adios_bp_buffer_struct_v1 * b = 0;
    uint32_t version = 0;

    b = malloc (sizeof (struct adios_bp_buffer_struct_v1));
    adios_buffer_struct_init (b);

    rc = adios_posix_open_read_internal (filename, "", b);
    if (!rc)
    {
        fprintf (stderr, "bpdump: file not found: %s\n", filename);

        return -1;
    }

    adios_posix_read_version (b);
    adios_parse_version (b, &version);
    version = version & ADIOS_VERSION_NUM_MASK;
    printf ("BP format version: %d\n", version);
    if (version < 2)
    {
        fprintf (stderr, "bpdump: This version of bpdump can only dump BP format version 2. "
                 "Use an older bpdump from adios 1.6 to dump this file.\n");
        adios_posix_close_internal (b);
        return -1;
    }

    struct adios_index_process_group_struct_v1 * pg_root = 0;
    struct adios_index_process_group_struct_v1 * pg = 0;
    struct adios_index_var_struct_v1 * vars_root = 0;
    struct adios_index_attribute_struct_v1 * attrs_root = 0;

    printf (DIVIDER);
    printf ("Process Groups Index:\n");
    adios_posix_read_index_offsets (b);
    adios_parse_index_offsets_v1 (b);

    /*
    printf ("End of process groups       = %" PRIu64 "\n", b->end_of_pgs);
    printf ("Process Groups Index Offset = %" PRIu64 "\n", b->pg_index_offset);
    printf ("Process Groups Index Size   = %" PRIu64 "\n", b->pg_size);
    printf ("Variable Index Offset       = %" PRIu64 "\n", b->vars_index_offset);
    printf ("Variable Index Size         = %" PRIu64 "\n", b->vars_size);
    printf ("Attribute Index Offset      = %" PRIu64 "\n", b->attrs_index_offset);
    printf ("Attribute Index Size        = %" PRIu64 "\n", b->attrs_size);
    */

    adios_posix_read_process_group_index (b);
    adios_parse_process_group_index_v1 (b, &pg_root, NULL);
    print_process_group_index (pg_root);

    printf (DIVIDER);
    printf ("Vars Index:\n");
    adios_posix_read_vars_index (b);
    adios_parse_vars_index_v1 (b, &vars_root, NULL, NULL);
    print_vars_index (vars_root);

    printf (DIVIDER);
    printf ("Attributes Index:\n");
    adios_posix_read_attributes_index (b);
    adios_parse_attributes_index_v1 (b, &attrs_root);
    print_attributes_index (attrs_root);

    if (version & ADIOS_VERSION_HAVE_SUBFILE)
    {
        printf (DIVIDER);
        return 0;
    }


    uint64_t pg_num = 0;
    pg = pg_root;
    while (pg)
    {

        pg_num++;
        /* Avoid processing PG's whose offset is beyond the start of index (they are actually in subfiles) */
        /* Note: only variables have subfile index, PG's don't so we need to be this indirect in the test */
        if (pg->offset_in_file >= b->pg_index_offset) 
        {
            printf ("Process Group %" PRIu64 " offset is beyond the footer.", pg_num);
            if (have_subfiles) {
                printf (" It is probably in a subfile but bpdump does not process subfiles.\n");
            } else {
                printf (" Since there are no subfiles, this probably means the BP file is corrupt.\n"
                        "offset=%" PRIu64 "  footer starts at %" PRIu64 "\n", pg->offset_in_file, b->pg_index_offset);
            }
            pg = pg->next;
            continue;
        }

        int var_dims_count = 0;
        struct var_dim * var_dims = 0;

        printf (DIVIDER);

        struct adios_process_group_header_struct_v1 pg_header;
        struct adios_vars_header_struct_v1 vars_header;
        struct adios_attributes_header_struct_v1 attrs_header;

        struct adios_var_header_struct_v1 var_header;
        struct adios_var_payload_struct_v1 var_payload;
        struct adios_attribute_struct_v1 attribute;

        // setup where to read the process group from (and size)
        b->read_pg_offset = pg->offset_in_file;
        if (pg->next)
        {
            b->read_pg_size =   pg->next->offset_in_file
                              - pg->offset_in_file;
        }
        else
        {
            b->read_pg_size =   b->pg_index_offset
                              - pg->offset_in_file;
        }

        adios_posix_read_process_group (b);
        adios_parse_process_group_header_v1 (b, &pg_header);
        print_process_group_header (pg_num, &pg_header);
        //printf ("\tSize of group in fil: %" PRIu64 " bytes\n",  b->read_pg_size);

        adios_parse_vars_header_v1 (b, &vars_header);
        print_vars_header (&vars_header);

        dump.host_language_fortran = pg_header.host_language_fortran;
        int i;
        for (i = 0; i < vars_header.count; i++)
        {
            var_payload.payload = 0;

            adios_parse_var_data_header_v1 (b, &var_header);
            print_var_header (&var_header);

            if ( var_header.dims == 0 || 
                 ( dump.do_dump &&
                   dump.dump_var &&
                   !strcasecmp (dump.dump_var, var_header.name)
                 )
               )
            {
                // add one for string null terminators
                var_payload.payload = malloc (var_header.payload_size + 1);
                adios_parse_var_data_payload_v1 (b, &var_header, &var_payload
                                                ,var_header.payload_size
                                                );
            }
            else
            {
                adios_parse_var_data_payload_v1 (b, &var_header, NULL, 0);
            }

            if (var_header.is_dim == adios_flag_yes)
            {
                var_dims = realloc (var_dims,   (var_dims_count + 1)
                                              * sizeof (struct var_dim)
                                   );

                var_dims [var_dims_count].id = var_header.id;
                var_dims [var_dims_count].rank = *(unsigned int *)
                                                        var_payload.payload;
                var_dims_count++;
            }

            if (dump.do_dump)
            {
                // make sure the buffer is big enough or send in null
                print_var_payload (&var_header, &var_payload, &dump
                                  ,var_dims_count, var_dims
                                  );
            }
            if (var_payload.payload)
            {
                free (var_payload.payload);
                var_payload.payload = 0;
            }
            printf ("\n");
        }

        adios_parse_attributes_header_v1 (b, &attrs_header);
        print_attrs_header (&attrs_header);

        for (i = 0; i < attrs_header.count; i++)
        {
            adios_parse_attribute_v1 (b, &attribute);
            print_attribute (&attribute);
            printf ("\n");
        }

        var_dims_count = 0;
        if (var_dims)
            free (var_dims);
        pg = pg->next;
    }
    printf (DIVIDER);
    printf ("End of %s\n", filename);

    adios_posix_close_internal (b);

    return 0;
}
Exemplo n.º 3
0
int main (int argc, char ** argv)
{
    char * filename;
    int rc = 0;

    if (argc < 2 || argc > 3)
    {
        print_usage (argc, argv);
        return -1;
    }

    if (argv [1][0] && argv [1][0] == '-')
    {
        if (   !strcmp (argv [1], "-f")
            || !strcmp (argv [1], "--force")
           )
        {
            do_write_index = 1;
            filename = argv [2];
        }
        else
        {
            print_usage (argc, argv);
            return -1;
        }
    }
    else
    {
        filename = argv [1];
        do_write_index = 0;
    }

    have_subfiles = 0;
    struct adios_bp_buffer_struct_v1 * b = 0;
    uint32_t version = 0;

    b = malloc (sizeof (struct adios_bp_buffer_struct_v1));
    adios_buffer_struct_init (b);

    int flags = O_RDONLY;
    if (do_write_index)
        flags = O_RDWR;

    int fd = open (filename, flags);
    if (fd < 0) {
        fprintf (stderr, "recover: cannot open file %s\n", filename);
        if (errno)
            fprintf (stderr, "%s\n", strerror(errno));
        return -1;
    }

    struct stat statbuf; 
    fstat (fd, &statbuf);
    uint64_t file_size = statbuf.st_size;

    b->f = fd; 

    printf ("File size in bytes: %llu\n", file_size);

    /* Variables to build new index */
    struct adios_index_struct_v1 * index = adios_alloc_index_v1(1);
    //struct adios_index_process_group_struct_v1 * pg_root = 0;
    struct adios_index_process_group_struct_v1 * pg = 0;
    //struct adios_index_var_struct_v1 * vars_root = 0;
    //struct adios_index_attribute_struct_v1 * attrs_root = 0;


    uint64_t pg_num = 0L;
    uint64_t curr_offset = 0L;
    uint64_t new_offset = 0L;
    uint64_t pgsize_reported; // size of current pg (as indicated in PG header (wrongly))
    uint64_t pgsize_actual; // size of current pg based on processing (accurate)
    int found_pg = 0;

    printf (DIVIDER);
    found_pg = find_pg (fd, new_offset, file_size, &pgsize_reported);

    // pass over the PGs from beginning of file
    while (found_pg)
    {
        curr_offset = new_offset;
        pg_num++;
        printf ("PG %llu found at offset %llu\n", pg_num, curr_offset);


        /* Let's process the group */
        /* Allocate PG index struct here to allow to be used below */
        pg = (struct adios_index_process_group_struct_v1 *)
                malloc (sizeof(struct adios_index_process_group_struct_v1));

        // setup where to read the process group from (and size)
        pg->offset_in_file = curr_offset;
        //b->read_pg_offset = pg->offset_in_file;
        b->read_pg_offset = curr_offset;
        b->read_pg_size = pgsize_reported;

        /* Temporary variables for parsing one PG */
        struct adios_process_group_header_struct_v1 pg_header;
        struct adios_vars_header_struct_v1 vars_header;
        struct adios_attributes_header_struct_v1 attrs_header;

        struct adios_var_header_struct_v1 var_header;
        struct adios_var_payload_struct_v1 var_payload;
        struct adios_attribute_struct_v1 attribute;

        init_dimensions ();  // store scalar values from this PG temporarily

        /* Read the whole PG into a buffer and start parsing */
        adios_posix_read_process_group (b);
        adios_parse_process_group_header_v1 (b, &pg_header);
        print_process_group_header (pg_num, &pg_header);

        add_pg_to_index (index, &pg_header, curr_offset);

        adios_parse_vars_header_v1 (b, &vars_header);
        print_vars_header (&vars_header);

        int i;
        for (i = 0; i < vars_header.count; i++)
        {
            var_payload.payload = 0;

            adios_parse_var_data_header_v1 (b, &var_header);
            print_var_header (&var_header);

            if ( var_header.dims == 0)
            {
                // Load scalars to save them for handling as dimension values
                var_payload.payload = malloc (var_header.payload_size + 1);
                adios_parse_var_data_payload_v1 (b, &var_header, &var_payload
                                                ,var_header.payload_size
                                                );
            }
            else
            {
                // Just parse to move the offset in buffer, don't read data
                adios_parse_var_data_payload_v1 (b, &var_header, NULL, 0);
            }

            store_scalar_dimensions (&var_header, &var_payload);
            add_var_to_index (index, &pg_header, &var_header, &var_payload);

            if (var_payload.payload)
            {
                free (var_payload.payload);
                var_payload.payload = 0;
            }
            //printf ("\n");

        }

        adios_parse_attributes_header_v1 (b, &attrs_header);
        print_attrs_header (&attrs_header);

        for (i = 0; i < attrs_header.count; i++)
        {
            adios_parse_attribute_v1 (b, &attribute);
            //print_attribute (&attribute);
            //printf ("\n");
        }

        pgsize_actual = b->offset;
        printf ("Actual size of group by processing: %llu bytes\n", pgsize_actual);

        pg = pg->next;
        

        printf (DIVIDER);
        found_pg = 0;
        if (curr_offset + pgsize_actual < file_size) 
        {
            new_offset =  curr_offset + pgsize_actual;
            found_pg = find_pg (fd, curr_offset+pgsize_actual, file_size, &pgsize_reported);
        }
        if (!found_pg && 
            pgsize_actual != pgsize_reported &&
            curr_offset + pgsize_reported < file_size) 
        {
            new_offset =  curr_offset + pgsize_reported;
            found_pg = find_pg (fd, curr_offset+pgsize_reported, file_size, &pgsize_reported);
        }
    }

    // The end of the last successfully processed PG
    // This will be the start of the index data
    curr_offset += pgsize_actual;

    printf (DIVIDER);
    printf ("Found %llu PGs to be processable\n", pg_num);

    write_index (fd, curr_offset, index);
    adios_posix_close_internal (b); // will close fd

    return 0;
}