void adios_posix1_init (const PairStruct * parameters ,struct adios_method_struct * method ) { struct adios_POSIX1_data_struct * p = 0; if (!adios_posix1_initialized) { adios_posix1_initialized = 1; } method->method_data = malloc (sizeof (struct adios_POSIX1_data_struct)); p = (struct adios_POSIX1_data_struct *) method->method_data; adios_buffer_struct_init (&p->b); p->index = adios_alloc_index_v1(1); // with hashtables p->vars_start = 0; p->vars_header_size = 0; }
void adios_posix_init (const PairStruct * parameters ,struct adios_method_struct * method ) { struct adios_POSIX_data_struct * p = 0; if (!adios_posix_initialized) { adios_posix_initialized = 1; } method->method_data = malloc (sizeof (struct adios_POSIX_data_struct)); p = (struct adios_POSIX_data_struct *) method->method_data; adios_buffer_struct_init (&p->b); p->index = adios_alloc_index_v1(1); // with hashtables p->vars_start = 0; p->vars_header_size = 0; #ifdef HAVE_MPI p->mf = 0; p->group_comm = MPI_COMM_NULL; p->rank = 0; p->size = 0; #endif }
int process_subfiles (int tid, int startidx, int endidx) { char fn[256]; uint32_t version = 0; int idx; int rc = 0; subindex[tid] = adios_alloc_index_v1(1); for (idx=startidx; idx<=endidx; idx++) { b[idx] = malloc (sizeof (struct adios_bp_buffer_struct_v1)); adios_buffer_struct_init (b[idx]); snprintf (fn, 256, "%s.dir/%s.%d", filename, filename, idx); rc = adios_posix_open_read_internal (fn, "", b[idx]); if (!rc) { fprintf (stderr, "bpmeta: file not found: %s\n", fn); return -1; } adios_posix_read_version (b[idx]); adios_parse_version (b[idx], &version); version = version & ADIOS_VERSION_NUM_MASK; if (verbose) { //printf (DIVIDER); printf ("Thread %d: Metadata of %s:\n", tid, fn); printf ("Thread %d: BP format version: %d\n", tid, version); } if (version < 2) { fprintf (stderr, "bpmeta: This version of bpmeta can only work with BP format version 2 and up. " "Use an older bpmeta from adios 1.6 to work with this file.\n"); adios_posix_close_internal (b[idx]); return -1; } struct adios_index_process_group_struct_v1 * new_pg_root = 0; struct adios_index_var_struct_v1 * new_vars_root = 0; struct adios_index_attribute_struct_v1 * new_attrs_root = 0; adios_posix_read_index_offsets (b[idx]); adios_parse_index_offsets_v1 (b[idx]); /* printf ("End of process groups = %llu\n", b->end_of_pgs); printf ("Process Groups Index Offset = %llu\n", b->pg_index_offset); printf ("Process Groups Index Size = %llu\n", b->pg_size); printf ("Variable Index Offset = %llu\n", b->vars_index_offset); printf ("Variable Index Size = %llu\n", b->vars_size); printf ("Attribute Index Offset = %llu\n", b->attrs_index_offset); printf ("Attribute Index Size = %llu\n", b->attrs_size); */ adios_posix_read_process_group_index (b[idx]); adios_parse_process_group_index_v1 (b[idx], &new_pg_root); print_pg_index (tid, new_pg_root); adios_posix_read_vars_index (b[idx]); adios_parse_vars_index_v1 (b[idx], &new_vars_root, NULL, NULL); print_variable_index (tid, new_vars_root); adios_posix_read_attributes_index (b[idx]); adios_parse_attributes_index_v1 (b[idx], &new_attrs_root); print_attribute_index (tid, new_attrs_root); adios_merge_index_v1 (subindex[tid], new_pg_root, new_vars_root, new_attrs_root); adios_posix_close_internal (b[idx]); adios_shared_buffer_free (b[idx]); } if (verbose>1) { //printf (DIVIDER); printf ("Thread %d: End of reading all subfiles\n", tid); } return 0; }
int main (int argc, char ** argv) { long int tmp; int c; while ((c = getopt_long(argc, argv, optstring, options, NULL)) != -1) { switch (c) { case 'n': case 't': errno = 0; tmp = strtol(optarg, (char **)NULL, 0); if (errno) { fprintf(stderr, "Error: could not convert -%c value: %s\n", c, optarg); return 1; } if (c == 'n') nsubfiles=tmp; else nthreads=tmp; break; case 'h': display_help(); return 0; break; case 'v': verbose++; break; default: printf("Unrecognized argument: %s\n", optarg); break; } } /* Check if we have a file defined */ if (optind >= argc) { printf ("Missing file name\n"); display_help(); return 1; } filename = strdup(argv[optind++]); if (nsubfiles < 1) nsubfiles = get_nsubfiles (filename); if (nsubfiles < 1) { printf ("Cannot determine the number of subfiles. To avoid this problem, " "provide the number of subfiles manually with the -n <N> option.\n"); return -1; } if (nthreads < 1) nthreads = 1; if (nthreads > nsubfiles) { printf ("Warning: asked for processing %d subfiles using %d threads. " "We will utilize only %d threads.\n", nsubfiles, nthreads, nsubfiles); nthreads = nsubfiles; } if (verbose>1) printf ("Create metadata file %s from %d subfiles using %d threads\n", filename, nsubfiles, nthreads); /* Initialize global variables */ b = malloc (nsubfiles * sizeof (struct adios_bp_buffer_struct_v1*)); subindex = malloc (nthreads * sizeof (struct adios_index_struct_v1*)); /* Split the processing work among T threads */ int tid; #if HAVE_PTHREAD pthread_t *thread = (pthread_t *) malloc (nthreads * sizeof(pthread_t)); struct thread_args *targs = (struct thread_args*) malloc (nthreads * sizeof(struct thread_args)); int K = nsubfiles/nthreads; // base number of files to be processed by one thread int L = nsubfiles%nthreads; // this many threads processes one more subfile int startidx, endidx; int rc; //printf ("K=%d L=%d\n", K, L); endidx = -1; for (tid=0; tid<nthreads; tid++) { startidx = endidx + 1; endidx = startidx + K - 1; targs[tid].tid = tid; targs[tid].startidx = startidx; targs[tid].endidx = endidx; if (tid < L) { endidx++; } if (verbose) printf ("Process subfiles from %d to %d with thread %d\n", targs[tid].startidx, targs[tid].endidx, targs[tid]); if (tid < nthreads-1) { /* Start worker thread. */ pthread_attr_t attr; pthread_attr_init (&attr); pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_JOINABLE); rc = pthread_create (&thread[tid], &attr, thread_main, &targs[tid]); if (rc) { printf ("ERROR: Thread %d: Cannot create thread, err code = %d\n", tid, rc); } pthread_attr_destroy(&attr); } else { // last "thread" is the main thread process_subfiles (tid, startidx, endidx); } } // wait here for everyone to finish for (tid=0; tid<nthreads-1; tid++) { void *status; rc = pthread_join (thread[tid], &status); if (rc) { printf ("ERROR: Thread %d: Cannot join thread, err code = %d\n", tid, rc); } else { if (verbose>1) printf ("Thread %d: Joined thread.\n", tid); } } free (targs); free (thread); #else /* non-threaded version */ nthreads = 1; process_subfiles (0, 0, nsubfiles-1); #endif /* Merge the T indexes into the global output index */ struct adios_index_struct_v1 * globalindex; globalindex = adios_alloc_index_v1(1); for (tid=0; tid<nthreads; tid++) { adios_merge_index_v1 (globalindex, subindex[tid]->pg_root, subindex[tid]->vars_root, subindex[tid]->attrs_root); } write_index (globalindex, filename); /* Clean-up */ adios_clear_index_v1 (globalindex); /*... already cleaned-up by globalindex clearing for (tid=0; tid<nthreads; tid++) { adios_clear_index_v1 (subindex[tid]); free (subindex[tid]); } */ free (subindex); free (globalindex); free (b); return 0; }
static void adios_posix_do_read (struct adios_file_struct * fd ,struct adios_method_struct * method ) { struct adios_POSIX_data_struct * p = (struct adios_POSIX_data_struct *) method->method_data; struct adios_var_struct * v = fd->group->vars; uint32_t version = 0; adios_posix_read_version (&p->b); adios_parse_version (&p->b, &version); version &= ADIOS_VERSION_NUM_MASK; switch (version) { case 1: case 2: { struct adios_index_struct_v1 * index = adios_alloc_index_v1(0); // no hashtables struct adios_index_process_group_struct_v1 * pg_root = index->pg_root; struct adios_index_process_group_struct_v1 * pg_root_temp = 0; adios_posix_read_index_offsets (&p->b); adios_parse_index_offsets_v1 (&p->b); adios_posix_read_process_group_index (&p->b); adios_parse_process_group_index_v1 (&p->b, &pg_root); #if 1 adios_posix_read_vars_index (&p->b); adios_parse_vars_index_v1 (&p->b, &index->vars_root, NULL, NULL); adios_posix_read_attributes_index (&p->b); adios_parse_attributes_index_v1 (&p->b, &index->attrs_root); #endif // the three section headers struct adios_process_group_header_struct_v1 pg_header; struct adios_vars_header_struct_v1 vars_header; struct adios_attributes_header_struct_v1 attrs_header; struct adios_var_header_struct_v1 var_header; struct adios_var_payload_struct_v1 var_payload; struct adios_attribute_struct_v1 attribute; int i; pg_root_temp = pg_root; while (pg_root_temp && pg_root_temp->next) pg_root_temp = pg_root_temp->next; p->b.read_pg_offset = pg_root_temp->offset_in_file; if (pg_root_temp->next) { p->b.read_pg_size = pg_root_temp->next->offset_in_file - pg_root_temp->offset_in_file; } else { p->b.read_pg_size = p->b.pg_index_offset - pg_root_temp->offset_in_file; } adios_posix_read_process_group (&p->b); adios_parse_process_group_header_v1 (&p->b, &pg_header); adios_parse_vars_header_v1 (&p->b, &vars_header); for (i = 0; i < vars_header.count; i++) { memset (&var_payload, 0 ,sizeof (struct adios_var_payload_struct_v1) ); adios_parse_var_data_header_v1 (&p->b, &var_header); struct adios_var_struct * v1 = v; while (v1) { if ( strcasecmp (var_header.name, v1->name) || strcasecmp (var_header.path, v1->path) ) { v1 = v1->next; } else break; } if (v1) { var_payload.payload = v1->data; adios_parse_var_data_payload_v1 (&p->b, &var_header ,&var_payload ,v1->data_size ); } else { adios_parse_var_data_payload_v1 (&p->b, &var_header ,NULL, 0 ); } adios_clear_var_header_v1 (&var_header); } #if 1 adios_parse_attributes_header_v1 (&p->b, &attrs_header); for (i = 0; i < attrs_header.count; i++) { adios_parse_attribute_v1 (&p->b, &attribute); adios_clear_attribute_v1 (&attribute); } #endif adios_clear_process_group_header_v1 (&pg_header); adios_clear_index_v1 (index); break; } default: fprintf (stderr, "POSIX read: file version unknown: %u\n", version); return; } adios_buffer_struct_clear (&p->b); }
int main (int argc, char ** argv) { char * filename; int rc = 0; if (argc < 2 || argc > 3) { print_usage (argc, argv); return -1; } if (argv [1][0] && argv [1][0] == '-') { if ( !strcmp (argv [1], "-f") || !strcmp (argv [1], "--force") ) { do_write_index = 1; filename = argv [2]; } else { print_usage (argc, argv); return -1; } } else { filename = argv [1]; do_write_index = 0; } have_subfiles = 0; struct adios_bp_buffer_struct_v1 * b = 0; uint32_t version = 0; b = malloc (sizeof (struct adios_bp_buffer_struct_v1)); adios_buffer_struct_init (b); int flags = O_RDONLY; if (do_write_index) flags = O_RDWR; int fd = open (filename, flags); if (fd < 0) { fprintf (stderr, "recover: cannot open file %s\n", filename); if (errno) fprintf (stderr, "%s\n", strerror(errno)); return -1; } struct stat statbuf; fstat (fd, &statbuf); uint64_t file_size = statbuf.st_size; b->f = fd; printf ("File size in bytes: %llu\n", file_size); /* Variables to build new index */ struct adios_index_struct_v1 * index = adios_alloc_index_v1(1); //struct adios_index_process_group_struct_v1 * pg_root = 0; struct adios_index_process_group_struct_v1 * pg = 0; //struct adios_index_var_struct_v1 * vars_root = 0; //struct adios_index_attribute_struct_v1 * attrs_root = 0; uint64_t pg_num = 0L; uint64_t curr_offset = 0L; uint64_t new_offset = 0L; uint64_t pgsize_reported; // size of current pg (as indicated in PG header (wrongly)) uint64_t pgsize_actual; // size of current pg based on processing (accurate) int found_pg = 0; printf (DIVIDER); found_pg = find_pg (fd, new_offset, file_size, &pgsize_reported); // pass over the PGs from beginning of file while (found_pg) { curr_offset = new_offset; pg_num++; printf ("PG %llu found at offset %llu\n", pg_num, curr_offset); /* Let's process the group */ /* Allocate PG index struct here to allow to be used below */ pg = (struct adios_index_process_group_struct_v1 *) malloc (sizeof(struct adios_index_process_group_struct_v1)); // setup where to read the process group from (and size) pg->offset_in_file = curr_offset; //b->read_pg_offset = pg->offset_in_file; b->read_pg_offset = curr_offset; b->read_pg_size = pgsize_reported; /* Temporary variables for parsing one PG */ struct adios_process_group_header_struct_v1 pg_header; struct adios_vars_header_struct_v1 vars_header; struct adios_attributes_header_struct_v1 attrs_header; struct adios_var_header_struct_v1 var_header; struct adios_var_payload_struct_v1 var_payload; struct adios_attribute_struct_v1 attribute; init_dimensions (); // store scalar values from this PG temporarily /* Read the whole PG into a buffer and start parsing */ adios_posix_read_process_group (b); adios_parse_process_group_header_v1 (b, &pg_header); print_process_group_header (pg_num, &pg_header); add_pg_to_index (index, &pg_header, curr_offset); adios_parse_vars_header_v1 (b, &vars_header); print_vars_header (&vars_header); int i; for (i = 0; i < vars_header.count; i++) { var_payload.payload = 0; adios_parse_var_data_header_v1 (b, &var_header); print_var_header (&var_header); if ( var_header.dims == 0) { // Load scalars to save them for handling as dimension values var_payload.payload = malloc (var_header.payload_size + 1); adios_parse_var_data_payload_v1 (b, &var_header, &var_payload ,var_header.payload_size ); } else { // Just parse to move the offset in buffer, don't read data adios_parse_var_data_payload_v1 (b, &var_header, NULL, 0); } store_scalar_dimensions (&var_header, &var_payload); add_var_to_index (index, &pg_header, &var_header, &var_payload); if (var_payload.payload) { free (var_payload.payload); var_payload.payload = 0; } //printf ("\n"); } adios_parse_attributes_header_v1 (b, &attrs_header); print_attrs_header (&attrs_header); for (i = 0; i < attrs_header.count; i++) { adios_parse_attribute_v1 (b, &attribute); //print_attribute (&attribute); //printf ("\n"); } pgsize_actual = b->offset; printf ("Actual size of group by processing: %llu bytes\n", pgsize_actual); pg = pg->next; printf (DIVIDER); found_pg = 0; if (curr_offset + pgsize_actual < file_size) { new_offset = curr_offset + pgsize_actual; found_pg = find_pg (fd, curr_offset+pgsize_actual, file_size, &pgsize_reported); } if (!found_pg && pgsize_actual != pgsize_reported && curr_offset + pgsize_reported < file_size) { new_offset = curr_offset + pgsize_reported; found_pg = find_pg (fd, curr_offset+pgsize_reported, file_size, &pgsize_reported); } } // The end of the last successfully processed PG // This will be the start of the index data curr_offset += pgsize_actual; printf (DIVIDER); printf ("Found %llu PGs to be processable\n", pg_num); write_index (fd, curr_offset, index); adios_posix_close_internal (b); // will close fd return 0; }