uint64_t estimateBytesOnVar(ADIOS_FILE* f, ADIOS_VARINFO* v) { if (v->ndim == 0) { return 0; } adios_inq_var_blockinfo(f, v); uint64_t result = 0; int i = 0; int j = 0; for (i=0; i<v->sum_nblocks; i++) { ADIOS_VARBLOCK curr = v->blockinfo[i]; uint64_t blockSize = fastbit_adios_util_getBlockSize(v, -1, i); result += blockSize; //uint64_t blockDataByteSize = adios_type_size (v->type, v->value) * blockSize0; /*uint64_t blockSize = 1; for (j=0; j<v->ndim; j++) { blockSize = blockSize* curr.count[j]; } result += blockSize; */ } return adios_type_size (v->type, v->value) * result; //int typeSize = adios_type_size(v->type, NULL); //return result * typeSize; }
void adios_checkpoint_verify_variables(ADIOS_FILE* fp, const char* name, RealType* origin) { ADIOS_VARINFO *vi; int count = 1; int size; vi = adios_inq_var(fp, name); adios_inq_var_blockinfo (fp, vi); if (vi->ndim > 0) { count*=vi->dims[0]; for (int j = 1; j < vi->ndim; j++) { count *= vi->dims[j]; } } size = count*adios_type_size(vi->type, vi->value); RealType *mem= (RealType *)malloc(size); adios_schedule_read(fp, NULL, name, 0, 1, mem); adios_perform_reads(fp, 1); for(int i=0; i<count; i++) { if(mem[i] == origin[i]) { cout<<name<<"["<<i<<"]verification passed "<<mem[i]<<endl; } else { cout<<name<<"["<<i<<"]verification not passed, readin: "<<mem[i]<<" writeout: "<<origin[i]<<endl; } } adios_free_varinfo (vi); free(mem); }
void adios_checkpoint_verify_random_variables(ADIOS_FILE* fp, const char* name, uint_type* origin) { ADIOS_VARINFO *vi; int count_int = 1; int size; uint64_t *start; uint64_t *count; vi = adios_inq_var(fp, name); adios_inq_var_blockinfo (fp, vi); if (vi->ndim > 0) { start = (uint64_t *)malloc(vi->ndim * sizeof(uint64_t)); count = (uint64_t *)malloc(vi->ndim * sizeof(uint64_t)); } for (int j=0; j<vi->nblocks[0]; j++) { if(j == OHMMS::Controller->rank()) { for (int k=0; k<vi->ndim; k++) { start[k] = vi->blockinfo[j].start[k]; count[k] = vi->blockinfo[j].count[k]; count_int *= count[k]; //cout<<OHMMS::Controller->rank()<<" count "<<start[k]<<" "<<count[k]<<endl; } } } size = count_int*adios_type_size(vi->type, vi->value); uint_type *mem= (uint_type*)malloc(size); ADIOS_SELECTION *sel = adios_selection_boundingbox(vi->ndim, start, count); adios_schedule_read(fp, sel, name, 0, 1, mem); adios_perform_reads(fp, 1); int flag = 0; for(int i=0; i<count_int; i++) { if(mem[i] == origin[i]) { //cout<<name<<"["<<i<<"]verification passed "<<mem[i]<<endl; } else { flag = 1; cout<<name<<"["<<i<<"]verification not passed, readin: "<<mem[i]<<" writeout: "<<origin[i]<<endl; } } if (flag == 0) cout<<name<<" verification passed "<<endl; else cout<<name<<" verification not passed "<<endl; adios_free_varinfo (vi); adios_selection_delete(sel); free(start); free(count); free(mem); }
int print_varinfo (ADIOS_FILE *f, int start_step) { ADIOS_VARINFO * v; int i,j,k; v = adios_inq_var (f, "t"); adios_inq_var_blockinfo (f, v); printf ("ndim = %d\n", v->ndim); printf ("dims[%llu]", v->dims[0]); if (v->dims[0] != gdims[start_step]) { printf ("\tERROR: expected [%llu]", gdims[start_step]); nerrors++; } printf("\n"); printf ("nsteps = %d\n", v->nsteps); printf ("sum_nblocks = %d\n", v->sum_nblocks); k = 0; // blockinfo is a contigous 1D array of elements from 0 to v->sum_nblocks-1 for (i = 0; i < v->nsteps; i++) { printf (" nblocks[%d] = %d\n", i, v->nblocks[i]); for (j = 0; j < v->nblocks[i]; j++) { printf(" block %2d: [%llu:%llu]", j, v->blockinfo[k].start[0], v->blockinfo[k].start[0] + v->blockinfo[k].count[0]-1); if (v->blockinfo[k].start[0] != block_offset [(start_step+i)*nblocks_per_step*size + j] || v->blockinfo[k].count[0] != block_count [(start_step+i)*nblocks_per_step*size + j] ) { nerrors++; printf ("\tERROR: expected [%llu:%llu]", block_offset [(start_step+i)*nblocks_per_step*size + j], block_offset [(start_step+i)*nblocks_per_step*size + j] + block_count [(start_step+i)*nblocks_per_step*size + j] -1 ); } printf("\n"); k++; } } adios_free_varinfo (v); }
void adios_trace_verify_local_variables(ADIOS_FILE* fp, const char* name, double* origin) { ADIOS_VARINFO *vi; int count = 1; unsigned long size = 1; vi = adios_inq_var(fp, name); adios_inq_var_blockinfo(fp, vi); for (int j=0; j<vi->nblocks[0]; j++) { if(OHMMS::Controller->rank() == j) { for (int k=0; k<vi->ndim; k++) { count *= vi->blockinfo[j].count[k]; } } } size = count * adios_type_size(vi->type, vi->value); double* mem = (double*)malloc(size); ADIOS_SELECTION *sel = adios_selection_writeblock(OHMMS::Controller->rank()); adios_schedule_read(fp, sel, name, 0, 1, mem); adios_perform_reads(fp, 1); int flag = 0; for(int i=0; i<count; i++) { if(mem[i] == origin[i]) { } else { flag = 1; cout<<OHMMS::Controller->rank()<<" "<<name<<"["<<i<<"]verification not passed, readin: "<<mem[i]<<" writeout: "<<origin[i]<<endl; } } if(flag == 0) cout<<OHMMS::Controller->rank()<<" "<<name<<" verification passed"<<endl; adios_selection_delete(sel); adios_free_varinfo (vi); free(mem); }
static void test_file_mode_reads_on_var(ADIOS_FILE *fp, const char *bp_filename, const char *varname) { int i; ADIOS_VARINFO *varinfo = adios_inq_var(fp, varname); MPI_Assert(COMM, varinfo); if (varinfo->value != NULL) { //if (rank == 0) fprintf(stderr, "(skipping scalar variable '%s')\n", varname); adios_free_varinfo(varinfo); return; } fprintf(stderr, "[rank %d/%d] Starting file-mode writeblock reads on %s:/%s\n", rank, size, bp_filename, varname); adios_inq_var_blockinfo(fp, varinfo); MPI_Assert(COMM, varinfo->blockinfo); const enum ADIOS_DATATYPES datatype = varinfo->type; const int datatypesize = adios_get_type_size(datatype, NULL); int timestep, timestep_blockidx, blockidx = 0; for (timestep = 0; timestep < varinfo->nsteps; ++timestep) { for (timestep_blockidx = 0; timestep_blockidx < varinfo->nblocks[timestep]; ++timestep_blockidx, ++blockidx) { if (blockidx % size != rank) continue; const ADIOS_VARBLOCK *vb = &varinfo->blockinfo[blockidx]; ADIOS_SELECTION *block_bb = adios_selection_boundingbox(varinfo->ndim, vb->start, vb->count); ADIOS_SELECTION *block_wb = adios_selection_writeblock(timestep_blockidx); ADIOS_SELECTION *block_abs_wb = adios_selection_writeblock(blockidx); block_abs_wb->u.block.is_absolute_index = 1; uint64_t blocksize = datatypesize; for (i = 0; i < varinfo->ndim; ++i) blocksize *= vb->count[i]; void *buf_bb = malloc(blocksize); void *buf_wb = malloc(blocksize); void *buf_abs_wb = malloc(blocksize); memset(buf_bb, 0, blocksize); memset(buf_wb, 1, blocksize); memset(buf_abs_wb, 2, blocksize); MPI_Assert(COMM, buf_bb && buf_wb && buf_abs_wb); adios_schedule_read(fp, block_bb, varname, timestep, 1, buf_bb ); adios_schedule_read(fp, block_wb, varname, timestep, 1, buf_wb ); adios_schedule_read(fp, block_abs_wb, varname, timestep, 1, buf_abs_wb); adios_perform_reads(fp, 1); fprintf(stderr, "[rank %d/%d] Checking file-mode blockidx %d BB vs. WB...\n", rank, size, blockidx); MPI_Assert(COMM, memcmp(buf_bb, buf_wb, blocksize) == 0); fprintf(stderr, "[rank %d/%d] Checking file-mode blockidx %d BB vs. abs-WB...\n", rank, size, blockidx); MPI_Assert(COMM, memcmp(buf_bb, buf_abs_wb, blocksize) == 0); free(buf_bb); free(buf_wb); free(buf_abs_wb); adios_selection_delete(block_bb); adios_selection_delete(block_wb); adios_selection_delete(block_abs_wb); } } adios_free_varinfo(varinfo); fprintf(stderr, "[rank %d/%d] Finished file-mode writeblock reads on %s:/%s\n", rank, size, bp_filename, varname); }
int main (int argc, char ** argv) { int rank, size, i, j; MPI_Comm comm = MPI_COMM_WORLD; ADIOS_FILE * f; ADIOS_VARINFO * v; ADIOS_SELECTION * sel; int steps = 0; int retval = 0; float timeout_sec = 1.0; void * data = NULL; uint64_t start[2], count[2]; MPI_Init (&argc, &argv); MPI_Comm_rank (comm, &rank); MPI_Comm_size (comm, &size); adios_read_init_method (ADIOS_READ_METHOD_BP, comm, "verbose=3"); f = adios_read_open ("adios_globaltime.bp", ADIOS_READ_METHOD_BP, comm, ADIOS_LOCKMODE_NONE, timeout_sec); if (adios_errno == err_file_not_found) { printf ("rank %d: Stream not found after waiting %f seconds: %s\n", rank, timeout_sec, adios_errmsg()); retval = adios_errno; } else if (adios_errno == err_end_of_stream) { printf ("rank %d: Stream terminated before open. %s\n", rank, adios_errmsg()); retval = adios_errno; } else if (f == NULL) { printf ("rank %d: Error at opening stream: %s\n", rank, adios_errmsg()); retval = adios_errno; } else { /* process file here... */ v = adios_inq_var (f, "temperature"); adios_inq_var_blockinfo (f, v); printf ("ndim = %d\n", v->ndim); //printf ("nsteps = %d\n", v->nsteps); printf ("dims[%llu][%llu]\n", v->dims[0], v->dims[1]); uint64_t slice_size = v->dims[0]/size; if (rank == size-1) slice_size = slice_size + v->dims[0]%size; start[0] = rank * slice_size; count[0] = slice_size; start[1] = 0; count[1] = v->dims[1]; data = malloc (slice_size * v->dims[1] * 8); /* Processing loop over the steps (we are already in the first one) */ while (adios_errno != err_end_of_stream) { steps++; // steps start counting from 1 sel = adios_selection_boundingbox (v->ndim, start, count); adios_schedule_read (f, sel, "temperature", 0, 1, data); adios_perform_reads (f, 1); if (rank == 0) printf ("--------- Step: %d --------------------------------\n", f->current_step); printf("rank=%d: [0:%lld,0:%lld] = [", rank, v->dims[0], v->dims[1]); for (i = 0; i < slice_size; i++) { printf (" ["); for (j = 0; j < v->dims[1]; j++) { printf ("%g ", *((double *)data + i * v->dims[1] + j)); } printf ("]"); } printf (" ]\n\n"); // advance to 1) next available step with 2) blocking wait adios_advance_step (f, 0, timeout_sec); if (adios_errno == err_step_notready) { printf ("rank %d: No new step arrived within the timeout. Quit. %s\n", rank, adios_errmsg()); break; // quit while loop } } adios_read_close (f); } if (rank==0) printf ("We have processed %d steps\n", steps); adios_read_finalize_method (ADIOS_READ_METHOD_BP); free (data); MPI_Finalize (); return retval; }
int print_scalar (ADIOS_FILE *f, char * name) { ADIOS_VARINFO * v; int i,j,k; v = adios_inq_var (f, name); adios_inq_var_blockinfo (f, v); printf ("Scalar '%s':\n", name); printf ("nsteps = %d\n", v->nsteps); printf ("nblocks per step = %d\n", v->nblocks[0]); int err; /* Read one writeblock across all timesteps */ int *data = (int*) calloc (v->nsteps, sizeof(int)); ADIOS_SELECTION *s; printf ("Read same instance across all timesteps:\n"); for (i=0; i < v->nblocks[0]; i++) { s = adios_selection_writeblock(i); err = adios_schedule_read_byid(f, s, v->varid, 0, v->nsteps, data); if (!err) { err = adios_perform_reads(f, 1); if (!err) { err = 0; printf (" block %d = [", i); for (j=0; j < v->nsteps; j++) { printf ("%d", data[j]); if (data[j] != block_offset [j*nblocks_per_step*size + i]) { err = 1; } if (j < v->nsteps-1) printf(","); } printf("]"); if (err) { nerrors++; printf ("\tERROR expected = ["); for (j=0; j < v->nsteps; j++) { printf ("%llu", block_offset [j*nblocks_per_step*size + i]); if (j < v->nsteps-1) printf(","); } printf("]"); } printf("\n"); } else { printf ("ERROR at reading scalar '%s': %s\n", name, adios_errmsg()); } } else { printf ("ERROR at scheduling read for scalar '%s': %s\n", name, adios_errmsg()); } adios_selection_delete(s); } /* Now read piecewise, one writeblock at a time */ printf ("Read each instance individually:\n"); for (j=0; j < v->nsteps; j++) { printf (" step %d: \n", j); for (i=0; i < v->nblocks[j]; i++) { s = adios_selection_writeblock(i); err = adios_schedule_read_byid(f, s, v->varid, j, 1, data); if (!err) { err = adios_perform_reads(f, 1); if (!err) { printf (" block %d = %d", i, data[0]); if (data[0] != block_offset [j*nblocks_per_step*size + i]) { printf ("\tERROR expected = %llu", block_offset [j*nblocks_per_step*size + i]); nerrors++; } printf ("\n"); } else { printf ("ERROR at reading scalar '%s': %s\n", name, adios_errmsg()); } } else { printf ("ERROR at scheduling read for scalar '%s': %s\n", name, adios_errmsg()); } adios_selection_delete(s); } } /* Now get them piecewise, but not with reading but through statistics */ printf ("Get each instance individually from available statistics:\n"); adios_inq_var_stat (f, v, 0, 1); if (v->statistics && v->statistics->blocks) { ADIOS_VARSTAT *stat = v->statistics; int blockid = 0; for (j=0; j < v->nsteps; j++) { printf (" step %d: \n", j); for (i=0; i < v->nblocks[j]; i++) { printf (" block %d = %d", i, *(int*)stat->blocks->mins[blockid]); if (*(int*)stat->blocks->mins[blockid] != block_offset [j*nblocks_per_step*size + i]) { printf ("\tERROR expected = %llu", block_offset [j*nblocks_per_step*size + i]); nerrors++; } printf ("\n"); blockid++; } } } adios_free_varinfo (v); free(data); }
int main (int argc, char ** argv) { int i, j, k,l; MPI_Comm comm_dummy = 0; /* MPI_Comm is defined through adios_read.h */ if (argc < 2) { printf("Usage: %s <BP-file>\n", argv[0]); return 1; } adios_read_init_method (ADIOS_READ_METHOD_BP, comm_dummy, "show_hidden_attrs"); ADIOS_FILE * f; f = adios_read_open_file (argv[1], ADIOS_READ_METHOD_BP, comm_dummy); if (f == NULL) { printf ("%s\n", adios_errmsg()); return -1; } /* For all variables */ printf(" Variables=%d:\n", f->nvars); for (i = 0; i < f->nvars; i++) { ADIOS_VARINFO * v = adios_inq_var_byid (f, i); adios_inq_var_stat (f, v, 0, 1); adios_inq_var_blockinfo (f, v); uint64_t total_size = adios_type_size (v->type, v->value); for (j = 0; j < v->ndim; j++) total_size *= v->dims[j]; printf(" %-9s %s", adios_type_to_string(v->type), f->var_namelist[i]); if (v->ndim == 0) { /* Scalars do not need to be read in, we get it from the metadata when using adios_inq_var */ printf(" = %s\n", value_to_string(v->type, v->value, 0)); } else { /* Arrays, print min/max statistics*/ printf("[%lld",v->dims[0]); for (j = 1; j < v->ndim; j++) printf(", %lld",v->dims[j]); //printf("] = \n"); if (v->type == adios_integer) printf("] : min=%d max=%d\n", (*(int*)v->statistics->min), (*(int*)v->statistics->max)); else if (v->type == adios_double) printf("] : min=%lg max=%lg\n", (*(double*)v->statistics->min), (*(double*)v->statistics->max)); /* Print block info */ for (l=0; l<v->nsteps; l++) { printf(" step %3d: \n", l); for (j=0; j<v->nblocks[l]; j++) { printf(" block %3d: [", j); for (k=0; k<v->ndim; k++) { printf("%3lld:%3lld", v->blockinfo[j].start[k], v->blockinfo[j].start[k]+v->blockinfo[j].count[k]-1); if (k<v->ndim-1) printf(", "); } printf("]\n"); } } } adios_free_varinfo (v); } /* variables */ /* For all attributes */ printf(" Attributes=%d:\n", f->nattrs); for (i = 0; i < f->nattrs; i++) { enum ADIOS_DATATYPES atype; int asize; void *adata; adios_get_attr_byid (f, i, &atype, &asize, &adata); printf(" %-9s %s = %s\n", adios_type_to_string(atype), f->attr_namelist[i], value_to_string(atype, adata, 0)); free(adata); } /* attributes */ adios_read_close (f); return 0; }
int worker(int argc, char* argv[]) { TAU_PROFILE_TIMER(timer, __func__, __FILE__, TAU_USER); TAU_PROFILE_START(timer); my_printf("%d of %d In worker B\n", myrank, commsize); static bool announced = false; /* validate input */ validate_input(argc, argv); my_printf("Worker B will execute until it sees n iterations.\n", iterations); /* ADIOS: These declarations are required to match the generated * gread_/gwrite_ functions. (And those functions are * generated by calling 'gpp.py adios_config.xml') ... * EXCEPT THAT THE generation of Reader code is broken. * So, we will write the reader code manually. */ uint64_t adios_groupsize; uint64_t adios_totalsize; uint64_t adios_handle; void * data = NULL; uint64_t start[2], count[2]; int i, j, steps = 0; int NX = 10; int NY = 1; double t[NX]; double p[NX]; /* ADIOS: Can duplicate, split the world, whatever. * This allows you to have P writers to N files. * With no splits, everyone shares 1 file, but * can write lock-free by using different areas. */ MPI_Comm adios_comm, adios_comm_b_to_c; adios_comm = MPI_COMM_WORLD; //MPI_Comm_dup(MPI_COMM_WORLD, &adios_comm); adios_comm_b_to_c = MPI_COMM_WORLD; //MPI_Comm_dup(MPI_COMM_WORLD, &adios_comm_b_to_c); enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_FLEXPATH; adios_read_init_method(method, adios_comm, "verbose=3"); if (adios_errno != err_no_error) { fprintf (stderr, "rank %d: Error %d at init: %s\n", myrank, adios_errno, adios_errmsg()); exit(4); } if (send_to_c) { adios_init("adios_config.xml", adios_comm); } /* ADIOS: Set up the adios communications and buffers, open the file. */ ADIOS_FILE *fp; // file handler ADIOS_VARINFO *vi; // information about one variable ADIOS_SELECTION * sel; char adios_filename_a_to_b[256]; char adios_filename_b_to_c[256]; enum ADIOS_LOCKMODE lock_mode = ADIOS_LOCKMODE_NONE; double timeout_sec = 1.0; sprintf(adios_filename_a_to_b, "adios_a_to_b.bp"); sprintf(adios_filename_b_to_c, "adios_b_to_c.bp"); my_printf ("rank %d: Worker B opening file: %s\n", myrank, adios_filename_a_to_b); fp = adios_read_open(adios_filename_a_to_b, method, adios_comm, lock_mode, timeout_sec); if (adios_errno == err_file_not_found) { fprintf (stderr, "rank %d: Stream not found after waiting %d seconds: %s\n", myrank, timeout_sec, adios_errmsg()); exit(1); } else if (adios_errno == err_end_of_stream) { // stream has been gone before we tried to open fprintf (stderr, "rank %d: Stream terminated before open. %s\n", myrank, adios_errmsg()); exit(2); } else if (fp == NULL) { // some other error happened fprintf (stderr, "rank %d: Error %d at opening: %s\n", myrank, adios_errno, adios_errmsg()); exit(3); } else { my_printf("Found file %s\n", adios_filename_a_to_b); my_printf ("File info:\n"); my_printf (" current step: %d\n", fp->current_step); my_printf (" last step: %d\n", fp->last_step); my_printf (" # of variables: %d:\n", fp->nvars); vi = adios_inq_var(fp, "temperature"); adios_inq_var_blockinfo(fp, vi); printf ("ndim = %d\n", vi->ndim); printf ("nsteps = %d\n", vi->nsteps); printf ("dims[%llu][%llu]\n", vi->dims[0], vi->dims[1]); uint64_t slice_size = vi->dims[0]/commsize; if (myrank == commsize-1) { slice_size = slice_size + vi->dims[0]%commsize; } start[0] = myrank * slice_size; count[0] = slice_size; start[1] = 0; count[1] = vi->dims[1]; data = malloc (slice_size * vi->dims[1] * 8); /* Processing loop over the steps (we are already in the first one) */ while (adios_errno != err_end_of_stream && steps < iterations) { steps++; // steps start counting from 1 TAU_PROFILE_TIMER(adios_recv_timer, "ADIOS recv", __FILE__, TAU_USER); TAU_PROFILE_START(adios_recv_timer); sel = adios_selection_boundingbox (vi->ndim, start, count); adios_schedule_read (fp, sel, "temperature", 0, 1, data); adios_perform_reads (fp, 1); if (myrank == 0) printf ("--------- B Step: %d --------------------------------\n", fp->current_step); #if 0 printf("B rank=%d: [0:%lld,0:%lld] = [", myrank, vi->dims[0], vi->dims[1]); for (i = 0; i < slice_size; i++) { printf (" ["); for (j = 0; j < vi->dims[1]; j++) { printf ("%g ", *((double *)data + i * vi->dims[1] + j)); } printf ("]"); } printf (" ]\n\n"); #endif // advance to 1) next available step with 2) blocking wait adios_advance_step (fp, 0, timeout_sec); if (adios_errno == err_step_notready) { printf ("B rank %d: No new step arrived within the timeout. Quit. %s\n", myrank, adios_errmsg()); break; // quit while loop } TAU_PROFILE_STOP(adios_recv_timer); /* Do some exchanges with neighbors */ //do_neighbor_exchange(); /* "Compute" */ compute(steps); for (i = 0; i < NX; i++) { t[i] = steps*100.0 + myrank*NX + i; } for (i = 0; i < NY; i++) { p[i] = steps*1000.0 + myrank*NY + i; } if (send_to_c) { TAU_PROFILE_TIMER(adios_send_timer, "ADIOS send", __FILE__, TAU_USER); TAU_PROFILE_START(adios_send_timer); /* ADIOS: write to the next application in the workflow */ if (steps == 0) { adios_open(&adios_handle, "b_to_c", adios_filename_b_to_c, "w", adios_comm_b_to_c); } else { adios_open(&adios_handle, "b_to_c", adios_filename_b_to_c, "a", adios_comm_b_to_c); } /* ADIOS: Actually write the data out. * Yes, this is the recommended method, and this way, changes in * configuration with the .XML file will, even in the worst-case * scenario, merely require running 'gpp.py adios_config.xml' * and typing 'make'. */ #include "gwrite_b_to_c.ch" /* ADIOS: Close out the file completely and finalize. * If MPI is being used, this must happen before MPI_Finalize(). */ adios_close(adios_handle); TAU_PROFILE_STOP(adios_send_timer); #if 1 if (!announced) { SOS_val foo; foo.i_val = NX; SOS_pack(example_pub, "NX", SOS_VAL_TYPE_INT, foo); SOS_announce(example_pub); SOS_publish(example_pub); announced = true; } #endif } MPI_Barrier(adios_comm_b_to_c); } MPI_Barrier(MPI_COMM_WORLD); adios_read_close(fp); /* ADIOS: Close out the file completely and finalize. * If MPI is being used, this must happen before MPI_Finalize(). */ adios_read_finalize_method(method); } if (send_to_c) { adios_finalize(myrank); } free(data); //MPI_Comm_free(&adios_comm); //MPI_Comm_free(&adios_comm_b_to_c); TAU_PROFILE_STOP(timer); /* exit */ return 0; }
int main (int argc, char** argv) { fastbit_init(0); fastbit_set_verbose_level(0); ADIOS_FILE * f; //MPI_Comm comm_dummy = 0; // MPI_Comm is defined through adios_read.h MPI_Comm comm_dummy = MPI_COMM_WORLD; int rank, size; MPI_Init (&argc, &argv); MPI_Comm_rank (comm_dummy, &rank); MPI_Comm_size (comm_dummy, &size); adios_init_noxml (comm_dummy); if (argc < 2) { printf("Usage: index_fastbit fileName (attrName)"); return 0; } f = adios_read_open_file (argv[1], ADIOS_READ_METHOD_BP, comm_dummy); if (f == NULL) { printf ("::%s\n", adios_errmsg()); return -1; } /* adios_allocate_buffer (ADIOS_BUFFER_ALLOC_NOW, (f->file_size)*2/1048576 + 5); // +5MB for extra room in buffer adios_declare_group (&gAdios_group, gGroupNameFastbitIdx, "", adios_flag_yes); adios_select_method (gAdios_group, "MPI", "", ""); */ gIdxFileName = fastbit_adios_util_getFastbitIndexFileName(argv[1]); unlink(gIdxFileName); adios_allocate_buffer (ADIOS_BUFFER_ALLOC_NOW, 500); // +5MB for extra room in buffer adios_declare_group (&gAdios_group, gGroupNameFastbitIdx, "", adios_flag_yes); adios_select_method (gAdios_group, "MPI", "", ""); adios_open (&gAdios_write_file, gGroupNameFastbitIdx, gIdxFileName, "w", MPI_COMM_WORLD); #ifdef MULTI_BLOCK int testid = adios_define_var (gAdios_group, "pack", "", adios_integer , 0, 0, 0); #endif #ifdef BOX int testid = adios_define_var (gAdios_group, "elements", "", adios_integer , 0, 0, 0); #endif //uint64_t estimatedbytes = (nb+nk+no)*adios_type_size(adios_double, NULL); int jobCounter = getJobCounter(f); uint64_t estimatedbytes = getByteEstimationOnFile(f, rank); if (size > 1) { int maxJobsPP = jobCounter/size + 1; estimatedbytes = estimatedbytes * maxJobsPP /jobCounter +1048576; } estimatedbytes += 1048576; uint64_t adios_totalsize; // adios_group_size needs to be call before any write_byid, Otherwise write_byid does nothing adios_group_size (gAdios_write_file, estimatedbytes , &adios_totalsize); printf("=> .. adios open output file: %s, rank %d allocated %" PRIu64 " bytes... \n", gIdxFileName, rank, adios_totalsize); // IMPORTANT: // can only call open/close once in a process // otherwise data is tangled or only the data in the last open/close call is recorded #ifdef MULTI_BLOCK adios_write_byid(gAdios_write_file, testid, &pack); #endif #ifdef BOX adios_write_byid(gAdios_write_file, testid, &recommended_index_ele); #endif sumLogTime(-1); sumLogTimeMillis(-1); if (argc >= 3) { int i=2; while (i<argc) { const char* varName = argv[i]; if(strstr(varName, "<binning prec") != NULL) { if (gBinningOption == NULL) { gBinningOption = argv[i]; } if (argc == 3) { buildIndexOnAllVar(f, rank, size); break; } i++; continue; } else { ADIOS_VARINFO * v = adios_inq_var(f, varName); if (v == NULL) { printf("No such variable: %s\n", varName); return 0; } printf("building fastbit index on variable: %s\n", varName); buildIndex_mpi(f, v, rank, size); adios_free_varinfo(v); i++; } } } else { buildIndexOnAllVar(f, rank, size); } sumLogTime(0); sumLogTimeMillis(0); adios_close(gAdios_write_file); adios_read_close(f); // // writing file clean up // // read back: f = adios_read_open_file (gIdxFileName, ADIOS_READ_METHOD_BP, comm_dummy); if (f == NULL) { printf("No such file: %s \n", gIdxFileName); return 0; } int numVars = f->nvars; int i=0; int k=0; int j=0; for (i=0; i<numVars; i++) { char* varName = f->var_namelist[i]; ADIOS_VARINFO* v = adios_inq_var(f, varName); adios_inq_var_blockinfo(f,v); int timestep = 0; for (k=0; k<v->sum_nblocks; k++) { verifyData(f, v, k, timestep); } adios_free_varinfo(v); } adios_read_close(f); if (rank == 0) { printf(" ==> index file is at: %s\n", gIdxFileName); } // clean up MPI_Barrier (comm_dummy); adios_finalize (rank); MPI_Finalize (); free (gIdxFileName); fastbit_cleanup(); return 0; }
void buildIndex_mpi(ADIOS_FILE* f, ADIOS_VARINFO* v, int rank, int size) { adios_inq_var_blockinfo(f, v); int i=0; int j=0; int k=0; printf("building index on variable %d, binning op=%s\n", v->varid, gBinningOption); int blockCounter = -1; FastBitDataType ft = fastbit_adios_util_getFastbitDataType(v->type); #ifdef SINGLE_BLOCK for (i=0; i < v->nsteps; i++) { int nblocks = v->nblocks[i]; for (j=0; j < nblocks; j++) { blockCounter++; if (blockCounter % size == rank) { onBlock(rank, f, v, i, j, blockCounter, ft); fastbit_cleanup(); } } printf(" rank %d, varid %d, timestep %d total index created = %" PRIu64 ", %" PRIu64 ", %" PRIu64 ", \n", rank, v->varid, i, sum_nb, sum_nk, sum_no); printf(" rank %d, varid %d, timestep %d total bytes = %" PRIu64 ", %" PRIu64 ", %" PRIu64 ", \n", rank, v->varid, i, adios_type_size(adios_unsigned_integer , NULL)*sum_nb, adios_type_size(adios_double, NULL)*sum_nk, adios_type_size(adios_long, NULL)*sum_no); printf("\n"); } #endif #ifdef MULTI_BLOCK for (i=0; i<v->nsteps; i++) { int nblocks = v->nblocks[i]; int blockStart = 0; int blockEnd = blockStart+pack; while (blockStart < nblocks) { if (blockEnd > nblocks) { blockEnd = nblocks; } printf("block start=%d, end=%d, max=%d\n", blockStart, blockEnd, nblocks); if (workCounter % size == rank) { printf("%ld mod %ld == %d \n", workCounter, size, rank); onMultiBlock(rank, f, v, i, blockStart, blockEnd, ft); fastbit_cleanup(); } workCounter ++; blockStart += pack; blockEnd = blockStart+pack; } fastbit_cleanup(); } #endif #ifdef BOX for (i=0; i<v->nsteps; i++) { uint64_t s = 0; uint64_t dataSize = 1; uint64_t start[v->ndim]; for (s=0; s<v->ndim; s++) { dataSize *= v->dims[s]; start[s] = 0; } uint64_t split = dataSize/recommended_index_ele; uint64_t startRef = 0; if (split == 0) { if (rank == 0) { onBox(rank, f, v, i, start, v->dims, ft); } } else { while (startRef < v->dims[0]) { uint64_t count[v->ndim]; start[0] = startRef; startRef += v->dims[0]/split; if (startRef >= v->dims[0]) { startRef = v->dims[0]; } count[0] = startRef - start[0]; for (s=1; s<v->ndim; s++) { start[s] = 0; count[s] = v->dims[s]; } if (workCounter % size == rank) { onBox(rank, f, v, i, start, count, ft); } workCounter ++; } } } #endif }
void ADIOS1CommonRead::InqVarBlockInfo(ADIOS_VARINFO *vi) { adios_inq_var_blockinfo(m_fh, vi); }
void performQuery(ADIOS_QUERY_TEST_INFO *queryInfo, ADIOS_FILE *f, int use_streaming, int print_points, int read_results) { int timestep = 0 ; ADIOS_VARINFO * tempVar = adios_inq_var(f, queryInfo->varName); adios_inq_var_blockinfo(f, tempVar); if (use_streaming) for (timestep = 0; timestep < queryInfo->fromStep; ++timestep) assert(adios_advance_step(f, 0, 0) == 0); fprintf(stderr,"times steps for variable is: [%d, %d], batch size is %" PRIu64 "\n", queryInfo->fromStep, queryInfo->fromStep + queryInfo->numSteps, queryInfo->batchSize); for (timestep = queryInfo->fromStep; timestep < queryInfo->fromStep + queryInfo->numSteps; timestep ++) { fprintf(stderr, "querying on timestep %d \n", timestep); while (1) { ADIOS_QUERY_RESULT *currBatch = NULL; currBatch = adios_query_evaluate( queryInfo->query, queryInfo->outputSelection, use_streaming ? 0 : timestep, queryInfo->batchSize ); if (currBatch == NULL) { fprintf(stderr, "Unexpected error status in querying evaluation, returned NULL as result. " "ADIOS error message: %s \n", adios_errmsg()); break; } if (currBatch->status == ADIOS_QUERY_RESULT_ERROR) { fprintf(stderr, "ERROR in querying evaluation: %s \n", adios_errmsg()); break; } if (currBatch->nselections == 0) { /* this is normal, we processed everything in previous loop or there is 0 total result */ break; } int n; if (currBatch->selections->type == ADIOS_SELECTION_POINTS) { for (n = 0; n < currBatch->nselections; n++) { const ADIOS_SELECTION_POINTS_STRUCT * retrievedPts = &(currBatch->selections[n].u.points); /* fprintf(stderr,"retrieved points %" PRIu64 " \n", retrievedPts->npoints); */ uint64_t * wboffs = calloc (retrievedPts->ndim, sizeof(uint64_t)); if (retrievedPts->container_selection && retrievedPts->container_selection->type == ADIOS_SELECTION_WRITEBLOCK) { int i; int blockidx = retrievedPts->container_selection->u.block.index; if (use_streaming) { adios_inq_var_blockinfo(f, tempVar); } else { for (i = 0; i < timestep-1; i++) blockidx += tempVar->nblocks[i]; } for (i = 0; i < retrievedPts->ndim; ++i) { wboffs[i] = tempVar->blockinfo[blockidx].start[i]; } } if (print_points) { printPoints(retrievedPts, timestep, wboffs); } free (wboffs); if (read_results) { int elmSize = adios_type_size(tempVar->type, NULL); void *data = malloc(retrievedPts->npoints * elmSize); // read returned temp data adios_schedule_read (f, &currBatch->selections[n], queryInfo->varName, use_streaming ? 0 : timestep, 1, data); adios_perform_reads(f, 1); free(data); } fprintf(stderr,"Total points retrieved %"PRIu64" in %d blocks\n", currBatch->npoints, currBatch->nselections); /* if (tempVar->type == adios_double) { */ /* for (i = 0; i < retrievedPts->npoints; i++) { */ /* fprintf(stderr,"%.6f\t", ((double*)data)[i]); */ /* } */ /* fprintf(stderr,"\n"); */ /* } */ /* else if (tempVar->type == adios_real) { */ /* for (i = 0; i < retrievedPts->npoints; i++) { */ /* fprintf(stderr,"%.6f\t", ((float*)data)[i]); */ /* } */ /* fprintf(stderr,"\n"); */ /* } */ } } else if (currBatch->selections->type == ADIOS_SELECTION_WRITEBLOCK) { fprintf(stderr,"Number of blocks retrieved: %d\n", currBatch->nselections); if (print_points) { for (n = 0; n < currBatch->nselections; n++) { fprintf(stdout,"%d %d\n", timestep, currBatch->selections[n].u.block.index); } } } free(currBatch->selections); if (currBatch->status == ADIOS_QUERY_NO_MORE_RESULTS) { free (currBatch); break; } free(currBatch); } if (use_streaming) { const int err = adios_advance_step(f, 0, 0); if (timestep < queryInfo->fromStep + queryInfo->numSteps - 1) { assert(err == 0); } else { assert(err == err_end_of_stream || err == err_step_notready); } } } adios_query_free(queryInfo->query); }