/// Read a sub-array. /// /// @Note The length of the buffer is assumed to be sufficiently large /// if the argument bufLen is less or equal to 0. The caller is /// responsible for allocating the buffer (@c buf) of the currect size. int64_t ADIOS_Var::readData(void* buf, int64_t bufLen, const uint64_t* start, const uint64_t* count) { uint64_t total_elm = 1; for (int i=0; i<getNumDimension(); i++) { total_elm *= count[i]; #ifdef DEBUG LOGGER(ibis::gVerbose > 5) << "ADIOS_Var::readData: [" << i << "] start=" << start[i] << ", count=" << count[i]; #endif } { uint64_t total_bytes = total_elm * adios_type_size(getType(), readValue()); if (bufLen > 0 && (uint64_t)bufLen < total_bytes) { LOGGER(ibis::gVerbose > 0) << "Warning -- ADIOS_Var::readData: bufLen (" << bufLen << ") < total_bytes (" << total_bytes << ")"; return -1; } } ADIOS_SELECTION *sel = adios_selection_boundingbox (m_handle->ndim, start, count); if (sel == 0) { LOGGER(ibis::gVerbose > 1) << "Warning -- ADIOS_Var::readData failed to create a selection"; return -2; } IBIS_BLOCK_GUARD(adios_selection_delete, sel); int ierr = adios_schedule_read_byid (getFile(), sel, index(), getFile()->current_step, 1, buf); if (ierr != 0) { LOGGER(ibis::gVerbose > 0) << "Warning -- ADIOS_Var::readData call to " "adios_schedule_read_byid failed due to " << adios_errmsg(); return -3; } ierr = adios_perform_reads(getFile(), 1); // 1 == blocking if (ierr != 0) { LOGGER(ibis::gVerbose > 0) << "Warning -- ADIOS_Var::readData call to adios_perform_reads on " << getFileName() << " failed due to " << adios_errmsg(); return -4; } LOGGER(ibis::gVerbose > 5) << "ADIOS_Var::readData: competeled reading " << total_elm << " element" << (total_elm>1?"s":"") << " for " << getName() << " from " << getFileName(); return total_elm; } // ADIOS_Var::readData
int read_stepbystep () { ADIOS_FILE * f; float timeout_sec = 0.0; int steps = 0; int retval = 0; MPI_Comm comm = MPI_COMM_SELF; adios_read_init_method (ADIOS_READ_METHOD_BP, comm, "verbose=3"); printf ("\n--------- Read as stream ------------\n"); f = adios_read_open (fname, ADIOS_READ_METHOD_BP, comm, ADIOS_LOCKMODE_NONE, timeout_sec); if (adios_errno == err_file_not_found) { printf ("Stream not found after waiting %f seconds: %s\n", timeout_sec, adios_errmsg()); retval = adios_errno; } else if (adios_errno == err_end_of_stream) { printf ("Stream terminated before open. %s\n", adios_errmsg()); retval = adios_errno; } else if (f == NULL) { printf ("Error at opening stream: %s\n", adios_errmsg()); retval = adios_errno; } else { /* Processing loop over the steps (we are already in the first one) */ while (adios_errno != err_end_of_stream) { steps++; // steps start counting from 1 printf ("Step: %d\n", f->current_step); print_varinfo (f, f->current_step); // advance to 1) next available step with 2) blocking wait adios_advance_step (f, 0, timeout_sec); if (adios_errno == err_step_notready) { //printf ("No new step arrived within the timeout. Quit. %s\n", // adios_errmsg()); break; // quit while loop } } adios_read_close (f); } adios_read_finalize_method (ADIOS_READ_METHOD_BP); //printf ("We have processed %d steps\n", steps); return retval; }
int read_file (char *fname) { ADIOS_SELECTION *sel; ADIOS_FILE * f; ADIOS_VARINFO * vi; int err=0,i,n; uint64_t start[1] = {0}; uint64_t count[2] = {NX}; uint64_t ndim; reset_rarrays(); log ("Read and check data in %s\n", fname); f = adios_read_open_file (fname, ADIOS_READ_METHOD_BP, comm); if (f == NULL) { printE ("Error at opening file: %s\n", adios_errmsg()); return 1; } sel = adios_selection_boundingbox (1, start, count); adios_schedule_read (f, sel, "t1", 0, 1, r1); adios_schedule_read (f, sel, "t2", 0, 1, r2); adios_perform_reads (f, 1); adios_selection_delete (sel); CHECK_ARRAY (t, r1, NX); CHECK_ARRAY (t, r2, NX); endread: adios_read_close(f); MPI_Barrier (comm); return err; }
/// The default constructor. It is private and can not be directly called /// by any user code. BPCommon::BPCommon() { int ierr = adios_init_noxml(); if (ierr != 0) { std::cerr << "BPCommon::ctor failed to initialize adios, " << adios_errmsg() << std::endl; throw "BPCommon failed to initialize adios"; } ierr = adios_allocate_buffer(ADIOS_BUFFER_ALLOC_NOW, FQ_ADIOS_DEFAULT_BUFFER_MB); if (ierr != 0) { std::cerr << "BPCommon::ctor failed to allocated " << FQ_ADIOS_DEFAULT_BUFFER_MB << " MB for ADIOS buffer, " << adios_errmsg() << std::endl; } }
int main (int argc, char ** argv) { int err; MPI_Init (&argc, &argv); MPI_Comm_rank (comm, &rank); MPI_Comm_size (comm, &size); init_vars(); adios_init_noxml (comm); err = adios_read_init_method(ADIOS_READ_METHOD_BP, comm, "verbose=2"); if (err) { printE ("%s\n", adios_errmsg()); } if (!err) err = declare_group (); if (!err) err = write_file ("reuse_dim.bp"); if (!err) err = read_file ("reuse_dim.bp"); adios_finalize (rank); fini_vars(); MPI_Finalize (); return err; }
AdiosCheckpointInput::AdiosCheckpointInput(std::string& checkpoint_file) { adios_read_init(method, OHMMS::Controller->getMPI(), "verbose=3;abort_on_error"); adios_file_handle = adios_read_open_file(checkpoint_file.c_str(), method, comm); if (adios_file_handle == NULL) { qmcplusplus::app_error() << adios_errmsg() << std::endl; } //Select for local values sel = adios_selection_writeblock (OHMMS::Controller->rank()); }
/// Destructor. BPCommon::~BPCommon() { int rank=0; int ierr; #ifndef FQ_NOMPI ierr = MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (ierr != MPI_SUCCESS) { std::cerr << "BPCommon::dtor failed to determine the MPI rank" << std::endl; exit(-1); } #endif ierr = adios_finalize(rank); if (ierr != 0) { std::cerr << "BPCommon::dtor failed to invoke adios_finalize, " << adios_errmsg() << std::endl; } }
/// Close the open file. This function frees the content of the ADIOS_FILE /// object under the control of this object. void ADIOS_File::close() { if (m_handle == 0) return; int ierr = adios_read_close(m_handle); if (ierr == 0) { LOGGER(ibis::gVerbose > 4) << "ADIOS_File::close successfully closed file \"" << m_fname << '"'; } else { LOGGER(ibis::gVerbose >= 0) << "Warning -- ADIOS_File::close failed to close file \"" << m_fname << "\" because of " << adios_errmsg(); } m_handle = 0; } // ADIOS_File::close
int read_all () { ADIOS_FILE * f; float timeout_sec = 0.0; int steps = 0; int retval = 0; MPI_Comm comm = MPI_COMM_SELF; adios_read_init_method (ADIOS_READ_METHOD_BP, comm, "verbose=3"); printf ("\n--------- Read as file ------------\n"); f = adios_read_open_file (fname, ADIOS_READ_METHOD_BP, comm); if (f == NULL) { printf ("Error at opening file: %s\n", adios_errmsg()); retval = adios_errno; } else { /* Processing all the steps at once */ print_varinfo (f, 0); adios_read_close (f); } adios_read_finalize_method (ADIOS_READ_METHOD_BP); return retval; }
int read_scalar () { ADIOS_FILE * f; float timeout_sec = 0.0; int steps = 0; int retval = 0; MPI_Comm comm = MPI_COMM_SELF; adios_read_init_method (ADIOS_READ_METHOD_BP, comm, "verbose=3"); printf ("\n--------- Read all instances of the scalar 'O' ------------\n"); f = adios_read_open_file (fname, ADIOS_READ_METHOD_BP, comm); if (f == NULL) { printf ("Error at opening file: %s\n", adios_errmsg()); retval = adios_errno; } else { /* Read the scalar O with writeblock selection */ print_scalar (f, "O"); adios_read_close (f); } adios_read_finalize_method (ADIOS_READ_METHOD_BP); return retval; }
int worker(int argc, char* argv[]) { TAU_PROFILE_TIMER(timer, __func__, __FILE__, TAU_USER); TAU_PROFILE_START(timer); my_printf("%d of %d In worker B\n", myrank, commsize); static bool announced = false; /* validate input */ validate_input(argc, argv); my_printf("Worker B will execute until it sees n iterations.\n", iterations); /* ADIOS: These declarations are required to match the generated * gread_/gwrite_ functions. (And those functions are * generated by calling 'gpp.py adios_config.xml') ... * EXCEPT THAT THE generation of Reader code is broken. * So, we will write the reader code manually. */ uint64_t adios_groupsize; uint64_t adios_totalsize; uint64_t adios_handle; void * data = NULL; uint64_t start[2], count[2]; int i, j, steps = 0; int NX = 10; int NY = 1; double t[NX]; double p[NX]; /* ADIOS: Can duplicate, split the world, whatever. * This allows you to have P writers to N files. * With no splits, everyone shares 1 file, but * can write lock-free by using different areas. */ MPI_Comm adios_comm, adios_comm_b_to_c; adios_comm = MPI_COMM_WORLD; //MPI_Comm_dup(MPI_COMM_WORLD, &adios_comm); adios_comm_b_to_c = MPI_COMM_WORLD; //MPI_Comm_dup(MPI_COMM_WORLD, &adios_comm_b_to_c); enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_FLEXPATH; adios_read_init_method(method, adios_comm, "verbose=3"); if (adios_errno != err_no_error) { fprintf (stderr, "rank %d: Error %d at init: %s\n", myrank, adios_errno, adios_errmsg()); exit(4); } if (send_to_c) { adios_init("adios_config.xml", adios_comm); } /* ADIOS: Set up the adios communications and buffers, open the file. */ ADIOS_FILE *fp; // file handler ADIOS_VARINFO *vi; // information about one variable ADIOS_SELECTION * sel; char adios_filename_a_to_b[256]; char adios_filename_b_to_c[256]; enum ADIOS_LOCKMODE lock_mode = ADIOS_LOCKMODE_NONE; double timeout_sec = 1.0; sprintf(adios_filename_a_to_b, "adios_a_to_b.bp"); sprintf(adios_filename_b_to_c, "adios_b_to_c.bp"); my_printf ("rank %d: Worker B opening file: %s\n", myrank, adios_filename_a_to_b); fp = adios_read_open(adios_filename_a_to_b, method, adios_comm, lock_mode, timeout_sec); if (adios_errno == err_file_not_found) { fprintf (stderr, "rank %d: Stream not found after waiting %d seconds: %s\n", myrank, timeout_sec, adios_errmsg()); exit(1); } else if (adios_errno == err_end_of_stream) { // stream has been gone before we tried to open fprintf (stderr, "rank %d: Stream terminated before open. %s\n", myrank, adios_errmsg()); exit(2); } else if (fp == NULL) { // some other error happened fprintf (stderr, "rank %d: Error %d at opening: %s\n", myrank, adios_errno, adios_errmsg()); exit(3); } else { my_printf("Found file %s\n", adios_filename_a_to_b); my_printf ("File info:\n"); my_printf (" current step: %d\n", fp->current_step); my_printf (" last step: %d\n", fp->last_step); my_printf (" # of variables: %d:\n", fp->nvars); vi = adios_inq_var(fp, "temperature"); adios_inq_var_blockinfo(fp, vi); printf ("ndim = %d\n", vi->ndim); printf ("nsteps = %d\n", vi->nsteps); printf ("dims[%llu][%llu]\n", vi->dims[0], vi->dims[1]); uint64_t slice_size = vi->dims[0]/commsize; if (myrank == commsize-1) { slice_size = slice_size + vi->dims[0]%commsize; } start[0] = myrank * slice_size; count[0] = slice_size; start[1] = 0; count[1] = vi->dims[1]; data = malloc (slice_size * vi->dims[1] * 8); /* Processing loop over the steps (we are already in the first one) */ while (adios_errno != err_end_of_stream && steps < iterations) { steps++; // steps start counting from 1 TAU_PROFILE_TIMER(adios_recv_timer, "ADIOS recv", __FILE__, TAU_USER); TAU_PROFILE_START(adios_recv_timer); sel = adios_selection_boundingbox (vi->ndim, start, count); adios_schedule_read (fp, sel, "temperature", 0, 1, data); adios_perform_reads (fp, 1); if (myrank == 0) printf ("--------- B Step: %d --------------------------------\n", fp->current_step); #if 0 printf("B rank=%d: [0:%lld,0:%lld] = [", myrank, vi->dims[0], vi->dims[1]); for (i = 0; i < slice_size; i++) { printf (" ["); for (j = 0; j < vi->dims[1]; j++) { printf ("%g ", *((double *)data + i * vi->dims[1] + j)); } printf ("]"); } printf (" ]\n\n"); #endif // advance to 1) next available step with 2) blocking wait adios_advance_step (fp, 0, timeout_sec); if (adios_errno == err_step_notready) { printf ("B rank %d: No new step arrived within the timeout. Quit. %s\n", myrank, adios_errmsg()); break; // quit while loop } TAU_PROFILE_STOP(adios_recv_timer); /* Do some exchanges with neighbors */ //do_neighbor_exchange(); /* "Compute" */ compute(steps); for (i = 0; i < NX; i++) { t[i] = steps*100.0 + myrank*NX + i; } for (i = 0; i < NY; i++) { p[i] = steps*1000.0 + myrank*NY + i; } if (send_to_c) { TAU_PROFILE_TIMER(adios_send_timer, "ADIOS send", __FILE__, TAU_USER); TAU_PROFILE_START(adios_send_timer); /* ADIOS: write to the next application in the workflow */ if (steps == 0) { adios_open(&adios_handle, "b_to_c", adios_filename_b_to_c, "w", adios_comm_b_to_c); } else { adios_open(&adios_handle, "b_to_c", adios_filename_b_to_c, "a", adios_comm_b_to_c); } /* ADIOS: Actually write the data out. * Yes, this is the recommended method, and this way, changes in * configuration with the .XML file will, even in the worst-case * scenario, merely require running 'gpp.py adios_config.xml' * and typing 'make'. */ #include "gwrite_b_to_c.ch" /* ADIOS: Close out the file completely and finalize. * If MPI is being used, this must happen before MPI_Finalize(). */ adios_close(adios_handle); TAU_PROFILE_STOP(adios_send_timer); #if 1 if (!announced) { SOS_val foo; foo.i_val = NX; SOS_pack(example_pub, "NX", SOS_VAL_TYPE_INT, foo); SOS_announce(example_pub); SOS_publish(example_pub); announced = true; } #endif } MPI_Barrier(adios_comm_b_to_c); } MPI_Barrier(MPI_COMM_WORLD); adios_read_close(fp); /* ADIOS: Close out the file completely and finalize. * If MPI is being used, this must happen before MPI_Finalize(). */ adios_read_finalize_method(method); } if (send_to_c) { adios_finalize(myrank); } free(data); //MPI_Comm_free(&adios_comm); //MPI_Comm_free(&adios_comm_b_to_c); TAU_PROFILE_STOP(timer); /* exit */ return 0; }
int main (int argc, char **argv){ int rank =0, size =0; int NX = 0; double *t = NULL; // this is an array we expect as a reference array double *t_ref = NULL; MPI_Comm comm = MPI_COMM_WORLD; diag_t diag = DIAG_OK; // to store the diagnostic information struct test_info test_result = {TEST_PASSED, "1D_arr_global_noxml"}; struct err_counts err = { 0, 0}; struct adios_tsprt_opts adios_opts; GET_ENTRY_OPTIONS(adios_opts, "Runs readers. It is recommended to run as many readers as writers."); // adios read initialization MPI_Init( &argc, &argv); MPI_Comm_rank (comm, &rank); // depending on the method SET_ERROR_IF_NOT_ZERO(adios_read_init_method(adios_opts.method, comm, adios_opts.adios_options), err.adios); RET_IF_ERROR(err.adios, rank); // I will be working with streams so the lock mode is necessary, // return immediately if the stream unavailable ADIOS_FILE *adios_handle = adios_read_open(FILE_NAME, adios_opts.method, comm, ADIOS_LOCKMODE_NONE, 0.0); if ( !adios_handle){ p_error("Quitting ... (%d) %s\n", adios_errno, adios_errmsg()); return DIAG_ERR; } // define portions of data how they will be read ADIOS_SELECTION *sel = NULL; ADIOS_VARINFO *avi = NULL; // for storing the variables char buf[STR_BUFFER_SIZE]; int step = 0; // read how many processors wrote that array avi = adios_inq_var (adios_handle, "size"); if (!avi){ p_error("rank %d: Quitting ... (%d) %s\n", rank, adios_errno, adios_errmsg()); diag = DIAG_ERR; goto close_adios; } size = *((int*)avi->value); adios_free_varinfo(avi); avi = NULL; // if I run the more readers than writers; just release // the excessive readers if (rank >= size){ p_info("rank %d: I am an excessive rank. Nothing to read ...\n", rank); diag = DIAG_OK; goto close_adios; } // read the size of the array avi = adios_inq_var (adios_handle, "NX"); if (!avi){ p_error("rank %d: Quitting ... (%d) %s\n", rank, adios_errno, adios_errmsg()); diag = DIAG_ERR; goto close_adios; } // I expect a scalar that will tell me the size of an array assert(0 == avi->ndim); assert(adios_integer == avi->type); NX = *((int*)avi->value); // I don't need that variable any more adios_free_varinfo(avi); assert(NX_DIM == NX); avi = NULL; // this will define the slice that we want to read; each rank should // read its own slice written by a corresponding writer rank uint64_t count[1] = { NX }; uint64_t start[1] = { 0 }; start[0] = rank*NX; sel = adios_selection_boundingbox(1,start, count); if( !sel ){ p_error("rank %d: Quitting ... (%d) %s\n", rank, adios_errno, adios_errmsg()); diag = DIAG_ERR; goto close_adios; } // make the reference array with reference values I expect to get t_ref = calloc(NX, sizeof(double)); if (gen_1D_array(t_ref, NX, rank) == DIAG_ERR){ p_error("Generating 1D array. Quitting ...\n"); diag = DIAG_ERR; goto close_adios; } // allocate the memory for the actual array to be read t = calloc(NX, sizeof(double)); if (adios_schedule_read(adios_handle, sel, "var_1d_array",0,1,t) != 0){ p_error("rank %d: Quitting ...(%d) %s\n", rank, adios_errno, adios_errmsg()); diag = DIAG_ERR; goto just_clean; } // not sure if this assumption is correct; difficult to find in the ADIOS sources if (adios_perform_reads(adios_handle, 1) != 0){ p_error("rank %d: Quitting ...(%d) %s\n", rank, adios_errno, adios_errmsg()); diag = DIAG_ERR; goto just_clean; } sprintf(buf, "Rank %d: var_1d_array: step %d: t: ", rank, step); int i = 0; for(i=0; i < NX; ++i){ if( t[i] != t_ref[i] ){ p_test_failed("%s: rank %d: for t[%d] (expected %.1f, got %.1f)\n", test_result.name, rank, i, t_ref[i], t[i] ); test_result.result = TEST_FAILED; break; } } if (TEST_PASSED == test_result.result) p_test_passed("%s: rank %d\n", test_result.name, rank); just_clean: // clean everything adios_selection_delete(sel); sel = NULL; free(t); t = NULL; free(t_ref); t_ref = NULL; close_adios: CLOSE_ADIOS_READER(adios_handle, adios_opts.method); if ((DIAG_OK == diag) && (TEST_PASSED == test_result.result)) { return 0; } else { return 1; } }
int read_scalar_stepbystep () { ADIOS_FILE * f; float timeout_sec = 0.0; int steps = 0; int retval = 0; MPI_Comm comm = MPI_COMM_SELF; adios_read_init_method (ADIOS_READ_METHOD_BP, comm, "verbose=3"); printf ("\n--------- Read scalar in stream using varinfo->value ------------\n"); f = adios_read_open (fname, ADIOS_READ_METHOD_BP, comm, ADIOS_LOCKMODE_NONE, timeout_sec); if (adios_errno == err_file_not_found) { printf ("Stream not found after waiting %f seconds: %s\n", timeout_sec, adios_errmsg()); retval = adios_errno; } else if (adios_errno == err_end_of_stream) { printf ("Stream terminated before open. %s\n", adios_errmsg()); retval = adios_errno; } else if (f == NULL) { printf ("Error at opening stream: %s\n", adios_errmsg()); retval = adios_errno; } else { /* Processing loop over the steps (we are already in the first one) */ while (adios_errno != err_end_of_stream) { steps++; // steps start counting from 1 printf ("Step: %d\n", f->current_step); /* Check the scalar O with varinfo->value */ ADIOS_VARINFO * v = adios_inq_var (f, "NX"); int value = *(int*)v->value; printf ("Scalar NX = %d", value); if (value != block_count [f->current_step*nblocks_per_step*size]) { printf ("\tERROR expected = %llu", block_count [f->current_step*nblocks_per_step*size]); nerrors++; } printf ("\n"); // advance to 1) next available step with 2) blocking wait adios_advance_step (f, 0, timeout_sec); if (adios_errno == err_step_notready) { //printf ("No new step arrived within the timeout. Quit. %s\n", // adios_errmsg()); break; // quit while loop } } adios_read_close (f); } adios_read_finalize_method (ADIOS_READ_METHOD_BP); //printf ("We have processed %d steps\n", steps); return retval; }
int main (int argc, char ** argv) { int err; int steps = 0, curr_step; int retval = 0; MPI_Init (&argc, &argv); comm = MPI_COMM_WORLD; MPI_Comm_rank (comm, &rank); MPI_Comm_size (comm, &numproc); if (processArgs(argc, argv)) { return 1; } print0("Input stream = %s\n", infilename); print0("Output stream = %s\n", outfilename); print0("Read method = %s (id=%d)\n", rmethodname, read_method); print0("Read method parameters = \"%s\"\n", rmethodparams); print0("Write method = %s\n", wmethodname); print0("Write method parameters = \"%s\"\n", wmethodparams); err = adios_read_init_method(read_method, comm, "max_chunk_size=100; " "app_id =32767; \n" "verbose= 3;" "poll_interval = 100;" ); if (!err) { print0 ("%s\n", adios_errmsg()); } adios_init_noxml(comm); print0 ("Waiting to open stream %s...\n", infilename); f = adios_read_open_stream (infilename, read_method, comm, ADIOS_LOCKMODE_ALL, timeout_sec); if (adios_errno == err_file_not_found) { print ("rank %d: Stream not found after waiting %d seconds: %s\n", rank, timeout_sec, adios_errmsg()); retval = adios_errno; } else if (adios_errno == err_end_of_stream) { print ("rank %d: Stream terminated before open. %s\n", rank, adios_errmsg()); retval = adios_errno; } else if (f == NULL) { print ("rank %d: Error at opening stream: %s\n", rank, adios_errmsg()); retval = adios_errno; } else { // process steps here... if (f->current_step != 0) { print ("rank %d: WARNING: First %d steps were missed by open.\n", rank, f->current_step); } while (1) { steps++; // start counting from 1 print0 ("File info:\n"); print0 (" current step: %d\n", f->current_step); print0 (" last step: %d\n", f->last_step); print0 (" # of variables: %d:\n", f->nvars); retval = process_metadata(steps); if (retval) break; retval = read_write(steps); if (retval) break; // advance to 1) next available step with 2) blocking wait curr_step = f->current_step; // save for final bye print adios_advance_step (f, 0, timeout_sec); if (adios_errno == err_end_of_stream) { break; // quit while loop } else if (adios_errno == err_step_notready) { print ("rank %d: No new step arrived within the timeout. Quit. %s\n", rank, adios_errmsg()); break; // quit while loop } else if (f->current_step != curr_step+1) { // we missed some steps print ("rank %d: WARNING: steps %d..%d were missed when advancing.\n", rank, curr_step+1, f->current_step-1); } } adios_read_close (f); } print0 ("Bye after processing %d steps\n", steps); adios_read_finalize_method (read_method); adios_finalize (rank); MPI_Finalize (); return retval; }
int print_scalar (ADIOS_FILE *f, char * name) { ADIOS_VARINFO * v; int i,j,k; v = adios_inq_var (f, name); adios_inq_var_blockinfo (f, v); printf ("Scalar '%s':\n", name); printf ("nsteps = %d\n", v->nsteps); printf ("nblocks per step = %d\n", v->nblocks[0]); int err; /* Read one writeblock across all timesteps */ int *data = (int*) calloc (v->nsteps, sizeof(int)); ADIOS_SELECTION *s; printf ("Read same instance across all timesteps:\n"); for (i=0; i < v->nblocks[0]; i++) { s = adios_selection_writeblock(i); err = adios_schedule_read_byid(f, s, v->varid, 0, v->nsteps, data); if (!err) { err = adios_perform_reads(f, 1); if (!err) { err = 0; printf (" block %d = [", i); for (j=0; j < v->nsteps; j++) { printf ("%d", data[j]); if (data[j] != block_offset [j*nblocks_per_step*size + i]) { err = 1; } if (j < v->nsteps-1) printf(","); } printf("]"); if (err) { nerrors++; printf ("\tERROR expected = ["); for (j=0; j < v->nsteps; j++) { printf ("%llu", block_offset [j*nblocks_per_step*size + i]); if (j < v->nsteps-1) printf(","); } printf("]"); } printf("\n"); } else { printf ("ERROR at reading scalar '%s': %s\n", name, adios_errmsg()); } } else { printf ("ERROR at scheduling read for scalar '%s': %s\n", name, adios_errmsg()); } adios_selection_delete(s); } /* Now read piecewise, one writeblock at a time */ printf ("Read each instance individually:\n"); for (j=0; j < v->nsteps; j++) { printf (" step %d: \n", j); for (i=0; i < v->nblocks[j]; i++) { s = adios_selection_writeblock(i); err = adios_schedule_read_byid(f, s, v->varid, j, 1, data); if (!err) { err = adios_perform_reads(f, 1); if (!err) { printf (" block %d = %d", i, data[0]); if (data[0] != block_offset [j*nblocks_per_step*size + i]) { printf ("\tERROR expected = %llu", block_offset [j*nblocks_per_step*size + i]); nerrors++; } printf ("\n"); } else { printf ("ERROR at reading scalar '%s': %s\n", name, adios_errmsg()); } } else { printf ("ERROR at scheduling read for scalar '%s': %s\n", name, adios_errmsg()); } adios_selection_delete(s); } } /* Now get them piecewise, but not with reading but through statistics */ printf ("Get each instance individually from available statistics:\n"); adios_inq_var_stat (f, v, 0, 1); if (v->statistics && v->statistics->blocks) { ADIOS_VARSTAT *stat = v->statistics; int blockid = 0; for (j=0; j < v->nsteps; j++) { printf (" step %d: \n", j); for (i=0; i < v->nblocks[j]; i++) { printf (" block %d = %d", i, *(int*)stat->blocks->mins[blockid]); if (*(int*)stat->blocks->mins[blockid] != block_offset [j*nblocks_per_step*size + i]) { printf ("\tERROR expected = %llu", block_offset [j*nblocks_per_step*size + i]); nerrors++; } printf ("\n"); blockid++; } } } adios_free_varinfo (v); free(data); }
int main (int argc, char ** argv) { int err, step ; int do_write = 1; int do_read = 1; int m = 0; char write_method[16]; MPI_Init (&argc, &argv); MPI_Comm_rank (comm, &rank); MPI_Comm_size (comm, &size); if (argc > 1) { m = strtol (argv[1], NULL, 10); if (errno) { printf("Invalid 1st argument %s\n", argv[1]); Usage(); return 1;} } if (argc > 2) { if (argv[2][0] == 'w' || argv[2][0] == 'W') { do_read = 0; } else if (argv[2][0] == 'r' || argv[2][0] == 'R') { do_write = 0; } else { printE ("Invalid command line argument %s. Allowed ones:\n" " w: do write only\n" " r: do read only\n", argv[2]); MPI_Finalize (); return 1; } } if (m==0) { read_method = ADIOS_READ_METHOD_BP; strcpy(write_method,"MPI"); } else { read_method = ADIOS_READ_METHOD_DATASPACES; strcpy(write_method,"DATASPACES"); } log ("Writing: %s method=%s\n" "Reading: %s method=%d\n", (do_write ? "yes" : "no"), write_method, (do_read ? "yes" : "no"), read_method); alloc_vars(); if (do_write) { adios_init_noxml (comm); adios_allocate_buffer (ADIOS_BUFFER_ALLOC_NOW, 10); } if (do_read) { err = adios_read_init_method(read_method, comm, "verbose=2"); if (err) { printE ("%s\n", adios_errmsg()); } } if (do_write) { adios_declare_group (&m_adios_group, "selections", "iter", adios_flag_yes); adios_select_method (m_adios_group, write_method, "verbose=2", ""); define_vars(); for (step=0; step<NSTEPS; step++) { if (!err) { set_vars (step); err = write_file (step); sleep(1); } } adios_free_group (m_adios_group); } if (!err && do_read) err = read_points (); //if (!err && do_read) // err = read_writerblocks (); if (do_read) { adios_read_finalize_method (read_method); } fini_vars(); if (do_write) { adios_finalize (rank); } MPI_Finalize (); return err; }
int main (int argc, char ** argv) { int rank, size, i, j, npl, token; MPI_Comm comm = MPI_COMM_WORLD; MPI_Status status; enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_BP; ADIOS_SELECTION * sel; void * data = NULL; uint64_t start[1], count[1]; MPI_Init (&argc, &argv); MPI_Comm_rank (comm, &rank); MPI_Comm_size (comm, &size); adios_read_init_method (method, comm, "verbose=3"); ADIOS_FILE * f = adios_read_open ("adios_global_no_xml.bp", method, comm, ADIOS_LOCKMODE_NONE, 0); if (f == NULL) { printf ("%s\n", adios_errmsg()); return -1; } ADIOS_VARINFO * v = adios_inq_var (f, "temperature"); /* Using less readers to read the global array back, i.e., non-uniform */ uint64_t slice_size = v->dims[0]/size; start[0] = slice_size * rank; if (rank == size-1) /* last rank may read more lines */ slice_size = slice_size + v->dims[0]%size; count[0] = slice_size; data = malloc (slice_size * sizeof (double)); if (data == NULL) { fprintf (stderr, "malloc failed.\n"); return -1; } /* Read a subset of the temperature array */ sel = adios_selection_boundingbox (v->ndim, start, count); adios_schedule_read (f, sel, "temperature", 0, 1, data); adios_perform_reads (f, 1); if (rank > 0) { MPI_Recv (&token, 1, MPI_INT, rank-1, 0, comm, &status); } printf (" ======== Rank %d ========== \n", rank); npl = 10; for (i = 0; i < slice_size; i+=npl) { printf ("[%4.4" PRIu64 "] ", rank*slice_size+i); for (j= 0; j < npl; j++) { printf (" %6.6g", * ((double *)data + i + j)); } printf ("\n"); } fflush(stdout); sleep(1); if (rank < size-1) { MPI_Send (&token, 1, MPI_INT, rank+1, 0, comm); } free (data); adios_selection_delete (sel); adios_free_varinfo (v); adios_read_close (f); MPI_Barrier (comm); adios_read_finalize_method (method); MPI_Finalize (); return 0; }
bool ADIOSFileObject::Open() { if (IsOpen()) return true; #ifdef PARALLEL fp = adios_fopen(fileName.c_str(), (MPI_Comm)VISIT_MPI_COMM); #else fp = adios_fopen(fileName.c_str(), 0); #endif char errmsg[1024]; if (fp == NULL) { sprintf(errmsg, "Error opening bp file %s:\n%s", fileName.c_str(), adios_errmsg()); EXCEPTION1(InvalidDBTypeException, errmsg); } ADIOS_VARINFO *avi; gps = (ADIOS_GROUP **) malloc(fp->groups_count * sizeof(ADIOS_GROUP *)); if (gps == NULL) EXCEPTION1(InvalidDBTypeException, "The file could not be opened. Not enough memory"); debug5 << "ADIOS BP file: " << fileName << endl; debug5 << "# of groups: " << fp->groups_count << endl; debug5 << "# of variables: " << fp->vars_count << endl; debug5 << "# of attributes:" << fp->attrs_count << endl; debug5 << "time steps: " << fp->ntimesteps << " from " << fp->tidx_start << endl; //Read in variables/scalars. variables.clear(); scalars.clear(); for (int gr=0; gr<fp->groups_count; gr++) { debug5 << " group " << fp->group_namelist[gr] << ":" << endl; gps[gr] = adios_gopen_byid(fp, gr); if (gps[gr] == NULL) { sprintf(errmsg, "Error opening group %s in bp file %s:\n%s", fp->group_namelist[gr], fileName.c_str(), adios_errmsg()); EXCEPTION1(InvalidDBTypeException, errmsg); } for (int vr=0; vr<gps[gr]->vars_count; vr++) { avi = adios_inq_var_byid(gps[gr], vr); if (avi == NULL) { sprintf(errmsg, "Error opening inquiring variable %s in group %s of bp file %s:\n%s", gps[gr]->var_namelist[vr], fp->group_namelist[gr], fileName.c_str(), adios_errmsg()); EXCEPTION1(InvalidDBTypeException, errmsg); } if (SupportedVariable(avi)) { //Scalar if (avi->ndim == 0) { ADIOSScalar s(gps[gr]->var_namelist[vr], avi); scalars[s.Name()] = s; debug5<<" added scalar "<<s<<endl; } //Variable else { // add variable to map, map id = variable path without the '/' in the beginning ADIOSVar v(gps[gr]->var_namelist[vr], gr, avi); variables[v.name] = v; debug5<<" added variable "<< v.name<<endl; } } else debug5<<"Skipping variable: "<<gps[gr]->var_namelist[vr]<<" dim= "<<avi->ndim <<" timedim= "<<avi->timedim <<" type= "<<adios_type_to_string(avi->type)<<endl; adios_free_varinfo(avi); } //Read in attributes. for (int a = 0; a < gps[gr]->attrs_count; a++) { int sz; void *data = NULL; ADIOS_DATATYPES attrType; if (adios_get_attr_byid(gps[gr], a, &attrType, &sz, &data) != 0) { debug5<<"Failed to get attr: "<<gps[gr]->attr_namelist[a]<<endl; continue; } ADIOSAttr attr(gps[gr]->attr_namelist[a], attrType, data); attributes[attr.Name()] = attr; free(data); } adios_gclose(gps[gr]); gps[gr] = NULL; } return true; }
int main (int argc, char ** argv) { int rank, size, i, j; MPI_Comm comm = MPI_COMM_WORLD; ADIOS_FILE * f; ADIOS_VARINFO * v; ADIOS_SELECTION * sel; int steps = 0; int retval = 0; float timeout_sec = 1.0; void * data = NULL; uint64_t start[2], count[2]; MPI_Init (&argc, &argv); MPI_Comm_rank (comm, &rank); MPI_Comm_size (comm, &size); adios_read_init_method (ADIOS_READ_METHOD_BP, comm, "verbose=3"); f = adios_read_open ("adios_globaltime.bp", ADIOS_READ_METHOD_BP, comm, ADIOS_LOCKMODE_NONE, timeout_sec); if (adios_errno == err_file_not_found) { printf ("rank %d: Stream not found after waiting %f seconds: %s\n", rank, timeout_sec, adios_errmsg()); retval = adios_errno; } else if (adios_errno == err_end_of_stream) { printf ("rank %d: Stream terminated before open. %s\n", rank, adios_errmsg()); retval = adios_errno; } else if (f == NULL) { printf ("rank %d: Error at opening stream: %s\n", rank, adios_errmsg()); retval = adios_errno; } else { /* process file here... */ v = adios_inq_var (f, "temperature"); adios_inq_var_blockinfo (f, v); printf ("ndim = %d\n", v->ndim); //printf ("nsteps = %d\n", v->nsteps); printf ("dims[%llu][%llu]\n", v->dims[0], v->dims[1]); uint64_t slice_size = v->dims[0]/size; if (rank == size-1) slice_size = slice_size + v->dims[0]%size; start[0] = rank * slice_size; count[0] = slice_size; start[1] = 0; count[1] = v->dims[1]; data = malloc (slice_size * v->dims[1] * 8); /* Processing loop over the steps (we are already in the first one) */ while (adios_errno != err_end_of_stream) { steps++; // steps start counting from 1 sel = adios_selection_boundingbox (v->ndim, start, count); adios_schedule_read (f, sel, "temperature", 0, 1, data); adios_perform_reads (f, 1); if (rank == 0) printf ("--------- Step: %d --------------------------------\n", f->current_step); printf("rank=%d: [0:%lld,0:%lld] = [", rank, v->dims[0], v->dims[1]); for (i = 0; i < slice_size; i++) { printf (" ["); for (j = 0; j < v->dims[1]; j++) { printf ("%g ", *((double *)data + i * v->dims[1] + j)); } printf ("]"); } printf (" ]\n\n"); // advance to 1) next available step with 2) blocking wait adios_advance_step (f, 0, timeout_sec); if (adios_errno == err_step_notready) { printf ("rank %d: No new step arrived within the timeout. Quit. %s\n", rank, adios_errmsg()); break; // quit while loop } } adios_read_close (f); } if (rank==0) printf ("We have processed %d steps\n", steps); adios_read_finalize_method (ADIOS_READ_METHOD_BP); free (data); MPI_Finalize (); return retval; }
int main (int argc, char ** argv) { int rank, size, i; MPI_Comm comm = MPI_COMM_WORLD; enum ADIOS_DATATYPES attr_type; enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_BP; int attr_size; void * data = NULL; MPI_Init (&argc, &argv); MPI_Comm_rank (comm, &rank); MPI_Comm_size (comm, &size); adios_read_init_method (method, comm, "verbose=3"); adios_logger_open ("log_read_C", rank); ADIOS_FILE * f = adios_read_open ("attributes_C.bp", method, comm, ADIOS_LOCKMODE_NONE, 0.0); if (f == NULL) { log_error ("%s\n", adios_errmsg()); return -1; } for (i = 0; i < f->nattrs; i++) { adios_get_attr (f, f->attr_namelist[i], &attr_type, &attr_size, &data); log_test("rank %d: attr: %s %s = ", rank, adios_type_to_string(attr_type), f->attr_namelist[i]); int type_size = adios_type_size (attr_type, data); int nelems = attr_size / type_size; int k; char *p = (char*)data; for (k=0; k<nelems; k++) { if (k>0) log_test(", "); switch (attr_type) { case adios_integer: log_test ("%d", *(int *)p); break; case adios_double: log_test ("%e", *(double *)p); break; case adios_string: log_test ("\"%s\"", (char *)p); break; case adios_string_array: log_test ("\"%s\"", *(char **)p); break; default: log_test ("??????\n"); } p=p+type_size; } log_test("\n"); free (data); data = 0; } adios_read_close (f); MPI_Barrier (comm); adios_read_finalize_method (ADIOS_READ_METHOD_BP); adios_logger_close(); MPI_Finalize (); return 0; }
int process_metadata(int step) { int retval = 0; int i, j; char gdims[256], ldims[256], offs[256]; uint64_t sum_count; ADIOS_VARINFO *v; // shortcut pointer if (step > 1) { // right now, nothing to prepare in later steps print("Step %d. return immediately\n",step); return 0; } /* First step processing */ // get groupname of stream, then declare for output adios_get_grouplist(f, &group_namelist); print0("Group name is %s\n", group_namelist[0]); adios_declare_group(&gh,group_namelist[0],"",adios_flag_yes); varinfo = (VarInfo *) malloc (sizeof(VarInfo) * f->nvars); if (!varinfo) { print("ERROR: rank %d cannot allocate %lu bytes\n", rank, sizeof(VarInfo)*f->nvars); return 1; } write_total = 0; largest_block = 0; // Decompose each variable and calculate output buffer size for (i=0; i<f->nvars; i++) { print0 ("Get info on variable %d: %s\n", i, f->var_namelist[i]); varinfo[i].v = adios_inq_var_byid (f, i); v = varinfo[i].v; // just a shortcut if (v == NULL) { print ("rank %d: ERROR: Variable %s inquiry failed: %s\n", rank, f->var_namelist[i], adios_errmsg()); return 1; } // print variable type and dimensions print0(" %-9s %s", adios_type_to_string(v->type), f->var_namelist[i]); if (v->ndim > 0) { print0("[%llu", v->dims[0]); for (j = 1; j < v->ndim; j++) print0(", %llu", v->dims[j]); print0("] :\n"); } else { print0("\tscalar\n"); } // determine subset we will write decompose (numproc, rank, v->ndim, v->dims, decomp_values, varinfo[i].count, varinfo[i].start, &sum_count); varinfo[i].writesize = sum_count * adios_type_size(v->type, v->value); if (varinfo[i].writesize != 0) { write_total += varinfo[i].writesize; if (largest_block < varinfo[i].writesize) largest_block = varinfo[i].writesize; } } // determine output buffer size and allocate it uint64_t bufsize = write_total + f->nvars*128 + f->nattrs*32 + 1024; if (bufsize > max_write_buffer_size) { print ("ERROR: rank %d: write buffer size needs to hold about %llu bytes, " "but max is set to %d\n", rank, bufsize, max_write_buffer_size); return 1; } print0 ("Rank %d: allocate %llu MB for output buffer\n", rank, bufsize/1048576+1); adios_allocate_buffer (ADIOS_BUFFER_ALLOC_NOW, bufsize/1048576+1); // allocate read buffer bufsize = largest_block + 128; if (bufsize > max_read_buffer_size) { print ("ERROR: rank %d: read buffer size needs to hold at least %llu bytes, " "but max is set to %d\n", rank, bufsize, max_read_buffer_size); return 1; } print0 ("Rank %d: allocate %g MB for input buffer\n", rank, (double)bufsize/1048576.0); readbuf = (char *) malloc ((size_t)bufsize); if (!readbuf) { print ("ERROR: rank %d: cannot allocate %llu bytes for read buffer\n", rank, bufsize); return 1; } // Select output method adios_select_method (gh, wmethodname, wmethodparams, ""); // Define variables for output based on decomposition char *vpath, *vname; for (i=0; i<f->nvars; i++) { v = varinfo[i].v; if (varinfo[i].writesize != 0) { // define variable for ADIOS writes getbasename (f->var_namelist[i], &vpath, &vname); if (v->ndim > 0) { int64s_to_str (v->ndim, v->dims, gdims); int64s_to_str (v->ndim, varinfo[i].count, ldims); int64s_to_str (v->ndim, varinfo[i].start, offs); print ("rank %d: Define variable path=\"%s\" name=\"%s\" " "gdims=%s ldims=%s offs=%s\n", rank, vpath, vname, gdims, ldims, offs); adios_define_var (gh, vname, vpath, v->type, ldims, gdims, offs); } else { print ("rank %d: Define scalar path=\"%s\" name=\"%s\"\n", rank, vpath, vname); adios_define_var (gh, vname, vpath, v->type, "", "", ""); } free(vpath); free(vname); } } if (rank == 0) { // get and define attributes enum ADIOS_DATATYPES attr_type; void * attr_value; char * attr_value_str; int attr_size; for (i=0; i<f->nattrs; i++) { adios_get_attr_byid (f, i, &attr_type, &attr_size, &attr_value); attr_value_str = (char *)value_to_string (attr_type, attr_value, 0); getbasename (f->attr_namelist[i], &vpath, &vname); if (vpath && !strcmp(vpath,"/__adios__")) { // skip on /__adios/... attributes print ("rank %d: Ignore this attribute path=\"%s\" name=\"%s\" value=\"%s\"\n", rank, vpath, vname, attr_value_str); } else { adios_define_attribute (gh, vname, vpath, attr_type, attr_value_str, ""); print ("rank %d: Define attribute path=\"%s\" name=\"%s\" value=\"%s\"\n", rank, vpath, vname, attr_value_str); free (attr_value); } } } return retval; }
int main (int argc, char ** argv) { char filename [256]; int rank, size, i, j, k, token; MPI_Comm comm = MPI_COMM_WORLD; MPI_Status status; enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_BP; ADIOS_SELECTION * sel; void * data = NULL; uint64_t start[3], count[3], step = 0; MPI_Init (&argc, &argv); MPI_Comm_rank (comm, &rank); MPI_Comm_size (comm, &size); adios_read_init_method (method, comm, "verbose=3"); /* adios_read_open_file() allows for seeing all timesteps in the file */ ADIOS_FILE * f = adios_read_open_file ("adios_globaltime.bp", method, comm); if (f == NULL) { printf ("%s\n", adios_errmsg()); return -1; } ADIOS_VARINFO * v = adios_inq_var (f, "temperature"); // read in two timesteps data = malloc (2 * v->dims[0] * v->dims[1] * sizeof (double)); if (data == NULL) { fprintf (stderr, "malloc failed.\n"); return -1; } // read in timestep 'rank' (up to 12) step = rank % 13; start[0] = 0; count[0] = v->dims[0]; start[1] = 0; count[1] = v->dims[1]; /* Read a subset of the temperature array */ sel = adios_selection_boundingbox (v->ndim, start, count); /* 2 steps from 'step' */ adios_schedule_read (f, sel, "temperature", step, 2, data); adios_perform_reads (f, 1); if (rank == 0) printf ("Array size of temperature [0:%lld,0:%lld]\n", v->dims[0], v->dims[1]); if (rank > 0) { MPI_Recv (&token, 1, MPI_INT, rank-1, 0, comm, &status); } printf("------------------------------------------------\n", rank); printf("rank=%d: \n", rank); for (i = 0; i < 2; i++) { printf ("step %lld = [\n", step+i); for (j = 0; j < v->dims[0]; j++) { printf (" ["); for (k = 0; k < v->dims[1]; k++) { printf ("%g ", ((double *)data) [ i * v->dims[0] * v->dims[1] + j * v->dims[1] + k]); } printf ("]\n"); } printf ("]\n"); } printf ("\n"); if (rank < size-1) { MPI_Send (&token, 1, MPI_INT, rank+1, 0, comm); } free (data); adios_free_varinfo (v); adios_read_close (f); MPI_Barrier (comm); adios_read_finalize_method (method); MPI_Finalize (); return 0; }
//Reads an adios restart file :: can only be instantiated if EAVL was built //with MPI enabled! eavlXGCParticleImporter::eavlXGCParticleImporter( const string &filename, ADIOS_READ_METHOD method, MPI_Comm communicator, ADIOS_LOCKMODE mode, int timeout_sec ) { timestep = 0; maxnum = 0; enphase = 0; inphase = 0; emaxgid = 0; imaxgid = 0; nvars = 0; time = 0; retVal = 0; totalIParticles = 0; totalEParticles = 0; fp = NULL; fp = NULL; getR = true; getZ = true; getPhi = true; getRho = true; getW1 = true; getW2 = true; getMu = true; getW0 = true; getF0 = true; getOriginNode = true; comm = communicator; std::string key (".restart"); std::size_t found = filename.find(key); if(found != std::string::npos) readingRestartFile = 1; else readingRestartFile = 0; char hostname[MPI_MAX_PROCESSOR_NAME]; char str [256]; int len = 0; //Set local mpi vars so we know how many minions there are, and wich we are MPI_Comm_size(comm,&numMPITasks); MPI_Comm_rank(comm,&mpiRank); MPI_Get_processor_name(hostname, &len); if(method == ADIOS_READ_METHOD_BP) //do not open stream if not staging fp = adios_read_open_file(filename.c_str(), method, comm); else fp = adios_read_open(filename.c_str(), method, comm, mode, timeout_sec); if (fp == NULL) { cerr << __LINE__ << endl; if(adios_errno == err_end_of_stream) { printf ("End of stream, no more steps expected. Quit. %s\n", adios_errmsg() ); } else { printf ("No new step arrived within the timeout. Quit. %s\n", adios_errmsg() ); THROW(eavlException, "XGC variable file not found."); } } Initialize(); }
int main (int argc, char ** argv) { int rank, size, i, j; MPI_Comm comm = MPI_COMM_WORLD; enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_BP; ADIOS_SELECTION * sel; void * data = NULL; uint64_t start[2], count[2]; MPI_Init (&argc, &argv); MPI_Comm_rank (comm, &rank); MPI_Comm_size (comm, &size); adios_read_init_method (method, comm, "verbose=4"); adios_logger_open ("log_read_C", rank); ADIOS_FILE * f = adios_read_open ("global_array_C.bp", method, comm, ADIOS_LOCKMODE_NONE, 0); if (f == NULL) { log_error ("%s\n", adios_errmsg()); return -1; } ADIOS_VARINFO * v = adios_inq_var (f, "temperature"); /* Using less readers to read the global array back, i.e., non-uniform */ uint64_t slice_size = v->dims[0]/size; start[0] = slice_size * rank; if (rank == size-1) /* last rank may read more lines */ slice_size = slice_size + v->dims[0]%size; count[0] = slice_size; start[1] = 0; count[1] = v->dims[1]; data = malloc (slice_size * v->dims[1] * sizeof (double)); if (data == NULL) { log_error (stderr, "malloc failed.\n"); return -1; } /* Read a subset of the temperature array */ sel = adios_selection_boundingbox (v->ndim, start, count); adios_schedule_read (f, sel, "temperature", 0, 1, data); adios_perform_reads (f, 1); for (i = 0; i < slice_size; i++) { log_test ("rank %d: [%lld,%d:%lld]", rank, start[0]+i, 0, slice_size); for (j = 0; j < v->dims[1]; j++) log_test (" %6.6g", * ((double *)data + i * v->dims[1] + j)); log_test ("\n"); } free (data); adios_read_close (f); MPI_Barrier (comm); adios_read_finalize_method (method); adios_logger_close(); MPI_Finalize (); return 0; }
int main (int argc, char ** argv) { //For varriable definitions: //gbounds = global bounds string, lbounds = local bounds string, offs = offset string, tstring = temp string to hold temperary stuff char gbounds[1007], lbounds[1007], offs[1007],tstring[100]; //size = number of cores, gidx = adios group index int rank, size, gidx, i, j, k, ii; //data = pointer to read-in data void * data = NULL; uint64_t s[] = {0,0,0,0,0,0,0,0,0,0}; //starting offset uint64_t c[] = {1,1,1,1,1,1,1,1,1,1}; //chunk block array uint64_t bytes_read = 0; int element_size; int64_t new_adios_group, m_adios_file; uint64_t var_size; //portion_bound, uint64_t adios_groupsize, adios_totalsize; int read_buffer; //possible maximum size you the user would like for each chunk in MB int write_buffer = 1536; //actual buffer size you use in MB int itime; int WRITEME=1; uint64_t chunk_size; //chunk size in # of elements char *var_path, *var_name; // full path cut into dir path and name MPI_Init(&argc,&argv); MPI_Comm_rank(comm,&rank); MPI_Comm_size(comm,&size); // timing numbers // we will time: // 0: adios_open, adios_group_size // 1: the total time to read in the data // 2: times around each write (will only work if we do NOT buffer.... // 3: the time in the close // 4: fopen, fclose // 5: total time // timers: the total I/O time int timers = 6; double start_time[timers], end_time[timers], total_time[timers]; if (TIMING==100) { for (itime=0;itime<timers;itime++) { start_time[itime] = 0; end_time[itime] = 0; total_time[itime]=0; } //MPI_Barrier(MPI_COMM_WORLD); start_time[5] = MPI_Wtime(); } if(rank==0) printf("converting...\n"); if (argc < 5) { if (rank==0) printf("Usage: %s <BP-file> <ADIOS-file> read_buffer(MB) write_buffer(MB) METHOD (LUSTRE_strip_count) (LUSTRE_strip_size) (LUSTRE_block_size)\n", argv[0]); return 1; } if(TIMING==100) start_time[4] = MPI_Wtime(); ADIOS_FILE * f = adios_fopen (argv[1], MPI_COMM_SELF); if(TIMING==100){ end_time[4] = MPI_Wtime(); total_time[4] = end_time[4]-start_time[4]; } adios_init_noxml(comm); // no xml will be used to write the new adios file read_buffer = atoi(argv[3]); write_buffer = atoi(argv[4]); adios_allocate_buffer (ADIOS_BUFFER_ALLOC_NOW, write_buffer); // allocate MB buffer if (f == NULL) { printf("rank=%d, file cant be opened\n", rank); if (DEBUG) printf ("%s\n", adios_errmsg()); return -1; } for (gidx = 0; gidx < f->groups_count; gidx++) { //group part adios_groupsize = 0; ADIOS_GROUP * g = adios_gopen (f, f->group_namelist[gidx]); if (g == NULL) { if (DEBUG) printf ("%s\n", adios_errmsg()); printf("rank %d: group cannot be opened.\n", rank); return -1; } /* First create all of the groups */ // now I need to create this group in the file that will be written adios_declare_group(&new_adios_group,f->group_namelist[gidx],"",adios_flag_yes); if(strcmp(argv[5],"MPI_LUSTRE")!=0) //see whether or not the user uses MPI_LUSTRE method adios_select_method (new_adios_group, argv[5], "", ""); //non-MPI_LUSTRE methods... like MPI, POSIX.... else{ char lustre_pars[1000]; strcpy(lustre_pars, ""); strcat(lustre_pars, "stripe_count="); sprintf(tstring, "%d", atoi(argv[6])); strcat(lustre_pars, tstring); strcat(lustre_pars, ",stripe_size="); sprintf(tstring, "%d", atoi(argv[7])); strcat(lustre_pars, tstring); strcat(lustre_pars, ",block_size="); sprintf(tstring, "%d", atoi(argv[8])); strcat(lustre_pars, tstring); if(rank==0) printf("lustre_pars=%s\n", lustre_pars); adios_select_method (new_adios_group, argv[5], lustre_pars, ""); //Use MPI Lustre method } // variable definition part for (i = 0; i < g->vars_count; i++) { ADIOS_VARINFO * v = adios_inq_var_byid (g, i); getbasename (g->var_namelist[i], &var_path, &var_name); if (v->ndim == 0) { // scalars: every process does them the same. adios_define_var(new_adios_group,var_name,var_path,v->type,0,0,0); getTypeInfo( v->type, &element_size); //element_size is size per element based on its type if (v->type == adios_string) { //special case when the scalar is string. adios_groupsize += strlen(v->value); } else { adios_groupsize += element_size; } } else { // vector variables getTypeInfo( v->type, &element_size); var_size=1; for (ii=0;ii<v->ndim;ii++) { var_size*=v->dims[ii]; } uint64_t total_size = var_size; //total_size tells you the total number of elements in the current vector variable var_size*=element_size; //var_size tells you the size of the current vector variable in bytess //re-initialize the s and c variables for(j=0; j<v->ndim; j++){ s[j] = 0; c[j] = 1; } //find the approximate chunk_size you would like to use. chunk_size = calcChunkSize(total_size, read_buffer*1024*1024/element_size, size); //set the chunk block array with the total size as close to chunk_size as possible calcC(chunk_size, v, c); strcpy(lbounds,""); for(j=0; j<v->ndim; j++){ sprintf(tstring, "%" PRId64 ",", c[j]); strcat(lbounds, tstring); } printf("rank=%d, name=%s, chunk_size1=%" PRId64 " c[]=%s\n",rank,g->var_namelist[i],chunk_size,lbounds); chunk_size = 1; for(ii=0; ii<v->ndim; ii++) //reset chunk_size based on the created c. Now the chunk_size is exact. chunk_size *= c[ii]; //current step points to where the process is in processing the vector. First sets with respect to rank. uint64_t current_step = rank*chunk_size; //First advance the starting point s by current_step. Of course, you don't do it if the current_step exceeds total_size. if(current_step<total_size) rS(v, s, current_step, rank); uint64_t elements_defined = 0; //First, the number of elements you have defined is 0. //You (the process) process your part of the vector when your current_step is smaller than the total_size while(current_step < total_size) { //ts, temporary s, is introduced for the sake of the inner do while loop below. Copy s to ts. uint64_t ts[] = {0,0,0,0,0,0,0,0,0,0}; arrCopy(s, ts); //for every outer while iteration, you always have the whole chunk_size remained to process. uint64_t remain_chunk = chunk_size; if(current_step+chunk_size>total_size) //except when you are nearing the end of the vector.... remain_chunk = total_size-current_step; //tc, temporary c, is introduced for the sake of the inner do while loop below. Copy s to tc. uint64_t tc[] = {1,1,1,1,1,1,1,1,1,1}; arrCopy(c, tc); do{ //how much of the remain chunk you wanna process? initially you think you can do all of it.... uint64_t used_chunk = remain_chunk; //you feel like you should process the vector with tc block size, but given ts, you might go over bound. uint64_t uc[] = {1,1,1,1,1,1,1,1,1,1}; //so you verify it by setting a new legit chunck block uc, and getting a new remain_chunk. remain_chunk = checkBound(v, ts, tc, uc, remain_chunk); //you check whether or not ts+uc goes over the bound. This is just checking to make sure there's no error. //Thereotically, there should be no problem at all. checkOverflow(0, v, ts, uc); //the below code fragment simply calculates gbounds, and sets place holders for lbounds and offs. strcpy(gbounds,""); strcpy(lbounds,""); strcpy(offs,""); for(j=0; j<v->ndim-1; j++){ sprintf(tstring, "%d,", (int)v->dims[j]); strcat(gbounds, tstring); //sprintf(tstring, "ldim%d_%s,", j, var_name); sprintf(tstring, "ldim%d,", j); strcat(lbounds, tstring); //sprintf(tstring, "offs%d_%s,", j, var_name); sprintf(tstring, "offs%d,", j); strcat(offs, tstring); } sprintf(tstring, "%d", (int)v->dims[v->ndim-1]); strcat(gbounds, tstring); //sprintf(tstring, "ldim%d_%s", v->ndim-1, var_name); sprintf(tstring, "ldim%d", v->ndim-1); strcat(lbounds, tstring); //sprintf(tstring, "offs%d_%s", v->ndim-1, var_name); sprintf(tstring, "offs%d", v->ndim-1); strcat(offs, tstring); //sprintf(tstring, "%d", v->ndim); for(j=0; j<v->ndim; j++){ //sprintf(tstring, "ldim%d_%s", j, var_name); sprintf(tstring, "ldim%d", j); adios_define_var(new_adios_group, tstring, "bp2bp", adios_unsigned_long, 0, 0, 0); //sprintf(tstring, "offs%d_%s", j, var_name); sprintf(tstring, "offs%d", j); adios_define_var(new_adios_group, tstring, "bp2bp", adios_unsigned_long, 0, 0, 0); } adios_define_var(new_adios_group,var_name,var_path,v->type,lbounds,gbounds,offs); if (DEBUG){ strcpy(lbounds,""); strcpy(offs,""); for(j=0; j<v->ndim; j++){ sprintf(tstring, "%" PRId64 ",", ts[j]); strcat(offs, tstring); sprintf(tstring, "%" PRId64 ",", uc[j]); strcat(lbounds, tstring); } printf("rank=%d, name=%s, gbounds=%s: lbounds=%s: offs=%s \n",rank,g->var_namelist[i],gbounds, lbounds, offs); } used_chunk -= remain_chunk; //you get the actual used_chunk here. elements_defined += used_chunk; if(remain_chunk!=0){ rS(v, ts, used_chunk, rank); //advance ts by used_chunk. for(k=0; k<10; k++) tc[k] = 1; calcC(remain_chunk, v, tc); //based on the remain_chunk, calculate new tc chunk block remained to process. } adios_groupsize+= used_chunk*element_size+2*v->ndim*8; }while(remain_chunk!=0); current_step += size*chunk_size; //once a whole chunk_size is processed, advance the current_step in roll-robin manner. if(current_step<total_size){ //advance s in the same way. rS(v, s, size*chunk_size, rank); } } //beside checkOverflow above, here you check whether or not the total number of elements processed across processes matches //the total number of elements in the original vector. if(DEBUG){ uint64_t* sb = (uint64_t *) malloc(sizeof(uint64_t)); uint64_t* rb = (uint64_t *) malloc(sizeof(uint64_t)); sb[0] = elements_defined; MPI_Reduce(sb,rb,1,MPI_UNSIGNED_LONG_LONG,MPI_SUM,0, comm); if(rank==0 && rb[0]!=total_size) printf("some array define mismatch. please use debug mode\n"); free(sb); free(rb); } } free (var_name); free (var_path); } // finished declaring all of the variables // Now we can define the attributes.... for (i = 0; i < g->attrs_count; i++) { enum ADIOS_DATATYPES atype; int asize; void *adata; adios_get_attr_byid (g, i, &atype, &asize, &adata); // if (DEBUG) printf("attribute name=%s\n",g->attr_namelist[i]); adios_define_attribute(new_adios_group,g->attr_namelist[i],"",atype,adata,0); } /*------------------------------ NOW WE WRITE -------------------------------------------- */ // Now we have everything declared... now we need to write them out!!!!!! if (WRITEME==1) { // open up the file for writing.... if (DEBUG) printf("rank=%d, opening file = %s, with group %s, size=%" PRId64 "\n",rank,argv[2],f->group_namelist[gidx],adios_groupsize); if(TIMING==100) start_time[0] = MPI_Wtime(); adios_open(&m_adios_file, f->group_namelist[gidx],argv[2],"w",comm); adios_group_size( m_adios_file, adios_groupsize, &adios_totalsize); //get both the total adios_totalsize and total adios_groupsize summed across processes. uint64_t* sb = (uint64_t *) malloc(sizeof(uint64_t));; uint64_t* rb = (uint64_t *) malloc(sizeof(uint64_t)); sb[0] = adios_groupsize; MPI_Reduce(sb,rb,1,MPI_UNSIGNED_LONG_LONG,MPI_SUM,0, comm); uint64_t* sb2 = (uint64_t *) malloc(sizeof(uint64_t));; uint64_t* rb2 = (uint64_t *) malloc(sizeof(uint64_t)); sb2[0] = adios_totalsize; MPI_Reduce(sb2,rb2,1,MPI_UNSIGNED_LONG_LONG,MPI_SUM,0, comm); if(rank==0){ printf("total adios_totalsize = %" PRId64 "\n", *rb2); printf("total adios_groupsize = %" PRId64 "\n", *rb); } free(sb); free(rb); free(sb2); free(rb2); if (TIMING==100) { end_time[0] = MPI_Wtime(); total_time[0]+=end_time[0] - start_time[0]; //variable definition time taken } // now we have to write out the variables.... since they are all declared now // This will be the place we actually write out the data!!!!!!!! for (i = 0; i < g->vars_count; i++) { ADIOS_VARINFO * v = adios_inq_var_byid (g, i); getbasename (g->var_namelist[i], &var_path, &var_name); if (v->ndim == 0) { if (DEBUG) { printf ("ADIOS WRITE SCALAR: rank=%d, name=%s value=", rank,g->var_namelist[i]); print_data (v->value, 0, v->type); printf ("\n"); } if (TIMING==100) { start_time[2] = MPI_Wtime(); } adios_write(m_adios_file,g->var_namelist[i],v->value); if (TIMING==100) { end_time[2] = MPI_Wtime(); total_time[2]+=end_time[2] - start_time[2]; //IO write time... } } else { for(j=0; j<v->ndim; j++){ s[j] = 0; c[j] = 1; } getTypeInfo( v->type, &element_size); uint64_t total_size = 1; for (ii=0;ii<v->ndim;ii++) total_size*=v->dims[ii]; chunk_size = calcChunkSize(total_size, read_buffer*1024*1024/element_size, size); calcC(chunk_size, v, c); chunk_size = 1; for(ii=0; ii<v->ndim; ii++) chunk_size *= c[ii]; uint64_t current_step = rank*chunk_size; if(current_step<total_size) rS(v, s, current_step, rank); uint64_t elements_written = 0; while(current_step < total_size) { uint64_t ts[] = {0,0,0,0,0,0,0,0,0,0}; arrCopy(s, ts); uint64_t remain_chunk = chunk_size; if(current_step+chunk_size>total_size) remain_chunk = total_size-current_step; uint64_t tc[] = {1,1,1,1,1,1,1,1,1,1}; arrCopy(c, tc); do{ uint64_t uc[] = {1,1,1,1,1,1,1,1,1,1}; uint64_t used_chunk = remain_chunk; remain_chunk = checkBound(v, ts, tc, uc, remain_chunk); checkOverflow(1, v, ts, uc); used_chunk -= remain_chunk; elements_written += used_chunk; //allocated space for data read-in data = (void *) malloc(used_chunk*element_size); if (TIMING==100) { start_time[1] = MPI_Wtime(); } if(PERFORMANCE_CHECK) printf("rank=%d, read start\n",rank); bytes_read = adios_read_var_byid(g,v->varid,ts,uc,data); if(PERFORMANCE_CHECK) printf("rank=%d, read end\n",rank); if (TIMING==100) { end_time[1] = MPI_Wtime(); total_time[1]+=end_time[1] -start_time[1]; //IO read time } if (DEBUG) printf ("ADIOS WRITE: rank=%d, name=%s datasize=%" PRId64 "\n",rank,g->var_namelist[i],bytes_read); if (TIMING==100) { start_time[2] = MPI_Wtime(); } if (DEBUG){ printf("rank=%d, write ts=",rank); int k; for(k=0; k<v->ndim; k++) printf("%" PRId64 ",", ts[k]); printf(" uc="); for(k=0; k<v->ndim; k++) printf("%" PRId64 ",", uc[k]); printf("\n"); } //local bounds and offets placeholders are not written out with actual values. if(PERFORMANCE_CHECK) printf("rank=%d, adios write start\n", rank); for(k=0; k<v->ndim; k++){ //sprintf(tstring, "ldim%d_%s", k, var_name); sprintf(tstring, "ldim%d", k); if (DEBUG) { printf ("ADIOS WRITE DIMENSION: rank=%d, name=%s value=", rank,tstring); print_data (&uc[k], 0, adios_unsigned_long); printf ("\n"); } adios_write(m_adios_file, tstring, &uc[k]); //sprintf(tstring, "offs%d_%s", k, var_name); sprintf(tstring, "offs%d", k); if (DEBUG) { printf ("ADIOS WRITE OFFSET: rank=%d, name=%s value=", rank,tstring); print_data (&ts[k], 0, adios_unsigned_long); printf ("\n"); } adios_write(m_adios_file, tstring, &ts[k]); } adios_write(m_adios_file,g->var_namelist[i],data); if(PERFORMANCE_CHECK) printf("rank=%d, adios write end\n", rank); if (TIMING==100) { end_time[2] = MPI_Wtime(); total_time[2]+=end_time[2] - start_time[2]; //IO write time } free(data); if(remain_chunk!=0){ rS(v, ts, used_chunk, rank); for(k=0; k<10; k++) tc[k] = 1; calcC(remain_chunk, v, tc); } }while(remain_chunk!=0); current_step += size*chunk_size; if(current_step<total_size) rS(v, s, size*chunk_size,rank); } if(DEBUG){ uint64_t* sb = (uint64_t *) malloc(sizeof(uint64_t));; uint64_t* rb = (uint64_t *) malloc(sizeof(uint64_t)); sb[0] = elements_written; MPI_Reduce(sb,rb,1,MPI_UNSIGNED_LONG_LONG,MPI_SUM,0, comm); if(rank==0 && rb[0]!=total_size) printf("some array read mismatch. please use debug mode\n"); free(sb); free(rb); } } free (var_name); free (var_path); }// end of the writing of the variable.. if (TIMING==100) { start_time[3] = MPI_Wtime(); } if(PERFORMANCE_CHECK) printf("rank=%d, adios_close start\n", rank); adios_close(m_adios_file); if(PERFORMANCE_CHECK) printf("rank=%d, adios_close end\n", rank); if (TIMING==100) { end_time[3] = MPI_Wtime(); total_time[3]+=end_time[3] - start_time[3]; } adios_gclose(g); } //end of WRITEME } // end of all of the groups if(rank==0) printf("conversion done!\n"); if(TIMING==100) start_time[4] = MPI_Wtime(); adios_fclose(f); if(TIMING==100){ end_time[4] = MPI_Wtime(); total_time[4] = total_time[4]+end_time[4]-start_time[4]; } adios_finalize(rank); // now, we write out the timing data, for each category, we give max, min, avg, std, all in seconds, across all processes. if(TIMING==100){ // 0: adios_open, adios_group_size // 1: the total time to read in the data // 2: times around each write (will only work if we do NOT buffer.... // 3: the time in the close // 4: fopen, fclose // 5: total time end_time[5] = MPI_Wtime(); total_time[5] = end_time[5] - start_time[5]; double sb[7]; sb[0] = total_time[1]; sb[1] = total_time[4]; //read_var, fopen+fclose sb[2] = sb[0]+sb[1]; sb[3] = total_time[0]; sb[4] = total_time[2]+total_time[3]; //adios_open+adios_group_size, write+close sb[5] = sb[3]+sb[4]; sb[6] = total_time[5]; //total double * rb = NULL; if(rank==0) rb = (double *)malloc(size*7*sizeof(double)); //MPI_Barrier(comm); MPI_Gather(sb, 7, MPI_DOUBLE, rb, 7, MPI_DOUBLE, 0, comm); if(rank==0){ double read_avg1 = 0; double read_avg2 = 0; double tread_avg = 0; double write_avg1 = 0; double write_avg2 = 0; double twrite_avg = 0; double total_avg = 0; for(j=0; j<size; j++){ read_avg1 += rb[7*j]; read_avg2 += rb[7*j+1]; tread_avg += rb[7*j+2]; write_avg1 += rb[7*j+3]; write_avg2 += rb[7*j+4]; twrite_avg += rb[7*j+5]; total_avg += rb[7*j+6]; } read_avg1 /= size; read_avg2 /= size; tread_avg /= size; write_avg1 /= size; write_avg2 /= size; twrite_avg /= size; total_avg /= size; double read1_max = rb[0]; double read1_min = rb[0]; double read1_std = rb[0]-read_avg1; read1_std *= read1_std; double read2_max = rb[1]; double read2_min = rb[1]; double read2_std = rb[1]-read_avg2; read2_std *= read2_std; double tread_max = rb[2]; double tread_min = rb[2]; double tread_std = rb[2]-tread_avg; tread_std *= tread_std; double write1_max = rb[3]; double write1_min = rb[3]; double write1_std = rb[3]-write_avg1; write1_std *= write1_std; double write2_max = rb[4]; double write2_min = rb[4]; double write2_std = rb[4]-write_avg2; write2_std *= write2_std; double twrite_max = rb[5]; double twrite_min = rb[5]; double twrite_std = rb[5]-twrite_avg; twrite_std *= twrite_std; double total_max = rb[6]; double total_min = rb[6]; double total_std = rb[6]-total_avg; total_std *= total_std; for(j=1; j<size; j++){ if(rb[7*j]>read1_max) read1_max = rb[7*j]; else if(rb[7*j]<read1_min) read1_min = rb[7*j]; double std = rb[7*j]-read_avg1; std *= std; read1_std += std; if(rb[7*j+1]>read2_max) read2_max = rb[7*j+1]; else if(rb[7*j+1]<read2_min) read2_min = rb[7*j+1]; std = rb[7*j+1]-read_avg2; std *= std; read2_std += std; if(rb[7*j+2]>tread_max) tread_max = rb[7*j+2]; else if(rb[7*j+2]<tread_min) tread_min = rb[7*j+2]; std = rb[7*j+2]-tread_avg; std *= std; tread_std += std; if(rb[7*j+3]>write1_max) write1_max = rb[7*j+3]; else if(rb[7*j+3]<write1_min) write1_min = rb[7*j+3]; std = rb[7*j+3]-write_avg1; std *= std; write1_std += std; if(rb[7*j+4]>write2_max) write2_max = rb[7*j+4]; else if(rb[7*j+4]<write2_min) write2_min = rb[7*j+4]; std = rb[7*j+4]-write_avg2; std *= std; write2_std += std; if(rb[7*j+5]>twrite_max) twrite_max = rb[7*j+5]; else if(rb[7*j+5]<twrite_min) twrite_min = rb[7*j+5]; std = rb[7*j+5]-twrite_avg; std *= std; twrite_std += std; if(rb[7*j+6]>total_max) total_max = rb[7*j+6]; else if(rb[7*j+6]<total_min) total_min = rb[7*j+6]; std = rb[7*j+6]-total_avg; std *= std; total_std += std; } read1_std /= size; read1_std = sqrt(read1_std); read2_std /= size; read2_std = sqrt(read2_std); tread_std /= size; tread_std = sqrt(tread_std); write1_std /= size; write1_std = sqrt(write1_std); write2_std /= size; write2_std = sqrt(write2_std); twrite_std /= size; twrite_std = sqrt(twrite_std); total_std /= size; total_std = sqrt(total_std); printf("---type--- max\tmin\tavg\tstd\n"); printf("---read_var--- %lf\t%lf\t%lf\t%lf\n", read1_max, read1_min, read_avg1, read1_std); printf("---fopen+fclose--- %lf\t%lf\t%lf\t%lf\n", read2_max, read2_min, read_avg2, read2_std); printf("---total_read--- %lf\t%lf\t%lf\t%lf\n", tread_max, tread_min, tread_avg, tread_std); printf("---adios_open+adios_groupsize--- %lf\t%lf\t%lf\t%lf\n", write1_max, write1_min, write_avg1, write1_std); printf("---write+close--- %lf\t%lf\t%lf\t%lf\n", write2_max, write2_min, write_avg2, write2_std); printf("---total_write--- %lf\t%lf\t%lf\t%lf\n", twrite_max, twrite_min, twrite_avg, twrite_std); printf("---total--- %lf\t%lf\t%lf\t%lf\n", total_max, total_min, total_avg, total_std); free(rb); } } // if (TIMING==100 && rank==0) { // printf("------------------------------------------------------------------\n"); // printf("Define variables = %lf\n",total_time[0]); // printf("Read variables = %lf\n",total_time[1]); // printf("Write variables = %lf\n",total_time[2]); // printf("Close File for write = %lf\n",total_time[3]); // printf("Total write time = %lf\n",total_time[2] + total_time[3]); // for (itime=0;itime<timers-1;itime++) // total_time[timers-1]+=total_time[itime]; // printf("Total I/O time = %lf\n",total_time[timers-1]); // } MPI_Finalize(); return(0); }
int main (int argc, char ** argv) { int i, j, k,l; MPI_Comm comm_dummy = 0; /* MPI_Comm is defined through adios_read.h */ if (argc < 2) { printf("Usage: %s <BP-file>\n", argv[0]); return 1; } adios_read_init_method (ADIOS_READ_METHOD_BP, comm_dummy, "show_hidden_attrs"); ADIOS_FILE * f; f = adios_read_open_file (argv[1], ADIOS_READ_METHOD_BP, comm_dummy); if (f == NULL) { printf ("%s\n", adios_errmsg()); return -1; } /* For all variables */ printf(" Variables=%d:\n", f->nvars); for (i = 0; i < f->nvars; i++) { ADIOS_VARINFO * v = adios_inq_var_byid (f, i); adios_inq_var_stat (f, v, 0, 1); adios_inq_var_blockinfo (f, v); uint64_t total_size = adios_type_size (v->type, v->value); for (j = 0; j < v->ndim; j++) total_size *= v->dims[j]; printf(" %-9s %s", adios_type_to_string(v->type), f->var_namelist[i]); if (v->ndim == 0) { /* Scalars do not need to be read in, we get it from the metadata when using adios_inq_var */ printf(" = %s\n", value_to_string(v->type, v->value, 0)); } else { /* Arrays, print min/max statistics*/ printf("[%lld",v->dims[0]); for (j = 1; j < v->ndim; j++) printf(", %lld",v->dims[j]); //printf("] = \n"); if (v->type == adios_integer) printf("] : min=%d max=%d\n", (*(int*)v->statistics->min), (*(int*)v->statistics->max)); else if (v->type == adios_double) printf("] : min=%lg max=%lg\n", (*(double*)v->statistics->min), (*(double*)v->statistics->max)); /* Print block info */ for (l=0; l<v->nsteps; l++) { printf(" step %3d: \n", l); for (j=0; j<v->nblocks[l]; j++) { printf(" block %3d: [", j); for (k=0; k<v->ndim; k++) { printf("%3lld:%3lld", v->blockinfo[j].start[k], v->blockinfo[j].start[k]+v->blockinfo[j].count[k]-1); if (k<v->ndim-1) printf(", "); } printf("]\n"); } } } adios_free_varinfo (v); } /* variables */ /* For all attributes */ printf(" Attributes=%d:\n", f->nattrs); for (i = 0; i < f->nattrs; i++) { enum ADIOS_DATATYPES atype; int asize; void *adata; adios_get_attr_byid (f, i, &atype, &asize, &adata); printf(" %-9s %s = %s\n", adios_type_to_string(atype), f->attr_namelist[i], value_to_string(atype, adata, 0)); free(adata); } /* attributes */ adios_read_close (f); return 0; }
int main (int argc, char ** argv) { int rank, size, i, j; MPI_Comm comm = MPI_COMM_WORLD; enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_BP; MPI_Init (&argc, &argv); MPI_Comm_rank (comm, &rank); MPI_Comm_size (comm, &size); adios_read_init_method (method, comm, "verbose=3"); ADIOS_FILE * f = adios_read_open_file("adios_stat.bp", method, comm); if (f == NULL) { fprintf (stderr, "%s\n", adios_errmsg()); return -1; } ADIOS_VARINFO * v = adios_inq_var (f, "temperature"); if (v) { /* get statistics, for each individual time-step */ adios_inq_var_stat (f, v, 1, 0); printf("Global MIN of temperature: %lf\n", * (double *) v->statistics->min); printf("Global MAX of temperature: %lf\n", * (double *) v->statistics->max); printf("Global AVG of temperature: %lf\n", * (double *) v->statistics->avg); printf("Global STD DEV of temperature: %lf\n", * (double *) v->statistics->std_dev); printf("\n"); printf("---------------------------------------------------------------------------\n"); if (v->statistics->steps) { printf("MIN\t\tMAX\t\tAVG\t\tSTD_DEV\t\tHISTOGRAM\n"); for(i = 0; i < v->nsteps; i++) { if (v->statistics->steps->mins[i]) printf("%lf\t", * (double *) v->statistics->steps->mins[i]); else printf("--\t\t"); if (v->statistics->steps->maxs[i]) printf("%lf\t", * (double *) v->statistics->steps->maxs[i]); else printf("--\t\t"); if (v->statistics->steps->avgs[i]) printf("%lf\t", * (double *) v->statistics->steps->avgs[i]); else printf("--\t\t"); if (v->statistics->steps->std_devs[i]) printf("%lf\t", * (double *) v->statistics->steps->std_devs[i]); else printf("--\t\t"); if (v->statistics->histogram) { for(j = 0; j <= v->statistics->histogram->num_breaks; j++) { printf("%d ", v->statistics->histogram->frequencies[i][j]); } printf("\n"); } else { printf("--\t\t"); } printf("\n"); } } else { printf ("Per step statistics is missing\n"); } printf("---------------------------------------------------------------------------\n"); printf("\n"); if (v->statistics->histogram) { printf("Break points:\t\t\t"); for(j = 0; j < v->statistics->histogram->num_breaks; j++) printf("%6.2lf\t", v->statistics->histogram->breaks[j]); printf("\n"); printf("Frequencies:\t\t\t"); for(j = 0; j <= v->statistics->histogram->num_breaks; j++) printf("%6d\t", v->statistics->histogram->gfrequencies[j]); } printf("\n\n"); #if 0 printf ("Auto covariance of MIN values of temperature over time: %lf\n", adios_stat_cov (v, v, "min", 0, 12, 0)); printf ("Auto correlation of MAX values of temperature over time, with lag 2 units: %lf\n", adios_stat_cor (v, v, "max", 0, 8, 2)); printf("\n\n"); #endif adios_free_varinfo (v); } else { fprintf (stderr, "ERROR: Cannot inquire statistics of variable 'temperature': %s\n", adios_errmsg()); } v = adios_inq_var (f, "complex"); if (v) { adios_inq_var_stat (f, v, 1, 0); double *C = v->statistics->min; printf("Global Minimum of variable complex - Magnitude: %lf\n", C[0]); printf("Global Minimum of variable complex - Real part: %lf\n", C[1]); printf("Global Minimum of variable complex - Imaginary part: %lfi\n", C[2]); double ** Cmin; Cmin = (double **) v->statistics->steps->mins; printf("\nMagnitude\t\tReal\t\t\tImaginary\n"); for (j = 0; j < v->nsteps; j++) { printf ("%lf\t\t%lf\t\t%lf\n", Cmin[j][0], Cmin[j][1], Cmin[j][2]); } printf("\n"); adios_free_varinfo (v); } else { fprintf (stderr, "ERROR: Cannot inquire statistics of variable 'complex': %s\n", adios_errmsg()); } adios_read_close (f); MPI_Barrier (comm); adios_read_finalize_method (method); MPI_Finalize (); return 0; }
int main (int argc, char ** argv) { int gidx, i, j, k,l; MPI_Comm comm_dummy = 0; /* MPI_Comm is defined through adios_read.h */ void * data = NULL; uint64_t start[] = {0,0,0,0,0,0,0,0,0,0}; uint64_t count[10]; int64_t bytes_read = 0; if (argc < 2) { printf("Usage: %s <BP-file>\n", argv[0]); return 1; } ADIOS_FILE * f; //int step; //for (step=0; step < 2; step++) { f = adios_fopen (argv[1], comm_dummy); if (f == NULL) { printf ("%s\n", adios_errmsg()); return -1; } /* For all groups */ for (gidx = 0; gidx < f->groups_count; gidx++) { printf("Group %s:\n", f->group_namelist[gidx]); ADIOS_GROUP * g = adios_gopen (f, f->group_namelist[gidx]); if (g == NULL) { printf ("%s\n", adios_errmsg()); return -1; } /* For all variables */ printf(" Variables=%d:\n", g->vars_count); for (i = 0; i < g->vars_count; i++) { ADIOS_VARINFO * v = adios_inq_var_byid (g, i); uint64_t total_size = adios_type_size (v->type, v->value); for (j = 0; j < v->ndim; j++) total_size *= v->dims[j]; printf(" %-9s %s", adios_type_to_string(v->type), g->var_namelist[i]); if (v->ndim == 0) { /* Scalars do not need to be read in, we get it from the metadata when using adios_inq_var */ printf(" = %s\n", value_to_string(v->type, v->value, 0)); } else { /* Arrays have to be read in from the file */ printf("[%" PRIu64,v->dims[0]); for (j = 1; j < v->ndim; j++) printf(", %" PRIu64,v->dims[j]); //printf("] = \n"); if (v->type == adios_integer) printf("] = min=%d max=%d timedim=%d\n", (*(int*)v->gmin), (*(int*)v->gmax), v->timedim); else if (v->type == adios_double) printf("] = min=%lg max=%lg timedim=%d\n", (*(double*)v->gmin), (*(double*)v->gmax), v->timedim); if (total_size > 1024*1024*1024) { printf(" // too big, do not read in\n"); } else { data = malloc (total_size); if (data == NULL) { fprintf (stderr, "malloc failed.\n"); return -1; } for (j = 0; j < v->ndim; j++) count[j] = v->dims[j]; bytes_read = adios_read_var_byid (g, i, start, count, data); if (bytes_read < 0) { printf ("%s\n", adios_errmsg()); } else if (bytes_read > 1024*1024) { printf ("Too big to print\n"); } else if (v->ndim == 1) { printf (" ["); for (j = 0; j < v->dims[0]; j++) printf("%s ", value_to_string(v->type, data, j)); printf ("]\n"); } else if (v->ndim == 2) { for (j = 0; j < v->dims[0]; j++) { printf (" row %d: [", j); for (k = 0; k < v->dims[1]; k++) printf("%s ", value_to_string(v->type, data, j*v->dims[1] + k)); printf ("]\n"); } } else if (v->ndim == 3) { for (j = 0; j < v->dims[0]; j++) { printf (" block %d: \n", j); for (k = 0; k < v->dims[1]; k++) { printf (" row %d: [", k); for (l = 0; l < v->dims[2]; l++) { printf("%s ", value_to_string(v->type, data, j*v->dims[1]*v->dims[2] + k*v->dims[1] + l)); } printf ("]\n"); } printf ("\n"); } } else { printf (" cannot print arrays with >3 dimensions\n"); } free (data); } } adios_free_varinfo (v); } /* variables */ /* For all attributes */ printf(" Attributes=%d:\n", g->attrs_count); for (i = 0; i < g->attrs_count; i++) { enum ADIOS_DATATYPES atype; int asize; void *adata; adios_get_attr_byid (g, i, &atype, &asize, &adata); printf(" %-9s %s = %s\n", adios_type_to_string(atype), g->attr_namelist[i], value_to_string(atype, adata, 0)); free(adata); } /* attributes */ adios_gclose (g); } /* groups */ adios_fclose (f); //} /* loop 'step' */ return 0; }
int read_points () { ADIOS_SELECTION *sel0,*sel1,*sel2,*sel3; ADIOS_FILE * f; ADIOS_VARINFO * vi; int err=0,n,n1, i,j,k; int nsteps_a, nsteps_b, nsteps_c; int v; uint64_t *pts1; uint64_t *pts2; uint64_t *pts3; uint64_t start[3] = {offs1,offs2,offs3}; uint64_t count[3] = {ldim1,ldim2,ldim3}; pts1 = (uint64_t*) malloc (1*sizeof(uint64_t)*ldim1); pts2 = (uint64_t*) malloc (2*sizeof(uint64_t)*ldim1*ldim2); pts3 = (uint64_t*) malloc (3*sizeof(uint64_t)*ldim1*ldim2*ldim3); for (i=0; i<ldim1; i++) { pts1[i] = offs1 + i; for (j=0; j<ldim2; j++) { pts2[i*2*ldim2 + 2*j] = offs1 + i; pts2[i*2*ldim2 + 2*j+1] = j; for (k=0; k<ldim3; k++) { pts3[i*3*ldim2*ldim3+j*3*ldim3+3*k] = offs1 + i; pts3[i*3*ldim2*ldim3+j*3*ldim3+3*k+1] = j; pts3[i*3*ldim2*ldim3+j*3*ldim3+3*k+2] = k; } } } //Force error on very last point: pts3[3*ldim1*ldim2*ldim3-1] = 0; reset_readvars(); log ("Read and check data in %s using point selections\n", FILENAME); f = adios_read_open (FILENAME, read_method, comm, ADIOS_LOCKMODE_CURRENT, 0.0); if (f == NULL) { printE ("Error at opening file: %s\n", rank, adios_errmsg()); return 1; } sel1 = adios_selection_points (1, ldim1, pts1); sel2 = adios_selection_points (2, ldim1*ldim2, pts2); sel3 = adios_selection_points (3, ldim1*ldim2*ldim3, pts3); #ifdef DEBUG_PRINTS fprintf(stderr, "1D selection: {"); for (i=0; i<ldim1; i++) { fprintf (stderr, "%d ", i, i, pts1[i]); } fprintf(stderr, "}\n"); fprintf(stderr, "2D selection :\n"); for (i=0; i<ldim1; i++) { fprintf (stderr, "i=%d, idx=%2d- = ", i, i*2*ldim2); for (j=0; j<ldim2; j++) { n = i*2*ldim2+2*j; fprintf (stderr, "{%d,%d} ", pts2[n], pts2[n+1]); } fprintf(stderr, "\n"); } fprintf(stderr, "3D selection :\n"); for (i=0; i<ldim1; i++) { for (j=0; j<ldim2; j++) { fprintf (stderr, "i=%d, j=%d, idx=%3d- = ", i, j, i*3*ldim2*ldim3+j*3*ldim3); for (k=0; k<ldim3; k++) { n = i*3*ldim2*ldim3+j*3*ldim3+3*k; fprintf (stderr, "{%d,%d,%d} ", pts3[n], pts3[n+1], pts3[n+2]); } fprintf(stderr, "\n"); } } #endif n1=0; while (n1 < NSTEPS && adios_errno != err_end_of_stream) { n1++; log (" Step %d\n", f->current_step); log (" Check 1D variable a1...\n"); adios_schedule_read (f, sel1, "a1", 0, 1, r1); adios_perform_reads (f, 1); #ifdef DEBUG_PRINTS fprintf(stderr, "1D result: {"); for (i=0; i<ldim1; i++) { fprintf (stderr, "%d ", i, i, r1[i]); } fprintf(stderr, "}\n"); #endif for (i=0; i<ldim1; i++) { v = VALUE1D(rank,f->current_step,i); if (r1[i] != v) { printE ("Error: a1[%d]=%d != read=%d\n", i, v, r1[i]); //goto endread; } } log (" Check 2D variable a2...\n"); adios_schedule_read (f, sel2, "a2", 0, 1, r2); adios_perform_reads (f, 1); #ifdef DEBUG_PRINTS fprintf(stderr, "2D result :\n"); n = 0; for (i=0; i<ldim1; i++) { fprintf (stderr, "row=%2d- = {", i); for (j=0; j<ldim2; j++) { fprintf (stderr, "%d ", r2[n]); n++; } fprintf(stderr, "}\n"); } #endif n = 0; for (i=0; i<ldim1; i++) { for (j=0; j<ldim2; j++) { v = VALUE2D(rank,f->current_step,i,j); if (v != r2[n]) { printE ("Error: a2[%d,%d]=%d != read=%d\n", i, j, v, r2[n]); //goto endread; } n++; } } log (" Check 3D variable a3...\n"); adios_schedule_read (f, sel3, "a3", 0, 1, r3); adios_perform_reads (f, 1); #ifdef DEBUG_PRINTS fprintf(stderr, "3D selection :\n"); n=0; for (i=0; i<ldim1; i++) { for (j=0; j<ldim2; j++) { fprintf (stderr, "[%d,%d] = {", i, j); for (k=0; k<ldim3; k++) { fprintf (stderr, "%d ", r3[n]); n++; } fprintf(stderr, "}\n"); } } #endif n = 0; for (i=0; i<ldim1; i++) { for (j=0; j<ldim2; j++) { for (k=0; k<ldim3; k++) { v = VALUE3D(rank,f->current_step,i,j,k); if (v != r3[n]) { printE ("Error: a3[%d,%d,%d]=%d != read=%d\n", i, j, k, v, r3[n]); //goto endread; } n++; } } } if (n1 < NSTEPS) { adios_advance_step (f, 0, -1.0); } } endread: adios_selection_delete (sel1); adios_selection_delete (sel2); adios_selection_delete (sel3); free(pts1); free(pts2); free(pts3); adios_read_close(f); MPI_Barrier (comm); return err; }
int main (int argc, char** argv) { fastbit_init(0); fastbit_set_verbose_level(0); ADIOS_FILE * f; //MPI_Comm comm_dummy = 0; // MPI_Comm is defined through adios_read.h MPI_Comm comm_dummy = MPI_COMM_WORLD; int rank, size; MPI_Init (&argc, &argv); MPI_Comm_rank (comm_dummy, &rank); MPI_Comm_size (comm_dummy, &size); adios_init_noxml (comm_dummy); if (argc < 2) { printf("Usage: index_fastbit fileName (attrName)"); return 0; } f = adios_read_open_file (argv[1], ADIOS_READ_METHOD_BP, comm_dummy); if (f == NULL) { printf ("::%s\n", adios_errmsg()); return -1; } /* adios_allocate_buffer (ADIOS_BUFFER_ALLOC_NOW, (f->file_size)*2/1048576 + 5); // +5MB for extra room in buffer adios_declare_group (&gAdios_group, gGroupNameFastbitIdx, "", adios_flag_yes); adios_select_method (gAdios_group, "MPI", "", ""); */ gIdxFileName = fastbit_adios_util_getFastbitIndexFileName(argv[1]); unlink(gIdxFileName); adios_allocate_buffer (ADIOS_BUFFER_ALLOC_NOW, 500); // +5MB for extra room in buffer adios_declare_group (&gAdios_group, gGroupNameFastbitIdx, "", adios_flag_yes); adios_select_method (gAdios_group, "MPI", "", ""); adios_open (&gAdios_write_file, gGroupNameFastbitIdx, gIdxFileName, "w", MPI_COMM_WORLD); #ifdef MULTI_BLOCK int testid = adios_define_var (gAdios_group, "pack", "", adios_integer , 0, 0, 0); #endif #ifdef BOX int testid = adios_define_var (gAdios_group, "elements", "", adios_integer , 0, 0, 0); #endif //uint64_t estimatedbytes = (nb+nk+no)*adios_type_size(adios_double, NULL); int jobCounter = getJobCounter(f); uint64_t estimatedbytes = getByteEstimationOnFile(f, rank); if (size > 1) { int maxJobsPP = jobCounter/size + 1; estimatedbytes = estimatedbytes * maxJobsPP /jobCounter +1048576; } estimatedbytes += 1048576; uint64_t adios_totalsize; // adios_group_size needs to be call before any write_byid, Otherwise write_byid does nothing adios_group_size (gAdios_write_file, estimatedbytes , &adios_totalsize); printf("=> .. adios open output file: %s, rank %d allocated %" PRIu64 " bytes... \n", gIdxFileName, rank, adios_totalsize); // IMPORTANT: // can only call open/close once in a process // otherwise data is tangled or only the data in the last open/close call is recorded #ifdef MULTI_BLOCK adios_write_byid(gAdios_write_file, testid, &pack); #endif #ifdef BOX adios_write_byid(gAdios_write_file, testid, &recommended_index_ele); #endif sumLogTime(-1); sumLogTimeMillis(-1); if (argc >= 3) { int i=2; while (i<argc) { const char* varName = argv[i]; if(strstr(varName, "<binning prec") != NULL) { if (gBinningOption == NULL) { gBinningOption = argv[i]; } if (argc == 3) { buildIndexOnAllVar(f, rank, size); break; } i++; continue; } else { ADIOS_VARINFO * v = adios_inq_var(f, varName); if (v == NULL) { printf("No such variable: %s\n", varName); return 0; } printf("building fastbit index on variable: %s\n", varName); buildIndex_mpi(f, v, rank, size); adios_free_varinfo(v); i++; } } } else { buildIndexOnAllVar(f, rank, size); } sumLogTime(0); sumLogTimeMillis(0); adios_close(gAdios_write_file); adios_read_close(f); // // writing file clean up // // read back: f = adios_read_open_file (gIdxFileName, ADIOS_READ_METHOD_BP, comm_dummy); if (f == NULL) { printf("No such file: %s \n", gIdxFileName); return 0; } int numVars = f->nvars; int i=0; int k=0; int j=0; for (i=0; i<numVars; i++) { char* varName = f->var_namelist[i]; ADIOS_VARINFO* v = adios_inq_var(f, varName); adios_inq_var_blockinfo(f,v); int timestep = 0; for (k=0; k<v->sum_nblocks; k++) { verifyData(f, v, k, timestep); } adios_free_varinfo(v); } adios_read_close(f); if (rank == 0) { printf(" ==> index file is at: %s\n", gIdxFileName); } // clean up MPI_Barrier (comm_dummy); adios_finalize (rank); MPI_Finalize (); free (gIdxFileName); fastbit_cleanup(); return 0; }