/// Read a sub-array.
///
/// @Note The length of the buffer is assumed to be sufficiently large
/// if the argument bufLen is less or equal to 0. The caller is
/// responsible for allocating the buffer (@c buf) of the currect size.
int64_t
ADIOS_Var::readData(void* buf, int64_t bufLen,
const uint64_t* start,
const uint64_t* count)
{
uint64_t total_elm = 1;
for (int i=0; i<getNumDimension(); i++) {
total_elm *= count[i];
#ifdef DEBUG
LOGGER(ibis::gVerbose > 5)
<< "ADIOS_Var::readData: [" << i << "] start=" << start[i]
<< ", count=" << count[i];
#endif
}
{
uint64_t total_bytes = total_elm *
adios_type_size(getType(), readValue());
if (bufLen > 0 && (uint64_t)bufLen < total_bytes) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- ADIOS_Var::readData: bufLen (" << bufLen
<< ") < total_bytes (" << total_bytes << ")";
return -1;
}
}
ADIOS_SELECTION *sel = adios_selection_boundingbox
(m_handle->ndim, start, count);
if (sel == 0) {
LOGGER(ibis::gVerbose > 1)
<< "Warning -- ADIOS_Var::readData failed to create a selection";
return -2;
}
IBIS_BLOCK_GUARD(adios_selection_delete, sel);
int ierr = adios_schedule_read_byid
(getFile(), sel, index(), getFile()->current_step, 1, buf);
if (ierr != 0) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- ADIOS_Var::readData call to "
"adios_schedule_read_byid failed due to " << adios_errmsg();
return -3;
}
ierr = adios_perform_reads(getFile(), 1); // 1 == blocking
if (ierr != 0) {
LOGGER(ibis::gVerbose > 0)
<< "Warning -- ADIOS_Var::readData call to adios_perform_reads on "
<< getFileName() << " failed due to " << adios_errmsg();
return -4;
}
LOGGER(ibis::gVerbose > 5)
<< "ADIOS_Var::readData: competeled reading " << total_elm
<< " element" << (total_elm>1?"s":"") << " for " << getName()
<< " from " << getFileName();
return total_elm;
} // ADIOS_Var::readData
int read_stepbystep ()
{
ADIOS_FILE * f;
float timeout_sec = 0.0;
int steps = 0;
int retval = 0;
MPI_Comm comm = MPI_COMM_SELF;
adios_read_init_method (ADIOS_READ_METHOD_BP, comm, "verbose=3");
printf ("\n--------- Read as stream ------------\n");
f = adios_read_open (fname, ADIOS_READ_METHOD_BP,
comm, ADIOS_LOCKMODE_NONE, timeout_sec);
if (adios_errno == err_file_not_found)
{
printf ("Stream not found after waiting %f seconds: %s\n",
timeout_sec, adios_errmsg());
retval = adios_errno;
}
else if (adios_errno == err_end_of_stream)
{
printf ("Stream terminated before open. %s\n", adios_errmsg());
retval = adios_errno;
}
else if (f == NULL) {
printf ("Error at opening stream: %s\n", adios_errmsg());
retval = adios_errno;
}
else
{
/* Processing loop over the steps (we are already in the first one) */
while (adios_errno != err_end_of_stream) {
steps++; // steps start counting from 1
printf ("Step: %d\n", f->current_step);
print_varinfo (f, f->current_step);
// advance to 1) next available step with 2) blocking wait
adios_advance_step (f, 0, timeout_sec);
if (adios_errno == err_step_notready)
{
//printf ("No new step arrived within the timeout. Quit. %s\n",
// adios_errmsg());
break; // quit while loop
}
}
adios_read_close (f);
}
adios_read_finalize_method (ADIOS_READ_METHOD_BP);
//printf ("We have processed %d steps\n", steps);
return retval;
}
int read_file (char *fname)
{
ADIOS_SELECTION *sel;
ADIOS_FILE * f;
ADIOS_VARINFO * vi;
int err=0,i,n;
uint64_t start[1] = {0};
uint64_t count[2] = {NX};
uint64_t ndim;
reset_rarrays();
log ("Read and check data in %s\n", fname);
f = adios_read_open_file (fname, ADIOS_READ_METHOD_BP, comm);
if (f == NULL) {
printE ("Error at opening file: %s\n", adios_errmsg());
return 1;
}
sel = adios_selection_boundingbox (1, start, count);
adios_schedule_read (f, sel, "t1", 0, 1, r1);
adios_schedule_read (f, sel, "t2", 0, 1, r2);
adios_perform_reads (f, 1);
adios_selection_delete (sel);
CHECK_ARRAY (t, r1, NX);
CHECK_ARRAY (t, r2, NX);
endread:
adios_read_close(f);
MPI_Barrier (comm);
return err;
}
/// The default constructor. It is private and can not be directly called
/// by any user code.
BPCommon::BPCommon() {
int ierr = adios_init_noxml();
if (ierr != 0) {
std::cerr << "BPCommon::ctor failed to initialize adios, "
<< adios_errmsg() << std::endl;
throw "BPCommon failed to initialize adios";
}
ierr = adios_allocate_buffer(ADIOS_BUFFER_ALLOC_NOW,
FQ_ADIOS_DEFAULT_BUFFER_MB);
if (ierr != 0) {
std::cerr << "BPCommon::ctor failed to allocated "
<< FQ_ADIOS_DEFAULT_BUFFER_MB << " MB for ADIOS buffer, "
<< adios_errmsg() << std::endl;
}
}
int main (int argc, char ** argv)
{
int err;
MPI_Init (&argc, &argv);
MPI_Comm_rank (comm, &rank);
MPI_Comm_size (comm, &size);
init_vars();
adios_init_noxml (comm);
err = adios_read_init_method(ADIOS_READ_METHOD_BP, comm, "verbose=2");
if (err) {
printE ("%s\n", adios_errmsg());
}
if (!err)
err = declare_group ();
if (!err)
err = write_file ("reuse_dim.bp");
if (!err)
err = read_file ("reuse_dim.bp");
adios_finalize (rank);
fini_vars();
MPI_Finalize ();
return err;
}
AdiosCheckpointInput::AdiosCheckpointInput(std::string& checkpoint_file)
{
adios_read_init(method, OHMMS::Controller->getMPI(), "verbose=3;abort_on_error");
adios_file_handle = adios_read_open_file(checkpoint_file.c_str(), method,
comm);
if (adios_file_handle == NULL)
{
qmcplusplus::app_error() << adios_errmsg() << std::endl;
}
//Select for local values
sel = adios_selection_writeblock (OHMMS::Controller->rank());
}
/// Destructor.
BPCommon::~BPCommon() {
int rank=0;
int ierr;
#ifndef FQ_NOMPI
ierr = MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if (ierr != MPI_SUCCESS) {
std::cerr << "BPCommon::dtor failed to determine the MPI rank"
<< std::endl;
exit(-1);
}
#endif
ierr = adios_finalize(rank);
if (ierr != 0) {
std::cerr << "BPCommon::dtor failed to invoke adios_finalize, "
<< adios_errmsg() << std::endl;
}
}
/// Close the open file. This function frees the content of the ADIOS_FILE
/// object under the control of this object.
void ADIOS_File::close()
{
if (m_handle == 0) return;
int ierr = adios_read_close(m_handle);
if (ierr == 0) {
LOGGER(ibis::gVerbose > 4)
<< "ADIOS_File::close successfully closed file \"" << m_fname
<< '"';
}
else {
LOGGER(ibis::gVerbose >= 0)
<< "Warning -- ADIOS_File::close failed to close file \""
<< m_fname << "\" because of " << adios_errmsg();
}
m_handle = 0;
} // ADIOS_File::close
int read_all ()
{
ADIOS_FILE * f;
float timeout_sec = 0.0;
int steps = 0;
int retval = 0;
MPI_Comm comm = MPI_COMM_SELF;
adios_read_init_method (ADIOS_READ_METHOD_BP, comm, "verbose=3");
printf ("\n--------- Read as file ------------\n");
f = adios_read_open_file (fname, ADIOS_READ_METHOD_BP, comm);
if (f == NULL) {
printf ("Error at opening file: %s\n", adios_errmsg());
retval = adios_errno;
}
else
{
/* Processing all the steps at once */
print_varinfo (f, 0);
adios_read_close (f);
}
adios_read_finalize_method (ADIOS_READ_METHOD_BP);
return retval;
}
int read_scalar ()
{
ADIOS_FILE * f;
float timeout_sec = 0.0;
int steps = 0;
int retval = 0;
MPI_Comm comm = MPI_COMM_SELF;
adios_read_init_method (ADIOS_READ_METHOD_BP, comm, "verbose=3");
printf ("\n--------- Read all instances of the scalar 'O' ------------\n");
f = adios_read_open_file (fname, ADIOS_READ_METHOD_BP, comm);
if (f == NULL) {
printf ("Error at opening file: %s\n", adios_errmsg());
retval = adios_errno;
}
else
{
/* Read the scalar O with writeblock selection */
print_scalar (f, "O");
adios_read_close (f);
}
adios_read_finalize_method (ADIOS_READ_METHOD_BP);
return retval;
}
int worker(int argc, char* argv[]) {
TAU_PROFILE_TIMER(timer, __func__, __FILE__, TAU_USER);
TAU_PROFILE_START(timer);
my_printf("%d of %d In worker B\n", myrank, commsize);
static bool announced = false;
/* validate input */
validate_input(argc, argv);
my_printf("Worker B will execute until it sees n iterations.\n", iterations);
/* ADIOS: These declarations are required to match the generated
* gread_/gwrite_ functions. (And those functions are
* generated by calling 'gpp.py adios_config.xml') ...
* EXCEPT THAT THE generation of Reader code is broken.
* So, we will write the reader code manually.
*/
uint64_t adios_groupsize;
uint64_t adios_totalsize;
uint64_t adios_handle;
void * data = NULL;
uint64_t start[2], count[2];
int i, j, steps = 0;
int NX = 10;
int NY = 1;
double t[NX];
double p[NX];
/* ADIOS: Can duplicate, split the world, whatever.
* This allows you to have P writers to N files.
* With no splits, everyone shares 1 file, but
* can write lock-free by using different areas.
*/
MPI_Comm adios_comm, adios_comm_b_to_c;
adios_comm = MPI_COMM_WORLD;
//MPI_Comm_dup(MPI_COMM_WORLD, &adios_comm);
adios_comm_b_to_c = MPI_COMM_WORLD;
//MPI_Comm_dup(MPI_COMM_WORLD, &adios_comm_b_to_c);
enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_FLEXPATH;
adios_read_init_method(method, adios_comm, "verbose=3");
if (adios_errno != err_no_error) {
fprintf (stderr, "rank %d: Error %d at init: %s\n", myrank, adios_errno, adios_errmsg());
exit(4);
}
if (send_to_c) {
adios_init("adios_config.xml", adios_comm);
}
/* ADIOS: Set up the adios communications and buffers, open the file.
*/
ADIOS_FILE *fp; // file handler
ADIOS_VARINFO *vi; // information about one variable
ADIOS_SELECTION * sel;
char adios_filename_a_to_b[256];
char adios_filename_b_to_c[256];
enum ADIOS_LOCKMODE lock_mode = ADIOS_LOCKMODE_NONE;
double timeout_sec = 1.0;
sprintf(adios_filename_a_to_b, "adios_a_to_b.bp");
sprintf(adios_filename_b_to_c, "adios_b_to_c.bp");
my_printf ("rank %d: Worker B opening file: %s\n", myrank, adios_filename_a_to_b);
fp = adios_read_open(adios_filename_a_to_b, method, adios_comm, lock_mode, timeout_sec);
if (adios_errno == err_file_not_found) {
fprintf (stderr, "rank %d: Stream not found after waiting %d seconds: %s\n",
myrank, timeout_sec, adios_errmsg());
exit(1);
} else if (adios_errno == err_end_of_stream) {
// stream has been gone before we tried to open
fprintf (stderr, "rank %d: Stream terminated before open. %s\n", myrank, adios_errmsg());
exit(2);
} else if (fp == NULL) {
// some other error happened
fprintf (stderr, "rank %d: Error %d at opening: %s\n", myrank, adios_errno, adios_errmsg());
exit(3);
} else {
my_printf("Found file %s\n", adios_filename_a_to_b);
my_printf ("File info:\n");
my_printf (" current step: %d\n", fp->current_step);
my_printf (" last step: %d\n", fp->last_step);
my_printf (" # of variables: %d:\n", fp->nvars);
vi = adios_inq_var(fp, "temperature");
adios_inq_var_blockinfo(fp, vi);
printf ("ndim = %d\n", vi->ndim);
printf ("nsteps = %d\n", vi->nsteps);
printf ("dims[%llu][%llu]\n", vi->dims[0], vi->dims[1]);
uint64_t slice_size = vi->dims[0]/commsize;
if (myrank == commsize-1) {
slice_size = slice_size + vi->dims[0]%commsize;
}
start[0] = myrank * slice_size;
count[0] = slice_size;
start[1] = 0;
count[1] = vi->dims[1];
data = malloc (slice_size * vi->dims[1] * 8);
/* Processing loop over the steps (we are already in the first one) */
while (adios_errno != err_end_of_stream && steps < iterations) {
steps++; // steps start counting from 1
TAU_PROFILE_TIMER(adios_recv_timer, "ADIOS recv", __FILE__, TAU_USER);
TAU_PROFILE_START(adios_recv_timer);
sel = adios_selection_boundingbox (vi->ndim, start, count);
adios_schedule_read (fp, sel, "temperature", 0, 1, data);
adios_perform_reads (fp, 1);
if (myrank == 0)
printf ("--------- B Step: %d --------------------------------\n",
fp->current_step);
#if 0
printf("B rank=%d: [0:%lld,0:%lld] = [", myrank, vi->dims[0], vi->dims[1]);
for (i = 0; i < slice_size; i++) {
printf (" [");
for (j = 0; j < vi->dims[1]; j++) {
printf ("%g ", *((double *)data + i * vi->dims[1] + j));
}
printf ("]");
}
printf (" ]\n\n");
#endif
// advance to 1) next available step with 2) blocking wait
adios_advance_step (fp, 0, timeout_sec);
if (adios_errno == err_step_notready)
{
printf ("B rank %d: No new step arrived within the timeout. Quit. %s\n",
myrank, adios_errmsg());
break; // quit while loop
}
TAU_PROFILE_STOP(adios_recv_timer);
/* Do some exchanges with neighbors */
//do_neighbor_exchange();
/* "Compute" */
compute(steps);
for (i = 0; i < NX; i++) {
t[i] = steps*100.0 + myrank*NX + i;
}
for (i = 0; i < NY; i++) {
p[i] = steps*1000.0 + myrank*NY + i;
}
if (send_to_c) {
TAU_PROFILE_TIMER(adios_send_timer, "ADIOS send", __FILE__, TAU_USER);
TAU_PROFILE_START(adios_send_timer);
/* ADIOS: write to the next application in the workflow */
if (steps == 0) {
adios_open(&adios_handle, "b_to_c", adios_filename_b_to_c, "w", adios_comm_b_to_c);
} else {
adios_open(&adios_handle, "b_to_c", adios_filename_b_to_c, "a", adios_comm_b_to_c);
}
/* ADIOS: Actually write the data out.
* Yes, this is the recommended method, and this way, changes in
* configuration with the .XML file will, even in the worst-case
* scenario, merely require running 'gpp.py adios_config.xml'
* and typing 'make'.
*/
#include "gwrite_b_to_c.ch"
/* ADIOS: Close out the file completely and finalize.
* If MPI is being used, this must happen before MPI_Finalize().
*/
adios_close(adios_handle);
TAU_PROFILE_STOP(adios_send_timer);
#if 1
if (!announced) {
SOS_val foo;
foo.i_val = NX;
SOS_pack(example_pub, "NX", SOS_VAL_TYPE_INT, foo);
SOS_announce(example_pub);
SOS_publish(example_pub);
announced = true;
}
#endif
}
MPI_Barrier(adios_comm_b_to_c);
}
MPI_Barrier(MPI_COMM_WORLD);
adios_read_close(fp);
/* ADIOS: Close out the file completely and finalize.
* If MPI is being used, this must happen before MPI_Finalize().
*/
adios_read_finalize_method(method);
}
if (send_to_c) {
adios_finalize(myrank);
}
free(data);
//MPI_Comm_free(&adios_comm);
//MPI_Comm_free(&adios_comm_b_to_c);
TAU_PROFILE_STOP(timer);
/* exit */
return 0;
}
int main (int argc, char **argv){
int rank =0, size =0;
int NX = 0;
double *t = NULL;
// this is an array we expect as a reference array
double *t_ref = NULL;
MPI_Comm comm = MPI_COMM_WORLD;
diag_t diag = DIAG_OK; // to store the diagnostic information
struct test_info test_result = {TEST_PASSED, "1D_arr_global_noxml"};
struct err_counts err = { 0, 0};
struct adios_tsprt_opts adios_opts;
GET_ENTRY_OPTIONS(adios_opts, "Runs readers. It is recommended to run as many readers as writers.");
// adios read initialization
MPI_Init( &argc, &argv);
MPI_Comm_rank (comm, &rank);
// depending on the method
SET_ERROR_IF_NOT_ZERO(adios_read_init_method(adios_opts.method, comm, adios_opts.adios_options), err.adios);
RET_IF_ERROR(err.adios, rank);
// I will be working with streams so the lock mode is necessary,
// return immediately if the stream unavailable
ADIOS_FILE *adios_handle = adios_read_open(FILE_NAME, adios_opts.method, comm, ADIOS_LOCKMODE_NONE, 0.0);
if ( !adios_handle){
p_error("Quitting ... (%d) %s\n", adios_errno, adios_errmsg());
return DIAG_ERR;
}
// define portions of data how they will be read
ADIOS_SELECTION *sel = NULL;
ADIOS_VARINFO *avi = NULL;
// for storing the variables
char buf[STR_BUFFER_SIZE];
int step = 0;
// read how many processors wrote that array
avi = adios_inq_var (adios_handle, "size");
if (!avi){
p_error("rank %d: Quitting ... (%d) %s\n", rank, adios_errno, adios_errmsg());
diag = DIAG_ERR;
goto close_adios;
}
size = *((int*)avi->value);
adios_free_varinfo(avi);
avi = NULL;
// if I run the more readers than writers; just release
// the excessive readers
if (rank >= size){
p_info("rank %d: I am an excessive rank. Nothing to read ...\n", rank);
diag = DIAG_OK;
goto close_adios;
}
// read the size of the array
avi = adios_inq_var (adios_handle, "NX");
if (!avi){
p_error("rank %d: Quitting ... (%d) %s\n", rank, adios_errno, adios_errmsg());
diag = DIAG_ERR;
goto close_adios;
}
// I expect a scalar that will tell me the size of an array
assert(0 == avi->ndim);
assert(adios_integer == avi->type);
NX = *((int*)avi->value);
// I don't need that variable any more
adios_free_varinfo(avi);
assert(NX_DIM == NX);
avi = NULL;
// this will define the slice that we want to read; each rank should
// read its own slice written by a corresponding writer rank
uint64_t count[1] = { NX };
uint64_t start[1] = { 0 };
start[0] = rank*NX;
sel = adios_selection_boundingbox(1,start, count);
if( !sel ){
p_error("rank %d: Quitting ... (%d) %s\n", rank, adios_errno, adios_errmsg());
diag = DIAG_ERR;
goto close_adios;
}
// make the reference array with reference values I expect to get
t_ref = calloc(NX, sizeof(double));
if (gen_1D_array(t_ref, NX, rank) == DIAG_ERR){
p_error("Generating 1D array. Quitting ...\n");
diag = DIAG_ERR;
goto close_adios;
}
// allocate the memory for the actual array to be read
t = calloc(NX, sizeof(double));
if (adios_schedule_read(adios_handle, sel, "var_1d_array",0,1,t) != 0){
p_error("rank %d: Quitting ...(%d) %s\n", rank, adios_errno, adios_errmsg());
diag = DIAG_ERR;
goto just_clean;
}
// not sure if this assumption is correct; difficult to find in the ADIOS sources
if (adios_perform_reads(adios_handle, 1) != 0){
p_error("rank %d: Quitting ...(%d) %s\n", rank, adios_errno, adios_errmsg());
diag = DIAG_ERR;
goto just_clean;
}
sprintf(buf, "Rank %d: var_1d_array: step %d: t: ", rank, step);
int i = 0;
for(i=0; i < NX; ++i){
if( t[i] != t_ref[i] ){
p_test_failed("%s: rank %d: for t[%d] (expected %.1f, got %.1f)\n", test_result.name, rank, i, t_ref[i], t[i] );
test_result.result = TEST_FAILED;
break;
}
}
if (TEST_PASSED == test_result.result)
p_test_passed("%s: rank %d\n", test_result.name, rank);
just_clean:
// clean everything
adios_selection_delete(sel);
sel = NULL;
free(t);
t = NULL;
free(t_ref);
t_ref = NULL;
close_adios:
CLOSE_ADIOS_READER(adios_handle, adios_opts.method);
if ((DIAG_OK == diag) && (TEST_PASSED == test_result.result)) {
return 0;
} else {
return 1;
}
}
int read_scalar_stepbystep ()
{
ADIOS_FILE * f;
float timeout_sec = 0.0;
int steps = 0;
int retval = 0;
MPI_Comm comm = MPI_COMM_SELF;
adios_read_init_method (ADIOS_READ_METHOD_BP, comm, "verbose=3");
printf ("\n--------- Read scalar in stream using varinfo->value ------------\n");
f = adios_read_open (fname, ADIOS_READ_METHOD_BP,
comm, ADIOS_LOCKMODE_NONE, timeout_sec);
if (adios_errno == err_file_not_found)
{
printf ("Stream not found after waiting %f seconds: %s\n",
timeout_sec, adios_errmsg());
retval = adios_errno;
}
else if (adios_errno == err_end_of_stream)
{
printf ("Stream terminated before open. %s\n", adios_errmsg());
retval = adios_errno;
}
else if (f == NULL) {
printf ("Error at opening stream: %s\n", adios_errmsg());
retval = adios_errno;
}
else
{
/* Processing loop over the steps (we are already in the first one) */
while (adios_errno != err_end_of_stream) {
steps++; // steps start counting from 1
printf ("Step: %d\n", f->current_step);
/* Check the scalar O with varinfo->value */
ADIOS_VARINFO * v = adios_inq_var (f, "NX");
int value = *(int*)v->value;
printf ("Scalar NX = %d", value);
if (value !=
block_count [f->current_step*nblocks_per_step*size])
{
printf ("\tERROR expected = %llu",
block_count [f->current_step*nblocks_per_step*size]);
nerrors++;
}
printf ("\n");
// advance to 1) next available step with 2) blocking wait
adios_advance_step (f, 0, timeout_sec);
if (adios_errno == err_step_notready)
{
//printf ("No new step arrived within the timeout. Quit. %s\n",
// adios_errmsg());
break; // quit while loop
}
}
adios_read_close (f);
}
adios_read_finalize_method (ADIOS_READ_METHOD_BP);
//printf ("We have processed %d steps\n", steps);
return retval;
}
int main (int argc, char ** argv)
{
int err;
int steps = 0, curr_step;
int retval = 0;
MPI_Init (&argc, &argv);
comm = MPI_COMM_WORLD;
MPI_Comm_rank (comm, &rank);
MPI_Comm_size (comm, &numproc);
if (processArgs(argc, argv)) {
return 1;
}
print0("Input stream = %s\n", infilename);
print0("Output stream = %s\n", outfilename);
print0("Read method = %s (id=%d)\n", rmethodname, read_method);
print0("Read method parameters = \"%s\"\n", rmethodparams);
print0("Write method = %s\n", wmethodname);
print0("Write method parameters = \"%s\"\n", wmethodparams);
err = adios_read_init_method(read_method, comm,
"max_chunk_size=100; "
"app_id =32767; \n"
"verbose= 3;"
"poll_interval = 100;"
);
if (!err) {
print0 ("%s\n", adios_errmsg());
}
adios_init_noxml(comm);
print0 ("Waiting to open stream %s...\n", infilename);
f = adios_read_open_stream (infilename, read_method, comm,
ADIOS_LOCKMODE_ALL, timeout_sec);
if (adios_errno == err_file_not_found)
{
print ("rank %d: Stream not found after waiting %d seconds: %s\n",
rank, timeout_sec, adios_errmsg());
retval = adios_errno;
}
else if (adios_errno == err_end_of_stream)
{
print ("rank %d: Stream terminated before open. %s\n", rank, adios_errmsg());
retval = adios_errno;
}
else if (f == NULL) {
print ("rank %d: Error at opening stream: %s\n", rank, adios_errmsg());
retval = adios_errno;
}
else
{
// process steps here...
if (f->current_step != 0) {
print ("rank %d: WARNING: First %d steps were missed by open.\n",
rank, f->current_step);
}
while (1)
{
steps++; // start counting from 1
print0 ("File info:\n");
print0 (" current step: %d\n", f->current_step);
print0 (" last step: %d\n", f->last_step);
print0 (" # of variables: %d:\n", f->nvars);
retval = process_metadata(steps);
if (retval) break;
retval = read_write(steps);
if (retval) break;
// advance to 1) next available step with 2) blocking wait
curr_step = f->current_step; // save for final bye print
adios_advance_step (f, 0, timeout_sec);
if (adios_errno == err_end_of_stream)
{
break; // quit while loop
}
else if (adios_errno == err_step_notready)
{
print ("rank %d: No new step arrived within the timeout. Quit. %s\n",
rank, adios_errmsg());
break; // quit while loop
}
else if (f->current_step != curr_step+1)
{
// we missed some steps
print ("rank %d: WARNING: steps %d..%d were missed when advancing.\n",
rank, curr_step+1, f->current_step-1);
}
}
adios_read_close (f);
}
print0 ("Bye after processing %d steps\n", steps);
adios_read_finalize_method (read_method);
adios_finalize (rank);
MPI_Finalize ();
return retval;
}
int print_scalar (ADIOS_FILE *f, char * name)
{
ADIOS_VARINFO * v;
int i,j,k;
v = adios_inq_var (f, name);
adios_inq_var_blockinfo (f, v);
printf ("Scalar '%s':\n", name);
printf ("nsteps = %d\n", v->nsteps);
printf ("nblocks per step = %d\n", v->nblocks[0]);
int err;
/* Read one writeblock across all timesteps */
int *data = (int*) calloc (v->nsteps, sizeof(int));
ADIOS_SELECTION *s;
printf ("Read same instance across all timesteps:\n");
for (i=0; i < v->nblocks[0]; i++) {
s = adios_selection_writeblock(i);
err = adios_schedule_read_byid(f, s, v->varid, 0, v->nsteps, data);
if (!err)
{
err = adios_perform_reads(f, 1);
if (!err)
{
err = 0;
printf (" block %d = [", i);
for (j=0; j < v->nsteps; j++) {
printf ("%d", data[j]);
if (data[j] !=
block_offset [j*nblocks_per_step*size + i])
{
err = 1;
}
if (j < v->nsteps-1) printf(",");
}
printf("]");
if (err)
{
nerrors++;
printf ("\tERROR expected = [");
for (j=0; j < v->nsteps; j++) {
printf ("%llu", block_offset [j*nblocks_per_step*size + i]);
if (j < v->nsteps-1) printf(",");
}
printf("]");
}
printf("\n");
} else {
printf ("ERROR at reading scalar '%s': %s\n", name, adios_errmsg());
}
} else {
printf ("ERROR at scheduling read for scalar '%s': %s\n", name, adios_errmsg());
}
adios_selection_delete(s);
}
/* Now read piecewise, one writeblock at a time */
printf ("Read each instance individually:\n");
for (j=0; j < v->nsteps; j++) {
printf (" step %d: \n", j);
for (i=0; i < v->nblocks[j]; i++) {
s = adios_selection_writeblock(i);
err = adios_schedule_read_byid(f, s, v->varid, j, 1, data);
if (!err)
{
err = adios_perform_reads(f, 1);
if (!err)
{
printf (" block %d = %d", i, data[0]);
if (data[0] !=
block_offset [j*nblocks_per_step*size + i])
{
printf ("\tERROR expected = %llu",
block_offset [j*nblocks_per_step*size + i]);
nerrors++;
}
printf ("\n");
} else {
printf ("ERROR at reading scalar '%s': %s\n", name, adios_errmsg());
}
} else {
printf ("ERROR at scheduling read for scalar '%s': %s\n", name, adios_errmsg());
}
adios_selection_delete(s);
}
}
/* Now get them piecewise, but not with reading but through statistics */
printf ("Get each instance individually from available statistics:\n");
adios_inq_var_stat (f, v, 0, 1);
if (v->statistics && v->statistics->blocks) {
ADIOS_VARSTAT *stat = v->statistics;
int blockid = 0;
for (j=0; j < v->nsteps; j++) {
printf (" step %d: \n", j);
for (i=0; i < v->nblocks[j]; i++) {
printf (" block %d = %d", i, *(int*)stat->blocks->mins[blockid]);
if (*(int*)stat->blocks->mins[blockid] !=
block_offset [j*nblocks_per_step*size + i])
{
printf ("\tERROR expected = %llu",
block_offset [j*nblocks_per_step*size + i]);
nerrors++;
}
printf ("\n");
blockid++;
}
}
}
adios_free_varinfo (v);
free(data);
}
int main (int argc, char ** argv)
{
int err, step ;
int do_write = 1;
int do_read = 1;
int m = 0;
char write_method[16];
MPI_Init (&argc, &argv);
MPI_Comm_rank (comm, &rank);
MPI_Comm_size (comm, &size);
if (argc > 1) {
m = strtol (argv[1], NULL, 10);
if (errno) { printf("Invalid 1st argument %s\n", argv[1]); Usage(); return 1;}
}
if (argc > 2) {
if (argv[2][0] == 'w' || argv[2][0] == 'W') {
do_read = 0;
} else if (argv[2][0] == 'r' || argv[2][0] == 'R') {
do_write = 0;
} else {
printE ("Invalid command line argument %s. Allowed ones:\n"
" w: do write only\n"
" r: do read only\n", argv[2]);
MPI_Finalize ();
return 1;
}
}
if (m==0) {
read_method = ADIOS_READ_METHOD_BP;
strcpy(write_method,"MPI");
} else {
read_method = ADIOS_READ_METHOD_DATASPACES;
strcpy(write_method,"DATASPACES");
}
log ("Writing: %s method=%s\n"
"Reading: %s method=%d\n",
(do_write ? "yes" : "no"), write_method,
(do_read ? "yes" : "no"), read_method);
alloc_vars();
if (do_write) {
adios_init_noxml (comm);
adios_allocate_buffer (ADIOS_BUFFER_ALLOC_NOW, 10);
}
if (do_read) {
err = adios_read_init_method(read_method, comm, "verbose=2");
if (err) {
printE ("%s\n", adios_errmsg());
}
}
if (do_write) {
adios_declare_group (&m_adios_group, "selections", "iter", adios_flag_yes);
adios_select_method (m_adios_group, write_method, "verbose=2", "");
define_vars();
for (step=0; step<NSTEPS; step++) {
if (!err) {
set_vars (step);
err = write_file (step);
sleep(1);
}
}
adios_free_group (m_adios_group);
}
if (!err && do_read)
err = read_points ();
//if (!err && do_read)
// err = read_writerblocks ();
if (do_read) {
adios_read_finalize_method (read_method);
}
fini_vars();
if (do_write) {
adios_finalize (rank);
}
MPI_Finalize ();
return err;
}
int main (int argc, char ** argv)
{
int rank, size, i, j, npl, token;
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Status status;
enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_BP;
ADIOS_SELECTION * sel;
void * data = NULL;
uint64_t start[1], count[1];
MPI_Init (&argc, &argv);
MPI_Comm_rank (comm, &rank);
MPI_Comm_size (comm, &size);
adios_read_init_method (method, comm, "verbose=3");
ADIOS_FILE * f = adios_read_open ("adios_global_no_xml.bp", method,
comm, ADIOS_LOCKMODE_NONE, 0);
if (f == NULL)
{
printf ("%s\n", adios_errmsg());
return -1;
}
ADIOS_VARINFO * v = adios_inq_var (f, "temperature");
/* Using less readers to read the global array back, i.e., non-uniform */
uint64_t slice_size = v->dims[0]/size;
start[0] = slice_size * rank;
if (rank == size-1) /* last rank may read more lines */
slice_size = slice_size + v->dims[0]%size;
count[0] = slice_size;
data = malloc (slice_size * sizeof (double));
if (data == NULL)
{
fprintf (stderr, "malloc failed.\n");
return -1;
}
/* Read a subset of the temperature array */
sel = adios_selection_boundingbox (v->ndim, start, count);
adios_schedule_read (f, sel, "temperature", 0, 1, data);
adios_perform_reads (f, 1);
if (rank > 0) {
MPI_Recv (&token, 1, MPI_INT, rank-1, 0, comm, &status);
}
printf (" ======== Rank %d ========== \n", rank);
npl = 10;
for (i = 0; i < slice_size; i+=npl) {
printf ("[%4.4" PRIu64 "] ", rank*slice_size+i);
for (j= 0; j < npl; j++) {
printf (" %6.6g", * ((double *)data + i + j));
}
printf ("\n");
}
fflush(stdout);
sleep(1);
if (rank < size-1) {
MPI_Send (&token, 1, MPI_INT, rank+1, 0, comm);
}
free (data);
adios_selection_delete (sel);
adios_free_varinfo (v);
adios_read_close (f);
MPI_Barrier (comm);
adios_read_finalize_method (method);
MPI_Finalize ();
return 0;
}
bool
ADIOSFileObject::Open()
{
if (IsOpen())
return true;
#ifdef PARALLEL
fp = adios_fopen(fileName.c_str(), (MPI_Comm)VISIT_MPI_COMM);
#else
fp = adios_fopen(fileName.c_str(), 0);
#endif
char errmsg[1024];
if (fp == NULL)
{
sprintf(errmsg, "Error opening bp file %s:\n%s", fileName.c_str(), adios_errmsg());
EXCEPTION1(InvalidDBTypeException, errmsg);
}
ADIOS_VARINFO *avi;
gps = (ADIOS_GROUP **) malloc(fp->groups_count * sizeof(ADIOS_GROUP *));
if (gps == NULL)
EXCEPTION1(InvalidDBTypeException, "The file could not be opened. Not enough memory");
debug5 << "ADIOS BP file: " << fileName << endl;
debug5 << "# of groups: " << fp->groups_count << endl;
debug5 << "# of variables: " << fp->vars_count << endl;
debug5 << "# of attributes:" << fp->attrs_count << endl;
debug5 << "time steps: " << fp->ntimesteps << " from " << fp->tidx_start << endl;
//Read in variables/scalars.
variables.clear();
scalars.clear();
for (int gr=0; gr<fp->groups_count; gr++)
{
debug5 << " group " << fp->group_namelist[gr] << ":" << endl;
gps[gr] = adios_gopen_byid(fp, gr);
if (gps[gr] == NULL)
{
sprintf(errmsg, "Error opening group %s in bp file %s:\n%s", fp->group_namelist[gr], fileName.c_str(), adios_errmsg());
EXCEPTION1(InvalidDBTypeException, errmsg);
}
for (int vr=0; vr<gps[gr]->vars_count; vr++)
{
avi = adios_inq_var_byid(gps[gr], vr);
if (avi == NULL)
{
sprintf(errmsg, "Error opening inquiring variable %s in group %s of bp file %s:\n%s",
gps[gr]->var_namelist[vr], fp->group_namelist[gr], fileName.c_str(), adios_errmsg());
EXCEPTION1(InvalidDBTypeException, errmsg);
}
if (SupportedVariable(avi))
{
//Scalar
if (avi->ndim == 0)
{
ADIOSScalar s(gps[gr]->var_namelist[vr], avi);
scalars[s.Name()] = s;
debug5<<" added scalar "<<s<<endl;
}
//Variable
else
{
// add variable to map, map id = variable path without the '/' in the beginning
ADIOSVar v(gps[gr]->var_namelist[vr], gr, avi);
variables[v.name] = v;
debug5<<" added variable "<< v.name<<endl;
}
}
else
debug5<<"Skipping variable: "<<gps[gr]->var_namelist[vr]<<" dim= "<<avi->ndim
<<" timedim= "<<avi->timedim
<<" type= "<<adios_type_to_string(avi->type)<<endl;
adios_free_varinfo(avi);
}
//Read in attributes.
for (int a = 0; a < gps[gr]->attrs_count; a++)
{
int sz;
void *data = NULL;
ADIOS_DATATYPES attrType;
if (adios_get_attr_byid(gps[gr], a, &attrType, &sz, &data) != 0)
{
debug5<<"Failed to get attr: "<<gps[gr]->attr_namelist[a]<<endl;
continue;
}
ADIOSAttr attr(gps[gr]->attr_namelist[a], attrType, data);
attributes[attr.Name()] = attr;
free(data);
}
adios_gclose(gps[gr]);
gps[gr] = NULL;
}
return true;
}
int main (int argc, char ** argv)
{
int rank, size, i, j;
MPI_Comm comm = MPI_COMM_WORLD;
ADIOS_FILE * f;
ADIOS_VARINFO * v;
ADIOS_SELECTION * sel;
int steps = 0;
int retval = 0;
float timeout_sec = 1.0;
void * data = NULL;
uint64_t start[2], count[2];
MPI_Init (&argc, &argv);
MPI_Comm_rank (comm, &rank);
MPI_Comm_size (comm, &size);
adios_read_init_method (ADIOS_READ_METHOD_BP, comm, "verbose=3");
f = adios_read_open ("adios_globaltime.bp", ADIOS_READ_METHOD_BP,
comm, ADIOS_LOCKMODE_NONE, timeout_sec);
if (adios_errno == err_file_not_found)
{
printf ("rank %d: Stream not found after waiting %f seconds: %s\n",
rank, timeout_sec, adios_errmsg());
retval = adios_errno;
}
else if (adios_errno == err_end_of_stream)
{
printf ("rank %d: Stream terminated before open. %s\n", rank, adios_errmsg());
retval = adios_errno;
}
else if (f == NULL) {
printf ("rank %d: Error at opening stream: %s\n", rank, adios_errmsg());
retval = adios_errno;
}
else
{
/* process file here... */
v = adios_inq_var (f, "temperature");
adios_inq_var_blockinfo (f, v);
printf ("ndim = %d\n", v->ndim);
//printf ("nsteps = %d\n", v->nsteps);
printf ("dims[%llu][%llu]\n", v->dims[0], v->dims[1]);
uint64_t slice_size = v->dims[0]/size;
if (rank == size-1)
slice_size = slice_size + v->dims[0]%size;
start[0] = rank * slice_size;
count[0] = slice_size;
start[1] = 0;
count[1] = v->dims[1];
data = malloc (slice_size * v->dims[1] * 8);
/* Processing loop over the steps (we are already in the first one) */
while (adios_errno != err_end_of_stream) {
steps++; // steps start counting from 1
sel = adios_selection_boundingbox (v->ndim, start, count);
adios_schedule_read (f, sel, "temperature", 0, 1, data);
adios_perform_reads (f, 1);
if (rank == 0)
printf ("--------- Step: %d --------------------------------\n",
f->current_step);
printf("rank=%d: [0:%lld,0:%lld] = [", rank, v->dims[0], v->dims[1]);
for (i = 0; i < slice_size; i++) {
printf (" [");
for (j = 0; j < v->dims[1]; j++) {
printf ("%g ", *((double *)data + i * v->dims[1] + j));
}
printf ("]");
}
printf (" ]\n\n");
// advance to 1) next available step with 2) blocking wait
adios_advance_step (f, 0, timeout_sec);
if (adios_errno == err_step_notready)
{
printf ("rank %d: No new step arrived within the timeout. Quit. %s\n",
rank, adios_errmsg());
break; // quit while loop
}
}
adios_read_close (f);
}
if (rank==0)
printf ("We have processed %d steps\n", steps);
adios_read_finalize_method (ADIOS_READ_METHOD_BP);
free (data);
MPI_Finalize ();
return retval;
}
int main (int argc, char ** argv)
{
int rank, size, i;
MPI_Comm comm = MPI_COMM_WORLD;
enum ADIOS_DATATYPES attr_type;
enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_BP;
int attr_size;
void * data = NULL;
MPI_Init (&argc, &argv);
MPI_Comm_rank (comm, &rank);
MPI_Comm_size (comm, &size);
adios_read_init_method (method, comm, "verbose=3");
adios_logger_open ("log_read_C", rank);
ADIOS_FILE * f = adios_read_open ("attributes_C.bp", method, comm, ADIOS_LOCKMODE_NONE, 0.0);
if (f == NULL)
{
log_error ("%s\n", adios_errmsg());
return -1;
}
for (i = 0; i < f->nattrs; i++)
{
adios_get_attr (f, f->attr_namelist[i], &attr_type, &attr_size, &data);
log_test("rank %d: attr: %s %s = ", rank, adios_type_to_string(attr_type), f->attr_namelist[i]);
int type_size = adios_type_size (attr_type, data);
int nelems = attr_size / type_size;
int k;
char *p = (char*)data;
for (k=0; k<nelems; k++)
{
if (k>0) log_test(", ");
switch (attr_type)
{
case adios_integer:
log_test ("%d", *(int *)p);
break;
case adios_double:
log_test ("%e", *(double *)p);
break;
case adios_string:
log_test ("\"%s\"", (char *)p);
break;
case adios_string_array:
log_test ("\"%s\"", *(char **)p);
break;
default:
log_test ("??????\n");
}
p=p+type_size;
}
log_test("\n");
free (data);
data = 0;
}
adios_read_close (f);
MPI_Barrier (comm);
adios_read_finalize_method (ADIOS_READ_METHOD_BP);
adios_logger_close();
MPI_Finalize ();
return 0;
}
int process_metadata(int step)
{
int retval = 0;
int i, j;
char gdims[256], ldims[256], offs[256];
uint64_t sum_count;
ADIOS_VARINFO *v; // shortcut pointer
if (step > 1)
{
// right now, nothing to prepare in later steps
print("Step %d. return immediately\n",step);
return 0;
}
/* First step processing */
// get groupname of stream, then declare for output
adios_get_grouplist(f, &group_namelist);
print0("Group name is %s\n", group_namelist[0]);
adios_declare_group(&gh,group_namelist[0],"",adios_flag_yes);
varinfo = (VarInfo *) malloc (sizeof(VarInfo) * f->nvars);
if (!varinfo) {
print("ERROR: rank %d cannot allocate %lu bytes\n", rank, sizeof(VarInfo)*f->nvars);
return 1;
}
write_total = 0;
largest_block = 0;
// Decompose each variable and calculate output buffer size
for (i=0; i<f->nvars; i++)
{
print0 ("Get info on variable %d: %s\n", i, f->var_namelist[i]);
varinfo[i].v = adios_inq_var_byid (f, i);
v = varinfo[i].v; // just a shortcut
if (v == NULL) {
print ("rank %d: ERROR: Variable %s inquiry failed: %s\n",
rank, f->var_namelist[i], adios_errmsg());
return 1;
}
// print variable type and dimensions
print0(" %-9s %s", adios_type_to_string(v->type), f->var_namelist[i]);
if (v->ndim > 0) {
print0("[%llu", v->dims[0]);
for (j = 1; j < v->ndim; j++)
print0(", %llu", v->dims[j]);
print0("] :\n");
} else {
print0("\tscalar\n");
}
// determine subset we will write
decompose (numproc, rank, v->ndim, v->dims, decomp_values,
varinfo[i].count, varinfo[i].start, &sum_count);
varinfo[i].writesize = sum_count * adios_type_size(v->type, v->value);
if (varinfo[i].writesize != 0) {
write_total += varinfo[i].writesize;
if (largest_block < varinfo[i].writesize)
largest_block = varinfo[i].writesize;
}
}
// determine output buffer size and allocate it
uint64_t bufsize = write_total + f->nvars*128 + f->nattrs*32 + 1024;
if (bufsize > max_write_buffer_size) {
print ("ERROR: rank %d: write buffer size needs to hold about %llu bytes, "
"but max is set to %d\n", rank, bufsize, max_write_buffer_size);
return 1;
}
print0 ("Rank %d: allocate %llu MB for output buffer\n", rank, bufsize/1048576+1);
adios_allocate_buffer (ADIOS_BUFFER_ALLOC_NOW, bufsize/1048576+1);
// allocate read buffer
bufsize = largest_block + 128;
if (bufsize > max_read_buffer_size) {
print ("ERROR: rank %d: read buffer size needs to hold at least %llu bytes, "
"but max is set to %d\n", rank, bufsize, max_read_buffer_size);
return 1;
}
print0 ("Rank %d: allocate %g MB for input buffer\n", rank, (double)bufsize/1048576.0);
readbuf = (char *) malloc ((size_t)bufsize);
if (!readbuf) {
print ("ERROR: rank %d: cannot allocate %llu bytes for read buffer\n",
rank, bufsize);
return 1;
}
// Select output method
adios_select_method (gh, wmethodname, wmethodparams, "");
// Define variables for output based on decomposition
char *vpath, *vname;
for (i=0; i<f->nvars; i++)
{
v = varinfo[i].v;
if (varinfo[i].writesize != 0) {
// define variable for ADIOS writes
getbasename (f->var_namelist[i], &vpath, &vname);
if (v->ndim > 0)
{
int64s_to_str (v->ndim, v->dims, gdims);
int64s_to_str (v->ndim, varinfo[i].count, ldims);
int64s_to_str (v->ndim, varinfo[i].start, offs);
print ("rank %d: Define variable path=\"%s\" name=\"%s\" "
"gdims=%s ldims=%s offs=%s\n",
rank, vpath, vname, gdims, ldims, offs);
adios_define_var (gh, vname, vpath, v->type, ldims, gdims, offs);
}
else
{
print ("rank %d: Define scalar path=\"%s\" name=\"%s\"\n",
rank, vpath, vname);
adios_define_var (gh, vname, vpath, v->type, "", "", "");
}
free(vpath);
free(vname);
}
}
if (rank == 0)
{
// get and define attributes
enum ADIOS_DATATYPES attr_type;
void * attr_value;
char * attr_value_str;
int attr_size;
for (i=0; i<f->nattrs; i++)
{
adios_get_attr_byid (f, i, &attr_type, &attr_size, &attr_value);
attr_value_str = (char *)value_to_string (attr_type, attr_value, 0);
getbasename (f->attr_namelist[i], &vpath, &vname);
if (vpath && !strcmp(vpath,"/__adios__")) {
// skip on /__adios/... attributes
print ("rank %d: Ignore this attribute path=\"%s\" name=\"%s\" value=\"%s\"\n",
rank, vpath, vname, attr_value_str);
} else {
adios_define_attribute (gh, vname, vpath,
attr_type, attr_value_str, "");
print ("rank %d: Define attribute path=\"%s\" name=\"%s\" value=\"%s\"\n",
rank, vpath, vname, attr_value_str);
free (attr_value);
}
}
}
return retval;
}
int main (int argc, char ** argv)
{
char filename [256];
int rank, size, i, j, k, token;
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Status status;
enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_BP;
ADIOS_SELECTION * sel;
void * data = NULL;
uint64_t start[3], count[3], step = 0;
MPI_Init (&argc, &argv);
MPI_Comm_rank (comm, &rank);
MPI_Comm_size (comm, &size);
adios_read_init_method (method, comm, "verbose=3");
/* adios_read_open_file() allows for seeing all timesteps in the file */
ADIOS_FILE * f = adios_read_open_file ("adios_globaltime.bp", method, comm);
if (f == NULL)
{
printf ("%s\n", adios_errmsg());
return -1;
}
ADIOS_VARINFO * v = adios_inq_var (f, "temperature");
// read in two timesteps
data = malloc (2 * v->dims[0] * v->dims[1] * sizeof (double));
if (data == NULL)
{
fprintf (stderr, "malloc failed.\n");
return -1;
}
// read in timestep 'rank' (up to 12)
step = rank % 13;
start[0] = 0;
count[0] = v->dims[0];
start[1] = 0;
count[1] = v->dims[1];
/* Read a subset of the temperature array */
sel = adios_selection_boundingbox (v->ndim, start, count);
/* 2 steps from 'step' */
adios_schedule_read (f, sel, "temperature", step, 2, data);
adios_perform_reads (f, 1);
if (rank == 0)
printf ("Array size of temperature [0:%lld,0:%lld]\n", v->dims[0], v->dims[1]);
if (rank > 0) {
MPI_Recv (&token, 1, MPI_INT, rank-1, 0, comm, &status);
}
printf("------------------------------------------------\n", rank);
printf("rank=%d: \n", rank);
for (i = 0; i < 2; i++) {
printf ("step %lld = [\n", step+i);
for (j = 0; j < v->dims[0]; j++) {
printf (" [");
for (k = 0; k < v->dims[1]; k++) {
printf ("%g ", ((double *)data) [ i * v->dims[0] * v->dims[1] + j * v->dims[1] + k]);
}
printf ("]\n");
}
printf ("]\n");
}
printf ("\n");
if (rank < size-1) {
MPI_Send (&token, 1, MPI_INT, rank+1, 0, comm);
}
free (data);
adios_free_varinfo (v);
adios_read_close (f);
MPI_Barrier (comm);
adios_read_finalize_method (method);
MPI_Finalize ();
return 0;
}
//Reads an adios restart file :: can only be instantiated if EAVL was built
//with MPI enabled!
eavlXGCParticleImporter::eavlXGCParticleImporter( const string &filename,
ADIOS_READ_METHOD method,
MPI_Comm communicator,
ADIOS_LOCKMODE mode,
int timeout_sec
)
{
timestep = 0;
maxnum = 0;
enphase = 0;
inphase = 0;
emaxgid = 0;
imaxgid = 0;
nvars = 0;
time = 0;
retVal = 0;
totalIParticles = 0;
totalEParticles = 0;
fp = NULL;
fp = NULL;
getR = true;
getZ = true;
getPhi = true;
getRho = true;
getW1 = true;
getW2 = true;
getMu = true;
getW0 = true;
getF0 = true;
getOriginNode = true;
comm = communicator;
std::string key (".restart");
std::size_t found = filename.find(key);
if(found != std::string::npos)
readingRestartFile = 1;
else
readingRestartFile = 0;
char hostname[MPI_MAX_PROCESSOR_NAME];
char str [256];
int len = 0;
//Set local mpi vars so we know how many minions there are, and wich we are
MPI_Comm_size(comm,&numMPITasks);
MPI_Comm_rank(comm,&mpiRank);
MPI_Get_processor_name(hostname, &len);
if(method == ADIOS_READ_METHOD_BP) //do not open stream if not staging
fp = adios_read_open_file(filename.c_str(), method, comm);
else
fp = adios_read_open(filename.c_str(), method, comm, mode, timeout_sec);
if (fp == NULL)
{
cerr << __LINE__ << endl;
if(adios_errno == err_end_of_stream)
{
printf ("End of stream, no more steps expected. Quit. %s\n",
adios_errmsg()
);
}
else
{
printf ("No new step arrived within the timeout. Quit. %s\n",
adios_errmsg()
);
THROW(eavlException, "XGC variable file not found.");
}
}
Initialize();
}
int main (int argc, char ** argv)
{
int rank, size, i, j;
MPI_Comm comm = MPI_COMM_WORLD;
enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_BP;
ADIOS_SELECTION * sel;
void * data = NULL;
uint64_t start[2], count[2];
MPI_Init (&argc, &argv);
MPI_Comm_rank (comm, &rank);
MPI_Comm_size (comm, &size);
adios_read_init_method (method, comm, "verbose=4");
adios_logger_open ("log_read_C", rank);
ADIOS_FILE * f = adios_read_open ("global_array_C.bp", method, comm, ADIOS_LOCKMODE_NONE, 0);
if (f == NULL)
{
log_error ("%s\n", adios_errmsg());
return -1;
}
ADIOS_VARINFO * v = adios_inq_var (f, "temperature");
/* Using less readers to read the global array back, i.e., non-uniform */
uint64_t slice_size = v->dims[0]/size;
start[0] = slice_size * rank;
if (rank == size-1) /* last rank may read more lines */
slice_size = slice_size + v->dims[0]%size;
count[0] = slice_size;
start[1] = 0;
count[1] = v->dims[1];
data = malloc (slice_size * v->dims[1] * sizeof (double));
if (data == NULL)
{
log_error (stderr, "malloc failed.\n");
return -1;
}
/* Read a subset of the temperature array */
sel = adios_selection_boundingbox (v->ndim, start, count);
adios_schedule_read (f, sel, "temperature", 0, 1, data);
adios_perform_reads (f, 1);
for (i = 0; i < slice_size; i++) {
log_test ("rank %d: [%lld,%d:%lld]", rank, start[0]+i, 0, slice_size);
for (j = 0; j < v->dims[1]; j++)
log_test (" %6.6g", * ((double *)data + i * v->dims[1] + j));
log_test ("\n");
}
free (data);
adios_read_close (f);
MPI_Barrier (comm);
adios_read_finalize_method (method);
adios_logger_close();
MPI_Finalize ();
return 0;
}
int main (int argc, char ** argv) {
//For varriable definitions:
//gbounds = global bounds string, lbounds = local bounds string, offs = offset string, tstring = temp string to hold temperary stuff
char gbounds[1007], lbounds[1007], offs[1007],tstring[100];
//size = number of cores, gidx = adios group index
int rank, size, gidx, i, j, k, ii;
//data = pointer to read-in data
void * data = NULL;
uint64_t s[] = {0,0,0,0,0,0,0,0,0,0}; //starting offset
uint64_t c[] = {1,1,1,1,1,1,1,1,1,1}; //chunk block array
uint64_t bytes_read = 0;
int element_size;
int64_t new_adios_group, m_adios_file;
uint64_t var_size; //portion_bound,
uint64_t adios_groupsize, adios_totalsize;
int read_buffer; //possible maximum size you the user would like for each chunk in MB
int write_buffer = 1536; //actual buffer size you use in MB
int itime;
int WRITEME=1;
uint64_t chunk_size; //chunk size in # of elements
char *var_path, *var_name; // full path cut into dir path and name
MPI_Init(&argc,&argv);
MPI_Comm_rank(comm,&rank);
MPI_Comm_size(comm,&size);
// timing numbers
// we will time:
// 0: adios_open, adios_group_size
// 1: the total time to read in the data
// 2: times around each write (will only work if we do NOT buffer....
// 3: the time in the close
// 4: fopen, fclose
// 5: total time
// timers: the total I/O time
int timers = 6;
double start_time[timers], end_time[timers], total_time[timers];
if (TIMING==100) {
for (itime=0;itime<timers;itime++) {
start_time[itime] = 0;
end_time[itime] = 0;
total_time[itime]=0;
}
//MPI_Barrier(MPI_COMM_WORLD);
start_time[5] = MPI_Wtime();
}
if(rank==0)
printf("converting...\n");
if (argc < 5) {
if (rank==0) printf("Usage: %s <BP-file> <ADIOS-file> read_buffer(MB) write_buffer(MB) METHOD (LUSTRE_strip_count) (LUSTRE_strip_size) (LUSTRE_block_size)\n", argv[0]);
return 1;
}
if(TIMING==100)
start_time[4] = MPI_Wtime();
ADIOS_FILE * f = adios_fopen (argv[1], MPI_COMM_SELF);
if(TIMING==100){
end_time[4] = MPI_Wtime();
total_time[4] = end_time[4]-start_time[4];
}
adios_init_noxml(comm); // no xml will be used to write the new adios file
read_buffer = atoi(argv[3]);
write_buffer = atoi(argv[4]);
adios_allocate_buffer (ADIOS_BUFFER_ALLOC_NOW, write_buffer); // allocate MB buffer
if (f == NULL) {
printf("rank=%d, file cant be opened\n", rank);
if (DEBUG) printf ("%s\n", adios_errmsg());
return -1;
}
for (gidx = 0; gidx < f->groups_count; gidx++) { //group part
adios_groupsize = 0;
ADIOS_GROUP * g = adios_gopen (f, f->group_namelist[gidx]);
if (g == NULL) {
if (DEBUG) printf ("%s\n", adios_errmsg());
printf("rank %d: group cannot be opened.\n", rank);
return -1;
}
/* First create all of the groups */
// now I need to create this group in the file that will be written
adios_declare_group(&new_adios_group,f->group_namelist[gidx],"",adios_flag_yes);
if(strcmp(argv[5],"MPI_LUSTRE")!=0) //see whether or not the user uses MPI_LUSTRE method
adios_select_method (new_adios_group, argv[5], "", ""); //non-MPI_LUSTRE methods... like MPI, POSIX....
else{
char lustre_pars[1000];
strcpy(lustre_pars, "");
strcat(lustre_pars, "stripe_count=");
sprintf(tstring, "%d", atoi(argv[6]));
strcat(lustre_pars, tstring);
strcat(lustre_pars, ",stripe_size=");
sprintf(tstring, "%d", atoi(argv[7]));
strcat(lustre_pars, tstring);
strcat(lustre_pars, ",block_size=");
sprintf(tstring, "%d", atoi(argv[8]));
strcat(lustre_pars, tstring);
if(rank==0)
printf("lustre_pars=%s\n", lustre_pars);
adios_select_method (new_adios_group, argv[5], lustre_pars, ""); //Use MPI Lustre method
}
// variable definition part
for (i = 0; i < g->vars_count; i++) {
ADIOS_VARINFO * v = adios_inq_var_byid (g, i);
getbasename (g->var_namelist[i], &var_path, &var_name);
if (v->ndim == 0)
{
// scalars: every process does them the same.
adios_define_var(new_adios_group,var_name,var_path,v->type,0,0,0);
getTypeInfo( v->type, &element_size); //element_size is size per element based on its type
if (v->type == adios_string) { //special case when the scalar is string.
adios_groupsize += strlen(v->value);
} else {
adios_groupsize += element_size;
}
}
else
{
// vector variables
getTypeInfo( v->type, &element_size);
var_size=1;
for (ii=0;ii<v->ndim;ii++) {
var_size*=v->dims[ii];
}
uint64_t total_size = var_size; //total_size tells you the total number of elements in the current vector variable
var_size*=element_size; //var_size tells you the size of the current vector variable in bytess
//re-initialize the s and c variables
for(j=0; j<v->ndim; j++){
s[j] = 0;
c[j] = 1;
}
//find the approximate chunk_size you would like to use.
chunk_size = calcChunkSize(total_size, read_buffer*1024*1024/element_size, size);
//set the chunk block array with the total size as close to chunk_size as possible
calcC(chunk_size, v, c);
strcpy(lbounds,"");
for(j=0; j<v->ndim; j++){
sprintf(tstring, "%" PRId64 ",", c[j]);
strcat(lbounds, tstring);
}
printf("rank=%d, name=%s, chunk_size1=%" PRId64 " c[]=%s\n",rank,g->var_namelist[i],chunk_size,lbounds);
chunk_size = 1;
for(ii=0; ii<v->ndim; ii++) //reset chunk_size based on the created c. Now the chunk_size is exact.
chunk_size *= c[ii];
//current step points to where the process is in processing the vector. First sets with respect to rank.
uint64_t current_step = rank*chunk_size;
//First advance the starting point s by current_step. Of course, you don't do it if the current_step exceeds total_size.
if(current_step<total_size)
rS(v, s, current_step, rank);
uint64_t elements_defined = 0; //First, the number of elements you have defined is 0.
//You (the process) process your part of the vector when your current_step is smaller than the total_size
while(current_step < total_size)
{
//ts, temporary s, is introduced for the sake of the inner do while loop below. Copy s to ts.
uint64_t ts[] = {0,0,0,0,0,0,0,0,0,0};
arrCopy(s, ts);
//for every outer while iteration, you always have the whole chunk_size remained to process.
uint64_t remain_chunk = chunk_size;
if(current_step+chunk_size>total_size) //except when you are nearing the end of the vector....
remain_chunk = total_size-current_step;
//tc, temporary c, is introduced for the sake of the inner do while loop below. Copy s to tc.
uint64_t tc[] = {1,1,1,1,1,1,1,1,1,1};
arrCopy(c, tc);
do{
//how much of the remain chunk you wanna process? initially you think you can do all of it....
uint64_t used_chunk = remain_chunk;
//you feel like you should process the vector with tc block size, but given ts, you might go over bound.
uint64_t uc[] = {1,1,1,1,1,1,1,1,1,1};
//so you verify it by setting a new legit chunck block uc, and getting a new remain_chunk.
remain_chunk = checkBound(v, ts, tc, uc, remain_chunk);
//you check whether or not ts+uc goes over the bound. This is just checking to make sure there's no error.
//Thereotically, there should be no problem at all.
checkOverflow(0, v, ts, uc);
//the below code fragment simply calculates gbounds, and sets place holders for lbounds and offs.
strcpy(gbounds,"");
strcpy(lbounds,"");
strcpy(offs,"");
for(j=0; j<v->ndim-1; j++){
sprintf(tstring, "%d,", (int)v->dims[j]);
strcat(gbounds, tstring);
//sprintf(tstring, "ldim%d_%s,", j, var_name);
sprintf(tstring, "ldim%d,", j);
strcat(lbounds, tstring);
//sprintf(tstring, "offs%d_%s,", j, var_name);
sprintf(tstring, "offs%d,", j);
strcat(offs, tstring);
}
sprintf(tstring, "%d", (int)v->dims[v->ndim-1]);
strcat(gbounds, tstring);
//sprintf(tstring, "ldim%d_%s", v->ndim-1, var_name);
sprintf(tstring, "ldim%d", v->ndim-1);
strcat(lbounds, tstring);
//sprintf(tstring, "offs%d_%s", v->ndim-1, var_name);
sprintf(tstring, "offs%d", v->ndim-1);
strcat(offs, tstring);
//sprintf(tstring, "%d", v->ndim);
for(j=0; j<v->ndim; j++){
//sprintf(tstring, "ldim%d_%s", j, var_name);
sprintf(tstring, "ldim%d", j);
adios_define_var(new_adios_group, tstring, "bp2bp", adios_unsigned_long, 0, 0, 0);
//sprintf(tstring, "offs%d_%s", j, var_name);
sprintf(tstring, "offs%d", j);
adios_define_var(new_adios_group, tstring, "bp2bp", adios_unsigned_long, 0, 0, 0);
}
adios_define_var(new_adios_group,var_name,var_path,v->type,lbounds,gbounds,offs);
if (DEBUG){
strcpy(lbounds,"");
strcpy(offs,"");
for(j=0; j<v->ndim; j++){
sprintf(tstring, "%" PRId64 ",", ts[j]);
strcat(offs, tstring);
sprintf(tstring, "%" PRId64 ",", uc[j]);
strcat(lbounds, tstring);
}
printf("rank=%d, name=%s, gbounds=%s: lbounds=%s: offs=%s \n",rank,g->var_namelist[i],gbounds, lbounds, offs);
}
used_chunk -= remain_chunk; //you get the actual used_chunk here.
elements_defined += used_chunk;
if(remain_chunk!=0){
rS(v, ts, used_chunk, rank); //advance ts by used_chunk.
for(k=0; k<10; k++)
tc[k] = 1;
calcC(remain_chunk, v, tc); //based on the remain_chunk, calculate new tc chunk block remained to process.
}
adios_groupsize+= used_chunk*element_size+2*v->ndim*8;
}while(remain_chunk!=0);
current_step += size*chunk_size; //once a whole chunk_size is processed, advance the current_step in roll-robin manner.
if(current_step<total_size){ //advance s in the same way.
rS(v, s, size*chunk_size, rank);
}
}
//beside checkOverflow above, here you check whether or not the total number of elements processed across processes matches
//the total number of elements in the original vector.
if(DEBUG){
uint64_t* sb = (uint64_t *) malloc(sizeof(uint64_t));
uint64_t* rb = (uint64_t *) malloc(sizeof(uint64_t));
sb[0] = elements_defined;
MPI_Reduce(sb,rb,1,MPI_UNSIGNED_LONG_LONG,MPI_SUM,0, comm);
if(rank==0 && rb[0]!=total_size)
printf("some array define mismatch. please use debug mode\n");
free(sb); free(rb);
}
}
free (var_name);
free (var_path);
} // finished declaring all of the variables
// Now we can define the attributes....
for (i = 0; i < g->attrs_count; i++) {
enum ADIOS_DATATYPES atype;
int asize;
void *adata;
adios_get_attr_byid (g, i, &atype, &asize, &adata);
// if (DEBUG) printf("attribute name=%s\n",g->attr_namelist[i]);
adios_define_attribute(new_adios_group,g->attr_namelist[i],"",atype,adata,0);
}
/*------------------------------ NOW WE WRITE -------------------------------------------- */
// Now we have everything declared... now we need to write them out!!!!!!
if (WRITEME==1) {
// open up the file for writing....
if (DEBUG) printf("rank=%d, opening file = %s, with group %s, size=%" PRId64 "\n",rank,argv[2],f->group_namelist[gidx],adios_groupsize);
if(TIMING==100)
start_time[0] = MPI_Wtime();
adios_open(&m_adios_file, f->group_namelist[gidx],argv[2],"w",comm);
adios_group_size( m_adios_file, adios_groupsize, &adios_totalsize);
//get both the total adios_totalsize and total adios_groupsize summed across processes.
uint64_t* sb = (uint64_t *) malloc(sizeof(uint64_t));;
uint64_t* rb = (uint64_t *) malloc(sizeof(uint64_t));
sb[0] = adios_groupsize;
MPI_Reduce(sb,rb,1,MPI_UNSIGNED_LONG_LONG,MPI_SUM,0, comm);
uint64_t* sb2 = (uint64_t *) malloc(sizeof(uint64_t));;
uint64_t* rb2 = (uint64_t *) malloc(sizeof(uint64_t));
sb2[0] = adios_totalsize;
MPI_Reduce(sb2,rb2,1,MPI_UNSIGNED_LONG_LONG,MPI_SUM,0, comm);
if(rank==0){
printf("total adios_totalsize = %" PRId64 "\n", *rb2);
printf("total adios_groupsize = %" PRId64 "\n", *rb);
}
free(sb); free(rb); free(sb2); free(rb2);
if (TIMING==100) {
end_time[0] = MPI_Wtime();
total_time[0]+=end_time[0] - start_time[0]; //variable definition time taken
}
// now we have to write out the variables.... since they are all declared now
// This will be the place we actually write out the data!!!!!!!!
for (i = 0; i < g->vars_count; i++) {
ADIOS_VARINFO * v = adios_inq_var_byid (g, i);
getbasename (g->var_namelist[i], &var_path, &var_name);
if (v->ndim == 0)
{
if (DEBUG) {
printf ("ADIOS WRITE SCALAR: rank=%d, name=%s value=",
rank,g->var_namelist[i]);
print_data (v->value, 0, v->type);
printf ("\n");
}
if (TIMING==100) {
start_time[2] = MPI_Wtime();
}
adios_write(m_adios_file,g->var_namelist[i],v->value);
if (TIMING==100) {
end_time[2] = MPI_Wtime();
total_time[2]+=end_time[2] - start_time[2]; //IO write time...
}
}
else
{
for(j=0; j<v->ndim; j++){
s[j] = 0;
c[j] = 1;
}
getTypeInfo( v->type, &element_size);
uint64_t total_size = 1;
for (ii=0;ii<v->ndim;ii++)
total_size*=v->dims[ii];
chunk_size = calcChunkSize(total_size, read_buffer*1024*1024/element_size, size);
calcC(chunk_size, v, c);
chunk_size = 1;
for(ii=0; ii<v->ndim; ii++)
chunk_size *= c[ii];
uint64_t current_step = rank*chunk_size;
if(current_step<total_size)
rS(v, s, current_step, rank);
uint64_t elements_written = 0;
while(current_step < total_size)
{
uint64_t ts[] = {0,0,0,0,0,0,0,0,0,0};
arrCopy(s, ts);
uint64_t remain_chunk = chunk_size;
if(current_step+chunk_size>total_size)
remain_chunk = total_size-current_step;
uint64_t tc[] = {1,1,1,1,1,1,1,1,1,1};
arrCopy(c, tc);
do{
uint64_t uc[] = {1,1,1,1,1,1,1,1,1,1};
uint64_t used_chunk = remain_chunk;
remain_chunk = checkBound(v, ts, tc, uc, remain_chunk);
checkOverflow(1, v, ts, uc);
used_chunk -= remain_chunk;
elements_written += used_chunk;
//allocated space for data read-in
data = (void *) malloc(used_chunk*element_size);
if (TIMING==100) {
start_time[1] = MPI_Wtime();
}
if(PERFORMANCE_CHECK) printf("rank=%d, read start\n",rank);
bytes_read = adios_read_var_byid(g,v->varid,ts,uc,data);
if(PERFORMANCE_CHECK) printf("rank=%d, read end\n",rank);
if (TIMING==100) {
end_time[1] = MPI_Wtime();
total_time[1]+=end_time[1] -start_time[1]; //IO read time
}
if (DEBUG)
printf ("ADIOS WRITE: rank=%d, name=%s datasize=%" PRId64 "\n",rank,g->var_namelist[i],bytes_read);
if (TIMING==100) {
start_time[2] = MPI_Wtime();
}
if (DEBUG){
printf("rank=%d, write ts=",rank);
int k;
for(k=0; k<v->ndim; k++)
printf("%" PRId64 ",", ts[k]);
printf(" uc=");
for(k=0; k<v->ndim; k++)
printf("%" PRId64 ",", uc[k]);
printf("\n");
}
//local bounds and offets placeholders are not written out with actual values.
if(PERFORMANCE_CHECK) printf("rank=%d, adios write start\n", rank);
for(k=0; k<v->ndim; k++){
//sprintf(tstring, "ldim%d_%s", k, var_name);
sprintf(tstring, "ldim%d", k);
if (DEBUG) {
printf ("ADIOS WRITE DIMENSION: rank=%d, name=%s value=",
rank,tstring);
print_data (&uc[k], 0, adios_unsigned_long);
printf ("\n");
}
adios_write(m_adios_file, tstring, &uc[k]);
//sprintf(tstring, "offs%d_%s", k, var_name);
sprintf(tstring, "offs%d", k);
if (DEBUG) {
printf ("ADIOS WRITE OFFSET: rank=%d, name=%s value=",
rank,tstring);
print_data (&ts[k], 0, adios_unsigned_long);
printf ("\n");
}
adios_write(m_adios_file, tstring, &ts[k]);
}
adios_write(m_adios_file,g->var_namelist[i],data);
if(PERFORMANCE_CHECK) printf("rank=%d, adios write end\n", rank);
if (TIMING==100) {
end_time[2] = MPI_Wtime();
total_time[2]+=end_time[2] - start_time[2]; //IO write time
}
free(data);
if(remain_chunk!=0){
rS(v, ts, used_chunk, rank);
for(k=0; k<10; k++)
tc[k] = 1;
calcC(remain_chunk, v, tc);
}
}while(remain_chunk!=0);
current_step += size*chunk_size;
if(current_step<total_size)
rS(v, s, size*chunk_size,rank);
}
if(DEBUG){
uint64_t* sb = (uint64_t *) malloc(sizeof(uint64_t));;
uint64_t* rb = (uint64_t *) malloc(sizeof(uint64_t));
sb[0] = elements_written;
MPI_Reduce(sb,rb,1,MPI_UNSIGNED_LONG_LONG,MPI_SUM,0, comm);
if(rank==0 && rb[0]!=total_size)
printf("some array read mismatch. please use debug mode\n");
free(sb); free(rb);
}
}
free (var_name);
free (var_path);
}// end of the writing of the variable..
if (TIMING==100) {
start_time[3] = MPI_Wtime();
}
if(PERFORMANCE_CHECK) printf("rank=%d, adios_close start\n", rank);
adios_close(m_adios_file);
if(PERFORMANCE_CHECK) printf("rank=%d, adios_close end\n", rank);
if (TIMING==100) {
end_time[3] = MPI_Wtime();
total_time[3]+=end_time[3] - start_time[3];
}
adios_gclose(g);
} //end of WRITEME
} // end of all of the groups
if(rank==0)
printf("conversion done!\n");
if(TIMING==100)
start_time[4] = MPI_Wtime();
adios_fclose(f);
if(TIMING==100){
end_time[4] = MPI_Wtime();
total_time[4] = total_time[4]+end_time[4]-start_time[4];
}
adios_finalize(rank);
// now, we write out the timing data, for each category, we give max, min, avg, std, all in seconds, across all processes.
if(TIMING==100){
// 0: adios_open, adios_group_size
// 1: the total time to read in the data
// 2: times around each write (will only work if we do NOT buffer....
// 3: the time in the close
// 4: fopen, fclose
// 5: total time
end_time[5] = MPI_Wtime();
total_time[5] = end_time[5] - start_time[5];
double sb[7];
sb[0] = total_time[1]; sb[1] = total_time[4]; //read_var, fopen+fclose
sb[2] = sb[0]+sb[1];
sb[3] = total_time[0]; sb[4] = total_time[2]+total_time[3]; //adios_open+adios_group_size, write+close
sb[5] = sb[3]+sb[4];
sb[6] = total_time[5]; //total
double * rb = NULL;
if(rank==0)
rb = (double *)malloc(size*7*sizeof(double));
//MPI_Barrier(comm);
MPI_Gather(sb, 7, MPI_DOUBLE, rb, 7, MPI_DOUBLE, 0, comm);
if(rank==0){
double read_avg1 = 0;
double read_avg2 = 0;
double tread_avg = 0;
double write_avg1 = 0;
double write_avg2 = 0;
double twrite_avg = 0;
double total_avg = 0;
for(j=0; j<size; j++){
read_avg1 += rb[7*j];
read_avg2 += rb[7*j+1];
tread_avg += rb[7*j+2];
write_avg1 += rb[7*j+3];
write_avg2 += rb[7*j+4];
twrite_avg += rb[7*j+5];
total_avg += rb[7*j+6];
}
read_avg1 /= size;
read_avg2 /= size;
tread_avg /= size;
write_avg1 /= size;
write_avg2 /= size;
twrite_avg /= size;
total_avg /= size;
double read1_max = rb[0];
double read1_min = rb[0];
double read1_std = rb[0]-read_avg1; read1_std *= read1_std;
double read2_max = rb[1];
double read2_min = rb[1];
double read2_std = rb[1]-read_avg2; read2_std *= read2_std;
double tread_max = rb[2];
double tread_min = rb[2];
double tread_std = rb[2]-tread_avg; tread_std *= tread_std;
double write1_max = rb[3];
double write1_min = rb[3];
double write1_std = rb[3]-write_avg1; write1_std *= write1_std;
double write2_max = rb[4];
double write2_min = rb[4];
double write2_std = rb[4]-write_avg2; write2_std *= write2_std;
double twrite_max = rb[5];
double twrite_min = rb[5];
double twrite_std = rb[5]-twrite_avg; twrite_std *= twrite_std;
double total_max = rb[6];
double total_min = rb[6];
double total_std = rb[6]-total_avg; total_std *= total_std;
for(j=1; j<size; j++){
if(rb[7*j]>read1_max)
read1_max = rb[7*j];
else if(rb[7*j]<read1_min)
read1_min = rb[7*j];
double std = rb[7*j]-read_avg1; std *= std;
read1_std += std;
if(rb[7*j+1]>read2_max)
read2_max = rb[7*j+1];
else if(rb[7*j+1]<read2_min)
read2_min = rb[7*j+1];
std = rb[7*j+1]-read_avg2; std *= std;
read2_std += std;
if(rb[7*j+2]>tread_max)
tread_max = rb[7*j+2];
else if(rb[7*j+2]<tread_min)
tread_min = rb[7*j+2];
std = rb[7*j+2]-tread_avg; std *= std;
tread_std += std;
if(rb[7*j+3]>write1_max)
write1_max = rb[7*j+3];
else if(rb[7*j+3]<write1_min)
write1_min = rb[7*j+3];
std = rb[7*j+3]-write_avg1; std *= std;
write1_std += std;
if(rb[7*j+4]>write2_max)
write2_max = rb[7*j+4];
else if(rb[7*j+4]<write2_min)
write2_min = rb[7*j+4];
std = rb[7*j+4]-write_avg2; std *= std;
write2_std += std;
if(rb[7*j+5]>twrite_max)
twrite_max = rb[7*j+5];
else if(rb[7*j+5]<twrite_min)
twrite_min = rb[7*j+5];
std = rb[7*j+5]-twrite_avg; std *= std;
twrite_std += std;
if(rb[7*j+6]>total_max)
total_max = rb[7*j+6];
else if(rb[7*j+6]<total_min)
total_min = rb[7*j+6];
std = rb[7*j+6]-total_avg; std *= std;
total_std += std;
}
read1_std /= size; read1_std = sqrt(read1_std);
read2_std /= size; read2_std = sqrt(read2_std);
tread_std /= size; tread_std = sqrt(tread_std);
write1_std /= size; write1_std = sqrt(write1_std);
write2_std /= size; write2_std = sqrt(write2_std);
twrite_std /= size; twrite_std = sqrt(twrite_std);
total_std /= size; total_std = sqrt(total_std);
printf("---type--- max\tmin\tavg\tstd\n");
printf("---read_var--- %lf\t%lf\t%lf\t%lf\n", read1_max, read1_min, read_avg1, read1_std);
printf("---fopen+fclose--- %lf\t%lf\t%lf\t%lf\n", read2_max, read2_min, read_avg2, read2_std);
printf("---total_read--- %lf\t%lf\t%lf\t%lf\n", tread_max, tread_min, tread_avg, tread_std);
printf("---adios_open+adios_groupsize--- %lf\t%lf\t%lf\t%lf\n", write1_max, write1_min, write_avg1, write1_std);
printf("---write+close--- %lf\t%lf\t%lf\t%lf\n", write2_max, write2_min, write_avg2, write2_std);
printf("---total_write--- %lf\t%lf\t%lf\t%lf\n", twrite_max, twrite_min, twrite_avg, twrite_std);
printf("---total--- %lf\t%lf\t%lf\t%lf\n", total_max, total_min, total_avg, total_std);
free(rb);
}
}
// if (TIMING==100 && rank==0) {
// printf("------------------------------------------------------------------\n");
// printf("Define variables = %lf\n",total_time[0]);
// printf("Read variables = %lf\n",total_time[1]);
// printf("Write variables = %lf\n",total_time[2]);
// printf("Close File for write = %lf\n",total_time[3]);
// printf("Total write time = %lf\n",total_time[2] + total_time[3]);
// for (itime=0;itime<timers-1;itime++)
// total_time[timers-1]+=total_time[itime];
// printf("Total I/O time = %lf\n",total_time[timers-1]);
// }
MPI_Finalize();
return(0);
}
int main (int argc, char ** argv)
{
int i, j, k,l;
MPI_Comm comm_dummy = 0; /* MPI_Comm is defined through adios_read.h */
if (argc < 2) {
printf("Usage: %s <BP-file>\n", argv[0]);
return 1;
}
adios_read_init_method (ADIOS_READ_METHOD_BP, comm_dummy, "show_hidden_attrs");
ADIOS_FILE * f;
f = adios_read_open_file (argv[1], ADIOS_READ_METHOD_BP, comm_dummy);
if (f == NULL) {
printf ("%s\n", adios_errmsg());
return -1;
}
/* For all variables */
printf(" Variables=%d:\n", f->nvars);
for (i = 0; i < f->nvars; i++) {
ADIOS_VARINFO * v = adios_inq_var_byid (f, i);
adios_inq_var_stat (f, v, 0, 1);
adios_inq_var_blockinfo (f, v);
uint64_t total_size = adios_type_size (v->type, v->value);
for (j = 0; j < v->ndim; j++)
total_size *= v->dims[j];
printf(" %-9s %s", adios_type_to_string(v->type), f->var_namelist[i]);
if (v->ndim == 0) {
/* Scalars do not need to be read in, we get it from the metadata
when using adios_inq_var */
printf(" = %s\n", value_to_string(v->type, v->value, 0));
} else {
/* Arrays, print min/max statistics*/
printf("[%lld",v->dims[0]);
for (j = 1; j < v->ndim; j++)
printf(", %lld",v->dims[j]);
//printf("] = \n");
if (v->type == adios_integer)
printf("] : min=%d max=%d\n",
(*(int*)v->statistics->min), (*(int*)v->statistics->max));
else if (v->type == adios_double)
printf("] : min=%lg max=%lg\n",
(*(double*)v->statistics->min), (*(double*)v->statistics->max));
/* Print block info */
for (l=0; l<v->nsteps; l++) {
printf(" step %3d: \n", l);
for (j=0; j<v->nblocks[l]; j++) {
printf(" block %3d: [", j);
for (k=0; k<v->ndim; k++) {
printf("%3lld:%3lld", v->blockinfo[j].start[k],
v->blockinfo[j].start[k]+v->blockinfo[j].count[k]-1);
if (k<v->ndim-1)
printf(", ");
}
printf("]\n");
}
}
}
adios_free_varinfo (v);
} /* variables */
/* For all attributes */
printf(" Attributes=%d:\n", f->nattrs);
for (i = 0; i < f->nattrs; i++) {
enum ADIOS_DATATYPES atype;
int asize;
void *adata;
adios_get_attr_byid (f, i, &atype, &asize, &adata);
printf(" %-9s %s = %s\n", adios_type_to_string(atype),
f->attr_namelist[i], value_to_string(atype, adata, 0));
free(adata);
} /* attributes */
adios_read_close (f);
return 0;
}
int main (int argc, char ** argv)
{
int rank, size, i, j;
MPI_Comm comm = MPI_COMM_WORLD;
enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_BP;
MPI_Init (&argc, &argv);
MPI_Comm_rank (comm, &rank);
MPI_Comm_size (comm, &size);
adios_read_init_method (method, comm, "verbose=3");
ADIOS_FILE * f = adios_read_open_file("adios_stat.bp", method, comm);
if (f == NULL)
{
fprintf (stderr, "%s\n", adios_errmsg());
return -1;
}
ADIOS_VARINFO * v = adios_inq_var (f, "temperature");
if (v) {
/* get statistics, for each individual time-step */
adios_inq_var_stat (f, v, 1, 0);
printf("Global MIN of temperature: %lf\n", * (double *) v->statistics->min);
printf("Global MAX of temperature: %lf\n", * (double *) v->statistics->max);
printf("Global AVG of temperature: %lf\n", * (double *) v->statistics->avg);
printf("Global STD DEV of temperature: %lf\n", * (double *) v->statistics->std_dev);
printf("\n");
printf("---------------------------------------------------------------------------\n");
if (v->statistics->steps) {
printf("MIN\t\tMAX\t\tAVG\t\tSTD_DEV\t\tHISTOGRAM\n");
for(i = 0; i < v->nsteps; i++)
{
if (v->statistics->steps->mins[i])
printf("%lf\t", * (double *) v->statistics->steps->mins[i]);
else
printf("--\t\t");
if (v->statistics->steps->maxs[i])
printf("%lf\t", * (double *) v->statistics->steps->maxs[i]);
else
printf("--\t\t");
if (v->statistics->steps->avgs[i])
printf("%lf\t", * (double *) v->statistics->steps->avgs[i]);
else
printf("--\t\t");
if (v->statistics->steps->std_devs[i])
printf("%lf\t", * (double *) v->statistics->steps->std_devs[i]);
else
printf("--\t\t");
if (v->statistics->histogram) {
for(j = 0; j <= v->statistics->histogram->num_breaks; j++) {
printf("%d ", v->statistics->histogram->frequencies[i][j]);
}
printf("\n");
} else {
printf("--\t\t");
}
printf("\n");
}
} else {
printf ("Per step statistics is missing\n");
}
printf("---------------------------------------------------------------------------\n");
printf("\n");
if (v->statistics->histogram) {
printf("Break points:\t\t\t");
for(j = 0; j < v->statistics->histogram->num_breaks; j++)
printf("%6.2lf\t", v->statistics->histogram->breaks[j]);
printf("\n");
printf("Frequencies:\t\t\t");
for(j = 0; j <= v->statistics->histogram->num_breaks; j++)
printf("%6d\t", v->statistics->histogram->gfrequencies[j]);
}
printf("\n\n");
#if 0
printf ("Auto covariance of MIN values of temperature over time: %lf\n", adios_stat_cov (v, v, "min", 0, 12, 0));
printf ("Auto correlation of MAX values of temperature over time, with lag 2 units: %lf\n", adios_stat_cor (v, v, "max", 0, 8, 2));
printf("\n\n");
#endif
adios_free_varinfo (v);
} else {
fprintf (stderr, "ERROR: Cannot inquire statistics of variable 'temperature': %s\n", adios_errmsg());
}
v = adios_inq_var (f, "complex");
if (v) {
adios_inq_var_stat (f, v, 1, 0);
double *C = v->statistics->min;
printf("Global Minimum of variable complex - Magnitude: %lf\n", C[0]);
printf("Global Minimum of variable complex - Real part: %lf\n", C[1]);
printf("Global Minimum of variable complex - Imaginary part: %lfi\n", C[2]);
double ** Cmin;
Cmin = (double **) v->statistics->steps->mins;
printf("\nMagnitude\t\tReal\t\t\tImaginary\n");
for (j = 0; j < v->nsteps; j++)
{
printf ("%lf\t\t%lf\t\t%lf\n", Cmin[j][0], Cmin[j][1], Cmin[j][2]);
}
printf("\n");
adios_free_varinfo (v);
} else {
fprintf (stderr, "ERROR: Cannot inquire statistics of variable 'complex': %s\n", adios_errmsg());
}
adios_read_close (f);
MPI_Barrier (comm);
adios_read_finalize_method (method);
MPI_Finalize ();
return 0;
}
int main (int argc, char ** argv)
{
int gidx, i, j, k,l;
MPI_Comm comm_dummy = 0; /* MPI_Comm is defined through adios_read.h */
void * data = NULL;
uint64_t start[] = {0,0,0,0,0,0,0,0,0,0};
uint64_t count[10];
int64_t bytes_read = 0;
if (argc < 2) {
printf("Usage: %s <BP-file>\n", argv[0]);
return 1;
}
ADIOS_FILE * f;
//int step;
//for (step=0; step < 2; step++) {
f = adios_fopen (argv[1], comm_dummy);
if (f == NULL) {
printf ("%s\n", adios_errmsg());
return -1;
}
/* For all groups */
for (gidx = 0; gidx < f->groups_count; gidx++) {
printf("Group %s:\n", f->group_namelist[gidx]);
ADIOS_GROUP * g = adios_gopen (f, f->group_namelist[gidx]);
if (g == NULL) {
printf ("%s\n", adios_errmsg());
return -1;
}
/* For all variables */
printf(" Variables=%d:\n", g->vars_count);
for (i = 0; i < g->vars_count; i++) {
ADIOS_VARINFO * v = adios_inq_var_byid (g, i);
uint64_t total_size = adios_type_size (v->type, v->value);
for (j = 0; j < v->ndim; j++)
total_size *= v->dims[j];
printf(" %-9s %s", adios_type_to_string(v->type), g->var_namelist[i]);
if (v->ndim == 0) {
/* Scalars do not need to be read in, we get it from the metadata
when using adios_inq_var */
printf(" = %s\n", value_to_string(v->type, v->value, 0));
} else {
/* Arrays have to be read in from the file */
printf("[%" PRIu64,v->dims[0]);
for (j = 1; j < v->ndim; j++)
printf(", %" PRIu64,v->dims[j]);
//printf("] = \n");
if (v->type == adios_integer)
printf("] = min=%d max=%d timedim=%d\n", (*(int*)v->gmin), (*(int*)v->gmax), v->timedim);
else if (v->type == adios_double)
printf("] = min=%lg max=%lg timedim=%d\n", (*(double*)v->gmin), (*(double*)v->gmax), v->timedim);
if (total_size > 1024*1024*1024) {
printf(" // too big, do not read in\n");
} else {
data = malloc (total_size);
if (data == NULL) {
fprintf (stderr, "malloc failed.\n");
return -1;
}
for (j = 0; j < v->ndim; j++)
count[j] = v->dims[j];
bytes_read = adios_read_var_byid (g, i, start, count, data);
if (bytes_read < 0) {
printf ("%s\n", adios_errmsg());
} else if (bytes_read > 1024*1024) {
printf ("Too big to print\n");
} else if (v->ndim == 1) {
printf (" [");
for (j = 0; j < v->dims[0]; j++)
printf("%s ", value_to_string(v->type, data, j));
printf ("]\n");
} else if (v->ndim == 2) {
for (j = 0; j < v->dims[0]; j++) {
printf (" row %d: [", j);
for (k = 0; k < v->dims[1]; k++)
printf("%s ", value_to_string(v->type, data, j*v->dims[1] + k));
printf ("]\n");
}
} else if (v->ndim == 3) {
for (j = 0; j < v->dims[0]; j++) {
printf (" block %d: \n", j);
for (k = 0; k < v->dims[1]; k++) {
printf (" row %d: [", k);
for (l = 0; l < v->dims[2]; l++) {
printf("%s ", value_to_string(v->type, data, j*v->dims[1]*v->dims[2] + k*v->dims[1] + l));
}
printf ("]\n");
}
printf ("\n");
}
} else {
printf (" cannot print arrays with >3 dimensions\n");
}
free (data);
}
}
adios_free_varinfo (v);
} /* variables */
/* For all attributes */
printf(" Attributes=%d:\n", g->attrs_count);
for (i = 0; i < g->attrs_count; i++) {
enum ADIOS_DATATYPES atype;
int asize;
void *adata;
adios_get_attr_byid (g, i, &atype, &asize, &adata);
printf(" %-9s %s = %s\n", adios_type_to_string(atype),
g->attr_namelist[i], value_to_string(atype, adata, 0));
free(adata);
} /* attributes */
adios_gclose (g);
} /* groups */
adios_fclose (f);
//} /* loop 'step' */
return 0;
}
int read_points ()
{
ADIOS_SELECTION *sel0,*sel1,*sel2,*sel3;
ADIOS_FILE * f;
ADIOS_VARINFO * vi;
int err=0,n,n1, i,j,k;
int nsteps_a, nsteps_b, nsteps_c;
int v;
uint64_t *pts1;
uint64_t *pts2;
uint64_t *pts3;
uint64_t start[3] = {offs1,offs2,offs3};
uint64_t count[3] = {ldim1,ldim2,ldim3};
pts1 = (uint64_t*) malloc (1*sizeof(uint64_t)*ldim1);
pts2 = (uint64_t*) malloc (2*sizeof(uint64_t)*ldim1*ldim2);
pts3 = (uint64_t*) malloc (3*sizeof(uint64_t)*ldim1*ldim2*ldim3);
for (i=0; i<ldim1; i++)
{
pts1[i] = offs1 + i;
for (j=0; j<ldim2; j++)
{
pts2[i*2*ldim2 + 2*j] = offs1 + i;
pts2[i*2*ldim2 + 2*j+1] = j;
for (k=0; k<ldim3; k++)
{
pts3[i*3*ldim2*ldim3+j*3*ldim3+3*k] = offs1 + i;
pts3[i*3*ldim2*ldim3+j*3*ldim3+3*k+1] = j;
pts3[i*3*ldim2*ldim3+j*3*ldim3+3*k+2] = k;
}
}
}
//Force error on very last point: pts3[3*ldim1*ldim2*ldim3-1] = 0;
reset_readvars();
log ("Read and check data in %s using point selections\n", FILENAME);
f = adios_read_open (FILENAME, read_method, comm,
ADIOS_LOCKMODE_CURRENT, 0.0);
if (f == NULL) {
printE ("Error at opening file: %s\n", rank, adios_errmsg());
return 1;
}
sel1 = adios_selection_points (1, ldim1, pts1);
sel2 = adios_selection_points (2, ldim1*ldim2, pts2);
sel3 = adios_selection_points (3, ldim1*ldim2*ldim3, pts3);
#ifdef DEBUG_PRINTS
fprintf(stderr, "1D selection: {");
for (i=0; i<ldim1; i++) {
fprintf (stderr, "%d ", i, i, pts1[i]);
}
fprintf(stderr, "}\n");
fprintf(stderr, "2D selection :\n");
for (i=0; i<ldim1; i++) {
fprintf (stderr, "i=%d, idx=%2d- = ", i, i*2*ldim2);
for (j=0; j<ldim2; j++) {
n = i*2*ldim2+2*j;
fprintf (stderr, "{%d,%d} ", pts2[n], pts2[n+1]);
}
fprintf(stderr, "\n");
}
fprintf(stderr, "3D selection :\n");
for (i=0; i<ldim1; i++) {
for (j=0; j<ldim2; j++) {
fprintf (stderr, "i=%d, j=%d, idx=%3d- = ", i, j, i*3*ldim2*ldim3+j*3*ldim3);
for (k=0; k<ldim3; k++) {
n = i*3*ldim2*ldim3+j*3*ldim3+3*k;
fprintf (stderr, "{%d,%d,%d} ", pts3[n], pts3[n+1], pts3[n+2]);
}
fprintf(stderr, "\n");
}
}
#endif
n1=0;
while (n1 < NSTEPS && adios_errno != err_end_of_stream) {
n1++;
log (" Step %d\n", f->current_step);
log (" Check 1D variable a1...\n");
adios_schedule_read (f, sel1, "a1", 0, 1, r1);
adios_perform_reads (f, 1);
#ifdef DEBUG_PRINTS
fprintf(stderr, "1D result: {");
for (i=0; i<ldim1; i++) {
fprintf (stderr, "%d ", i, i, r1[i]);
}
fprintf(stderr, "}\n");
#endif
for (i=0; i<ldim1; i++) {
v = VALUE1D(rank,f->current_step,i);
if (r1[i] != v) {
printE ("Error: a1[%d]=%d != read=%d\n", i, v, r1[i]);
//goto endread;
}
}
log (" Check 2D variable a2...\n");
adios_schedule_read (f, sel2, "a2", 0, 1, r2);
adios_perform_reads (f, 1);
#ifdef DEBUG_PRINTS
fprintf(stderr, "2D result :\n");
n = 0;
for (i=0; i<ldim1; i++) {
fprintf (stderr, "row=%2d- = {", i);
for (j=0; j<ldim2; j++) {
fprintf (stderr, "%d ", r2[n]);
n++;
}
fprintf(stderr, "}\n");
}
#endif
n = 0;
for (i=0; i<ldim1; i++) {
for (j=0; j<ldim2; j++) {
v = VALUE2D(rank,f->current_step,i,j);
if (v != r2[n]) {
printE ("Error: a2[%d,%d]=%d != read=%d\n", i, j, v, r2[n]);
//goto endread;
}
n++;
}
}
log (" Check 3D variable a3...\n");
adios_schedule_read (f, sel3, "a3", 0, 1, r3);
adios_perform_reads (f, 1);
#ifdef DEBUG_PRINTS
fprintf(stderr, "3D selection :\n");
n=0;
for (i=0; i<ldim1; i++) {
for (j=0; j<ldim2; j++) {
fprintf (stderr, "[%d,%d] = {", i, j);
for (k=0; k<ldim3; k++) {
fprintf (stderr, "%d ", r3[n]);
n++;
}
fprintf(stderr, "}\n");
}
}
#endif
n = 0;
for (i=0; i<ldim1; i++) {
for (j=0; j<ldim2; j++) {
for (k=0; k<ldim3; k++) {
v = VALUE3D(rank,f->current_step,i,j,k);
if (v != r3[n]) {
printE ("Error: a3[%d,%d,%d]=%d != read=%d\n", i, j, k, v, r3[n]);
//goto endread;
}
n++;
}
}
}
if (n1 < NSTEPS)
{
adios_advance_step (f, 0, -1.0);
}
}
endread:
adios_selection_delete (sel1);
adios_selection_delete (sel2);
adios_selection_delete (sel3);
free(pts1);
free(pts2);
free(pts3);
adios_read_close(f);
MPI_Barrier (comm);
return err;
}
int main (int argc, char** argv)
{
fastbit_init(0);
fastbit_set_verbose_level(0);
ADIOS_FILE * f;
//MPI_Comm comm_dummy = 0; // MPI_Comm is defined through adios_read.h
MPI_Comm comm_dummy = MPI_COMM_WORLD;
int rank, size;
MPI_Init (&argc, &argv);
MPI_Comm_rank (comm_dummy, &rank);
MPI_Comm_size (comm_dummy, &size);
adios_init_noxml (comm_dummy);
if (argc < 2) {
printf("Usage: index_fastbit fileName (attrName)");
return 0;
}
f = adios_read_open_file (argv[1], ADIOS_READ_METHOD_BP, comm_dummy);
if (f == NULL) {
printf ("::%s\n", adios_errmsg());
return -1;
}
/*
adios_allocate_buffer (ADIOS_BUFFER_ALLOC_NOW, (f->file_size)*2/1048576 + 5); // +5MB for extra room in buffer
adios_declare_group (&gAdios_group, gGroupNameFastbitIdx, "", adios_flag_yes);
adios_select_method (gAdios_group, "MPI", "", "");
*/
gIdxFileName = fastbit_adios_util_getFastbitIndexFileName(argv[1]);
unlink(gIdxFileName);
adios_allocate_buffer (ADIOS_BUFFER_ALLOC_NOW, 500); // +5MB for extra room in buffer
adios_declare_group (&gAdios_group, gGroupNameFastbitIdx, "", adios_flag_yes);
adios_select_method (gAdios_group, "MPI", "", "");
adios_open (&gAdios_write_file, gGroupNameFastbitIdx, gIdxFileName, "w", MPI_COMM_WORLD);
#ifdef MULTI_BLOCK
int testid = adios_define_var (gAdios_group, "pack", "", adios_integer , 0, 0, 0);
#endif
#ifdef BOX
int testid = adios_define_var (gAdios_group, "elements", "", adios_integer , 0, 0, 0);
#endif
//uint64_t estimatedbytes = (nb+nk+no)*adios_type_size(adios_double, NULL);
int jobCounter = getJobCounter(f);
uint64_t estimatedbytes = getByteEstimationOnFile(f, rank);
if (size > 1) {
int maxJobsPP = jobCounter/size + 1;
estimatedbytes = estimatedbytes * maxJobsPP /jobCounter +1048576;
}
estimatedbytes += 1048576;
uint64_t adios_totalsize; // adios_group_size needs to be call before any write_byid, Otherwise write_byid does nothing
adios_group_size (gAdios_write_file, estimatedbytes , &adios_totalsize);
printf("=> .. adios open output file: %s, rank %d allocated %" PRIu64 " bytes... \n", gIdxFileName, rank, adios_totalsize);
// IMPORTANT:
// can only call open/close once in a process
// otherwise data is tangled or only the data in the last open/close call is recorded
#ifdef MULTI_BLOCK
adios_write_byid(gAdios_write_file, testid, &pack);
#endif
#ifdef BOX
adios_write_byid(gAdios_write_file, testid, &recommended_index_ele);
#endif
sumLogTime(-1);
sumLogTimeMillis(-1);
if (argc >= 3) {
int i=2;
while (i<argc) {
const char* varName = argv[i];
if(strstr(varName, "<binning prec") != NULL) {
if (gBinningOption == NULL) {
gBinningOption = argv[i];
}
if (argc == 3) {
buildIndexOnAllVar(f, rank, size);
break;
}
i++;
continue;
} else {
ADIOS_VARINFO * v = adios_inq_var(f, varName);
if (v == NULL) {
printf("No such variable: %s\n", varName);
return 0;
}
printf("building fastbit index on variable: %s\n", varName);
buildIndex_mpi(f, v, rank, size);
adios_free_varinfo(v);
i++;
}
}
} else {
buildIndexOnAllVar(f, rank, size);
}
sumLogTime(0);
sumLogTimeMillis(0);
adios_close(gAdios_write_file);
adios_read_close(f);
//
// writing file clean up
//
// read back:
f = adios_read_open_file (gIdxFileName, ADIOS_READ_METHOD_BP, comm_dummy);
if (f == NULL) {
printf("No such file: %s \n", gIdxFileName);
return 0;
}
int numVars = f->nvars;
int i=0;
int k=0;
int j=0;
for (i=0; i<numVars; i++) {
char* varName = f->var_namelist[i];
ADIOS_VARINFO* v = adios_inq_var(f, varName);
adios_inq_var_blockinfo(f,v);
int timestep = 0;
for (k=0; k<v->sum_nblocks; k++) {
verifyData(f, v, k, timestep);
}
adios_free_varinfo(v);
}
adios_read_close(f);
if (rank == 0) {
printf(" ==> index file is at: %s\n", gIdxFileName);
}
// clean up
MPI_Barrier (comm_dummy);
adios_finalize (rank);
MPI_Finalize ();
free (gIdxFileName);
fastbit_cleanup();
return 0;
}
