void ToolsAdiosParallel::listAvailableDatasets()
{
ADIOS_VARINFO *pVarInfo;
//available data sets in this file
m_outStream << "Number of available data sets: ";
m_outStream << pFile->nvars << std::endl;
for(int i = 0; i < pFile->nvars; i++)
{
m_outStream << pFile->var_namelist[i] << " ";
pVarInfo = adios_inq_var(pFile, pFile->var_namelist[i]);
if(pVarInfo->ndim > 0)
{
// print number of elements per dimension to m_outstream
m_outStream << "[";
for(int j = 0; j < pVarInfo->ndim; j++)
{
m_outStream << pVarInfo->dims[j];
if(j < pVarInfo->ndim-1)
{
m_outStream << ",";
}
}
m_outStream << "]" << std::endl;
}
else
{
m_outStream << "[scalar]" << std::endl;
}
adios_free_varinfo(pVarInfo);
}
}
int64_t getByteEstimation(ADIOS_FILE* f, int rank, int argc, char** argv)
{
uint64_t bytes = 0;
if (argc >= 3) {
int i=2;
while (i<argc) {
const char* varName = argv[i];
if(strstr(varName, "<binning prec") != NULL) {
if (gBinningOption == NULL) {
gBinningOption = argv[i];
}
if (argc == 3) {
return getByteEstimationOnFile(f, rank);
}
i++;
continue;
}
ADIOS_VARINFO * v = adios_inq_var(f, varName);
if (v == NULL) {
printf("Invalid variable: %s\n", varName);
return -1;
}
uint64_t varSize = estimateBytesOnVar(f, v);
printf(" var: %s has size: %" PRId64 "\n", varName, varSize);
bytes += varSize;
adios_free_varinfo(v);
i++;
}
return bytes;
} else {
return getByteEstimationOnFile(f,rank);
}
}
void adios_checkpoint_verify_variables(ADIOS_FILE* fp, const char* name, int* origin)
{
ADIOS_VARINFO *vi;
int count = 1;
int size;
vi = adios_inq_var(fp, name);
if (vi->ndim > 0)
{
cout<<name<<" verification not passed, not a scalar"<<endl;
return;
}
size = count*adios_type_size(vi->type, vi->value);
int * mem= (int * )malloc(size);
ADIOS_SELECTION *sel = adios_selection_writeblock(OHMMS::Controller->rank());
adios_schedule_read(fp, sel, name, 0, 1, mem);
//adios_schedule_read(fp, NULL, name, 0, 1, mem);
adios_perform_reads(fp, 1);
if(mem[0] == *origin)
{
cout<<name<<" verification passed "<<mem[0]<<endl;
}
else
{
cout<<name<<" verification not passed, readin: "<<mem[0]<<" writeout: "<<*origin<<endl;
}
adios_free_varinfo (vi);
adios_selection_delete(sel);
free(mem);
}
int getJobCounter(ADIOS_FILE* f)
{
int numVars = f->nvars;
int counter = 0;
int i=0, k=0;
for (i=0; i<numVars; i++) {
char* varName = f->var_namelist[i];
ADIOS_VARINFO* v = adios_inq_var(f, varName);
if (v->ndim == 0) {
continue;
}
for (k=0; k<v->nsteps; k++) {
#ifdef MULTI_BOX
int nblocks = v->nblocks[k];
printf("var = %s, timestep = %ld, nblocks=%ld\n", varName, k, nblocks);
int remainder = nblocks % pack;
counter += nblocks/pack;
#endif
#ifdef BOX
int j=0;
uint64_t totalElements =1;
for (j=0; j<v->ndim; j++) {
totalElements *= v->dims[j];
}
int remainder = totalElements % recommended_index_ele;
counter += totalElements/recommended_index_ele;
#endif
if (remainder > 0) {
counter += 1;
}
}
}
return counter;
}
void adios_checkpoint_verify_variables(ADIOS_FILE* fp, const char* name, RealType* origin)
{
ADIOS_VARINFO *vi;
int count = 1;
int size;
vi = adios_inq_var(fp, name);
adios_inq_var_blockinfo (fp, vi);
if (vi->ndim > 0)
{
count*=vi->dims[0];
for (int j = 1; j < vi->ndim; j++)
{
count *= vi->dims[j];
}
}
size = count*adios_type_size(vi->type, vi->value);
RealType *mem= (RealType *)malloc(size);
adios_schedule_read(fp, NULL, name, 0, 1, mem);
adios_perform_reads(fp, 1);
for(int i=0; i<count; i++)
{
if(mem[i] == origin[i])
{
cout<<name<<"["<<i<<"]verification passed "<<mem[i]<<endl;
}
else
{
cout<<name<<"["<<i<<"]verification not passed, readin: "<<mem[i]<<" writeout: "<<origin[i]<<endl;
}
}
adios_free_varinfo (vi);
free(mem);
}
void adios_checkpoint_verify_variables(ADIOS_FILE* fp, const char* name, unsigned long origin)
{
ADIOS_VARINFO *vi;
int count = 1;
int size;
vi = adios_inq_var(fp, name);
if (vi->ndim > 0)
{
cout<<name<<" verification not passed, not a scalar"<<endl;
return;
}
size = count*adios_type_size(vi->type, vi->value);
unsigned long* mem= (unsigned long* )malloc(size);
adios_schedule_read(fp, NULL, name, 0, 1, mem);
adios_perform_reads(fp, 1);
if(mem[0] == origin)
{
cout<<name<<" verification passed"<<mem[0]<<endl;
}
else
{
cout<<name<<" verification not passed, readin: "<<mem[0]<<" writeout: "<<origin<<endl;
}
adios_free_varinfo (vi);
free(mem);
}
/// Constructor.
ADIOS_Var::ADIOS_Var(ADIOS_File* afile, const char* vn)
: m_file(afile), m_name(vn), m_handle(0)
{
if (afile != 0 && afile->getHandle() != 0)
m_handle = adios_inq_var(afile->getHandle(), vn);
LOGGER(ibis::gVerbose > 8)
<< "adios_inq_var is called for " << m_name << ": m_handle="
<< m_handle;
}
std::string AdiosCheckpointInput::getScalar(const std::string& var_name)
{
//Scalars are all stored in the metadata so we can read them without disk access
ADIOS_VARINFO* adios_inq = adios_inq_var(adios_file_handle, var_name.c_str());
int string_size = adios_type_size(adios_inq->type, adios_inq->value);
std::string string_val = static_cast<char*>(*adios_inq->value);
adios_free_varinfo(adios_inq);
return string_val;
}
void adios_checkpoint_verify_random_variables(ADIOS_FILE* fp, const char* name, uint_type* origin)
{
ADIOS_VARINFO *vi;
int count_int = 1;
int size;
uint64_t *start;
uint64_t *count;
vi = adios_inq_var(fp, name);
adios_inq_var_blockinfo (fp, vi);
if (vi->ndim > 0)
{
start = (uint64_t *)malloc(vi->ndim * sizeof(uint64_t));
count = (uint64_t *)malloc(vi->ndim * sizeof(uint64_t));
}
for (int j=0; j<vi->nblocks[0]; j++)
{
if(j == OHMMS::Controller->rank())
{
for (int k=0; k<vi->ndim; k++)
{
start[k] = vi->blockinfo[j].start[k];
count[k] = vi->blockinfo[j].count[k];
count_int *= count[k];
//cout<<OHMMS::Controller->rank()<<" count "<<start[k]<<" "<<count[k]<<endl;
}
}
}
size = count_int*adios_type_size(vi->type, vi->value);
uint_type *mem= (uint_type*)malloc(size);
ADIOS_SELECTION *sel = adios_selection_boundingbox(vi->ndim, start, count);
adios_schedule_read(fp, sel, name, 0, 1, mem);
adios_perform_reads(fp, 1);
int flag = 0;
for(int i=0; i<count_int; i++)
{
if(mem[i] == origin[i])
{
//cout<<name<<"["<<i<<"]verification passed "<<mem[i]<<endl;
}
else
{
flag = 1;
cout<<name<<"["<<i<<"]verification not passed, readin: "<<mem[i]<<" writeout: "<<origin[i]<<endl;
}
}
if (flag == 0) cout<<name<<" verification passed "<<endl;
else cout<<name<<" verification not passed "<<endl;
adios_free_varinfo (vi);
adios_selection_delete(sel);
free(start);
free(count);
free(mem);
}
void ADIOS1CommonRead::ScheduleReadCommon(const std::string &name,
const Dims &offs, const Dims &ldims,
const int fromStep, const int nSteps,
const bool readAsLocalValue,
const bool readAsJoinedArray,
void *data)
{
if (readAsLocalValue)
{
/* Get all the requested values from metadata now */
ADIOS_VARINFO *vi = adios_inq_var(m_fh, name.c_str());
if (vi)
{
adios_inq_var_stat(m_fh, vi, 0, 1);
int elemsize = adios_type_size(vi->type, nullptr);
long long blockidx = 0;
for (int i = 0; i < fromStep; i++)
{
blockidx += vi->nblocks[i];
}
char *dest = (char *)data;
for (int i = fromStep; i < fromStep + nSteps; i++)
{
for (int j = 0; j < vi->nblocks[i]; j++)
{
memcpy(dest, vi->statistics->blocks->mins[blockidx],
elemsize);
++blockidx;
dest += elemsize;
}
}
adios_free_varinfo(vi);
}
}
else
{
uint64_t start[32], count[32];
for (int i = 0; i < ldims.size(); i++)
{
start[i] = (uint64_t)offs[i];
count[i] = (uint64_t)ldims[i];
}
ADIOS_SELECTION *sel = nullptr;
if (ldims.size() > 0)
{
sel = adios_selection_boundingbox(ldims.size(), start, count);
}
adios_schedule_read(m_fh, sel, name.c_str(), (int)fromStep, (int)nSteps,
data);
adios_selection_delete(sel);
}
}
T AdiosCheckpointInput::getScalar(const std::string& var_name)
{
//Scalars are all stored in the metadata so we can read them without disk access
ADIOS_VARINFO* adios_inq = adios_inq_var(adios_file_handle, var_name.c_str());
if (adios_type_size(adios_inq->type, adios_inq->value) != sizeof(T))
{
qmcplusplus::app_error() << "Data type does not match data type found in file: " << std::endl;
return ;
}
T scalar_value = static_cast<T>(*adios_inq->value);
adios_free_varinfo(adios_inq);
return scalar_value;
}
void ToolsAdiosParallel::convertToText()
{
if(m_options.data.size() == 0)
throw std::runtime_error("No datasets requested");
for (size_t i = 0; i < m_options.data.size(); ++i)
{
ADIOS_VARINFO *pVarInfo;
//get name of dataset to print
std::string nodeName = m_options.data[i];
uint8_t *P;
int varElement = 1;
int varTypeSize = 0;
adios_read_init_method(ADIOS_READ_METHOD_BP, comm, nodeName.c_str());
pVarInfo = adios_inq_var(pFile, nodeName.c_str());
varTypeSize = adios_type_size(pVarInfo->type, NULL);
// get number of elements combined in a dataset
for(int j = 0; j < pVarInfo->ndim; j++)
{
varElement = varElement * pVarInfo->dims[j];
}
// allocate memory
P = (uint8_t*) malloc (sizeof(uint8_t) * varTypeSize * varElement);
adios_schedule_read(pFile, NULL, nodeName.c_str(), 0, 1, P);
adios_perform_reads(pFile, 1);
if(pVarInfo->ndim > 0)
{
for(int k = 0; k < varElement; k++)
{
printValue(pVarInfo->type, &P[k*varTypeSize]);
}
}
else
{
printValue(pVarInfo->type, pVarInfo->value);
}
adios_free_varinfo(pVarInfo);
}
}
/** Read the skalar field and optionally the attribute into the values referenced by the pointers */
void operator()(ThreadParams& params,
const std::string& name, T_Scalar* value,
const std::string& attrName = "", T_Attribute* attribute = nullptr)
{
log<picLog::INPUT_OUTPUT> ("ADIOS: read %1%D scalars: %2%") % simDim % name;
std::string datasetName = params.adiosBasePath + name;
ADIOS_VARINFO* varInfo;
ADIOS_CMD_EXPECT_NONNULL( varInfo = adios_inq_var(params.fp, datasetName.c_str()) );
if(varInfo->ndim != simDim)
throw std::runtime_error(std::string("Invalid dimensionality for ") + name);
if(varInfo->type != traits::PICToAdios<T_Scalar>().type)
throw std::runtime_error(std::string("Invalid type for ") + name);
DataSpace<simDim> gridPos = Environment<simDim>::get().GridController().getPosition();
uint64_t start[varInfo->ndim];
uint64_t count[varInfo->ndim];
for(int d = 0; d < varInfo->ndim; ++d)
{
/* \see adios_define_var: z,y,x in C-order */
start[d] = gridPos.revert()[d];
count[d] = 1;
}
ADIOS_SELECTION* fSel = adios_selection_boundingbox(varInfo->ndim, start, count);
// avoid deadlock between not finished pmacc tasks and mpi calls in adios
__getTransactionEvent().waitForFinished();
/* specify what we want to read, but start reading at below at `adios_perform_reads` */
/* magic parameters (0, 1): `from_step` (not used in streams), `nsteps` to read (must be 1 for stream) */
log<picLog::INPUT_OUTPUT > ("ADIOS: Schedule read skalar %1%)") % datasetName;
ADIOS_CMD( adios_schedule_read(params.fp, fSel, datasetName.c_str(), 0, 1, (void*)value) );
/* start a blocking read of all scheduled variables */
ADIOS_CMD( adios_perform_reads(params.fp, 1) );
adios_selection_delete(fSel);
adios_free_varinfo(varInfo);
if(!attrName.empty())
{
log<picLog::INPUT_OUTPUT> ("ADIOS: read attribute %1% for scalars: %2%") % attrName % name;
*attribute = readAttribute<T_Attribute>(params.fp, datasetName, attrName);
}
}
int print_varinfo (ADIOS_FILE *f, int start_step)
{
ADIOS_VARINFO * v;
int i,j,k;
v = adios_inq_var (f, "t");
adios_inq_var_blockinfo (f, v);
printf ("ndim = %d\n", v->ndim);
printf ("dims[%llu]", v->dims[0]);
if (v->dims[0] != gdims[start_step])
{
printf ("\tERROR: expected [%llu]", gdims[start_step]);
nerrors++;
}
printf("\n");
printf ("nsteps = %d\n", v->nsteps);
printf ("sum_nblocks = %d\n", v->sum_nblocks);
k = 0; // blockinfo is a contigous 1D array of elements from 0 to v->sum_nblocks-1
for (i = 0; i < v->nsteps; i++) {
printf (" nblocks[%d] = %d\n", i, v->nblocks[i]);
for (j = 0; j < v->nblocks[i]; j++) {
printf(" block %2d: [%llu:%llu]", j,
v->blockinfo[k].start[0],
v->blockinfo[k].start[0] + v->blockinfo[k].count[0]-1);
if (v->blockinfo[k].start[0] != block_offset [(start_step+i)*nblocks_per_step*size + j] ||
v->blockinfo[k].count[0] != block_count [(start_step+i)*nblocks_per_step*size + j] )
{
nerrors++;
printf ("\tERROR: expected [%llu:%llu]",
block_offset [(start_step+i)*nblocks_per_step*size + j],
block_offset [(start_step+i)*nblocks_per_step*size + j] +
block_count [(start_step+i)*nblocks_per_step*size + j] -1
);
}
printf("\n");
k++;
}
}
adios_free_varinfo (v);
}
static void check_xform_var(ADIOS_GROUP *ag, double *arr) {
int64_t readlen;
double *readarr = malloc(N * sizeof(double));
memset(readarr, 0, N*sizeof(double));
// Check the transformed data
ADIOS_VARINFO *vi_xform = adios_inq_var(ag, XFORM_VAR);
assert(vi_xform);
assert(vi_xform->type == adios_double);
assert(vi_xform->ndim == 1); // Apparently the "dummy" dimension has gone away for timed arrays that are written only once
assert(vi_xform->dims[0] == N);
readlen = adios_read_var_byid(ag, vi_xform->varid, (uint64_t[]){0}, vi_xform->dims, readarr);
assert(readlen == N * sizeof(double));
assert(memcmp(readarr, arr, N * sizeof(double)) == 0);
// Cleanup
adios_free_varinfo(vi_xform);
free(readarr);
}
void AdiosCheckpointInput::getVector(const std::string& var_name, vector<T>& buffer)
{
ADIOS_VARINFO* adios_inq = adios_inq_var(adios_file_handle, var_name.c_str());
//Check to make sure the T is the same size as the data type on the disk
if (adios_type_size(adios_inq->type, NULL) != sizeof(T))
{
qmcplusplus::app_error() << "Data type does not match data type found in file: " << std::endl;
APP_ABORT("qmcapp");
return ;
}
uint64_t total_size = 1;
for (int i = 0; i < adios_inq->ndim; i++)
total_size *= adios_inq->dims[i];
//make sure we have enough space to copy all the data from the file
buffer.reserve(total_size);
//schedule the read
adios_schedule_read(adios_file_handle, sel, var_name.c_str(), 0, 1, &(buffer[0]));
//perform the read
adios_perform_reads(adios_file_handle, 1);
//Don't need the information about the variable anymore
adios_free_varinfo(adios_inq);
}
int64_t getByteEstimationOnFile(ADIOS_FILE* f, int rank)
{
// check all vars
int numVars = f->nvars;
uint64_t bytes = 0;
int i=0;
for (i=0; i<numVars; i++) {
char* varName = f->var_namelist[i];
ADIOS_VARINFO* v = adios_inq_var(f, varName);
uint64_t varSize = estimateBytesOnVar(f, v);
if (rank == 0) {
printf(" var: %s has size: %" PRId64 "\n", varName, varSize);
}
bytes += varSize;
adios_free_varinfo(v);
}
return bytes;
}
void AdiosCheckpointInput::performReads()
{
for (int i = 0; i < read_queue.size(); i++)
{
const char* var_name = read_queue[i].c_str();
//grab the information from the metadata about this variable
ADIOS_VARINFO* adios_inq = adios_inq_var(adios_file_handle, var_name);
//Calculate the total size of the vector based on the metadata
uint64_t total_size = adios_type_size(adios_inq->type, NULL);
for (int i = 0; i < adios_inq->ndim; i++)
total_size *= adios_inq->dims[i];
//allocate and then store for when retrieve
void* buff = malloc(total_size);
buffers.push_back(buff);
//schedule the read
adios_schedule_read(adios_file_handle, sel, var_name, 0, 1, buff);
//Don't need the information about the variable anymore
adios_free_varinfo(adios_inq);
}
//perform the read
adios_perform_reads(adios_file_handle, 1);
}
void adios_trace_verify_local_variables(ADIOS_FILE* fp, const char* name, double* origin)
{
ADIOS_VARINFO *vi;
int count = 1;
unsigned long size = 1;
vi = adios_inq_var(fp, name);
adios_inq_var_blockinfo(fp, vi);
for (int j=0; j<vi->nblocks[0]; j++)
{
if(OHMMS::Controller->rank() == j)
{
for (int k=0; k<vi->ndim; k++)
{
count *= vi->blockinfo[j].count[k];
}
}
}
size = count * adios_type_size(vi->type, vi->value);
double* mem = (double*)malloc(size);
ADIOS_SELECTION *sel = adios_selection_writeblock(OHMMS::Controller->rank());
adios_schedule_read(fp, sel, name, 0, 1, mem);
adios_perform_reads(fp, 1);
int flag = 0;
for(int i=0; i<count; i++)
{
if(mem[i] == origin[i])
{
}
else
{
flag = 1;
cout<<OHMMS::Controller->rank()<<" "<<name<<"["<<i<<"]verification not passed, readin: "<<mem[i]<<" writeout: "<<origin[i]<<endl;
}
}
if(flag == 0) cout<<OHMMS::Controller->rank()<<" "<<name<<" verification passed"<<endl;
adios_selection_delete(sel);
adios_free_varinfo (vi);
free(mem);
}
void buildIndexOnAllVar(ADIOS_FILE* f, int rank, int size)
{
int numVars = f->nvars;
int i=0;
for (i=0; i<numVars; i++) {
char* varName = f->var_namelist[i];
ADIOS_VARINFO* v = adios_inq_var(f, varName);
if (rank == 0) {
printf("\n==> building fastbit index on %dth variable: %s, %s ", i, varName, gBinningOption);
}
if (v->ndim > 0) {
buildIndex_mpi(f, v, rank, size);
} else {
if (rank == 0) {
printf("\t ... skipping scalar ...\n");
}
}
adios_free_varinfo(v);
}
}
static void test_file_mode_reads_on_var(ADIOS_FILE *fp, const char *bp_filename, const char *varname) {
int i;
ADIOS_VARINFO *varinfo = adios_inq_var(fp, varname);
MPI_Assert(COMM, varinfo);
if (varinfo->value != NULL) {
//if (rank == 0) fprintf(stderr, "(skipping scalar variable '%s')\n", varname);
adios_free_varinfo(varinfo);
return;
}
fprintf(stderr, "[rank %d/%d] Starting file-mode writeblock reads on %s:/%s\n", rank, size, bp_filename, varname);
adios_inq_var_blockinfo(fp, varinfo);
MPI_Assert(COMM, varinfo->blockinfo);
const enum ADIOS_DATATYPES datatype = varinfo->type;
const int datatypesize = adios_get_type_size(datatype, NULL);
int timestep, timestep_blockidx, blockidx = 0;
for (timestep = 0; timestep < varinfo->nsteps; ++timestep) {
for (timestep_blockidx = 0; timestep_blockidx < varinfo->nblocks[timestep]; ++timestep_blockidx, ++blockidx) {
if (blockidx % size != rank) continue;
const ADIOS_VARBLOCK *vb = &varinfo->blockinfo[blockidx];
ADIOS_SELECTION *block_bb = adios_selection_boundingbox(varinfo->ndim, vb->start, vb->count);
ADIOS_SELECTION *block_wb = adios_selection_writeblock(timestep_blockidx);
ADIOS_SELECTION *block_abs_wb = adios_selection_writeblock(blockidx);
block_abs_wb->u.block.is_absolute_index = 1;
uint64_t blocksize = datatypesize;
for (i = 0; i < varinfo->ndim; ++i)
blocksize *= vb->count[i];
void *buf_bb = malloc(blocksize);
void *buf_wb = malloc(blocksize);
void *buf_abs_wb = malloc(blocksize);
memset(buf_bb, 0, blocksize);
memset(buf_wb, 1, blocksize);
memset(buf_abs_wb, 2, blocksize);
MPI_Assert(COMM, buf_bb && buf_wb && buf_abs_wb);
adios_schedule_read(fp, block_bb, varname, timestep, 1, buf_bb );
adios_schedule_read(fp, block_wb, varname, timestep, 1, buf_wb );
adios_schedule_read(fp, block_abs_wb, varname, timestep, 1, buf_abs_wb);
adios_perform_reads(fp, 1);
fprintf(stderr, "[rank %d/%d] Checking file-mode blockidx %d BB vs. WB...\n", rank, size, blockidx);
MPI_Assert(COMM, memcmp(buf_bb, buf_wb, blocksize) == 0);
fprintf(stderr, "[rank %d/%d] Checking file-mode blockidx %d BB vs. abs-WB...\n", rank, size, blockidx);
MPI_Assert(COMM, memcmp(buf_bb, buf_abs_wb, blocksize) == 0);
free(buf_bb); free(buf_wb); free(buf_abs_wb);
adios_selection_delete(block_bb);
adios_selection_delete(block_wb);
adios_selection_delete(block_abs_wb);
}
}
adios_free_varinfo(varinfo);
fprintf(stderr, "[rank %d/%d] Finished file-mode writeblock reads on %s:/%s\n", rank, size, bp_filename, varname);
}
int main (int argc, char ** argv)
{
int rank, size, i, j;
MPI_Comm comm = MPI_COMM_WORLD;
enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_BP;
MPI_Init (&argc, &argv);
MPI_Comm_rank (comm, &rank);
MPI_Comm_size (comm, &size);
adios_read_init_method (method, comm, "verbose=3");
ADIOS_FILE * f = adios_read_open_file("adios_stat.bp", method, comm);
if (f == NULL)
{
fprintf (stderr, "%s\n", adios_errmsg());
return -1;
}
ADIOS_VARINFO * v = adios_inq_var (f, "temperature");
if (v) {
/* get statistics, for each individual time-step */
adios_inq_var_stat (f, v, 1, 0);
printf("Global MIN of temperature: %lf\n", * (double *) v->statistics->min);
printf("Global MAX of temperature: %lf\n", * (double *) v->statistics->max);
printf("Global AVG of temperature: %lf\n", * (double *) v->statistics->avg);
printf("Global STD DEV of temperature: %lf\n", * (double *) v->statistics->std_dev);
printf("\n");
printf("---------------------------------------------------------------------------\n");
if (v->statistics->steps) {
printf("MIN\t\tMAX\t\tAVG\t\tSTD_DEV\t\tHISTOGRAM\n");
for(i = 0; i < v->nsteps; i++)
{
if (v->statistics->steps->mins[i])
printf("%lf\t", * (double *) v->statistics->steps->mins[i]);
else
printf("--\t\t");
if (v->statistics->steps->maxs[i])
printf("%lf\t", * (double *) v->statistics->steps->maxs[i]);
else
printf("--\t\t");
if (v->statistics->steps->avgs[i])
printf("%lf\t", * (double *) v->statistics->steps->avgs[i]);
else
printf("--\t\t");
if (v->statistics->steps->std_devs[i])
printf("%lf\t", * (double *) v->statistics->steps->std_devs[i]);
else
printf("--\t\t");
if (v->statistics->histogram) {
for(j = 0; j <= v->statistics->histogram->num_breaks; j++) {
printf("%d ", v->statistics->histogram->frequencies[i][j]);
}
printf("\n");
} else {
printf("--\t\t");
}
printf("\n");
}
} else {
printf ("Per step statistics is missing\n");
}
printf("---------------------------------------------------------------------------\n");
printf("\n");
if (v->statistics->histogram) {
printf("Break points:\t\t\t");
for(j = 0; j < v->statistics->histogram->num_breaks; j++)
printf("%6.2lf\t", v->statistics->histogram->breaks[j]);
printf("\n");
printf("Frequencies:\t\t\t");
for(j = 0; j <= v->statistics->histogram->num_breaks; j++)
printf("%6d\t", v->statistics->histogram->gfrequencies[j]);
}
printf("\n\n");
#if 0
printf ("Auto covariance of MIN values of temperature over time: %lf\n", adios_stat_cov (v, v, "min", 0, 12, 0));
printf ("Auto correlation of MAX values of temperature over time, with lag 2 units: %lf\n", adios_stat_cor (v, v, "max", 0, 8, 2));
printf("\n\n");
#endif
adios_free_varinfo (v);
} else {
fprintf (stderr, "ERROR: Cannot inquire statistics of variable 'temperature': %s\n", adios_errmsg());
}
v = adios_inq_var (f, "complex");
if (v) {
adios_inq_var_stat (f, v, 1, 0);
double *C = v->statistics->min;
printf("Global Minimum of variable complex - Magnitude: %lf\n", C[0]);
printf("Global Minimum of variable complex - Real part: %lf\n", C[1]);
printf("Global Minimum of variable complex - Imaginary part: %lfi\n", C[2]);
double ** Cmin;
Cmin = (double **) v->statistics->steps->mins;
printf("\nMagnitude\t\tReal\t\t\tImaginary\n");
for (j = 0; j < v->nsteps; j++)
{
printf ("%lf\t\t%lf\t\t%lf\n", Cmin[j][0], Cmin[j][1], Cmin[j][2]);
}
printf("\n");
adios_free_varinfo (v);
} else {
fprintf (stderr, "ERROR: Cannot inquire statistics of variable 'complex': %s\n", adios_errmsg());
}
adios_read_close (f);
MPI_Barrier (comm);
adios_read_finalize_method (method);
MPI_Finalize ();
return 0;
}
int main (int argc, char ** argv)
{
int rank, size, i, j;
MPI_Comm comm = MPI_COMM_WORLD;
ADIOS_FILE * f;
ADIOS_VARINFO * v;
ADIOS_SELECTION * sel;
int steps = 0;
int retval = 0;
float timeout_sec = 1.0;
void * data = NULL;
uint64_t start[2], count[2];
MPI_Init (&argc, &argv);
MPI_Comm_rank (comm, &rank);
MPI_Comm_size (comm, &size);
adios_read_init_method (ADIOS_READ_METHOD_BP, comm, "verbose=3");
f = adios_read_open ("adios_globaltime.bp", ADIOS_READ_METHOD_BP,
comm, ADIOS_LOCKMODE_NONE, timeout_sec);
if (adios_errno == err_file_not_found)
{
printf ("rank %d: Stream not found after waiting %f seconds: %s\n",
rank, timeout_sec, adios_errmsg());
retval = adios_errno;
}
else if (adios_errno == err_end_of_stream)
{
printf ("rank %d: Stream terminated before open. %s\n", rank, adios_errmsg());
retval = adios_errno;
}
else if (f == NULL) {
printf ("rank %d: Error at opening stream: %s\n", rank, adios_errmsg());
retval = adios_errno;
}
else
{
/* process file here... */
v = adios_inq_var (f, "temperature");
adios_inq_var_blockinfo (f, v);
printf ("ndim = %d\n", v->ndim);
//printf ("nsteps = %d\n", v->nsteps);
printf ("dims[%llu][%llu]\n", v->dims[0], v->dims[1]);
uint64_t slice_size = v->dims[0]/size;
if (rank == size-1)
slice_size = slice_size + v->dims[0]%size;
start[0] = rank * slice_size;
count[0] = slice_size;
start[1] = 0;
count[1] = v->dims[1];
data = malloc (slice_size * v->dims[1] * 8);
/* Processing loop over the steps (we are already in the first one) */
while (adios_errno != err_end_of_stream) {
steps++; // steps start counting from 1
sel = adios_selection_boundingbox (v->ndim, start, count);
adios_schedule_read (f, sel, "temperature", 0, 1, data);
adios_perform_reads (f, 1);
if (rank == 0)
printf ("--------- Step: %d --------------------------------\n",
f->current_step);
printf("rank=%d: [0:%lld,0:%lld] = [", rank, v->dims[0], v->dims[1]);
for (i = 0; i < slice_size; i++) {
printf (" [");
for (j = 0; j < v->dims[1]; j++) {
printf ("%g ", *((double *)data + i * v->dims[1] + j));
}
printf ("]");
}
printf (" ]\n\n");
// advance to 1) next available step with 2) blocking wait
adios_advance_step (f, 0, timeout_sec);
if (adios_errno == err_step_notready)
{
printf ("rank %d: No new step arrived within the timeout. Quit. %s\n",
rank, adios_errmsg());
break; // quit while loop
}
}
adios_read_close (f);
}
if (rank==0)
printf ("We have processed %d steps\n", steps);
adios_read_finalize_method (ADIOS_READ_METHOD_BP);
free (data);
MPI_Finalize ();
return retval;
}
void
eavlXGCParticleImporter::Initialize()
{
ephase.clear();
iphase.clear();
egid.clear();
igid.clear();
last_available_timestep = fp->last_step;
if(readingRestartFile) //check for restart file format
{
if(fp->nvars <= 25)
readingRestartFile = 0;
}
if(readingRestartFile)
{
int numVarsPerGroup = 25;
nvars = fp->nvars/numVarsPerGroup;
if(nvars <= mpiRank)
{
printf("Warning! :: Thread[%i] is wasting cycles :: too many processors for data\n", mpiRank);
return;
}
//----Set indexes for each reader if there is more than one
int endIndex;
int startIndex = (nvars / numMPITasks) * mpiRank;
if (nvars % numMPITasks > mpiRank)
{
startIndex += mpiRank;
endIndex = startIndex + (nvars / numMPITasks) + 1;
}
else
{
startIndex += nvars % numMPITasks;
endIndex = startIndex + (nvars / numMPITasks);
}
startIndex *= numVarsPerGroup;
endIndex *= numVarsPerGroup;
//--
for(int i = startIndex; i < endIndex; i++)
{
ADIOS_VARINFO *avi = adios_inq_var_byid(fp, i);
cerr << __LINE__ << endl;
string longvarNm(&fp->var_namelist[i][0]); //!!This changed to not remove first char
cerr << __LINE__ << endl;
string varNm = longvarNm.substr(longvarNm.find("/",1,1)+1,longvarNm.length());
if(varNm == "ephase")
{
ephase[longvarNm] = avi;
ephaseAvail = 1;
}
else if(varNm == "egid")
{
egid[longvarNm] = avi;
}
else if(varNm == "iphase")
{
iphase[longvarNm] = avi;
iphaseAvail = 1;
}
else if(varNm == "igid")
{
igid[longvarNm] = avi;
}
else if(varNm == "inum")
{
int ipart;
AdiosGetValue (fp, i, ipart);
totalIParticles += ipart;
//totalIParticles += (int)(*(int *)avi->value);
adios_free_varinfo(avi);
}
else if(varNm == "enum")
{
int epart;
AdiosGetValue (fp, i, epart);
totalEParticles += epart;
//totalEParticles += (int)(*(int *)avi->value);
adios_free_varinfo(avi);
}
else if(i < startIndex + numVarsPerGroup)
{
if (varNm == "timestep")
{
//timestep = (int)(*(int *)avi->value);
AdiosGetValue (fp, i, timestep);
}
else if(varNm == "time")
{
AdiosGetValue (fp, i, time);
//time = (double)(*(double *)avi->value);
}
else if(varNm == "maxnum")
{
AdiosGetValue (fp, i, maxnum);
//maxnum = (int)(*(int *)avi->value);
}
else if (varNm == "inphase")
{
AdiosGetValue (fp, i, inphase);
//inphase = (int)(*(int *)avi->value);
}
else if(varNm == "enphase")
{
AdiosGetValue (fp, i, enphase);
//enphase = (int)(*(int *)avi->value);
}
else if(varNm == "emaxgid")
{
AdiosGetValue (fp, i, emaxgid);
//emaxgid = (long long)(*(long long *)avi->value);
}
else if(varNm == "imaxgid")
{
AdiosGetValue (fp, i, imaxgid);
//imaxgid = (long long)(*(long long *)avi->value);
}
adios_free_varinfo(avi);
}
else
{
adios_free_varinfo(avi);
}
}
}
else //reading particle file or aggregated restart file
{
for(int i = 0; i < fp->nvars; i++)
{
string varNm(&fp->var_namelist[i][0]);
if(varNm == "ephase")
{
ephaseAvail = 1;
//----Set indexes for each reader for the ELECTRONS
ADIOS_VARINFO *avi = adios_inq_var(fp, "ephase");
nvars = avi->dims[0];
int endIndex;
int startIndex = (nvars / numMPITasks) * mpiRank;
if (nvars % numMPITasks > mpiRank)
{
startIndex += mpiRank;
endIndex = startIndex + (nvars / numMPITasks) + 1;
}
else
{
startIndex += nvars % numMPITasks;
endIndex = startIndex + (nvars / numMPITasks);
}
ELECTRONendIndex = endIndex;
ELECTRONstartIndex = startIndex;
adios_free_varinfo(avi);
//--
}
else if(varNm == "iphase")
{
iphaseAvail = 1;
//----Set indexes for each reader for the IONS
ADIOS_VARINFO *avi = adios_inq_var(fp, "iphase");
nvars = avi->dims[0];
int endIndex;
int startIndex = (nvars / numMPITasks) * mpiRank;
if (nvars % numMPITasks > mpiRank)
{
startIndex += mpiRank;
endIndex = startIndex + (nvars / numMPITasks) + 1;
}
else
{
startIndex += nvars % numMPITasks;
endIndex = startIndex + (nvars / numMPITasks);
}
IONendIndex = endIndex;
IONstartIndex = startIndex;
adios_free_varinfo(avi);
//--
}
}
for(int i = 0; i < fp->nvars; i++)
{
ADIOS_VARINFO *avi = adios_inq_var_byid(fp, i);
string varNm(&fp->var_namelist[i][0]);
if(varNm == "ephase")
{
ephase[varNm] = avi;
}
else if(varNm == "egid")
{
egid[varNm] = avi;
}
else if(varNm == "iphase")
{
iphase[varNm] = avi;
}
else if(varNm == "igid")
{
igid[varNm] = avi;
}
else if(varNm == "inum")
{
totalIParticles = IONendIndex - IONstartIndex;
adios_free_varinfo(avi);
}
else if(varNm == "enum")
{
totalEParticles = ELECTRONendIndex - ELECTRONstartIndex;
adios_free_varinfo(avi);
}
else if (varNm == "timestep")
{
//timestep = (int)(*(int *)avi->value);
AdiosGetValue (fp, i, timestep);
adios_free_varinfo(avi);
}
else if(varNm == "time")
{
AdiosGetValue (fp, i, time);
//time = (double)(*(double *)avi->value);
adios_free_varinfo(avi);
}
else if (varNm == "inphase")
{
AdiosGetValue (fp, i, inphase);
//inphase = (int)(*(int *)avi->value);
adios_free_varinfo(avi);
}
else if(varNm == "enphase")
{
AdiosGetValue (fp, i, enphase);
//enphase = (int)(*(int *)avi->value);
adios_free_varinfo(avi);
}
else if(varNm == "enum_total")
{
AdiosGetValue (fp, i, emaxgid);
//emaxgid = (long long)(*(long long *)avi->value);
adios_free_varinfo(avi);
}
else if(varNm == "inum_total")
{
AdiosGetValue (fp, i, imaxgid);
//imaxgid = (long long)(*(long long *)avi->value);
adios_free_varinfo(avi);
}
else
{
adios_free_varinfo(avi);
}
}
}
}
int read_scalar_stepbystep ()
{
ADIOS_FILE * f;
float timeout_sec = 0.0;
int steps = 0;
int retval = 0;
MPI_Comm comm = MPI_COMM_SELF;
adios_read_init_method (ADIOS_READ_METHOD_BP, comm, "verbose=3");
printf ("\n--------- Read scalar in stream using varinfo->value ------------\n");
f = adios_read_open (fname, ADIOS_READ_METHOD_BP,
comm, ADIOS_LOCKMODE_NONE, timeout_sec);
if (adios_errno == err_file_not_found)
{
printf ("Stream not found after waiting %f seconds: %s\n",
timeout_sec, adios_errmsg());
retval = adios_errno;
}
else if (adios_errno == err_end_of_stream)
{
printf ("Stream terminated before open. %s\n", adios_errmsg());
retval = adios_errno;
}
else if (f == NULL) {
printf ("Error at opening stream: %s\n", adios_errmsg());
retval = adios_errno;
}
else
{
/* Processing loop over the steps (we are already in the first one) */
while (adios_errno != err_end_of_stream) {
steps++; // steps start counting from 1
printf ("Step: %d\n", f->current_step);
/* Check the scalar O with varinfo->value */
ADIOS_VARINFO * v = adios_inq_var (f, "NX");
int value = *(int*)v->value;
printf ("Scalar NX = %d", value);
if (value !=
block_count [f->current_step*nblocks_per_step*size])
{
printf ("\tERROR expected = %llu",
block_count [f->current_step*nblocks_per_step*size]);
nerrors++;
}
printf ("\n");
// advance to 1) next available step with 2) blocking wait
adios_advance_step (f, 0, timeout_sec);
if (adios_errno == err_step_notready)
{
//printf ("No new step arrived within the timeout. Quit. %s\n",
// adios_errmsg());
break; // quit while loop
}
}
adios_read_close (f);
}
adios_read_finalize_method (ADIOS_READ_METHOD_BP);
//printf ("We have processed %d steps\n", steps);
return retval;
}
int print_scalar (ADIOS_FILE *f, char * name)
{
ADIOS_VARINFO * v;
int i,j,k;
v = adios_inq_var (f, name);
adios_inq_var_blockinfo (f, v);
printf ("Scalar '%s':\n", name);
printf ("nsteps = %d\n", v->nsteps);
printf ("nblocks per step = %d\n", v->nblocks[0]);
int err;
/* Read one writeblock across all timesteps */
int *data = (int*) calloc (v->nsteps, sizeof(int));
ADIOS_SELECTION *s;
printf ("Read same instance across all timesteps:\n");
for (i=0; i < v->nblocks[0]; i++) {
s = adios_selection_writeblock(i);
err = adios_schedule_read_byid(f, s, v->varid, 0, v->nsteps, data);
if (!err)
{
err = adios_perform_reads(f, 1);
if (!err)
{
err = 0;
printf (" block %d = [", i);
for (j=0; j < v->nsteps; j++) {
printf ("%d", data[j]);
if (data[j] !=
block_offset [j*nblocks_per_step*size + i])
{
err = 1;
}
if (j < v->nsteps-1) printf(",");
}
printf("]");
if (err)
{
nerrors++;
printf ("\tERROR expected = [");
for (j=0; j < v->nsteps; j++) {
printf ("%llu", block_offset [j*nblocks_per_step*size + i]);
if (j < v->nsteps-1) printf(",");
}
printf("]");
}
printf("\n");
} else {
printf ("ERROR at reading scalar '%s': %s\n", name, adios_errmsg());
}
} else {
printf ("ERROR at scheduling read for scalar '%s': %s\n", name, adios_errmsg());
}
adios_selection_delete(s);
}
/* Now read piecewise, one writeblock at a time */
printf ("Read each instance individually:\n");
for (j=0; j < v->nsteps; j++) {
printf (" step %d: \n", j);
for (i=0; i < v->nblocks[j]; i++) {
s = adios_selection_writeblock(i);
err = adios_schedule_read_byid(f, s, v->varid, j, 1, data);
if (!err)
{
err = adios_perform_reads(f, 1);
if (!err)
{
printf (" block %d = %d", i, data[0]);
if (data[0] !=
block_offset [j*nblocks_per_step*size + i])
{
printf ("\tERROR expected = %llu",
block_offset [j*nblocks_per_step*size + i]);
nerrors++;
}
printf ("\n");
} else {
printf ("ERROR at reading scalar '%s': %s\n", name, adios_errmsg());
}
} else {
printf ("ERROR at scheduling read for scalar '%s': %s\n", name, adios_errmsg());
}
adios_selection_delete(s);
}
}
/* Now get them piecewise, but not with reading but through statistics */
printf ("Get each instance individually from available statistics:\n");
adios_inq_var_stat (f, v, 0, 1);
if (v->statistics && v->statistics->blocks) {
ADIOS_VARSTAT *stat = v->statistics;
int blockid = 0;
for (j=0; j < v->nsteps; j++) {
printf (" step %d: \n", j);
for (i=0; i < v->nblocks[j]; i++) {
printf (" block %d = %d", i, *(int*)stat->blocks->mins[blockid]);
if (*(int*)stat->blocks->mins[blockid] !=
block_offset [j*nblocks_per_step*size + i])
{
printf ("\tERROR expected = %llu",
block_offset [j*nblocks_per_step*size + i]);
nerrors++;
}
printf ("\n");
blockid++;
}
}
}
adios_free_varinfo (v);
free(data);
}
int main (int argc, char ** argv)
{
int i, j, datasize;
MPI_Comm comm = MPI_COMM_WORLD;
enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_BP;
ADIOS_SELECTION * sel1;
double * data = NULL;
uint64_t start[2], count[2];
MPI_Init (&argc, &argv);
#ifdef WITH_NCSU_TIMER
timer_init();
#endif
adios_read_init_method (method, comm, NULL);
ADIOS_FILE * f = adios_read_open_file ("adios_global.bp", method, comm);
ADIOS_VARINFO * varinfo = adios_inq_var (f, "temperature");
if (varinfo)
{
int nranks;
assert(varinfo->ndim == 2);
nranks = varinfo->dims[0];
assert(nranks % 4 == 0);
assert(varinfo->dims[1] == 10);
datasize = (nranks / 2) * varinfo->dims[1] * sizeof(double);
data = malloc (datasize);
start[0] = nranks / 4;
start[1] = 2;
count[0] = nranks / 2;
count[1] = 6;
sel1 = adios_selection_boundingbox (varinfo->ndim, start, count);
adios_schedule_read (f, sel1, "temperature", 0, 1, data);
adios_perform_reads (f, 1);
printf("Subvolume at (%" PRIu64 ",%" PRIu64 ") of size (%" PRIu64 ",%" PRIu64 "):\n", start[0], start[1], count[0], count[1]);
for (i = 0; i < count[0]; i++) {
printf("[ ");
for (j = 0; j < count[1]; j++) {
printf("%.0lf ", data[i * count[1] + j]);
}
printf("]\n");
}
adios_selection_delete (sel1);
}
adios_free_varinfo (varinfo);
adios_read_close (f);
adios_read_finalize_method (ADIOS_READ_METHOD_BP);
#ifdef WITH_NCSU_TIMER
printf("[TIMERS] ");
timer_result_t *results = timer_get_results_sorted();
for (i = 0; i < timer_get_num_timers(); i++) {
printf("%s: %0.4lf ", results[i].name, results[i].time);
}
printf("\n");
free(results);
#endif
#ifdef WITH_NCSU_TIMER
timer_finalize();
#endif
MPI_Finalize ();
return 0;
}
int main (int argc, char ** argv)
{
char filename [256] = "stream.bp";
int rank, size;
int NX, NY;
int len, off;
double *t = NULL;
MPI_Comm comm = MPI_COMM_WORLD;
int64_t adios_handle;
uint64_t adios_groupsize, adios_totalsize;
uint64_t start[2], count[2];
ADIOS_SELECTION *sel;
int steps = 0;
MPI_Init (&argc, &argv);
MPI_Comm_rank (comm, &rank);
MPI_Comm_size (comm, &size);
// ADIOS read init
adios_read_init_method (ADIOS_READ_METHOD_BP, comm, "verbose=3");
ADIOS_FILE* fp = adios_read_open_file ("kstar.bp",
ADIOS_READ_METHOD_BP,
comm);
assert(fp != NULL);
ADIOS_VARINFO* nx_info = adios_inq_var( fp, "N");
ADIOS_VARINFO* ny_info = adios_inq_var( fp, "L");
NX = *((int *)nx_info->value);
NY= *((int*)ny_info->value);
len = NX / size;
off = len * rank;
if (rank == size-1)
len = len + NX % size;
printf("\trank=%d: NX,NY,len,off = %d\t%d\t%d\t%d\n", rank, NX, NY, len, off);
assert(len > 0);
t = (double *) malloc(sizeof(double) * len * NY);
memset(t, '\0', sizeof(double) * len * NY);
assert(t != NULL);
start[0] = off;
start[1] = 0;
count[0] = len;
count[1] = NY;
sel = adios_selection_boundingbox (2, start, count);
// ADIOS write init
adios_init ("adios.xml", comm);
remove (filename);
//int ii;
//for(ii = 0; ii<10; ii++){
// for (i = 0; i < len * NY; i++)
// t[i] = ii*1000 + rank;
while(adios_errno != err_end_of_stream && adios_errno != err_step_notready)
{
steps++;
// Reading
adios_schedule_read (fp, sel, "var", 0, 1, t);
adios_perform_reads (fp, 1);
// Debugging
//for (i = 0; i < len*NY; i++) t[i] = off * NY + i;
printf("step=%d\trank=%d\t[%d,%d]\n", steps, rank, len, NY);
// Writing
adios_open (&adios_handle, "writer", filename, "a", comm);
adios_groupsize = 4*4 + 8*len*NY;
adios_group_size (adios_handle, adios_groupsize, &adios_totalsize);
adios_write (adios_handle, "NX", &NX);
adios_write (adios_handle, "NY", &NY);
adios_write (adios_handle, "len", &len);
adios_write (adios_handle, "off", &off);
adios_write (adios_handle, "var_2d_array", t);
adios_close (adios_handle);
// Advance
MPI_Barrier (comm);
adios_advance_step(fp, 0, TIMEOUT_SEC);
}
free(t);
MPI_Barrier (comm);
adios_read_close(fp);
if (rank==0)
printf ("We have processed %d steps\n", steps);
MPI_Barrier (comm);
adios_read_finalize_method(ADIOS_READ_METHOD_BP);
adios_finalize (rank);
MPI_Finalize ();
return 0;
}
int main (int argc, char ** argv)
{
int rank, size;
int NX, NY;
int len, off;
double *t = NULL;
MPI_Comm comm = MPI_COMM_WORLD;
uint64_t start[2], count[2];
ADIOS_SELECTION *sel;
int steps = 0;
#ifdef _USE_GNUPLOT
int i, j;
double *tmp;
FILE *pipe;
#else
// Variables for ADIOS write
int64_t adios_handle;
uint64_t adios_groupsize, adios_totalsize;
char outfn[256];
#endif
MPI_Init (&argc, &argv);
MPI_Comm_rank (comm, &rank);
MPI_Comm_size (comm, &size);
adios_read_init_method(ADIOS_READ_METHOD_FLEXPATH, comm, "");
ADIOS_FILE* fp = adios_read_open("stream.bp",
ADIOS_READ_METHOD_FLEXPATH,
comm, ADIOS_LOCKMODE_NONE, 0.0);
assert(fp != NULL);
ADIOS_VARINFO* nx_info = adios_inq_var( fp, "NX");
ADIOS_VARINFO* ny_info = adios_inq_var( fp, "NY");
NX = *((int *)nx_info->value);
NY= *((int*)ny_info->value);
len = NX / size;
off = len * rank;
if (rank == size-1)
len = len + NX % size;
printf("\trank=%d: NX,NY,len,off = %d\t%d\t%d\t%d\n", rank, NX, NY, len, off);
assert(len > 0);
t = (double *) malloc(sizeof(double) * len * NY);
memset(t, '\0', sizeof(double) * len * NY);
assert(t != NULL);
start[0] = off;
start[1] = 0;
count[0] = len;
count[1] = NY;
// Not working ...
//sel = adios_selection_boundingbox (2, start, count);
sel = malloc(sizeof(ADIOS_SELECTION));
sel->type=ADIOS_SELECTION_WRITEBLOCK;
sel->u.block.index = rank;
#ifdef _USE_GNUPLOT
if ((NX % size) > 0)
{
fprintf(stderr, "Equal distribution is required\n");
return -1;
}
if (rank == 0) {
pipe = popen("gnuplot", "w");
fprintf(pipe, "set view map\n");
fprintf(pipe, "set xrange [0:%d]\n", NX-1);
tmp = (double *) malloc(sizeof(double) * NX * NY);
assert(tmp != NULL);
}
#else
// ADIOS write init
adios_init ("adios.xml", comm);
#endif
//while(adios_errno != err_end_of_stream && adios_errno != err_step_notready)
while(1)
{
steps++;
// Reading
adios_schedule_read (fp, sel, "var_2d_array", 0, 1, t);
adios_perform_reads (fp, 1);
printf("step=%d\trank=%d\tfp->current_step=%d\t[%d,%d]\n",
steps, rank, fp->current_step, len, NY);
/*
// Debugging
for (i=0; i<len; i++) {
printf("%d: rank=%d: t[%d,0:4] = ", steps, rank, off+i);
for (j=0; j<5; j++) {
printf(", %g", t[i*NY + j]);
}
printf(" ...\n");
}
*/
// Do something
#ifdef _USE_GNUPLOT // Option 1: plotting
MPI_Gather(t, len * NY, MPI_DOUBLE, tmp, len * NY, MPI_DOUBLE, 0, comm);
if (rank == 0)
{
fprintf(pipe, "set title 'Soft X-Rray Signal (shot #%d)'\n", steps);
fprintf(pipe, "set xlabel 'Channel#'\n");
fprintf(pipe, "set ylabel 'Timesteps'\n");
fprintf(pipe, "set cblabel 'Voltage (eV)'\n");
# ifndef _GNUPLOT_INTERACTIVE
fprintf(pipe, "set terminal png\n");
fprintf(pipe, "set output 'fig%03d.png'\n", steps);
# endif
fprintf(pipe, "splot '-' matrix with image\n");
//fprintf(pipe, "plot '-' with lines, '-' with lines, '-' with lines\n");
double *sum = calloc(NX, sizeof(double));
for (j = 0; j < NY; j++) {
for (i = 0; i < NX; i++) {
sum[i] += tmp[i * NY + j];
}
}
for (j = 0; j < NY; j++) {
for (i = 0; i < NX; i++) {
fprintf (pipe, "%g ", (-tmp[i * NY + j] + sum[i]/NY)/3276.8);
}
fprintf(pipe, "\n");
}
fprintf(pipe, "e\n");
fprintf(pipe, "e\n");
fflush (pipe);
# ifdef _GNUPLOT_INTERACTIVE
printf ("Press [Enter] to continue . . .");
fflush (stdout);
getchar ();
# endif
free(sum);
}
#else // Option 2: BP writing
snprintf (outfn, sizeof(outfn), "reader_%3.3d.bp", steps);
adios_open (&adios_handle, "reader", outfn, "w", comm);
adios_groupsize = 4 * sizeof(int) + sizeof(double) * len * NY;
adios_group_size (adios_handle, adios_groupsize, &adios_totalsize);
adios_write (adios_handle, "NX", &NX);
adios_write (adios_handle, "NY", &NY);
adios_write (adios_handle, "len", &len);
adios_write (adios_handle, "off", &off);
adios_write (adios_handle, "var", t);
adios_close (adios_handle);
#endif
// Advance
MPI_Barrier (comm);
adios_advance_step(fp, 0, TIMEOUT_SEC);
if (adios_errno == err_end_of_stream)
{
printf("rank %d, Stream terminated. Quit\n", rank);
break; // quit while loop
}
else if (adios_errno == err_step_notready)
{
printf ("rank %d: No new step arrived within the timeout. Quit.\n", rank);
break; // quit while loop
}
else if (adios_errno != err_no_error) {
printf("ADIOS returned code=%d, msg:%s\n",
adios_errno, adios_get_last_errmsg());
break; // quit while loop
}
}
//
free(t);
adios_read_close(fp);
//printf("rank %d, Successfully closed stream\n", rank);
adios_read_finalize_method(ADIOS_READ_METHOD_FLEXPATH);
//printf("rank %d, Successfully finalized read method\n", rank);
#ifndef _USE_GNUPLOT
adios_finalize (rank);
//printf("rank %d, Successfully finalized adios\n", rank);
#else
if (rank==0) {
free(tmp);
pclose(pipe);
}
#endif
MPI_Finalize ();
return 0;
}
int main(int argc, char ** argv)
{
int rank, size, varid, numvars;
int bins, step, mod;
char *filename, *in_stream, *data_var_name;
MPI_Comm comm = MPI_COMM_WORLD;
enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_FLEXPATH;
//enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_BP;
ADIOS_SELECTION * global_range_select;
double *data;
uint64_t tstep, global_size, mysize, mystart, sz;
MPI_Init (&argc, &argv);
MPI_Comm_rank(comm, &rank);
MPI_Comm_size(comm, &size);
/* Command line parsing */
if (rank == 0 && argc < 4) {
fprintf(stderr, "\nHistogram usage: <exec> input-stream-name num-bins"
" arr1 [arr2] [arr3] [...]\n"
"\t where arr1, arr2, arr3 ... are the names of the arrays to be analyzed.\n");
MPI_Abort(comm, -1);
}
MPI_Barrier(comm);
in_stream = argv[1];
//Parse cmd line
bins = atoi(argv[2]);
numvars = argc - 3;
const char *vars[numvars];
for (varid=0; varid < numvars; varid++)
{
vars[varid] = argv[varid + 3];
}
/* Adios open and init */
adios_read_init_method (method, comm, "verbose=1");
ADIOS_FILE * f = adios_read_open (in_stream, method, comm, ADIOS_LOCKMODE_ALL, -1);
step = 0; //not used now
while (adios_errno != err_end_of_stream){
//resource monitor
/*loop over different arrays inside stream*/
for (varid = 0; varid < numvars; varid++){
#ifdef ENABLE_MONITOR
//double t1 = wfgettimeofday();
lib_mem_init();
ind_timer_start(0, "whole timestep");
#endif
//Init variables....
global_size = 0; tstep = 0;
mod = 0; mysize = 0; mystart = 0;
adios_schedule_read (f, NULL, "ntimestep", 0, 1, &tstep);
adios_perform_reads (f, 1);
ADIOS_VARINFO * glob_info = adios_inq_var (f, vars[varid]);
global_size = glob_info->dims[0];
//printf("[DEBUG] global_size = %" PRIu64 " ntimestep = %" PRIu64 "\n",
// global_size, tstep);
//printf("[HIST%d] received data for timestep %" PRIu64 " with ndim: %d and globalsize:%"
// PRIu64 " \n", rank, tstep, ndim, global_size);
//sleep(800);
//debug
//Array slice computation
mod = global_size % size;//size = MPI size
if (mod == 0){
mysize = global_size / size;
mystart = mysize * rank;
}
else {
mysize = global_size / (size);
if (rank < mod){
mysize++;
mystart = mysize * rank;
}
else {
mystart = (mod * (mysize + 1)) +
((rank - mod) * mysize);
}
}
#ifdef ENABLE_MONITOR
nohandler_mem(rank);
#endif
//printf("[HISTO%d]: mysize = %" PRIu64" mystart = %" PRIu64 "\n", rank, mysize, mystart);
//debug
//if (step == 0) sleep(800);
uint64_t starts[] = {mystart};
uint64_t counts[] = {mysize};
global_range_select = adios_selection_boundingbox (1, starts, counts);
//Allocate space for arrays
uint64_t msize = ((uint64_t) sizeof(double) * mysize);
//printf("[DEBUG] mysize = %" PRIu64 " msize= %" PRIu64" \n", mysize, msize);
//data = (double *) malloc(sizeof(double) * mysize);
data = new double[mysize];
if (data == NULL){
//printf("DEBUG: malloc returned NULL, size was %d\n", msize);
}
else {
if (rank == 0)
printf("[HIST0] DEBUG: malloc successful, size was %d\n", mysize);
}
//memset (data, 0, sizeof(double) * mysize);
//Read data
adios_schedule_read (f,
global_range_select,
vars[varid],
0, 1, data);
adios_perform_reads
(f, 1);
#ifdef ENABLE_MONITOR
nohandler_mem(rank);
#endif
//printf("PERFORM_READS success of variable: %s\n", vars[varid]);
/*
Data check
if (step == 4) {
FILE *fp;
char *log;
asprintf(&log, "histo-input%d-%d.log", step, rank);
fp = fopen(log, "w");
fprintf(fp, "timestep: %" PRIu64 " mysize: %"PRIu64 "\n",
tstep, mysize);
for (i=0; i<(int)mysize; i++){
fprintf(fp, "%lf\n", data[i]);
}
fclose(fp);
sleep(800);
}
*/
// find max and min
sz = 0;
sz = mysize;
double min = data[0];
double max = data[0];
for (uint64_t i = 1; i < sz; ++i)
{
if (data[i] > max) max = data[i];
if (data[i] < min) min = data[i];
}//local max, min found.
//local data should just use shared mem.
double g_min, g_max;
// Find the global max/min
MPI_Allreduce (&min, &g_min, 1, MPI_DOUBLE, MPI_MIN, comm);
MPI_Allreduce (&max, &g_max, 1, MPI_DOUBLE, MPI_MAX, comm);
//printf("[HIST%d] glob-min: %f, glob-max: %f\n", rank, g_min, g_max);
nohandler_mem(rank);
double width = (g_max - g_min)/bins;
std::vector<uint64_t> hist(bins);
for (uint64_t i = 0; i < sz; ++i)//fill local bins
{
//printf("[HISTO%d] local filling adding index %" PRIu64 "\n", rank, i);
int idx = int((data[i] - g_min)/width);//discover index
if (idx == bins) // we hit the max
--idx;
//printf("[%d]: %f -> %d\n", rank, data[i], idx);
++hist[idx];
}
delete[] data;
// Global reduce histograms
std::vector<uint64_t> g_hist(bins);
MPI_Reduce(&hist[0], &g_hist[0], bins, MPI_UINT64_T, MPI_SUM, 0, comm);
//debug
//printf("[Completed histogram routine]\n");
if (rank == 0) //print histogram to file
{
FILE *fp;
const char *log = "histograms.log";
fp = fopen(log, "a");
fprintf(fp, "Histogram for %s, timestep %" PRIu64"\n", vars[varid], tstep);
for (int i = 0; i < bins; ++i)
fprintf(fp, " %f-%f: %" PRIu64 "\n", g_min + i*width, g_min + (i+1)*width, g_hist[i]);
fclose (fp);
}
#ifdef ENABLE_MONITOR
nohandler_mem(rank);
#endif
if (rank == 0) //print histogram to terminal
{
printf("Histogram for %s, timestep %" PRIu64"\n", vars[varid], tstep);
for (int i = 0; i < bins; ++i)
printf(" %f-%f: %" PRIu64 "\n",
g_min + i*width, g_min + (i+1)*width, g_hist[i]);
}
//resource monitor
#ifdef ENABLE_MONITOR
//double t2 = wfgettimeofday();
ind_timer_end(0);
char monitor_title[40];
sprintf(monitor_title, "histogram-%s", vars[varid]);
monitor_out (rank, size, tstep, msize, t1, t2, comm, monitor_title);
#endif
}
//end of read + analysis for 3 variables
adios_release_step(f);
//delete[] data;
if (rank == 0) printf("[HIST%d] read and wrote data for timestep %" PRIu64 "\n", rank, tstep);
step++;
adios_advance_step(f, 0, -1);
/*
if (step == 6){
double t1 = wfgettimeofday();
FILE *tfp;
tfp = fopen("time.log", "a");
fprintf(tfp, "rank %d histogram end time: %f\n", rank, t1);
fclose(tfp);
}
*/
}//end of adios stream while loop
if (rank == 0) printf("[HIST%d] out of read loop\n", rank);
/* performance measurement */
/*
if (rank == 0){
double t3 = wfgettimeofday();
FILE *tfp;
tfp = fopen("time.log", "a");
fprintf(tfp, "master histogram end time: %f\n", t3);
fclose(tfp);
}
*/
#ifdef ENABLE_MONITOR
outer_timer_end(rank, "histogram");
#endif
adios_read_close(f);
adios_read_finalize_method(method);
MPI_Finalize();
return 0;
}