void adios_checkpoint_verify_variables(ADIOS_FILE* fp, const char* name, int* origin)
{
ADIOS_VARINFO *vi;
int count = 1;
int size;
vi = adios_inq_var(fp, name);
if (vi->ndim > 0)
{
cout<<name<<" verification not passed, not a scalar"<<endl;
return;
}
size = count*adios_type_size(vi->type, vi->value);
int * mem= (int * )malloc(size);
ADIOS_SELECTION *sel = adios_selection_writeblock(OHMMS::Controller->rank());
adios_schedule_read(fp, sel, name, 0, 1, mem);
//adios_schedule_read(fp, NULL, name, 0, 1, mem);
adios_perform_reads(fp, 1);
if(mem[0] == *origin)
{
cout<<name<<" verification passed "<<mem[0]<<endl;
}
else
{
cout<<name<<" verification not passed, readin: "<<mem[0]<<" writeout: "<<*origin<<endl;
}
adios_free_varinfo (vi);
adios_selection_delete(sel);
free(mem);
}
int read_file (char *fname)
{
ADIOS_SELECTION *sel;
ADIOS_FILE * f;
ADIOS_VARINFO * vi;
int err=0,i,n;
uint64_t start[1] = {0};
uint64_t count[2] = {NX};
uint64_t ndim;
reset_rarrays();
log ("Read and check data in %s\n", fname);
f = adios_read_open_file (fname, ADIOS_READ_METHOD_BP, comm);
if (f == NULL) {
printE ("Error at opening file: %s\n", adios_errmsg());
return 1;
}
sel = adios_selection_boundingbox (1, start, count);
adios_schedule_read (f, sel, "t1", 0, 1, r1);
adios_schedule_read (f, sel, "t2", 0, 1, r2);
adios_perform_reads (f, 1);
adios_selection_delete (sel);
CHECK_ARRAY (t, r1, NX);
CHECK_ARRAY (t, r2, NX);
endread:
adios_read_close(f);
MPI_Barrier (comm);
return err;
}
// k is numbered from 1 to sum_nblocks
void verifyData(ADIOS_FILE* f, ADIOS_VARINFO* v, int k, int timestep)
{
uint64_t blockBytes = adios_type_size (v->type, v->value);
int j=0;
if (v->ndim <= 0) {
return;
}
//printf("verify block[%d]: ", k);
for (j=0; j<v->ndim; j++)
{
blockBytes *= v->blockinfo[k].count[j];
//printf("%" PRIu64 ":%" PRIu64 " ", v->blockinfo[k].start[j], v->blockinfo[k].count[j]);
}
void* data = NULL;
data = malloc(blockBytes);
ADIOS_SELECTION* sel = adios_selection_boundingbox (v->ndim, v->blockinfo[k].start, v->blockinfo[k].count);
int err = adios_schedule_read_byid(f, sel, v->varid, timestep, 1, data);
if (!err) {
err = adios_perform_reads(f, 1);
}
//fastbit_adios_util_printData(data, v->type, blockBytes/adios_type_size(v->type, v->value));
adios_selection_delete(sel);
free(data);
data = NULL;
}
eavlField *
eavlXGCImporter::GetField(const string &name, const string &mesh, int chunk)
{
map<string, ADIOS_VARINFO*>::const_iterator it = variables.find(name);
if (it == variables.end())
THROW(eavlException, string("Variable not found: ")+name);
uint64_t s[3], c[3];
ADIOS_SELECTION *sel = MakeSelection(it->second, s, c);
int nt = 1;
for (int i = 0; i < it->second->ndim; i++)
nt *= c[i];
double *buff = new double[nt];
adios_schedule_read_byid(fp, sel, it->second->varid, 0, 1, buff);
int retval = adios_perform_reads(fp, 1);
adios_selection_delete(sel);
eavlFloatArray *arr = new eavlFloatArray(name, 1);
arr->SetNumberOfTuples(nt);
for (int i = 0; i < nt; i++)
arr->SetComponentFromDouble(i, 0, buff[i]);
delete [] buff;
eavlField *field = new eavlField(1, arr, eavlField::ASSOC_POINTS);
//field->PrintSummary(cout);
return field;
}
void onBlock(int rank, ADIOS_FILE* f, ADIOS_VARINFO* v, int i, int j, int blockCounter, FastBitDataType ft)
{
char bmsVarName[100];
char keyVarName[100];
char offsetName[100];
int64_t var_ids_bms[v->nblocks[i]];
int64_t var_ids_key[v->nblocks[i]];
int64_t var_ids_offset[v->nblocks[i]];
sprintf(bmsVarName, "bms-%d-%d-%d", v->varid, i, j);
sprintf(keyVarName, "key-%d-%d-%d", v->varid, i, j);
sprintf(offsetName, "offset-%d-%d-%d", v->varid, i, j);
uint64_t blockSize = fastbit_adios_util_getBlockSize(v, i, j);
uint64_t blockDataByteSize = adios_type_size (v->type, v->value) * blockSize;
char notes[100];
logTime(NULL); logTimeMillis(NULL);
sprintf(notes, " reading data from adios on varid=%d, time=%d, block: %d, size=%ld bytes=%ld", v->varid, i, j, blockSize, blockDataByteSize);
logTime(notes); logTimeMillis(notes);
localtime(&indexRefresh);
//printf(" %d th block / (%d), size= %" PRIu64 " bytes=%" PRIu64, j, blockSize, blockCounter, blockDataByteSize);
void* data = malloc (blockDataByteSize);
ADIOS_SELECTION* blockSel = adios_selection_writeblock(j);
//adios_selcton_writeblock(num), 0 <= num < nblocks[timestep]
//ADIOS_SELECTION* blockSel = adios_selection_writeblock(blockCounter);
int err = adios_schedule_read_byid(f, blockSel, v->varid, i, 1, data);
if (!err) {
err = adios_perform_reads(f, 1);
} else {
printf("Unable to read block %d at timestep: %d \n", j, i);
return;
//break;
}
//fastbit_adios_util_printData(data, v->type, blockSize);
char selName[20];
sprintf(selName, "block-%d", j);
processData(data, blockSize, rank, i, selName, ft, v);
//processData(void* data, uint64_t dataCount, int rank, int timestep, char* selName, FastBitDataType ft, ADIOS_VARINFO* v)
adios_selection_delete(blockSel);
verifyData(f, v, blockCounter, i);
} // onblock
void adios_checkpoint_verify_random_variables(ADIOS_FILE* fp, const char* name, uint_type* origin)
{
ADIOS_VARINFO *vi;
int count_int = 1;
int size;
uint64_t *start;
uint64_t *count;
vi = adios_inq_var(fp, name);
adios_inq_var_blockinfo (fp, vi);
if (vi->ndim > 0)
{
start = (uint64_t *)malloc(vi->ndim * sizeof(uint64_t));
count = (uint64_t *)malloc(vi->ndim * sizeof(uint64_t));
}
for (int j=0; j<vi->nblocks[0]; j++)
{
if(j == OHMMS::Controller->rank())
{
for (int k=0; k<vi->ndim; k++)
{
start[k] = vi->blockinfo[j].start[k];
count[k] = vi->blockinfo[j].count[k];
count_int *= count[k];
//cout<<OHMMS::Controller->rank()<<" count "<<start[k]<<" "<<count[k]<<endl;
}
}
}
size = count_int*adios_type_size(vi->type, vi->value);
uint_type *mem= (uint_type*)malloc(size);
ADIOS_SELECTION *sel = adios_selection_boundingbox(vi->ndim, start, count);
adios_schedule_read(fp, sel, name, 0, 1, mem);
adios_perform_reads(fp, 1);
int flag = 0;
for(int i=0; i<count_int; i++)
{
if(mem[i] == origin[i])
{
//cout<<name<<"["<<i<<"]verification passed "<<mem[i]<<endl;
}
else
{
flag = 1;
cout<<name<<"["<<i<<"]verification not passed, readin: "<<mem[i]<<" writeout: "<<origin[i]<<endl;
}
}
if (flag == 0) cout<<name<<" verification passed "<<endl;
else cout<<name<<" verification not passed "<<endl;
adios_free_varinfo (vi);
adios_selection_delete(sel);
free(start);
free(count);
free(mem);
}
void ADIOS1CommonRead::ScheduleReadCommon(const std::string &name,
const Dims &offs, const Dims &ldims,
const int fromStep, const int nSteps,
const bool readAsLocalValue,
const bool readAsJoinedArray,
void *data)
{
if (readAsLocalValue)
{
/* Get all the requested values from metadata now */
ADIOS_VARINFO *vi = adios_inq_var(m_fh, name.c_str());
if (vi)
{
adios_inq_var_stat(m_fh, vi, 0, 1);
int elemsize = adios_type_size(vi->type, nullptr);
long long blockidx = 0;
for (int i = 0; i < fromStep; i++)
{
blockidx += vi->nblocks[i];
}
char *dest = (char *)data;
for (int i = fromStep; i < fromStep + nSteps; i++)
{
for (int j = 0; j < vi->nblocks[i]; j++)
{
memcpy(dest, vi->statistics->blocks->mins[blockidx],
elemsize);
++blockidx;
dest += elemsize;
}
}
adios_free_varinfo(vi);
}
}
else
{
uint64_t start[32], count[32];
for (int i = 0; i < ldims.size(); i++)
{
start[i] = (uint64_t)offs[i];
count[i] = (uint64_t)ldims[i];
}
ADIOS_SELECTION *sel = nullptr;
if (ldims.size() > 0)
{
sel = adios_selection_boundingbox(ldims.size(), start, count);
}
adios_schedule_read(m_fh, sel, name.c_str(), (int)fromStep, (int)nSteps,
data);
adios_selection_delete(sel);
}
}
/** Read the skalar field and optionally the attribute into the values referenced by the pointers */
void operator()(ThreadParams& params,
const std::string& name, T_Scalar* value,
const std::string& attrName = "", T_Attribute* attribute = nullptr)
{
log<picLog::INPUT_OUTPUT> ("ADIOS: read %1%D scalars: %2%") % simDim % name;
std::string datasetName = params.adiosBasePath + name;
ADIOS_VARINFO* varInfo;
ADIOS_CMD_EXPECT_NONNULL( varInfo = adios_inq_var(params.fp, datasetName.c_str()) );
if(varInfo->ndim != simDim)
throw std::runtime_error(std::string("Invalid dimensionality for ") + name);
if(varInfo->type != traits::PICToAdios<T_Scalar>().type)
throw std::runtime_error(std::string("Invalid type for ") + name);
DataSpace<simDim> gridPos = Environment<simDim>::get().GridController().getPosition();
uint64_t start[varInfo->ndim];
uint64_t count[varInfo->ndim];
for(int d = 0; d < varInfo->ndim; ++d)
{
/* \see adios_define_var: z,y,x in C-order */
start[d] = gridPos.revert()[d];
count[d] = 1;
}
ADIOS_SELECTION* fSel = adios_selection_boundingbox(varInfo->ndim, start, count);
// avoid deadlock between not finished pmacc tasks and mpi calls in adios
__getTransactionEvent().waitForFinished();
/* specify what we want to read, but start reading at below at `adios_perform_reads` */
/* magic parameters (0, 1): `from_step` (not used in streams), `nsteps` to read (must be 1 for stream) */
log<picLog::INPUT_OUTPUT > ("ADIOS: Schedule read skalar %1%)") % datasetName;
ADIOS_CMD( adios_schedule_read(params.fp, fSel, datasetName.c_str(), 0, 1, (void*)value) );
/* start a blocking read of all scheduled variables */
ADIOS_CMD( adios_perform_reads(params.fp, 1) );
adios_selection_delete(fSel);
adios_free_varinfo(varInfo);
if(!attrName.empty())
{
log<picLog::INPUT_OUTPUT> ("ADIOS: read attribute %1% for scalars: %2%") % attrName % name;
*attribute = readAttribute<T_Attribute>(params.fp, datasetName, attrName);
}
}
void adios_trace_verify_local_variables(ADIOS_FILE* fp, const char* name, double* origin)
{
ADIOS_VARINFO *vi;
int count = 1;
unsigned long size = 1;
vi = adios_inq_var(fp, name);
adios_inq_var_blockinfo(fp, vi);
for (int j=0; j<vi->nblocks[0]; j++)
{
if(OHMMS::Controller->rank() == j)
{
for (int k=0; k<vi->ndim; k++)
{
count *= vi->blockinfo[j].count[k];
}
}
}
size = count * adios_type_size(vi->type, vi->value);
double* mem = (double*)malloc(size);
ADIOS_SELECTION *sel = adios_selection_writeblock(OHMMS::Controller->rank());
adios_schedule_read(fp, sel, name, 0, 1, mem);
adios_perform_reads(fp, 1);
int flag = 0;
for(int i=0; i<count; i++)
{
if(mem[i] == origin[i])
{
}
else
{
flag = 1;
cout<<OHMMS::Controller->rank()<<" "<<name<<"["<<i<<"]verification not passed, readin: "<<mem[i]<<" writeout: "<<origin[i]<<endl;
}
}
if(flag == 0) cout<<OHMMS::Controller->rank()<<" "<<name<<" verification passed"<<endl;
adios_selection_delete(sel);
adios_free_varinfo (vi);
free(mem);
}
static void test_file_mode_reads_on_var(ADIOS_FILE *fp, const char *bp_filename, const char *varname) {
int i;
ADIOS_VARINFO *varinfo = adios_inq_var(fp, varname);
MPI_Assert(COMM, varinfo);
if (varinfo->value != NULL) {
//if (rank == 0) fprintf(stderr, "(skipping scalar variable '%s')\n", varname);
adios_free_varinfo(varinfo);
return;
}
fprintf(stderr, "[rank %d/%d] Starting file-mode writeblock reads on %s:/%s\n", rank, size, bp_filename, varname);
adios_inq_var_blockinfo(fp, varinfo);
MPI_Assert(COMM, varinfo->blockinfo);
const enum ADIOS_DATATYPES datatype = varinfo->type;
const int datatypesize = adios_get_type_size(datatype, NULL);
int timestep, timestep_blockidx, blockidx = 0;
for (timestep = 0; timestep < varinfo->nsteps; ++timestep) {
for (timestep_blockidx = 0; timestep_blockidx < varinfo->nblocks[timestep]; ++timestep_blockidx, ++blockidx) {
if (blockidx % size != rank) continue;
const ADIOS_VARBLOCK *vb = &varinfo->blockinfo[blockidx];
ADIOS_SELECTION *block_bb = adios_selection_boundingbox(varinfo->ndim, vb->start, vb->count);
ADIOS_SELECTION *block_wb = adios_selection_writeblock(timestep_blockidx);
ADIOS_SELECTION *block_abs_wb = adios_selection_writeblock(blockidx);
block_abs_wb->u.block.is_absolute_index = 1;
uint64_t blocksize = datatypesize;
for (i = 0; i < varinfo->ndim; ++i)
blocksize *= vb->count[i];
void *buf_bb = malloc(blocksize);
void *buf_wb = malloc(blocksize);
void *buf_abs_wb = malloc(blocksize);
memset(buf_bb, 0, blocksize);
memset(buf_wb, 1, blocksize);
memset(buf_abs_wb, 2, blocksize);
MPI_Assert(COMM, buf_bb && buf_wb && buf_abs_wb);
adios_schedule_read(fp, block_bb, varname, timestep, 1, buf_bb );
adios_schedule_read(fp, block_wb, varname, timestep, 1, buf_wb );
adios_schedule_read(fp, block_abs_wb, varname, timestep, 1, buf_abs_wb);
adios_perform_reads(fp, 1);
fprintf(stderr, "[rank %d/%d] Checking file-mode blockidx %d BB vs. WB...\n", rank, size, blockidx);
MPI_Assert(COMM, memcmp(buf_bb, buf_wb, blocksize) == 0);
fprintf(stderr, "[rank %d/%d] Checking file-mode blockidx %d BB vs. abs-WB...\n", rank, size, blockidx);
MPI_Assert(COMM, memcmp(buf_bb, buf_abs_wb, blocksize) == 0);
free(buf_bb); free(buf_wb); free(buf_abs_wb);
adios_selection_delete(block_bb);
adios_selection_delete(block_wb);
adios_selection_delete(block_abs_wb);
}
}
adios_free_varinfo(varinfo);
fprintf(stderr, "[rank %d/%d] Finished file-mode writeblock reads on %s:/%s\n", rank, size, bp_filename, varname);
}
int read_points ()
{
ADIOS_SELECTION *sel0,*sel1,*sel2,*sel3;
ADIOS_FILE * f;
ADIOS_VARINFO * vi;
int err=0,n,n1, i,j,k;
int nsteps_a, nsteps_b, nsteps_c;
int v;
uint64_t *pts1;
uint64_t *pts2;
uint64_t *pts3;
uint64_t start[3] = {offs1,offs2,offs3};
uint64_t count[3] = {ldim1,ldim2,ldim3};
pts1 = (uint64_t*) malloc (1*sizeof(uint64_t)*ldim1);
pts2 = (uint64_t*) malloc (2*sizeof(uint64_t)*ldim1*ldim2);
pts3 = (uint64_t*) malloc (3*sizeof(uint64_t)*ldim1*ldim2*ldim3);
for (i=0; i<ldim1; i++)
{
pts1[i] = offs1 + i;
for (j=0; j<ldim2; j++)
{
pts2[i*2*ldim2 + 2*j] = offs1 + i;
pts2[i*2*ldim2 + 2*j+1] = j;
for (k=0; k<ldim3; k++)
{
pts3[i*3*ldim2*ldim3+j*3*ldim3+3*k] = offs1 + i;
pts3[i*3*ldim2*ldim3+j*3*ldim3+3*k+1] = j;
pts3[i*3*ldim2*ldim3+j*3*ldim3+3*k+2] = k;
}
}
}
//Force error on very last point: pts3[3*ldim1*ldim2*ldim3-1] = 0;
reset_readvars();
log ("Read and check data in %s using point selections\n", FILENAME);
f = adios_read_open (FILENAME, read_method, comm,
ADIOS_LOCKMODE_CURRENT, 0.0);
if (f == NULL) {
printE ("Error at opening file: %s\n", rank, adios_errmsg());
return 1;
}
sel1 = adios_selection_points (1, ldim1, pts1);
sel2 = adios_selection_points (2, ldim1*ldim2, pts2);
sel3 = adios_selection_points (3, ldim1*ldim2*ldim3, pts3);
#ifdef DEBUG_PRINTS
fprintf(stderr, "1D selection: {");
for (i=0; i<ldim1; i++) {
fprintf (stderr, "%d ", i, i, pts1[i]);
}
fprintf(stderr, "}\n");
fprintf(stderr, "2D selection :\n");
for (i=0; i<ldim1; i++) {
fprintf (stderr, "i=%d, idx=%2d- = ", i, i*2*ldim2);
for (j=0; j<ldim2; j++) {
n = i*2*ldim2+2*j;
fprintf (stderr, "{%d,%d} ", pts2[n], pts2[n+1]);
}
fprintf(stderr, "\n");
}
fprintf(stderr, "3D selection :\n");
for (i=0; i<ldim1; i++) {
for (j=0; j<ldim2; j++) {
fprintf (stderr, "i=%d, j=%d, idx=%3d- = ", i, j, i*3*ldim2*ldim3+j*3*ldim3);
for (k=0; k<ldim3; k++) {
n = i*3*ldim2*ldim3+j*3*ldim3+3*k;
fprintf (stderr, "{%d,%d,%d} ", pts3[n], pts3[n+1], pts3[n+2]);
}
fprintf(stderr, "\n");
}
}
#endif
n1=0;
while (n1 < NSTEPS && adios_errno != err_end_of_stream) {
n1++;
log (" Step %d\n", f->current_step);
log (" Check 1D variable a1...\n");
adios_schedule_read (f, sel1, "a1", 0, 1, r1);
adios_perform_reads (f, 1);
#ifdef DEBUG_PRINTS
fprintf(stderr, "1D result: {");
for (i=0; i<ldim1; i++) {
fprintf (stderr, "%d ", i, i, r1[i]);
}
fprintf(stderr, "}\n");
#endif
for (i=0; i<ldim1; i++) {
v = VALUE1D(rank,f->current_step,i);
if (r1[i] != v) {
printE ("Error: a1[%d]=%d != read=%d\n", i, v, r1[i]);
//goto endread;
}
}
log (" Check 2D variable a2...\n");
adios_schedule_read (f, sel2, "a2", 0, 1, r2);
adios_perform_reads (f, 1);
#ifdef DEBUG_PRINTS
fprintf(stderr, "2D result :\n");
n = 0;
for (i=0; i<ldim1; i++) {
fprintf (stderr, "row=%2d- = {", i);
for (j=0; j<ldim2; j++) {
fprintf (stderr, "%d ", r2[n]);
n++;
}
fprintf(stderr, "}\n");
}
#endif
n = 0;
for (i=0; i<ldim1; i++) {
for (j=0; j<ldim2; j++) {
v = VALUE2D(rank,f->current_step,i,j);
if (v != r2[n]) {
printE ("Error: a2[%d,%d]=%d != read=%d\n", i, j, v, r2[n]);
//goto endread;
}
n++;
}
}
log (" Check 3D variable a3...\n");
adios_schedule_read (f, sel3, "a3", 0, 1, r3);
adios_perform_reads (f, 1);
#ifdef DEBUG_PRINTS
fprintf(stderr, "3D selection :\n");
n=0;
for (i=0; i<ldim1; i++) {
for (j=0; j<ldim2; j++) {
fprintf (stderr, "[%d,%d] = {", i, j);
for (k=0; k<ldim3; k++) {
fprintf (stderr, "%d ", r3[n]);
n++;
}
fprintf(stderr, "}\n");
}
}
#endif
n = 0;
for (i=0; i<ldim1; i++) {
for (j=0; j<ldim2; j++) {
for (k=0; k<ldim3; k++) {
v = VALUE3D(rank,f->current_step,i,j,k);
if (v != r3[n]) {
printE ("Error: a3[%d,%d,%d]=%d != read=%d\n", i, j, k, v, r3[n]);
//goto endread;
}
n++;
}
}
}
if (n1 < NSTEPS)
{
adios_advance_step (f, 0, -1.0);
}
}
endread:
adios_selection_delete (sel1);
adios_selection_delete (sel2);
adios_selection_delete (sel3);
free(pts1);
free(pts2);
free(pts3);
adios_read_close(f);
MPI_Barrier (comm);
return err;
}
eavlField *
eavlXGCParticleImporter::GetField(const string &name, const string &mesh, int chunk)
{
int idx = 0;
eavlField *field;
uint64_t s[3], c[3];
map<string, ADIOS_VARINFO*>::const_iterator it;
if(name.compare("ephase") == 0 && ephaseAvail)
{
eavlFloatArray *arr = new eavlFloatArray(name, 1);
arr->SetNumberOfTuples(totalEParticles*9);
for(it = ephase.begin(); it != ephase.end(); it++)
{
int nt = 1;
ADIOS_SELECTION *sel;
if(readingRestartFile)
sel = MakeSelection(it->second, s, c);
else
sel = MakeLimitedSelection(it->second, s, c, 0);
for(int i = 0; i < it->second->ndim; i++)
nt *= c[i];
double *buff = new double[nt];
adios_schedule_read_byid(fp, sel, it->second->varid, 0, 1, buff);
int retval = adios_perform_reads(fp, 1);
adios_selection_delete(sel);
for(int i = 0; i < nt; i++)
{
arr->SetComponentFromDouble(idx, 0, buff[i]);
idx++;
}
delete [] buff;
}
field = new eavlField(1, arr, eavlField::ASSOC_POINTS);
}
else if(name.compare("iphase") == 0 && iphaseAvail)
{
eavlFloatArray *arr = new eavlFloatArray(name, 1);
arr->SetNumberOfTuples(totalIParticles*9);
for(it = iphase.begin(); it != iphase.end(); it++)
{
int nt = 1;
ADIOS_SELECTION *sel;
if(readingRestartFile)
sel = MakeSelection(it->second, s, c);
else
sel = MakeLimitedSelection(it->second, s, c, 1);
for (int i = 0; i < it->second->ndim; i++)
nt *= c[i];
double *buff = new double[nt];
adios_schedule_read_byid(fp, sel, it->second->varid, 0, 1, buff);
int retval = adios_perform_reads(fp, 1);
adios_selection_delete(sel);
for(int i = 0; i < nt; i++)
{
arr->SetComponentFromDouble(idx, 0, buff[i]);
idx++;
}
delete [] buff;
}
field = new eavlField(1, arr, eavlField::ASSOC_POINTS);
}
else if(name.compare("igid") == 0 && iphaseAvail)
{
eavlIntArray *arr = new eavlIntArray(name, 1);
arr->SetNumberOfTuples(totalIParticles);
for(it = igid.begin(); it != igid.end(); it++)
{
int nt = 1;
ADIOS_SELECTION *sel;
if(readingRestartFile)
sel = MakeSelection(it->second, s, c);
else
sel = MakeLimitedSelection(it->second, s, c, 1);
for(int i = 0; i < it->second->ndim; i++)
nt *= c[i];
long long *buff = new long long[nt];
adios_schedule_read_byid(fp, sel, it->second->varid, 0, 1, buff);
int retval = adios_perform_reads(fp, 1);
adios_selection_delete(sel);
for(int i = 0; i < nt; i++)
{
arr->SetValue(idx, (int)buff[i]);
idx++;
}
delete [] buff;
}
field = new eavlField(1, arr, eavlField::ASSOC_POINTS);
}
else if((name.compare("egid") == 0) && ephaseAvail )
{
eavlIntArray *arr = new eavlIntArray(name, 1);
arr->SetNumberOfTuples(totalEParticles);
for (it = egid.begin(); it != egid.end(); it++)
{
int nt = 1;
ADIOS_SELECTION *sel;
if(readingRestartFile)
sel = MakeSelection(it->second, s, c);
else
sel = MakeLimitedSelection(it->second, s, c, 0);
for(int i = 0; i < it->second->ndim; i++)
nt *= c[i];
long long *buff = new long long[nt];
adios_schedule_read_byid(fp, sel, it->second->varid, 0, 1, buff);
int retval = adios_perform_reads(fp, 1);
adios_selection_delete(sel);
for(int i = 0; i < nt; i++)
{
arr->SetValue(idx, (int)buff[i]);
idx++;
}
delete [] buff;
}
field = new eavlField(1, arr, eavlField::ASSOC_POINTS);
}
else
{
THROW(eavlException, string("Variable not found: ")+name);
}
return field;
}
eavlDataSet *
eavlXGCParticleImporter::GetMesh(const string &name, int chunk)
{
eavlDataSet *ds = new eavlDataSet;
if(name == "iMesh" && iphaseAvail)
{
ds->SetNumPoints(totalIParticles);
eavlCoordinatesCartesian *coords = new eavlCoordinatesCartesian(NULL,
eavlCoordinatesCartesian::X,
eavlCoordinatesCartesian::Y,
eavlCoordinatesCartesian::Z);
ds->AddCoordinateSystem(coords);
coords->SetAxis(0, new eavlCoordinateAxisField("xcoords", 0));
coords->SetAxis(1, new eavlCoordinateAxisField("ycoords", 0));
coords->SetAxis(2, new eavlCoordinateAxisField("zcoords", 0));
eavlArray *axisValues[3] = {
new eavlFloatArray("xcoords", 1),
new eavlFloatArray("ycoords", 1),
new eavlFloatArray("zcoords", 1)
};
axisValues[0]->SetNumberOfTuples(totalIParticles);
axisValues[1]->SetNumberOfTuples(totalIParticles);
axisValues[2]->SetNumberOfTuples(totalIParticles);
//Set all of the axis values to the x, y, z coordinates of the
//iphase particles; set computational node origin
eavlIntArray *originNode;
if(getOriginNode && readingRestartFile)
originNode = new eavlIntArray("originNode", 1, totalIParticles);
eavlFloatArray *r;
if(getR)
r = new eavlFloatArray("R", 1, totalIParticles);
eavlFloatArray *z;
if(getZ)
z = new eavlFloatArray("Z", 1, totalIParticles);
eavlFloatArray *phi;
if(getPhi)
phi = new eavlFloatArray("phi", 1, totalIParticles);
eavlFloatArray *rho;
if(getRho)
rho = new eavlFloatArray("rho", 1, totalIParticles);
eavlFloatArray *w1;
if(getW1)
w1 = new eavlFloatArray("w1", 1, totalIParticles);
eavlFloatArray *w2;
if(getW2)
w2 = new eavlFloatArray("w2", 1, totalIParticles);
eavlFloatArray *mu;
if(getMu)
mu = new eavlFloatArray("mu", 1, totalIParticles);
eavlFloatArray *w0;
if(getW0)
w0 = new eavlFloatArray("w0", 1, totalIParticles);
eavlFloatArray *f0;
if(getF0)
f0 = new eavlFloatArray("f0", 1, totalIParticles);
uint64_t s[3], c[3];
double *buff;
int nt = 1, idx = 0;
map<string, ADIOS_VARINFO*>::const_iterator it;
for(it = iphase.begin(); it != iphase.end(); it++)
{
ADIOS_SELECTION *sel;
if(readingRestartFile)
sel = MakeSelection(it->second, s, c);
else
sel = MakeLimitedSelection(it->second, s, c, 1);
nt = 1;
for (int i = 0; i < it->second->ndim; i++)
nt *= c[i];
buff = new double[nt];
adios_schedule_read_byid(fp, sel, it->second->varid, 0, 1, buff);
adios_perform_reads(fp, 1);
adios_selection_delete(sel);
string nodeNum;
if(getOriginNode && readingRestartFile) nodeNum = it->first.substr(it->first.find("_",1,1)+1, 5);
for(int i = 0; i < nt; i+=9)
{
if(getR) r->SetValue(idx, buff[i]);
if(getZ) z->SetValue(idx, buff[i+1]);
if(getPhi) phi->SetValue(idx, buff[i+2]);
if(getRho) rho->SetValue(idx, buff[i+3]);
if(getW1) w1->SetValue(idx, buff[i+4]);
if(getW2) w2->SetValue(idx, buff[i+5]);
if(getMu) mu->SetValue(idx, buff[i+6]);
if(getW0) w0->SetValue(idx, buff[i+7]);
if(getF0) f0->SetValue(idx, buff[i+8]);
axisValues[0]->SetComponentFromDouble(idx, 0, r->GetValue(idx)*cos(phi->GetValue(idx)));
axisValues[1]->SetComponentFromDouble(idx, 0, r->GetValue(idx)*sin(phi->GetValue(idx)));
axisValues[2]->SetComponentFromDouble(idx, 0, z->GetValue(idx));
if(getOriginNode && readingRestartFile) originNode->SetValue(idx, atoi(nodeNum.c_str()));
idx++;
}
delete [] buff;
}
ds->AddField(new eavlField(1, axisValues[0], eavlField::ASSOC_POINTS));
ds->AddField(new eavlField(1, axisValues[1], eavlField::ASSOC_POINTS));
ds->AddField(new eavlField(1, axisValues[2], eavlField::ASSOC_POINTS));
if(getOriginNode && readingRestartFile)
ds->AddField(new eavlField(1, originNode, eavlField::ASSOC_POINTS));
if(getR) ds->AddField(new eavlField(1, r, eavlField::ASSOC_POINTS));
if(getZ) ds->AddField(new eavlField(1, z, eavlField::ASSOC_POINTS));
if(getPhi) ds->AddField(new eavlField(1, phi, eavlField::ASSOC_POINTS));
if(getRho) ds->AddField(new eavlField(1, rho, eavlField::ASSOC_POINTS));
if(getW1) ds->AddField(new eavlField(1, w1, eavlField::ASSOC_POINTS));
if(getW2) ds->AddField(new eavlField(1, w2, eavlField::ASSOC_POINTS));
if(getMu) ds->AddField(new eavlField(1, mu, eavlField::ASSOC_POINTS));
if(getW0) ds->AddField(new eavlField(1, w0, eavlField::ASSOC_POINTS));
if(getF0) ds->AddField(new eavlField(1, f0, eavlField::ASSOC_POINTS));
eavlCellSet *cellSet = new eavlCellSetAllPoints(name + "_cells", totalIParticles);
ds->AddCellSet(cellSet);
//-- END set axis values
//----Set the ids of all axis values
idx = 0;
long long *idBuff;
eavlIntArray *axisIds = new eavlIntArray("id", 1, totalIParticles);
for(it = igid.begin(); it != igid.end(); it++)
{
ADIOS_SELECTION *sel;
if(readingRestartFile)
sel = MakeSelection(it->second, s, c);
else
sel = MakeLimitedSelection(it->second, s, c, 1);
nt = 1;
for (int i = 0; i < it->second->ndim; i++)
nt *= c[i];
idBuff = new long long[nt];
adios_schedule_read_byid(fp, sel, it->second->varid, 0, 1, idBuff);
adios_perform_reads(fp, 1);
adios_selection_delete(sel);
for(int i = 0; i < nt; i++)
{
axisIds->SetValue(idx, (int)idBuff[i]);
idx++;
}
delete [] idBuff;
}
ds->AddField(new eavlField(1, axisIds, eavlField::ASSOC_POINTS));
//-- END set ids
}
else if(name == "eMesh" && ephaseAvail)
{
ds->SetNumPoints(totalEParticles);
eavlCoordinatesCartesian *coords = new eavlCoordinatesCartesian(NULL,
eavlCoordinatesCartesian::X,
eavlCoordinatesCartesian::Y,
eavlCoordinatesCartesian::Z);
ds->AddCoordinateSystem(coords);
coords->SetAxis(0, new eavlCoordinateAxisField("xcoords", 0));
coords->SetAxis(1, new eavlCoordinateAxisField("ycoords", 0));
coords->SetAxis(2, new eavlCoordinateAxisField("zcoords", 0));
eavlArray *axisValues[3] = {
new eavlFloatArray("xcoords", 1),
new eavlFloatArray("ycoords", 1),
new eavlFloatArray("zcoords", 1)
};
axisValues[0]->SetNumberOfTuples(totalEParticles);
axisValues[1]->SetNumberOfTuples(totalEParticles);
axisValues[2]->SetNumberOfTuples(totalEParticles);
//Set all of the axis values to the x, y, z coordinates of the
//ephase particles; set computational node origin
eavlIntArray *originNode;
if(getOriginNode && readingRestartFile)
originNode = new eavlIntArray("originNode", 1, totalIParticles);
eavlFloatArray *r;
if(getR)
r = new eavlFloatArray("R", 1, totalIParticles);
eavlFloatArray *z;
if(getZ)
z = new eavlFloatArray("Z", 1, totalIParticles);
eavlFloatArray *phi;
if(getPhi)
phi = new eavlFloatArray("phi", 1, totalIParticles);
eavlFloatArray *rho;
if(getRho)
rho = new eavlFloatArray("rho", 1, totalIParticles);
eavlFloatArray *w1;
if(getW1)
w1 = new eavlFloatArray("w1", 1, totalIParticles);
eavlFloatArray *w2;
if(getW2)
w2 = new eavlFloatArray("w2", 1, totalIParticles);
eavlFloatArray *mu;
if(getMu)
mu = new eavlFloatArray("mu", 1, totalIParticles);
eavlFloatArray *w0;
if(getW0)
w0 = new eavlFloatArray("w0", 1, totalIParticles);
eavlFloatArray *f0;
if(getF0)
f0 = new eavlFloatArray("f0", 1, totalIParticles);
uint64_t s[3], c[3];
int nt = 1, idx = 0;
double *buff;
map<string, ADIOS_VARINFO*>::const_iterator it;
for(it = ephase.begin(); it != ephase.end(); it++)
{
ADIOS_SELECTION *sel;
if(readingRestartFile)
sel = MakeSelection(it->second, s, c);
else
sel = MakeLimitedSelection(it->second, s, c, 0);
nt = 1;
for(int i = 0; i < it->second->ndim; i++)
nt *= c[i];
buff = new double[nt];
adios_schedule_read_byid(fp, sel, it->second->varid, 0, 1, buff);
adios_perform_reads(fp, 1);
adios_selection_delete(sel);
string nodeNum;
if(getOriginNode && readingRestartFile) nodeNum = it->first.substr(it->first.find("_",1,1)+1, 5);
for(int i = 0; i < nt; i+=9)
{
if(getR) r->SetValue(idx, buff[i]);
if(getZ) z->SetValue(idx, buff[i+1]);
if(getPhi) phi->SetValue(idx, buff[i+2]);
if(getRho) rho->SetValue(idx, buff[i+3]);
if(getW1) w1->SetValue(idx, buff[i+4]);
if(getW2) w2->SetValue(idx, buff[i+5]);
if(getMu) mu->SetValue(idx, buff[i+6]);
if(getW0) w0->SetValue(idx, buff[i+7]);
if(getF0) f0->SetValue(idx, buff[i+8]);
axisValues[0]->SetComponentFromDouble(idx, 0, r->GetValue(idx)*cos(phi->GetValue(idx)));
axisValues[1]->SetComponentFromDouble(idx, 0, r->GetValue(idx)*sin(phi->GetValue(idx)));
axisValues[2]->SetComponentFromDouble(idx, 0, z->GetValue(idx));
if(getOriginNode && readingRestartFile) originNode->SetValue(idx, atoi(nodeNum.c_str()));
idx++;
}
delete [] buff;
}
ds->AddField(new eavlField(1, axisValues[0], eavlField::ASSOC_POINTS));
ds->AddField(new eavlField(1, axisValues[1], eavlField::ASSOC_POINTS));
ds->AddField(new eavlField(1, axisValues[2], eavlField::ASSOC_POINTS));
if(getOriginNode && readingRestartFile)
ds->AddField(new eavlField(1, originNode, eavlField::ASSOC_POINTS));
if(getR) ds->AddField(new eavlField(1, r, eavlField::ASSOC_POINTS));
if(getZ) ds->AddField(new eavlField(1, z, eavlField::ASSOC_POINTS));
if(getPhi) ds->AddField(new eavlField(1, phi, eavlField::ASSOC_POINTS));
if(getRho) ds->AddField(new eavlField(1, rho, eavlField::ASSOC_POINTS));
if(getW1) ds->AddField(new eavlField(1, w1, eavlField::ASSOC_POINTS));
if(getW2) ds->AddField(new eavlField(1, w2, eavlField::ASSOC_POINTS));
if(getMu) ds->AddField(new eavlField(1, mu, eavlField::ASSOC_POINTS));
if(getW0) ds->AddField(new eavlField(1, w0, eavlField::ASSOC_POINTS));
if(getF0) ds->AddField(new eavlField(1, f0, eavlField::ASSOC_POINTS));
eavlCellSet *cellSet = new eavlCellSetAllPoints(name + "_cells", totalEParticles);
ds->AddCellSet(cellSet);
//-- END set axis values
//----Set the ids of all axis values
idx = 0;
long long *idBuff;
eavlIntArray *axisIds = new eavlIntArray("id", 1, totalEParticles);
for(it = egid.begin(); it != egid.end(); it++)
{
ADIOS_SELECTION *sel;
if(readingRestartFile)
sel = MakeSelection(it->second, s, c);
else
sel = MakeLimitedSelection(it->second, s, c, 0);
nt = 1;
for (int i = 0; i < it->second->ndim; i++)
nt *= c[i];
idBuff = new long long[nt];
adios_schedule_read_byid(fp, sel, it->second->varid, 0, 1, idBuff);
adios_perform_reads(fp, 1);
adios_selection_delete(sel);
for(int i = 0; i < nt; i++)
{
axisIds->SetValue(idx, (int)idBuff[i]);
idx++;
}
delete [] idBuff;
}
ds->AddField(new eavlField(1, axisIds, eavlField::ASSOC_POINTS));
}
return ds;
}
int main (int argc, char ** argv)
{
int rank, size, i, j, npl, token;
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Status status;
enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_BP;
ADIOS_SELECTION * sel;
void * data = NULL;
uint64_t start[1], count[1];
MPI_Init (&argc, &argv);
MPI_Comm_rank (comm, &rank);
MPI_Comm_size (comm, &size);
adios_read_init_method (method, comm, "verbose=3");
ADIOS_FILE * f = adios_read_open ("adios_global_no_xml.bp", method,
comm, ADIOS_LOCKMODE_NONE, 0);
if (f == NULL)
{
printf ("%s\n", adios_errmsg());
return -1;
}
ADIOS_VARINFO * v = adios_inq_var (f, "temperature");
/* Using less readers to read the global array back, i.e., non-uniform */
uint64_t slice_size = v->dims[0]/size;
start[0] = slice_size * rank;
if (rank == size-1) /* last rank may read more lines */
slice_size = slice_size + v->dims[0]%size;
count[0] = slice_size;
data = malloc (slice_size * sizeof (double));
if (data == NULL)
{
fprintf (stderr, "malloc failed.\n");
return -1;
}
/* Read a subset of the temperature array */
sel = adios_selection_boundingbox (v->ndim, start, count);
adios_schedule_read (f, sel, "temperature", 0, 1, data);
adios_perform_reads (f, 1);
if (rank > 0) {
MPI_Recv (&token, 1, MPI_INT, rank-1, 0, comm, &status);
}
printf (" ======== Rank %d ========== \n", rank);
npl = 10;
for (i = 0; i < slice_size; i+=npl) {
printf ("[%4.4" PRIu64 "] ", rank*slice_size+i);
for (j= 0; j < npl; j++) {
printf (" %6.6g", * ((double *)data + i + j));
}
printf ("\n");
}
fflush(stdout);
sleep(1);
if (rank < size-1) {
MPI_Send (&token, 1, MPI_INT, rank+1, 0, comm);
}
free (data);
adios_selection_delete (sel);
adios_free_varinfo (v);
adios_read_close (f);
MPI_Barrier (comm);
adios_read_finalize_method (method);
MPI_Finalize ();
return 0;
}
int print_scalar (ADIOS_FILE *f, char * name)
{
ADIOS_VARINFO * v;
int i,j,k;
v = adios_inq_var (f, name);
adios_inq_var_blockinfo (f, v);
printf ("Scalar '%s':\n", name);
printf ("nsteps = %d\n", v->nsteps);
printf ("nblocks per step = %d\n", v->nblocks[0]);
int err;
/* Read one writeblock across all timesteps */
int *data = (int*) calloc (v->nsteps, sizeof(int));
ADIOS_SELECTION *s;
printf ("Read same instance across all timesteps:\n");
for (i=0; i < v->nblocks[0]; i++) {
s = adios_selection_writeblock(i);
err = adios_schedule_read_byid(f, s, v->varid, 0, v->nsteps, data);
if (!err)
{
err = adios_perform_reads(f, 1);
if (!err)
{
err = 0;
printf (" block %d = [", i);
for (j=0; j < v->nsteps; j++) {
printf ("%d", data[j]);
if (data[j] !=
block_offset [j*nblocks_per_step*size + i])
{
err = 1;
}
if (j < v->nsteps-1) printf(",");
}
printf("]");
if (err)
{
nerrors++;
printf ("\tERROR expected = [");
for (j=0; j < v->nsteps; j++) {
printf ("%llu", block_offset [j*nblocks_per_step*size + i]);
if (j < v->nsteps-1) printf(",");
}
printf("]");
}
printf("\n");
} else {
printf ("ERROR at reading scalar '%s': %s\n", name, adios_errmsg());
}
} else {
printf ("ERROR at scheduling read for scalar '%s': %s\n", name, adios_errmsg());
}
adios_selection_delete(s);
}
/* Now read piecewise, one writeblock at a time */
printf ("Read each instance individually:\n");
for (j=0; j < v->nsteps; j++) {
printf (" step %d: \n", j);
for (i=0; i < v->nblocks[j]; i++) {
s = adios_selection_writeblock(i);
err = adios_schedule_read_byid(f, s, v->varid, j, 1, data);
if (!err)
{
err = adios_perform_reads(f, 1);
if (!err)
{
printf (" block %d = %d", i, data[0]);
if (data[0] !=
block_offset [j*nblocks_per_step*size + i])
{
printf ("\tERROR expected = %llu",
block_offset [j*nblocks_per_step*size + i]);
nerrors++;
}
printf ("\n");
} else {
printf ("ERROR at reading scalar '%s': %s\n", name, adios_errmsg());
}
} else {
printf ("ERROR at scheduling read for scalar '%s': %s\n", name, adios_errmsg());
}
adios_selection_delete(s);
}
}
/* Now get them piecewise, but not with reading but through statistics */
printf ("Get each instance individually from available statistics:\n");
adios_inq_var_stat (f, v, 0, 1);
if (v->statistics && v->statistics->blocks) {
ADIOS_VARSTAT *stat = v->statistics;
int blockid = 0;
for (j=0; j < v->nsteps; j++) {
printf (" step %d: \n", j);
for (i=0; i < v->nblocks[j]; i++) {
printf (" block %d = %d", i, *(int*)stat->blocks->mins[blockid]);
if (*(int*)stat->blocks->mins[blockid] !=
block_offset [j*nblocks_per_step*size + i])
{
printf ("\tERROR expected = %llu",
block_offset [j*nblocks_per_step*size + i]);
nerrors++;
}
printf ("\n");
blockid++;
}
}
}
adios_free_varinfo (v);
free(data);
}
int main (int argc, char ** argv)
{
int i, j, datasize;
MPI_Comm comm = MPI_COMM_WORLD;
enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_BP;
ADIOS_SELECTION * sel1;
double * data = NULL;
uint64_t start[2], count[2];
MPI_Init (&argc, &argv);
#ifdef WITH_NCSU_TIMER
timer_init();
#endif
adios_read_init_method (method, comm, NULL);
ADIOS_FILE * f = adios_read_open_file ("adios_global.bp", method, comm);
ADIOS_VARINFO * varinfo = adios_inq_var (f, "temperature");
if (varinfo)
{
int nranks;
assert(varinfo->ndim == 2);
nranks = varinfo->dims[0];
assert(nranks % 4 == 0);
assert(varinfo->dims[1] == 10);
datasize = (nranks / 2) * varinfo->dims[1] * sizeof(double);
data = malloc (datasize);
start[0] = nranks / 4;
start[1] = 2;
count[0] = nranks / 2;
count[1] = 6;
sel1 = adios_selection_boundingbox (varinfo->ndim, start, count);
adios_schedule_read (f, sel1, "temperature", 0, 1, data);
adios_perform_reads (f, 1);
printf("Subvolume at (%" PRIu64 ",%" PRIu64 ") of size (%" PRIu64 ",%" PRIu64 "):\n", start[0], start[1], count[0], count[1]);
for (i = 0; i < count[0]; i++) {
printf("[ ");
for (j = 0; j < count[1]; j++) {
printf("%.0lf ", data[i * count[1] + j]);
}
printf("]\n");
}
adios_selection_delete (sel1);
}
adios_free_varinfo (varinfo);
adios_read_close (f);
adios_read_finalize_method (ADIOS_READ_METHOD_BP);
#ifdef WITH_NCSU_TIMER
printf("[TIMERS] ");
timer_result_t *results = timer_get_results_sorted();
for (i = 0; i < timer_get_num_timers(); i++) {
printf("%s: %0.4lf ", results[i].name, results[i].time);
}
printf("\n");
free(results);
#endif
#ifdef WITH_NCSU_TIMER
timer_finalize();
#endif
MPI_Finalize ();
return 0;
}
int main (int argc, char **argv){
int rank =0, size =0;
int NX = 0;
double *t = NULL;
// this is an array we expect as a reference array
double *t_ref = NULL;
MPI_Comm comm = MPI_COMM_WORLD;
diag_t diag = DIAG_OK; // to store the diagnostic information
struct test_info test_result = {TEST_PASSED, "1D_arr_global_noxml"};
struct err_counts err = { 0, 0};
struct adios_tsprt_opts adios_opts;
GET_ENTRY_OPTIONS(adios_opts, "Runs readers. It is recommended to run as many readers as writers.");
// adios read initialization
MPI_Init( &argc, &argv);
MPI_Comm_rank (comm, &rank);
// depending on the method
SET_ERROR_IF_NOT_ZERO(adios_read_init_method(adios_opts.method, comm, adios_opts.adios_options), err.adios);
RET_IF_ERROR(err.adios, rank);
// I will be working with streams so the lock mode is necessary,
// return immediately if the stream unavailable
ADIOS_FILE *adios_handle = adios_read_open(FILE_NAME, adios_opts.method, comm, ADIOS_LOCKMODE_NONE, 0.0);
if ( !adios_handle){
p_error("Quitting ... (%d) %s\n", adios_errno, adios_errmsg());
return DIAG_ERR;
}
// define portions of data how they will be read
ADIOS_SELECTION *sel = NULL;
ADIOS_VARINFO *avi = NULL;
// for storing the variables
char buf[STR_BUFFER_SIZE];
int step = 0;
// read how many processors wrote that array
avi = adios_inq_var (adios_handle, "size");
if (!avi){
p_error("rank %d: Quitting ... (%d) %s\n", rank, adios_errno, adios_errmsg());
diag = DIAG_ERR;
goto close_adios;
}
size = *((int*)avi->value);
adios_free_varinfo(avi);
avi = NULL;
// if I run the more readers than writers; just release
// the excessive readers
if (rank >= size){
p_info("rank %d: I am an excessive rank. Nothing to read ...\n", rank);
diag = DIAG_OK;
goto close_adios;
}
// read the size of the array
avi = adios_inq_var (adios_handle, "NX");
if (!avi){
p_error("rank %d: Quitting ... (%d) %s\n", rank, adios_errno, adios_errmsg());
diag = DIAG_ERR;
goto close_adios;
}
// I expect a scalar that will tell me the size of an array
assert(0 == avi->ndim);
assert(adios_integer == avi->type);
NX = *((int*)avi->value);
// I don't need that variable any more
adios_free_varinfo(avi);
assert(NX_DIM == NX);
avi = NULL;
// this will define the slice that we want to read; each rank should
// read its own slice written by a corresponding writer rank
uint64_t count[1] = { NX };
uint64_t start[1] = { 0 };
start[0] = rank*NX;
sel = adios_selection_boundingbox(1,start, count);
if( !sel ){
p_error("rank %d: Quitting ... (%d) %s\n", rank, adios_errno, adios_errmsg());
diag = DIAG_ERR;
goto close_adios;
}
// make the reference array with reference values I expect to get
t_ref = calloc(NX, sizeof(double));
if (gen_1D_array(t_ref, NX, rank) == DIAG_ERR){
p_error("Generating 1D array. Quitting ...\n");
diag = DIAG_ERR;
goto close_adios;
}
// allocate the memory for the actual array to be read
t = calloc(NX, sizeof(double));
if (adios_schedule_read(adios_handle, sel, "var_1d_array",0,1,t) != 0){
p_error("rank %d: Quitting ...(%d) %s\n", rank, adios_errno, adios_errmsg());
diag = DIAG_ERR;
goto just_clean;
}
// not sure if this assumption is correct; difficult to find in the ADIOS sources
if (adios_perform_reads(adios_handle, 1) != 0){
p_error("rank %d: Quitting ...(%d) %s\n", rank, adios_errno, adios_errmsg());
diag = DIAG_ERR;
goto just_clean;
}
sprintf(buf, "Rank %d: var_1d_array: step %d: t: ", rank, step);
int i = 0;
for(i=0; i < NX; ++i){
if( t[i] != t_ref[i] ){
p_test_failed("%s: rank %d: for t[%d] (expected %.1f, got %.1f)\n", test_result.name, rank, i, t_ref[i], t[i] );
test_result.result = TEST_FAILED;
break;
}
}
if (TEST_PASSED == test_result.result)
p_test_passed("%s: rank %d\n", test_result.name, rank);
just_clean:
// clean everything
adios_selection_delete(sel);
sel = NULL;
free(t);
t = NULL;
free(t_ref);
t_ref = NULL;
close_adios:
CLOSE_ADIOS_READER(adios_handle, adios_opts.method);
if ((DIAG_OK == diag) && (TEST_PASSED == test_result.result)) {
return 0;
} else {
return 1;
}
}
eavlDataSet *
eavlXGCImporter::GetMesh(const string &name, int chunk)
{
eavlDataSet *ds = new eavlDataSet;
if (name == "mesh2D")
{
ds->SetNumPoints(nNodes);
eavlCoordinatesCartesian *coords = new eavlCoordinatesCartesian(NULL,
eavlCoordinatesCartesian::X,
eavlCoordinatesCartesian::Y);
ds->AddCoordinateSystem(coords);
coords->SetAxis(0, new eavlCoordinateAxisField("xcoords", 0));
coords->SetAxis(1, new eavlCoordinateAxisField("ycoords", 0));
eavlArray *axisValues[2] = {new eavlFloatArray("xcoords", 1),
new eavlFloatArray("ycoords", 1)};
axisValues[0]->SetNumberOfTuples(nNodes);
axisValues[1]->SetNumberOfTuples(nNodes);
//read points
double *buff = new double[2*nNodes];
uint64_t s[3], c[3];
ADIOS_SELECTION *sel = MakeSelection(points, s, c);
adios_schedule_read_byid(mesh_fp, sel, points->varid, 0, 1, buff);
int retval = adios_perform_reads(mesh_fp, 1);
adios_selection_delete(sel);
for (int i = 0; i < nNodes; i++)
{
axisValues[0]->SetComponentFromDouble(i, 0, buff[i*2 +0]);
axisValues[1]->SetComponentFromDouble(i, 0, buff[i*2 +1]);
}
ds->AddField(new eavlField(1, axisValues[0], eavlField::ASSOC_POINTS));
ds->AddField(new eavlField(1, axisValues[1], eavlField::ASSOC_POINTS));
delete [] buff;
eavlCellSetExplicit *cellSet = new eavlCellSetExplicit(name + "_Cells", 2);
eavlExplicitConnectivity conn;
//read cells
int *nodeList = new int[nElems*3];
sel = MakeSelection(cells, s, c);
adios_schedule_read_byid(mesh_fp, sel, cells->varid, 0, 1, nodeList);
retval = adios_perform_reads(mesh_fp, 1);
adios_selection_delete(sel);
int nodes[3];
for (int i = 0; i < nElems; i++)
{
nodes[0] = nodeList[i*3+0];
nodes[1] = nodeList[i*3+1];
nodes[2] = nodeList[i*3+2];
conn.AddElement(EAVL_TRI, 3, nodes);
}
delete [] nodeList;
cellSet->SetCellNodeConnectivity(conn);
ds->AddCellSet(cellSet);
}
else if (name == "mesh3D")
{
int nPts = nNodes*nPlanes;
ds->SetNumPoints(nPts);
eavlCoordinatesCartesian *coords = new eavlCoordinatesCartesian(NULL,
eavlCoordinatesCartesian::X,
eavlCoordinatesCartesian::Y,
eavlCoordinatesCartesian::Z);
ds->AddCoordinateSystem(coords);
coords->SetAxis(0, new eavlCoordinateAxisField("xcoords", 0));
coords->SetAxis(1, new eavlCoordinateAxisField("ycoords", 0));
coords->SetAxis(2, new eavlCoordinateAxisField("zcoords", 0));
eavlArray *axisValues[3] = {new eavlFloatArray("xcoords", 1),
new eavlFloatArray("ycoords", 1),
new eavlFloatArray("zcoords", 1)};
axisValues[0]->SetNumberOfTuples(nPts);
axisValues[1]->SetNumberOfTuples(nPts);
axisValues[2]->SetNumberOfTuples(nPts);
//read points
double *buff = new double[2*nNodes];
uint64_t s[3], c[3];
ADIOS_SELECTION *sel = MakeSelection(points, s, c);
adios_schedule_read_byid(mesh_fp, sel, points->varid, 0, 1, buff);
int retval = adios_perform_reads(mesh_fp, 1);
adios_selection_delete(sel);
int idx = 0;
double dPhi = 2.0*M_PI/(double)nPlanes;
//double dPhi = 2.0*M_PI/(double)32;
for (int i = 0; i < nPlanes; i++)
{
double phi = (double)i * dPhi;
for (int j = 0; j < nNodes; j++)
{
double R = buff[j*2 +0];
double Z = buff[j*2 +1];
axisValues[0]->SetComponentFromDouble(idx, 0, R*cos(phi));
axisValues[1]->SetComponentFromDouble(idx, 0, R*sin(phi));
axisValues[2]->SetComponentFromDouble(idx, 0, Z);
idx++;
}
}
ds->AddField(new eavlField(1, axisValues[0], eavlField::ASSOC_POINTS));
ds->AddField(new eavlField(1, axisValues[1], eavlField::ASSOC_POINTS));
ds->AddField(new eavlField(1, axisValues[2], eavlField::ASSOC_POINTS));
delete [] buff;
eavlCellSetExplicit *cellSet = new eavlCellSetExplicit(name + "_Cells", 3);
eavlExplicitConnectivity conn;
//read cells
int *nodeList = new int[nElems*3];
sel = MakeSelection(cells, s, c);
adios_schedule_read_byid(mesh_fp, sel, cells->varid, 0, 1, nodeList);
retval = adios_perform_reads(mesh_fp, 1);
adios_selection_delete(sel);
int nodes[6];
for (int i = 0; i < nPlanes; i++)
{
int cellCnt = 0;
for (int j = 0; j < nElems*3; j+=3)
{
int off = i*nNodes;
nodes[0] = nodeList[j+0] + off;
nodes[1] = nodeList[j+1] + off;
nodes[2] = nodeList[j+2] + off;
off = ((i==nPlanes-1) ? 0 : (i+1)*nNodes);
nodes[3] = nodeList[j+0] + off;
nodes[4] = nodeList[j+1] + off;
nodes[5] = nodeList[j+2] + off;
conn.AddElement(EAVL_WEDGE, 6, nodes);
}
}
delete [] nodeList;
cellSet->SetCellNodeConnectivity(conn);
ds->AddCellSet(cellSet);
}
//ds->PrintSummary(cout);
return ds;
}
eavlDataSet *ReadPsiMesh(const string &filename)
{
MPI_Comm comm_dummy = 0;
ADIOS_FILE *fp = adios_read_open_file(filename.c_str(), ADIOS_READ_METHOD_BP, comm_dummy);
int nNodes = 0, nElems = 0, ptID = -1, elemID = -1;
ADIOS_VARINFO *points = NULL, *cells = NULL, *psi;
for (int i = 0; i < fp->nvars; i++)
{
ADIOS_VARINFO *avi = adios_inq_var_byid(fp, i);
string varNm(&fp->var_namelist[i][1]);
if (varNm == "n_t")
{
nElems = (int)(*(int *)avi->value);
adios_free_varinfo(avi);
}
else if (varNm == "n_n")
{
nNodes = (int)(*(int *)avi->value);
adios_free_varinfo(avi);
}
else if (varNm == "coordinates/values")
points = avi;
else if (varNm == "cell_set[0]/node_connect_list")
cells = avi;
else if (varNm == "psi")
psi = avi;
else
adios_free_varinfo(avi);
}
cout<<"nNodes= "<<nNodes<<" nTri= "<<nElems<<endl;
eavlDataSet *out = new eavlDataSet;
out->SetNumPoints(nNodes);
eavlCoordinatesCartesian *coords = new eavlCoordinatesCartesian(NULL,
eavlCoordinatesCartesian::X,
eavlCoordinatesCartesian::Y);
out->AddCoordinateSystem(coords);
coords->SetAxis(0, new eavlCoordinateAxisField("xcoords", 0));
coords->SetAxis(1, new eavlCoordinateAxisField("ycoords", 0));
eavlArray *axisValues[2] = {new eavlFloatArray("xcoords", 1),
new eavlFloatArray("ycoords", 1)};
axisValues[0]->SetNumberOfTuples(nNodes);
axisValues[1]->SetNumberOfTuples(nNodes);
//read points.
double *buff = new double[2*nNodes];
uint64_t s[3], c[3];
ADIOS_SELECTION *sel = MakeSelection(points, s, c);
adios_schedule_read_byid(fp, sel, points->varid, 0, 1, buff);
int retval = adios_perform_reads(fp, 1);
adios_selection_delete(sel);
adios_free_varinfo(points);
for (int i = 0; i < nNodes; i++)
{
axisValues[0]->SetComponentFromDouble(i, 0, buff[i*2 +0]);
axisValues[1]->SetComponentFromDouble(i, 0, buff[i*2 +1]);
}
out->AddField(new eavlField(1, axisValues[0], eavlField::ASSOC_POINTS));
out->AddField(new eavlField(1, axisValues[1], eavlField::ASSOC_POINTS));
delete [] buff;
eavlCellSetExplicit *cellSet = new eavlCellSetExplicit("2D_cells", 2);
eavlExplicitConnectivity conn;
//read cells
int *nodeList = new int[nElems*3];
sel = MakeSelection(cells, s, c);
adios_schedule_read_byid(fp, sel, cells->varid, 0, 1, nodeList);
retval = adios_perform_reads(fp, 1);
adios_selection_delete(sel);
int nodes[3];
for (int i = 0; i < nElems; i++)
{
nodes[0] = nodeList[i*3+0];
nodes[1] = nodeList[i*3+1];
nodes[2] = nodeList[i*3+2];
conn.AddElement(EAVL_TRI, 3, nodes);
}
delete [] nodeList;
cellSet->SetCellNodeConnectivity(conn);
out->AddCellSet(cellSet);
//read psi.
buff = new double[nNodes];
sel = MakeSelection(psi, s, c);
adios_schedule_read_byid(fp, sel, psi->varid, 0, 1, buff);
retval = adios_perform_reads(fp, 1);
adios_selection_delete(sel);
adios_free_varinfo(psi);
eavlArray *psiValues = new eavlFloatArray("psi", 1);
psiValues->SetNumberOfTuples(nNodes);
for (int i = 0; i < nNodes; i++)
psiValues->SetComponentFromDouble(i, 0, buff[i]);
out->AddField(new eavlField(1, psiValues, eavlField::ASSOC_POINTS));
delete [] buff;
return out;
}
void performQuery(ADIOS_QUERY_TEST_INFO *queryInfo, ADIOS_FILE *f, int use_streaming, int print_points, int read_results)
{
int timestep = 0 ;
ADIOS_VARINFO * tempVar = adios_inq_var(f, queryInfo->varName);
if (use_streaming)
for (timestep = 0; timestep < queryInfo->fromStep; ++timestep)
assert(adios_advance_step(f, 0, 0) == 0);
fprintf(stderr,"times steps for variable is: [%d, %d], batch size is %" PRIu64 "\n", queryInfo->fromStep, queryInfo->fromStep + queryInfo->numSteps, queryInfo->batchSize);
for (timestep = queryInfo->fromStep; timestep < queryInfo->fromStep + queryInfo->numSteps; timestep ++) {
fprintf(stderr, "querying on timestep %d \n", timestep);
ADIOS_SELECTION* currBatch = NULL;
while (adios_query_evaluate(queryInfo->query, queryInfo->outputSelection, use_streaming ? 0 : timestep, queryInfo->batchSize, &currBatch) >= 0) {
if (currBatch == NULL) {
break;
}
assert(currBatch->type ==ADIOS_SELECTION_POINTS);
const ADIOS_SELECTION_POINTS_STRUCT * retrievedPts = &(currBatch->u.points);
/* fprintf(stderr,"retrieved points %" PRIu64 " \n", retrievedPts->npoints); */
if (print_points) {
printPoints(retrievedPts, timestep);
}
if (read_results) {
int elmSize = adios_type_size(tempVar->type, NULL);
void *data = malloc(retrievedPts->npoints * elmSize);
// read returned temp data
adios_schedule_read (f, currBatch, queryInfo->varName, use_streaming ? 0 : timestep, 1, data);
adios_perform_reads(f, 1);
free(data);
}
fprintf(stderr,"Total data retrieved:%"PRIu64"\n", retrievedPts->npoints);
/* if (tempVar->type == adios_double) { */
/* for (i = 0; i < retrievedPts->npoints; i++) { */
/* fprintf(stderr,"%.6f\t", ((double*)data)[i]); */
/* } */
/* fprintf(stderr,"\n"); */
/* } */
/* else if (tempVar->type == adios_real) { */
/* for (i = 0; i < retrievedPts->npoints; i++) { */
/* fprintf(stderr,"%.6f\t", ((float*)data)[i]); */
/* } */
/* fprintf(stderr,"\n"); */
/* } */
adios_selection_delete(currBatch);
currBatch = NULL;
}
if (use_streaming) {
const int err = adios_advance_step(f, 0, 0);
if (timestep < queryInfo->fromStep + queryInfo->numSteps - 1) {
assert(err == 0);
} else {
assert(err == err_end_of_stream || err == err_step_notready);
}
}
}
adios_query_free(queryInfo->query);
}