void adios_checkpoint_verify_variables(ADIOS_FILE* fp, const char* name, int* origin)
{
ADIOS_VARINFO *vi;
int count = 1;
int size;
vi = adios_inq_var(fp, name);
if (vi->ndim > 0)
{
cout<<name<<" verification not passed, not a scalar"<<endl;
return;
}
size = count*adios_type_size(vi->type, vi->value);
int * mem= (int * )malloc(size);
ADIOS_SELECTION *sel = adios_selection_writeblock(OHMMS::Controller->rank());
adios_schedule_read(fp, sel, name, 0, 1, mem);
//adios_schedule_read(fp, NULL, name, 0, 1, mem);
adios_perform_reads(fp, 1);
if(mem[0] == *origin)
{
cout<<name<<" verification passed "<<mem[0]<<endl;
}
else
{
cout<<name<<" verification not passed, readin: "<<mem[0]<<" writeout: "<<*origin<<endl;
}
adios_free_varinfo (vi);
adios_selection_delete(sel);
free(mem);
}
AdiosCheckpointInput::AdiosCheckpointInput(std::string& checkpoint_file)
{
adios_read_init(method, OHMMS::Controller->getMPI(), "verbose=3;abort_on_error");
adios_file_handle = adios_read_open_file(checkpoint_file.c_str(), method,
comm);
if (adios_file_handle == NULL)
{
qmcplusplus::app_error() << adios_errmsg() << std::endl;
}
//Select for local values
sel = adios_selection_writeblock (OHMMS::Controller->rank());
}
void onBlock(int rank, ADIOS_FILE* f, ADIOS_VARINFO* v, int i, int j, int blockCounter, FastBitDataType ft)
{
char bmsVarName[100];
char keyVarName[100];
char offsetName[100];
int64_t var_ids_bms[v->nblocks[i]];
int64_t var_ids_key[v->nblocks[i]];
int64_t var_ids_offset[v->nblocks[i]];
sprintf(bmsVarName, "bms-%d-%d-%d", v->varid, i, j);
sprintf(keyVarName, "key-%d-%d-%d", v->varid, i, j);
sprintf(offsetName, "offset-%d-%d-%d", v->varid, i, j);
uint64_t blockSize = fastbit_adios_util_getBlockSize(v, i, j);
uint64_t blockDataByteSize = adios_type_size (v->type, v->value) * blockSize;
char notes[100];
logTime(NULL); logTimeMillis(NULL);
sprintf(notes, " reading data from adios on varid=%d, time=%d, block: %d, size=%ld bytes=%ld", v->varid, i, j, blockSize, blockDataByteSize);
logTime(notes); logTimeMillis(notes);
localtime(&indexRefresh);
//printf(" %d th block / (%d), size= %" PRIu64 " bytes=%" PRIu64, j, blockSize, blockCounter, blockDataByteSize);
void* data = malloc (blockDataByteSize);
ADIOS_SELECTION* blockSel = adios_selection_writeblock(j);
//adios_selcton_writeblock(num), 0 <= num < nblocks[timestep]
//ADIOS_SELECTION* blockSel = adios_selection_writeblock(blockCounter);
int err = adios_schedule_read_byid(f, blockSel, v->varid, i, 1, data);
if (!err) {
err = adios_perform_reads(f, 1);
} else {
printf("Unable to read block %d at timestep: %d \n", j, i);
return;
//break;
}
//fastbit_adios_util_printData(data, v->type, blockSize);
char selName[20];
sprintf(selName, "block-%d", j);
processData(data, blockSize, rank, i, selName, ft, v);
//processData(void* data, uint64_t dataCount, int rank, int timestep, char* selName, FastBitDataType ft, ADIOS_VARINFO* v)
adios_selection_delete(blockSel);
verifyData(f, v, blockCounter, i);
} // onblock
void adios_trace_verify_local_variables(ADIOS_FILE* fp, const char* name, double* origin)
{
ADIOS_VARINFO *vi;
int count = 1;
unsigned long size = 1;
vi = adios_inq_var(fp, name);
adios_inq_var_blockinfo(fp, vi);
for (int j=0; j<vi->nblocks[0]; j++)
{
if(OHMMS::Controller->rank() == j)
{
for (int k=0; k<vi->ndim; k++)
{
count *= vi->blockinfo[j].count[k];
}
}
}
size = count * adios_type_size(vi->type, vi->value);
double* mem = (double*)malloc(size);
ADIOS_SELECTION *sel = adios_selection_writeblock(OHMMS::Controller->rank());
adios_schedule_read(fp, sel, name, 0, 1, mem);
adios_perform_reads(fp, 1);
int flag = 0;
for(int i=0; i<count; i++)
{
if(mem[i] == origin[i])
{
}
else
{
flag = 1;
cout<<OHMMS::Controller->rank()<<" "<<name<<"["<<i<<"]verification not passed, readin: "<<mem[i]<<" writeout: "<<origin[i]<<endl;
}
}
if(flag == 0) cout<<OHMMS::Controller->rank()<<" "<<name<<" verification passed"<<endl;
adios_selection_delete(sel);
adios_free_varinfo (vi);
free(mem);
}
void onMultiBlock(int rank, ADIOS_FILE* f, ADIOS_VARINFO* v, int timestep, int blockStart, int blockEnd, FastBitDataType ft)
{
int i=0;
char selName[100];
sprintf(selName, "block-%d", blockStart, blockEnd);
uint64_t blockSizeArray[blockEnd-blockStart];
uint64_t totalCount = 0;
for (i = blockStart; i< blockEnd; i++) {
blockSizeArray[i-blockStart] = fastbit_adios_util_getBlockSize(v, timestep, i);
//totalBytes += blockSizeArray[i-blockStart] * adios_type_size (v->type, v->value);
totalCount += blockSizeArray[i-blockStart];
}
//void* data = malloc (totalBytes);
double* data = (double *) calloc (totalCount, sizeof (double));
uint64_t currStart = 0;
for (i = blockStart; i < blockEnd; i++) {
ADIOS_SELECTION* blockSel = adios_selection_writeblock(i);
int err = adios_schedule_read_byid(f, blockSel, v->varid, timestep, 1, &data[currStart]);
currStart += blockSizeArray[i-blockStart];
if (!err) {
err = adios_perform_reads(f, 1);
} else {
printf("Unable to read block %d at timestep: %d \n", i, timestep);
return;
//break;
}
}
processData(data, totalCount, rank, timestep, selName, ft, v);
//processData(void* data, uint64_t dataCount, int rank, int timestep, char* selName, FastBitDataType ft, ADIOS_VARINFO* v)
} // on multi block
static void test_file_mode_reads_on_var(ADIOS_FILE *fp, const char *bp_filename, const char *varname) {
int i;
ADIOS_VARINFO *varinfo = adios_inq_var(fp, varname);
MPI_Assert(COMM, varinfo);
if (varinfo->value != NULL) {
//if (rank == 0) fprintf(stderr, "(skipping scalar variable '%s')\n", varname);
adios_free_varinfo(varinfo);
return;
}
fprintf(stderr, "[rank %d/%d] Starting file-mode writeblock reads on %s:/%s\n", rank, size, bp_filename, varname);
adios_inq_var_blockinfo(fp, varinfo);
MPI_Assert(COMM, varinfo->blockinfo);
const enum ADIOS_DATATYPES datatype = varinfo->type;
const int datatypesize = adios_get_type_size(datatype, NULL);
int timestep, timestep_blockidx, blockidx = 0;
for (timestep = 0; timestep < varinfo->nsteps; ++timestep) {
for (timestep_blockidx = 0; timestep_blockidx < varinfo->nblocks[timestep]; ++timestep_blockidx, ++blockidx) {
if (blockidx % size != rank) continue;
const ADIOS_VARBLOCK *vb = &varinfo->blockinfo[blockidx];
ADIOS_SELECTION *block_bb = adios_selection_boundingbox(varinfo->ndim, vb->start, vb->count);
ADIOS_SELECTION *block_wb = adios_selection_writeblock(timestep_blockidx);
ADIOS_SELECTION *block_abs_wb = adios_selection_writeblock(blockidx);
block_abs_wb->u.block.is_absolute_index = 1;
uint64_t blocksize = datatypesize;
for (i = 0; i < varinfo->ndim; ++i)
blocksize *= vb->count[i];
void *buf_bb = malloc(blocksize);
void *buf_wb = malloc(blocksize);
void *buf_abs_wb = malloc(blocksize);
memset(buf_bb, 0, blocksize);
memset(buf_wb, 1, blocksize);
memset(buf_abs_wb, 2, blocksize);
MPI_Assert(COMM, buf_bb && buf_wb && buf_abs_wb);
adios_schedule_read(fp, block_bb, varname, timestep, 1, buf_bb );
adios_schedule_read(fp, block_wb, varname, timestep, 1, buf_wb );
adios_schedule_read(fp, block_abs_wb, varname, timestep, 1, buf_abs_wb);
adios_perform_reads(fp, 1);
fprintf(stderr, "[rank %d/%d] Checking file-mode blockidx %d BB vs. WB...\n", rank, size, blockidx);
MPI_Assert(COMM, memcmp(buf_bb, buf_wb, blocksize) == 0);
fprintf(stderr, "[rank %d/%d] Checking file-mode blockidx %d BB vs. abs-WB...\n", rank, size, blockidx);
MPI_Assert(COMM, memcmp(buf_bb, buf_abs_wb, blocksize) == 0);
free(buf_bb); free(buf_wb); free(buf_abs_wb);
adios_selection_delete(block_bb);
adios_selection_delete(block_wb);
adios_selection_delete(block_abs_wb);
}
}
adios_free_varinfo(varinfo);
fprintf(stderr, "[rank %d/%d] Finished file-mode writeblock reads on %s:/%s\n", rank, size, bp_filename, varname);
}
int print_scalar (ADIOS_FILE *f, char * name)
{
ADIOS_VARINFO * v;
int i,j,k;
v = adios_inq_var (f, name);
adios_inq_var_blockinfo (f, v);
printf ("Scalar '%s':\n", name);
printf ("nsteps = %d\n", v->nsteps);
printf ("nblocks per step = %d\n", v->nblocks[0]);
int err;
/* Read one writeblock across all timesteps */
int *data = (int*) calloc (v->nsteps, sizeof(int));
ADIOS_SELECTION *s;
printf ("Read same instance across all timesteps:\n");
for (i=0; i < v->nblocks[0]; i++) {
s = adios_selection_writeblock(i);
err = adios_schedule_read_byid(f, s, v->varid, 0, v->nsteps, data);
if (!err)
{
err = adios_perform_reads(f, 1);
if (!err)
{
err = 0;
printf (" block %d = [", i);
for (j=0; j < v->nsteps; j++) {
printf ("%d", data[j]);
if (data[j] !=
block_offset [j*nblocks_per_step*size + i])
{
err = 1;
}
if (j < v->nsteps-1) printf(",");
}
printf("]");
if (err)
{
nerrors++;
printf ("\tERROR expected = [");
for (j=0; j < v->nsteps; j++) {
printf ("%llu", block_offset [j*nblocks_per_step*size + i]);
if (j < v->nsteps-1) printf(",");
}
printf("]");
}
printf("\n");
} else {
printf ("ERROR at reading scalar '%s': %s\n", name, adios_errmsg());
}
} else {
printf ("ERROR at scheduling read for scalar '%s': %s\n", name, adios_errmsg());
}
adios_selection_delete(s);
}
/* Now read piecewise, one writeblock at a time */
printf ("Read each instance individually:\n");
for (j=0; j < v->nsteps; j++) {
printf (" step %d: \n", j);
for (i=0; i < v->nblocks[j]; i++) {
s = adios_selection_writeblock(i);
err = adios_schedule_read_byid(f, s, v->varid, j, 1, data);
if (!err)
{
err = adios_perform_reads(f, 1);
if (!err)
{
printf (" block %d = %d", i, data[0]);
if (data[0] !=
block_offset [j*nblocks_per_step*size + i])
{
printf ("\tERROR expected = %llu",
block_offset [j*nblocks_per_step*size + i]);
nerrors++;
}
printf ("\n");
} else {
printf ("ERROR at reading scalar '%s': %s\n", name, adios_errmsg());
}
} else {
printf ("ERROR at scheduling read for scalar '%s': %s\n", name, adios_errmsg());
}
adios_selection_delete(s);
}
}
/* Now get them piecewise, but not with reading but through statistics */
printf ("Get each instance individually from available statistics:\n");
adios_inq_var_stat (f, v, 0, 1);
if (v->statistics && v->statistics->blocks) {
ADIOS_VARSTAT *stat = v->statistics;
int blockid = 0;
for (j=0; j < v->nsteps; j++) {
printf (" step %d: \n", j);
for (i=0; i < v->nblocks[j]; i++) {
printf (" block %d = %d", i, *(int*)stat->blocks->mins[blockid]);
if (*(int*)stat->blocks->mins[blockid] !=
block_offset [j*nblocks_per_step*size + i])
{
printf ("\tERROR expected = %llu",
block_offset [j*nblocks_per_step*size + i]);
nerrors++;
}
printf ("\n");
blockid++;
}
}
}
adios_free_varinfo (v);
free(data);
}