uint64_t estimateBytesOnVar(ADIOS_FILE* f, ADIOS_VARINFO* v)
{
if (v->ndim == 0) {
return 0;
}
adios_inq_var_blockinfo(f, v);
uint64_t result = 0;
int i = 0;
int j = 0;
for (i=0; i<v->sum_nblocks; i++) {
ADIOS_VARBLOCK curr = v->blockinfo[i];
uint64_t blockSize = fastbit_adios_util_getBlockSize(v, -1, i);
result += blockSize;
//uint64_t blockDataByteSize = adios_type_size (v->type, v->value) * blockSize0;
/*uint64_t blockSize = 1;
for (j=0; j<v->ndim; j++) {
blockSize = blockSize* curr.count[j];
}
result += blockSize;
*/
}
return adios_type_size (v->type, v->value) * result;
//int typeSize = adios_type_size(v->type, NULL);
//return result * typeSize;
}
void adios_checkpoint_verify_variables(ADIOS_FILE* fp, const char* name, RealType* origin)
{
ADIOS_VARINFO *vi;
int count = 1;
int size;
vi = adios_inq_var(fp, name);
adios_inq_var_blockinfo (fp, vi);
if (vi->ndim > 0)
{
count*=vi->dims[0];
for (int j = 1; j < vi->ndim; j++)
{
count *= vi->dims[j];
}
}
size = count*adios_type_size(vi->type, vi->value);
RealType *mem= (RealType *)malloc(size);
adios_schedule_read(fp, NULL, name, 0, 1, mem);
adios_perform_reads(fp, 1);
for(int i=0; i<count; i++)
{
if(mem[i] == origin[i])
{
cout<<name<<"["<<i<<"]verification passed "<<mem[i]<<endl;
}
else
{
cout<<name<<"["<<i<<"]verification not passed, readin: "<<mem[i]<<" writeout: "<<origin[i]<<endl;
}
}
adios_free_varinfo (vi);
free(mem);
}
void adios_checkpoint_verify_random_variables(ADIOS_FILE* fp, const char* name, uint_type* origin)
{
ADIOS_VARINFO *vi;
int count_int = 1;
int size;
uint64_t *start;
uint64_t *count;
vi = adios_inq_var(fp, name);
adios_inq_var_blockinfo (fp, vi);
if (vi->ndim > 0)
{
start = (uint64_t *)malloc(vi->ndim * sizeof(uint64_t));
count = (uint64_t *)malloc(vi->ndim * sizeof(uint64_t));
}
for (int j=0; j<vi->nblocks[0]; j++)
{
if(j == OHMMS::Controller->rank())
{
for (int k=0; k<vi->ndim; k++)
{
start[k] = vi->blockinfo[j].start[k];
count[k] = vi->blockinfo[j].count[k];
count_int *= count[k];
//cout<<OHMMS::Controller->rank()<<" count "<<start[k]<<" "<<count[k]<<endl;
}
}
}
size = count_int*adios_type_size(vi->type, vi->value);
uint_type *mem= (uint_type*)malloc(size);
ADIOS_SELECTION *sel = adios_selection_boundingbox(vi->ndim, start, count);
adios_schedule_read(fp, sel, name, 0, 1, mem);
adios_perform_reads(fp, 1);
int flag = 0;
for(int i=0; i<count_int; i++)
{
if(mem[i] == origin[i])
{
//cout<<name<<"["<<i<<"]verification passed "<<mem[i]<<endl;
}
else
{
flag = 1;
cout<<name<<"["<<i<<"]verification not passed, readin: "<<mem[i]<<" writeout: "<<origin[i]<<endl;
}
}
if (flag == 0) cout<<name<<" verification passed "<<endl;
else cout<<name<<" verification not passed "<<endl;
adios_free_varinfo (vi);
adios_selection_delete(sel);
free(start);
free(count);
free(mem);
}
int print_varinfo (ADIOS_FILE *f, int start_step)
{
ADIOS_VARINFO * v;
int i,j,k;
v = adios_inq_var (f, "t");
adios_inq_var_blockinfo (f, v);
printf ("ndim = %d\n", v->ndim);
printf ("dims[%llu]", v->dims[0]);
if (v->dims[0] != gdims[start_step])
{
printf ("\tERROR: expected [%llu]", gdims[start_step]);
nerrors++;
}
printf("\n");
printf ("nsteps = %d\n", v->nsteps);
printf ("sum_nblocks = %d\n", v->sum_nblocks);
k = 0; // blockinfo is a contigous 1D array of elements from 0 to v->sum_nblocks-1
for (i = 0; i < v->nsteps; i++) {
printf (" nblocks[%d] = %d\n", i, v->nblocks[i]);
for (j = 0; j < v->nblocks[i]; j++) {
printf(" block %2d: [%llu:%llu]", j,
v->blockinfo[k].start[0],
v->blockinfo[k].start[0] + v->blockinfo[k].count[0]-1);
if (v->blockinfo[k].start[0] != block_offset [(start_step+i)*nblocks_per_step*size + j] ||
v->blockinfo[k].count[0] != block_count [(start_step+i)*nblocks_per_step*size + j] )
{
nerrors++;
printf ("\tERROR: expected [%llu:%llu]",
block_offset [(start_step+i)*nblocks_per_step*size + j],
block_offset [(start_step+i)*nblocks_per_step*size + j] +
block_count [(start_step+i)*nblocks_per_step*size + j] -1
);
}
printf("\n");
k++;
}
}
adios_free_varinfo (v);
}
void adios_trace_verify_local_variables(ADIOS_FILE* fp, const char* name, double* origin)
{
ADIOS_VARINFO *vi;
int count = 1;
unsigned long size = 1;
vi = adios_inq_var(fp, name);
adios_inq_var_blockinfo(fp, vi);
for (int j=0; j<vi->nblocks[0]; j++)
{
if(OHMMS::Controller->rank() == j)
{
for (int k=0; k<vi->ndim; k++)
{
count *= vi->blockinfo[j].count[k];
}
}
}
size = count * adios_type_size(vi->type, vi->value);
double* mem = (double*)malloc(size);
ADIOS_SELECTION *sel = adios_selection_writeblock(OHMMS::Controller->rank());
adios_schedule_read(fp, sel, name, 0, 1, mem);
adios_perform_reads(fp, 1);
int flag = 0;
for(int i=0; i<count; i++)
{
if(mem[i] == origin[i])
{
}
else
{
flag = 1;
cout<<OHMMS::Controller->rank()<<" "<<name<<"["<<i<<"]verification not passed, readin: "<<mem[i]<<" writeout: "<<origin[i]<<endl;
}
}
if(flag == 0) cout<<OHMMS::Controller->rank()<<" "<<name<<" verification passed"<<endl;
adios_selection_delete(sel);
adios_free_varinfo (vi);
free(mem);
}
static void test_file_mode_reads_on_var(ADIOS_FILE *fp, const char *bp_filename, const char *varname) {
int i;
ADIOS_VARINFO *varinfo = adios_inq_var(fp, varname);
MPI_Assert(COMM, varinfo);
if (varinfo->value != NULL) {
//if (rank == 0) fprintf(stderr, "(skipping scalar variable '%s')\n", varname);
adios_free_varinfo(varinfo);
return;
}
fprintf(stderr, "[rank %d/%d] Starting file-mode writeblock reads on %s:/%s\n", rank, size, bp_filename, varname);
adios_inq_var_blockinfo(fp, varinfo);
MPI_Assert(COMM, varinfo->blockinfo);
const enum ADIOS_DATATYPES datatype = varinfo->type;
const int datatypesize = adios_get_type_size(datatype, NULL);
int timestep, timestep_blockidx, blockidx = 0;
for (timestep = 0; timestep < varinfo->nsteps; ++timestep) {
for (timestep_blockidx = 0; timestep_blockidx < varinfo->nblocks[timestep]; ++timestep_blockidx, ++blockidx) {
if (blockidx % size != rank) continue;
const ADIOS_VARBLOCK *vb = &varinfo->blockinfo[blockidx];
ADIOS_SELECTION *block_bb = adios_selection_boundingbox(varinfo->ndim, vb->start, vb->count);
ADIOS_SELECTION *block_wb = adios_selection_writeblock(timestep_blockidx);
ADIOS_SELECTION *block_abs_wb = adios_selection_writeblock(blockidx);
block_abs_wb->u.block.is_absolute_index = 1;
uint64_t blocksize = datatypesize;
for (i = 0; i < varinfo->ndim; ++i)
blocksize *= vb->count[i];
void *buf_bb = malloc(blocksize);
void *buf_wb = malloc(blocksize);
void *buf_abs_wb = malloc(blocksize);
memset(buf_bb, 0, blocksize);
memset(buf_wb, 1, blocksize);
memset(buf_abs_wb, 2, blocksize);
MPI_Assert(COMM, buf_bb && buf_wb && buf_abs_wb);
adios_schedule_read(fp, block_bb, varname, timestep, 1, buf_bb );
adios_schedule_read(fp, block_wb, varname, timestep, 1, buf_wb );
adios_schedule_read(fp, block_abs_wb, varname, timestep, 1, buf_abs_wb);
adios_perform_reads(fp, 1);
fprintf(stderr, "[rank %d/%d] Checking file-mode blockidx %d BB vs. WB...\n", rank, size, blockidx);
MPI_Assert(COMM, memcmp(buf_bb, buf_wb, blocksize) == 0);
fprintf(stderr, "[rank %d/%d] Checking file-mode blockidx %d BB vs. abs-WB...\n", rank, size, blockidx);
MPI_Assert(COMM, memcmp(buf_bb, buf_abs_wb, blocksize) == 0);
free(buf_bb); free(buf_wb); free(buf_abs_wb);
adios_selection_delete(block_bb);
adios_selection_delete(block_wb);
adios_selection_delete(block_abs_wb);
}
}
adios_free_varinfo(varinfo);
fprintf(stderr, "[rank %d/%d] Finished file-mode writeblock reads on %s:/%s\n", rank, size, bp_filename, varname);
}
int main (int argc, char ** argv)
{
int rank, size, i, j;
MPI_Comm comm = MPI_COMM_WORLD;
ADIOS_FILE * f;
ADIOS_VARINFO * v;
ADIOS_SELECTION * sel;
int steps = 0;
int retval = 0;
float timeout_sec = 1.0;
void * data = NULL;
uint64_t start[2], count[2];
MPI_Init (&argc, &argv);
MPI_Comm_rank (comm, &rank);
MPI_Comm_size (comm, &size);
adios_read_init_method (ADIOS_READ_METHOD_BP, comm, "verbose=3");
f = adios_read_open ("adios_globaltime.bp", ADIOS_READ_METHOD_BP,
comm, ADIOS_LOCKMODE_NONE, timeout_sec);
if (adios_errno == err_file_not_found)
{
printf ("rank %d: Stream not found after waiting %f seconds: %s\n",
rank, timeout_sec, adios_errmsg());
retval = adios_errno;
}
else if (adios_errno == err_end_of_stream)
{
printf ("rank %d: Stream terminated before open. %s\n", rank, adios_errmsg());
retval = adios_errno;
}
else if (f == NULL) {
printf ("rank %d: Error at opening stream: %s\n", rank, adios_errmsg());
retval = adios_errno;
}
else
{
/* process file here... */
v = adios_inq_var (f, "temperature");
adios_inq_var_blockinfo (f, v);
printf ("ndim = %d\n", v->ndim);
//printf ("nsteps = %d\n", v->nsteps);
printf ("dims[%llu][%llu]\n", v->dims[0], v->dims[1]);
uint64_t slice_size = v->dims[0]/size;
if (rank == size-1)
slice_size = slice_size + v->dims[0]%size;
start[0] = rank * slice_size;
count[0] = slice_size;
start[1] = 0;
count[1] = v->dims[1];
data = malloc (slice_size * v->dims[1] * 8);
/* Processing loop over the steps (we are already in the first one) */
while (adios_errno != err_end_of_stream) {
steps++; // steps start counting from 1
sel = adios_selection_boundingbox (v->ndim, start, count);
adios_schedule_read (f, sel, "temperature", 0, 1, data);
adios_perform_reads (f, 1);
if (rank == 0)
printf ("--------- Step: %d --------------------------------\n",
f->current_step);
printf("rank=%d: [0:%lld,0:%lld] = [", rank, v->dims[0], v->dims[1]);
for (i = 0; i < slice_size; i++) {
printf (" [");
for (j = 0; j < v->dims[1]; j++) {
printf ("%g ", *((double *)data + i * v->dims[1] + j));
}
printf ("]");
}
printf (" ]\n\n");
// advance to 1) next available step with 2) blocking wait
adios_advance_step (f, 0, timeout_sec);
if (adios_errno == err_step_notready)
{
printf ("rank %d: No new step arrived within the timeout. Quit. %s\n",
rank, adios_errmsg());
break; // quit while loop
}
}
adios_read_close (f);
}
if (rank==0)
printf ("We have processed %d steps\n", steps);
adios_read_finalize_method (ADIOS_READ_METHOD_BP);
free (data);
MPI_Finalize ();
return retval;
}
int print_scalar (ADIOS_FILE *f, char * name)
{
ADIOS_VARINFO * v;
int i,j,k;
v = adios_inq_var (f, name);
adios_inq_var_blockinfo (f, v);
printf ("Scalar '%s':\n", name);
printf ("nsteps = %d\n", v->nsteps);
printf ("nblocks per step = %d\n", v->nblocks[0]);
int err;
/* Read one writeblock across all timesteps */
int *data = (int*) calloc (v->nsteps, sizeof(int));
ADIOS_SELECTION *s;
printf ("Read same instance across all timesteps:\n");
for (i=0; i < v->nblocks[0]; i++) {
s = adios_selection_writeblock(i);
err = adios_schedule_read_byid(f, s, v->varid, 0, v->nsteps, data);
if (!err)
{
err = adios_perform_reads(f, 1);
if (!err)
{
err = 0;
printf (" block %d = [", i);
for (j=0; j < v->nsteps; j++) {
printf ("%d", data[j]);
if (data[j] !=
block_offset [j*nblocks_per_step*size + i])
{
err = 1;
}
if (j < v->nsteps-1) printf(",");
}
printf("]");
if (err)
{
nerrors++;
printf ("\tERROR expected = [");
for (j=0; j < v->nsteps; j++) {
printf ("%llu", block_offset [j*nblocks_per_step*size + i]);
if (j < v->nsteps-1) printf(",");
}
printf("]");
}
printf("\n");
} else {
printf ("ERROR at reading scalar '%s': %s\n", name, adios_errmsg());
}
} else {
printf ("ERROR at scheduling read for scalar '%s': %s\n", name, adios_errmsg());
}
adios_selection_delete(s);
}
/* Now read piecewise, one writeblock at a time */
printf ("Read each instance individually:\n");
for (j=0; j < v->nsteps; j++) {
printf (" step %d: \n", j);
for (i=0; i < v->nblocks[j]; i++) {
s = adios_selection_writeblock(i);
err = adios_schedule_read_byid(f, s, v->varid, j, 1, data);
if (!err)
{
err = adios_perform_reads(f, 1);
if (!err)
{
printf (" block %d = %d", i, data[0]);
if (data[0] !=
block_offset [j*nblocks_per_step*size + i])
{
printf ("\tERROR expected = %llu",
block_offset [j*nblocks_per_step*size + i]);
nerrors++;
}
printf ("\n");
} else {
printf ("ERROR at reading scalar '%s': %s\n", name, adios_errmsg());
}
} else {
printf ("ERROR at scheduling read for scalar '%s': %s\n", name, adios_errmsg());
}
adios_selection_delete(s);
}
}
/* Now get them piecewise, but not with reading but through statistics */
printf ("Get each instance individually from available statistics:\n");
adios_inq_var_stat (f, v, 0, 1);
if (v->statistics && v->statistics->blocks) {
ADIOS_VARSTAT *stat = v->statistics;
int blockid = 0;
for (j=0; j < v->nsteps; j++) {
printf (" step %d: \n", j);
for (i=0; i < v->nblocks[j]; i++) {
printf (" block %d = %d", i, *(int*)stat->blocks->mins[blockid]);
if (*(int*)stat->blocks->mins[blockid] !=
block_offset [j*nblocks_per_step*size + i])
{
printf ("\tERROR expected = %llu",
block_offset [j*nblocks_per_step*size + i]);
nerrors++;
}
printf ("\n");
blockid++;
}
}
}
adios_free_varinfo (v);
free(data);
}
int main (int argc, char ** argv)
{
int i, j, k,l;
MPI_Comm comm_dummy = 0; /* MPI_Comm is defined through adios_read.h */
if (argc < 2) {
printf("Usage: %s <BP-file>\n", argv[0]);
return 1;
}
adios_read_init_method (ADIOS_READ_METHOD_BP, comm_dummy, "show_hidden_attrs");
ADIOS_FILE * f;
f = adios_read_open_file (argv[1], ADIOS_READ_METHOD_BP, comm_dummy);
if (f == NULL) {
printf ("%s\n", adios_errmsg());
return -1;
}
/* For all variables */
printf(" Variables=%d:\n", f->nvars);
for (i = 0; i < f->nvars; i++) {
ADIOS_VARINFO * v = adios_inq_var_byid (f, i);
adios_inq_var_stat (f, v, 0, 1);
adios_inq_var_blockinfo (f, v);
uint64_t total_size = adios_type_size (v->type, v->value);
for (j = 0; j < v->ndim; j++)
total_size *= v->dims[j];
printf(" %-9s %s", adios_type_to_string(v->type), f->var_namelist[i]);
if (v->ndim == 0) {
/* Scalars do not need to be read in, we get it from the metadata
when using adios_inq_var */
printf(" = %s\n", value_to_string(v->type, v->value, 0));
} else {
/* Arrays, print min/max statistics*/
printf("[%lld",v->dims[0]);
for (j = 1; j < v->ndim; j++)
printf(", %lld",v->dims[j]);
//printf("] = \n");
if (v->type == adios_integer)
printf("] : min=%d max=%d\n",
(*(int*)v->statistics->min), (*(int*)v->statistics->max));
else if (v->type == adios_double)
printf("] : min=%lg max=%lg\n",
(*(double*)v->statistics->min), (*(double*)v->statistics->max));
/* Print block info */
for (l=0; l<v->nsteps; l++) {
printf(" step %3d: \n", l);
for (j=0; j<v->nblocks[l]; j++) {
printf(" block %3d: [", j);
for (k=0; k<v->ndim; k++) {
printf("%3lld:%3lld", v->blockinfo[j].start[k],
v->blockinfo[j].start[k]+v->blockinfo[j].count[k]-1);
if (k<v->ndim-1)
printf(", ");
}
printf("]\n");
}
}
}
adios_free_varinfo (v);
} /* variables */
/* For all attributes */
printf(" Attributes=%d:\n", f->nattrs);
for (i = 0; i < f->nattrs; i++) {
enum ADIOS_DATATYPES atype;
int asize;
void *adata;
adios_get_attr_byid (f, i, &atype, &asize, &adata);
printf(" %-9s %s = %s\n", adios_type_to_string(atype),
f->attr_namelist[i], value_to_string(atype, adata, 0));
free(adata);
} /* attributes */
adios_read_close (f);
return 0;
}
int worker(int argc, char* argv[]) {
TAU_PROFILE_TIMER(timer, __func__, __FILE__, TAU_USER);
TAU_PROFILE_START(timer);
my_printf("%d of %d In worker B\n", myrank, commsize);
static bool announced = false;
/* validate input */
validate_input(argc, argv);
my_printf("Worker B will execute until it sees n iterations.\n", iterations);
/* ADIOS: These declarations are required to match the generated
* gread_/gwrite_ functions. (And those functions are
* generated by calling 'gpp.py adios_config.xml') ...
* EXCEPT THAT THE generation of Reader code is broken.
* So, we will write the reader code manually.
*/
uint64_t adios_groupsize;
uint64_t adios_totalsize;
uint64_t adios_handle;
void * data = NULL;
uint64_t start[2], count[2];
int i, j, steps = 0;
int NX = 10;
int NY = 1;
double t[NX];
double p[NX];
/* ADIOS: Can duplicate, split the world, whatever.
* This allows you to have P writers to N files.
* With no splits, everyone shares 1 file, but
* can write lock-free by using different areas.
*/
MPI_Comm adios_comm, adios_comm_b_to_c;
adios_comm = MPI_COMM_WORLD;
//MPI_Comm_dup(MPI_COMM_WORLD, &adios_comm);
adios_comm_b_to_c = MPI_COMM_WORLD;
//MPI_Comm_dup(MPI_COMM_WORLD, &adios_comm_b_to_c);
enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_FLEXPATH;
adios_read_init_method(method, adios_comm, "verbose=3");
if (adios_errno != err_no_error) {
fprintf (stderr, "rank %d: Error %d at init: %s\n", myrank, adios_errno, adios_errmsg());
exit(4);
}
if (send_to_c) {
adios_init("adios_config.xml", adios_comm);
}
/* ADIOS: Set up the adios communications and buffers, open the file.
*/
ADIOS_FILE *fp; // file handler
ADIOS_VARINFO *vi; // information about one variable
ADIOS_SELECTION * sel;
char adios_filename_a_to_b[256];
char adios_filename_b_to_c[256];
enum ADIOS_LOCKMODE lock_mode = ADIOS_LOCKMODE_NONE;
double timeout_sec = 1.0;
sprintf(adios_filename_a_to_b, "adios_a_to_b.bp");
sprintf(adios_filename_b_to_c, "adios_b_to_c.bp");
my_printf ("rank %d: Worker B opening file: %s\n", myrank, adios_filename_a_to_b);
fp = adios_read_open(adios_filename_a_to_b, method, adios_comm, lock_mode, timeout_sec);
if (adios_errno == err_file_not_found) {
fprintf (stderr, "rank %d: Stream not found after waiting %d seconds: %s\n",
myrank, timeout_sec, adios_errmsg());
exit(1);
} else if (adios_errno == err_end_of_stream) {
// stream has been gone before we tried to open
fprintf (stderr, "rank %d: Stream terminated before open. %s\n", myrank, adios_errmsg());
exit(2);
} else if (fp == NULL) {
// some other error happened
fprintf (stderr, "rank %d: Error %d at opening: %s\n", myrank, adios_errno, adios_errmsg());
exit(3);
} else {
my_printf("Found file %s\n", adios_filename_a_to_b);
my_printf ("File info:\n");
my_printf (" current step: %d\n", fp->current_step);
my_printf (" last step: %d\n", fp->last_step);
my_printf (" # of variables: %d:\n", fp->nvars);
vi = adios_inq_var(fp, "temperature");
adios_inq_var_blockinfo(fp, vi);
printf ("ndim = %d\n", vi->ndim);
printf ("nsteps = %d\n", vi->nsteps);
printf ("dims[%llu][%llu]\n", vi->dims[0], vi->dims[1]);
uint64_t slice_size = vi->dims[0]/commsize;
if (myrank == commsize-1) {
slice_size = slice_size + vi->dims[0]%commsize;
}
start[0] = myrank * slice_size;
count[0] = slice_size;
start[1] = 0;
count[1] = vi->dims[1];
data = malloc (slice_size * vi->dims[1] * 8);
/* Processing loop over the steps (we are already in the first one) */
while (adios_errno != err_end_of_stream && steps < iterations) {
steps++; // steps start counting from 1
TAU_PROFILE_TIMER(adios_recv_timer, "ADIOS recv", __FILE__, TAU_USER);
TAU_PROFILE_START(adios_recv_timer);
sel = adios_selection_boundingbox (vi->ndim, start, count);
adios_schedule_read (fp, sel, "temperature", 0, 1, data);
adios_perform_reads (fp, 1);
if (myrank == 0)
printf ("--------- B Step: %d --------------------------------\n",
fp->current_step);
#if 0
printf("B rank=%d: [0:%lld,0:%lld] = [", myrank, vi->dims[0], vi->dims[1]);
for (i = 0; i < slice_size; i++) {
printf (" [");
for (j = 0; j < vi->dims[1]; j++) {
printf ("%g ", *((double *)data + i * vi->dims[1] + j));
}
printf ("]");
}
printf (" ]\n\n");
#endif
// advance to 1) next available step with 2) blocking wait
adios_advance_step (fp, 0, timeout_sec);
if (adios_errno == err_step_notready)
{
printf ("B rank %d: No new step arrived within the timeout. Quit. %s\n",
myrank, adios_errmsg());
break; // quit while loop
}
TAU_PROFILE_STOP(adios_recv_timer);
/* Do some exchanges with neighbors */
//do_neighbor_exchange();
/* "Compute" */
compute(steps);
for (i = 0; i < NX; i++) {
t[i] = steps*100.0 + myrank*NX + i;
}
for (i = 0; i < NY; i++) {
p[i] = steps*1000.0 + myrank*NY + i;
}
if (send_to_c) {
TAU_PROFILE_TIMER(adios_send_timer, "ADIOS send", __FILE__, TAU_USER);
TAU_PROFILE_START(adios_send_timer);
/* ADIOS: write to the next application in the workflow */
if (steps == 0) {
adios_open(&adios_handle, "b_to_c", adios_filename_b_to_c, "w", adios_comm_b_to_c);
} else {
adios_open(&adios_handle, "b_to_c", adios_filename_b_to_c, "a", adios_comm_b_to_c);
}
/* ADIOS: Actually write the data out.
* Yes, this is the recommended method, and this way, changes in
* configuration with the .XML file will, even in the worst-case
* scenario, merely require running 'gpp.py adios_config.xml'
* and typing 'make'.
*/
#include "gwrite_b_to_c.ch"
/* ADIOS: Close out the file completely and finalize.
* If MPI is being used, this must happen before MPI_Finalize().
*/
adios_close(adios_handle);
TAU_PROFILE_STOP(adios_send_timer);
#if 1
if (!announced) {
SOS_val foo;
foo.i_val = NX;
SOS_pack(example_pub, "NX", SOS_VAL_TYPE_INT, foo);
SOS_announce(example_pub);
SOS_publish(example_pub);
announced = true;
}
#endif
}
MPI_Barrier(adios_comm_b_to_c);
}
MPI_Barrier(MPI_COMM_WORLD);
adios_read_close(fp);
/* ADIOS: Close out the file completely and finalize.
* If MPI is being used, this must happen before MPI_Finalize().
*/
adios_read_finalize_method(method);
}
if (send_to_c) {
adios_finalize(myrank);
}
free(data);
//MPI_Comm_free(&adios_comm);
//MPI_Comm_free(&adios_comm_b_to_c);
TAU_PROFILE_STOP(timer);
/* exit */
return 0;
}
int main (int argc, char** argv)
{
fastbit_init(0);
fastbit_set_verbose_level(0);
ADIOS_FILE * f;
//MPI_Comm comm_dummy = 0; // MPI_Comm is defined through adios_read.h
MPI_Comm comm_dummy = MPI_COMM_WORLD;
int rank, size;
MPI_Init (&argc, &argv);
MPI_Comm_rank (comm_dummy, &rank);
MPI_Comm_size (comm_dummy, &size);
adios_init_noxml (comm_dummy);
if (argc < 2) {
printf("Usage: index_fastbit fileName (attrName)");
return 0;
}
f = adios_read_open_file (argv[1], ADIOS_READ_METHOD_BP, comm_dummy);
if (f == NULL) {
printf ("::%s\n", adios_errmsg());
return -1;
}
/*
adios_allocate_buffer (ADIOS_BUFFER_ALLOC_NOW, (f->file_size)*2/1048576 + 5); // +5MB for extra room in buffer
adios_declare_group (&gAdios_group, gGroupNameFastbitIdx, "", adios_flag_yes);
adios_select_method (gAdios_group, "MPI", "", "");
*/
gIdxFileName = fastbit_adios_util_getFastbitIndexFileName(argv[1]);
unlink(gIdxFileName);
adios_allocate_buffer (ADIOS_BUFFER_ALLOC_NOW, 500); // +5MB for extra room in buffer
adios_declare_group (&gAdios_group, gGroupNameFastbitIdx, "", adios_flag_yes);
adios_select_method (gAdios_group, "MPI", "", "");
adios_open (&gAdios_write_file, gGroupNameFastbitIdx, gIdxFileName, "w", MPI_COMM_WORLD);
#ifdef MULTI_BLOCK
int testid = adios_define_var (gAdios_group, "pack", "", adios_integer , 0, 0, 0);
#endif
#ifdef BOX
int testid = adios_define_var (gAdios_group, "elements", "", adios_integer , 0, 0, 0);
#endif
//uint64_t estimatedbytes = (nb+nk+no)*adios_type_size(adios_double, NULL);
int jobCounter = getJobCounter(f);
uint64_t estimatedbytes = getByteEstimationOnFile(f, rank);
if (size > 1) {
int maxJobsPP = jobCounter/size + 1;
estimatedbytes = estimatedbytes * maxJobsPP /jobCounter +1048576;
}
estimatedbytes += 1048576;
uint64_t adios_totalsize; // adios_group_size needs to be call before any write_byid, Otherwise write_byid does nothing
adios_group_size (gAdios_write_file, estimatedbytes , &adios_totalsize);
printf("=> .. adios open output file: %s, rank %d allocated %" PRIu64 " bytes... \n", gIdxFileName, rank, adios_totalsize);
// IMPORTANT:
// can only call open/close once in a process
// otherwise data is tangled or only the data in the last open/close call is recorded
#ifdef MULTI_BLOCK
adios_write_byid(gAdios_write_file, testid, &pack);
#endif
#ifdef BOX
adios_write_byid(gAdios_write_file, testid, &recommended_index_ele);
#endif
sumLogTime(-1);
sumLogTimeMillis(-1);
if (argc >= 3) {
int i=2;
while (i<argc) {
const char* varName = argv[i];
if(strstr(varName, "<binning prec") != NULL) {
if (gBinningOption == NULL) {
gBinningOption = argv[i];
}
if (argc == 3) {
buildIndexOnAllVar(f, rank, size);
break;
}
i++;
continue;
} else {
ADIOS_VARINFO * v = adios_inq_var(f, varName);
if (v == NULL) {
printf("No such variable: %s\n", varName);
return 0;
}
printf("building fastbit index on variable: %s\n", varName);
buildIndex_mpi(f, v, rank, size);
adios_free_varinfo(v);
i++;
}
}
} else {
buildIndexOnAllVar(f, rank, size);
}
sumLogTime(0);
sumLogTimeMillis(0);
adios_close(gAdios_write_file);
adios_read_close(f);
//
// writing file clean up
//
// read back:
f = adios_read_open_file (gIdxFileName, ADIOS_READ_METHOD_BP, comm_dummy);
if (f == NULL) {
printf("No such file: %s \n", gIdxFileName);
return 0;
}
int numVars = f->nvars;
int i=0;
int k=0;
int j=0;
for (i=0; i<numVars; i++) {
char* varName = f->var_namelist[i];
ADIOS_VARINFO* v = adios_inq_var(f, varName);
adios_inq_var_blockinfo(f,v);
int timestep = 0;
for (k=0; k<v->sum_nblocks; k++) {
verifyData(f, v, k, timestep);
}
adios_free_varinfo(v);
}
adios_read_close(f);
if (rank == 0) {
printf(" ==> index file is at: %s\n", gIdxFileName);
}
// clean up
MPI_Barrier (comm_dummy);
adios_finalize (rank);
MPI_Finalize ();
free (gIdxFileName);
fastbit_cleanup();
return 0;
}
void buildIndex_mpi(ADIOS_FILE* f, ADIOS_VARINFO* v, int rank, int size)
{
adios_inq_var_blockinfo(f, v);
int i=0;
int j=0;
int k=0;
printf("building index on variable %d, binning op=%s\n", v->varid, gBinningOption);
int blockCounter = -1;
FastBitDataType ft = fastbit_adios_util_getFastbitDataType(v->type);
#ifdef SINGLE_BLOCK
for (i=0; i < v->nsteps; i++) {
int nblocks = v->nblocks[i];
for (j=0; j < nblocks; j++) {
blockCounter++;
if (blockCounter % size == rank) {
onBlock(rank, f, v, i, j, blockCounter, ft);
fastbit_cleanup();
}
}
printf(" rank %d, varid %d, timestep %d total index created = %" PRIu64 ", %" PRIu64 ", %" PRIu64 ", \n", rank, v->varid, i, sum_nb, sum_nk, sum_no);
printf(" rank %d, varid %d, timestep %d total bytes = %" PRIu64 ", %" PRIu64 ", %" PRIu64 ", \n", rank, v->varid, i,
adios_type_size(adios_unsigned_integer , NULL)*sum_nb,
adios_type_size(adios_double, NULL)*sum_nk,
adios_type_size(adios_long, NULL)*sum_no);
printf("\n");
}
#endif
#ifdef MULTI_BLOCK
for (i=0; i<v->nsteps; i++) {
int nblocks = v->nblocks[i];
int blockStart = 0;
int blockEnd = blockStart+pack;
while (blockStart < nblocks) {
if (blockEnd > nblocks) {
blockEnd = nblocks;
}
printf("block start=%d, end=%d, max=%d\n", blockStart, blockEnd, nblocks);
if (workCounter % size == rank) {
printf("%ld mod %ld == %d \n", workCounter, size, rank);
onMultiBlock(rank, f, v, i, blockStart, blockEnd, ft);
fastbit_cleanup();
}
workCounter ++;
blockStart += pack;
blockEnd = blockStart+pack;
}
fastbit_cleanup();
}
#endif
#ifdef BOX
for (i=0; i<v->nsteps; i++) {
uint64_t s = 0;
uint64_t dataSize = 1;
uint64_t start[v->ndim];
for (s=0; s<v->ndim; s++) {
dataSize *= v->dims[s];
start[s] = 0;
}
uint64_t split = dataSize/recommended_index_ele;
uint64_t startRef = 0;
if (split == 0) {
if (rank == 0) {
onBox(rank, f, v, i, start, v->dims, ft);
}
} else {
while (startRef < v->dims[0]) {
uint64_t count[v->ndim];
start[0] = startRef;
startRef += v->dims[0]/split;
if (startRef >= v->dims[0]) {
startRef = v->dims[0];
}
count[0] = startRef - start[0];
for (s=1; s<v->ndim; s++) {
start[s] = 0;
count[s] = v->dims[s];
}
if (workCounter % size == rank) {
onBox(rank, f, v, i, start, count, ft);
}
workCounter ++;
}
}
}
#endif
}
void ADIOS1CommonRead::InqVarBlockInfo(ADIOS_VARINFO *vi)
{
adios_inq_var_blockinfo(m_fh, vi);
}
void performQuery(ADIOS_QUERY_TEST_INFO *queryInfo, ADIOS_FILE *f, int use_streaming, int print_points, int read_results)
{
int timestep = 0 ;
ADIOS_VARINFO * tempVar = adios_inq_var(f, queryInfo->varName);
adios_inq_var_blockinfo(f, tempVar);
if (use_streaming)
for (timestep = 0; timestep < queryInfo->fromStep; ++timestep)
assert(adios_advance_step(f, 0, 0) == 0);
fprintf(stderr,"times steps for variable is: [%d, %d], batch size is %" PRIu64 "\n", queryInfo->fromStep, queryInfo->fromStep + queryInfo->numSteps, queryInfo->batchSize);
for (timestep = queryInfo->fromStep; timestep < queryInfo->fromStep + queryInfo->numSteps; timestep ++) {
fprintf(stderr, "querying on timestep %d \n", timestep);
while (1)
{
ADIOS_QUERY_RESULT *currBatch = NULL;
currBatch = adios_query_evaluate(
queryInfo->query,
queryInfo->outputSelection,
use_streaming ? 0 : timestep,
queryInfo->batchSize
);
if (currBatch == NULL) {
fprintf(stderr, "Unexpected error status in querying evaluation, returned NULL as result. "
"ADIOS error message: %s \n", adios_errmsg());
break;
}
if (currBatch->status == ADIOS_QUERY_RESULT_ERROR) {
fprintf(stderr, "ERROR in querying evaluation: %s \n", adios_errmsg());
break;
}
if (currBatch->nselections == 0) {
/* this is normal, we processed everything in previous loop or there is 0 total result */
break;
}
int n;
if (currBatch->selections->type == ADIOS_SELECTION_POINTS)
{
for (n = 0; n < currBatch->nselections; n++)
{
const ADIOS_SELECTION_POINTS_STRUCT * retrievedPts = &(currBatch->selections[n].u.points);
/* fprintf(stderr,"retrieved points %" PRIu64 " \n", retrievedPts->npoints); */
uint64_t * wboffs = calloc (retrievedPts->ndim, sizeof(uint64_t));
if (retrievedPts->container_selection &&
retrievedPts->container_selection->type == ADIOS_SELECTION_WRITEBLOCK)
{
int i;
int blockidx = retrievedPts->container_selection->u.block.index;
if (use_streaming) {
adios_inq_var_blockinfo(f, tempVar);
} else {
for (i = 0; i < timestep-1; i++)
blockidx += tempVar->nblocks[i];
}
for (i = 0; i < retrievedPts->ndim; ++i) {
wboffs[i] = tempVar->blockinfo[blockidx].start[i];
}
}
if (print_points) {
printPoints(retrievedPts, timestep, wboffs);
}
free (wboffs);
if (read_results) {
int elmSize = adios_type_size(tempVar->type, NULL);
void *data = malloc(retrievedPts->npoints * elmSize);
// read returned temp data
adios_schedule_read (f, &currBatch->selections[n], queryInfo->varName, use_streaming ? 0 : timestep, 1, data);
adios_perform_reads(f, 1);
free(data);
}
fprintf(stderr,"Total points retrieved %"PRIu64" in %d blocks\n",
currBatch->npoints, currBatch->nselections);
/* if (tempVar->type == adios_double) { */
/* for (i = 0; i < retrievedPts->npoints; i++) { */
/* fprintf(stderr,"%.6f\t", ((double*)data)[i]); */
/* } */
/* fprintf(stderr,"\n"); */
/* } */
/* else if (tempVar->type == adios_real) { */
/* for (i = 0; i < retrievedPts->npoints; i++) { */
/* fprintf(stderr,"%.6f\t", ((float*)data)[i]); */
/* } */
/* fprintf(stderr,"\n"); */
/* } */
}
}
else if (currBatch->selections->type == ADIOS_SELECTION_WRITEBLOCK)
{
fprintf(stderr,"Number of blocks retrieved: %d\n", currBatch->nselections);
if (print_points) {
for (n = 0; n < currBatch->nselections; n++)
{
fprintf(stdout,"%d %d\n", timestep, currBatch->selections[n].u.block.index);
}
}
}
free(currBatch->selections);
if (currBatch->status == ADIOS_QUERY_NO_MORE_RESULTS) {
free (currBatch);
break;
}
free(currBatch);
}
if (use_streaming) {
const int err = adios_advance_step(f, 0, 0);
if (timestep < queryInfo->fromStep + queryInfo->numSteps - 1) {
assert(err == 0);
} else {
assert(err == err_end_of_stream || err == err_step_notready);
}
}
}
adios_query_free(queryInfo->query);
}