void ToolsAdiosParallel::listAvailableDatasets() { ADIOS_VARINFO *pVarInfo; //available data sets in this file m_outStream << "Number of available data sets: "; m_outStream << pFile->nvars << std::endl; for(int i = 0; i < pFile->nvars; i++) { m_outStream << pFile->var_namelist[i] << " "; pVarInfo = adios_inq_var(pFile, pFile->var_namelist[i]); if(pVarInfo->ndim > 0) { // print number of elements per dimension to m_outstream m_outStream << "["; for(int j = 0; j < pVarInfo->ndim; j++) { m_outStream << pVarInfo->dims[j]; if(j < pVarInfo->ndim-1) { m_outStream << ","; } } m_outStream << "]" << std::endl; } else { m_outStream << "[scalar]" << std::endl; } adios_free_varinfo(pVarInfo); } }
int64_t getByteEstimation(ADIOS_FILE* f, int rank, int argc, char** argv) { uint64_t bytes = 0; if (argc >= 3) { int i=2; while (i<argc) { const char* varName = argv[i]; if(strstr(varName, "<binning prec") != NULL) { if (gBinningOption == NULL) { gBinningOption = argv[i]; } if (argc == 3) { return getByteEstimationOnFile(f, rank); } i++; continue; } ADIOS_VARINFO * v = adios_inq_var(f, varName); if (v == NULL) { printf("Invalid variable: %s\n", varName); return -1; } uint64_t varSize = estimateBytesOnVar(f, v); printf(" var: %s has size: %" PRId64 "\n", varName, varSize); bytes += varSize; adios_free_varinfo(v); i++; } return bytes; } else { return getByteEstimationOnFile(f,rank); } }
void adios_checkpoint_verify_variables(ADIOS_FILE* fp, const char* name, int* origin) { ADIOS_VARINFO *vi; int count = 1; int size; vi = adios_inq_var(fp, name); if (vi->ndim > 0) { cout<<name<<" verification not passed, not a scalar"<<endl; return; } size = count*adios_type_size(vi->type, vi->value); int * mem= (int * )malloc(size); ADIOS_SELECTION *sel = adios_selection_writeblock(OHMMS::Controller->rank()); adios_schedule_read(fp, sel, name, 0, 1, mem); //adios_schedule_read(fp, NULL, name, 0, 1, mem); adios_perform_reads(fp, 1); if(mem[0] == *origin) { cout<<name<<" verification passed "<<mem[0]<<endl; } else { cout<<name<<" verification not passed, readin: "<<mem[0]<<" writeout: "<<*origin<<endl; } adios_free_varinfo (vi); adios_selection_delete(sel); free(mem); }
int getJobCounter(ADIOS_FILE* f) { int numVars = f->nvars; int counter = 0; int i=0, k=0; for (i=0; i<numVars; i++) { char* varName = f->var_namelist[i]; ADIOS_VARINFO* v = adios_inq_var(f, varName); if (v->ndim == 0) { continue; } for (k=0; k<v->nsteps; k++) { #ifdef MULTI_BOX int nblocks = v->nblocks[k]; printf("var = %s, timestep = %ld, nblocks=%ld\n", varName, k, nblocks); int remainder = nblocks % pack; counter += nblocks/pack; #endif #ifdef BOX int j=0; uint64_t totalElements =1; for (j=0; j<v->ndim; j++) { totalElements *= v->dims[j]; } int remainder = totalElements % recommended_index_ele; counter += totalElements/recommended_index_ele; #endif if (remainder > 0) { counter += 1; } } } return counter; }
void adios_checkpoint_verify_variables(ADIOS_FILE* fp, const char* name, RealType* origin) { ADIOS_VARINFO *vi; int count = 1; int size; vi = adios_inq_var(fp, name); adios_inq_var_blockinfo (fp, vi); if (vi->ndim > 0) { count*=vi->dims[0]; for (int j = 1; j < vi->ndim; j++) { count *= vi->dims[j]; } } size = count*adios_type_size(vi->type, vi->value); RealType *mem= (RealType *)malloc(size); adios_schedule_read(fp, NULL, name, 0, 1, mem); adios_perform_reads(fp, 1); for(int i=0; i<count; i++) { if(mem[i] == origin[i]) { cout<<name<<"["<<i<<"]verification passed "<<mem[i]<<endl; } else { cout<<name<<"["<<i<<"]verification not passed, readin: "<<mem[i]<<" writeout: "<<origin[i]<<endl; } } adios_free_varinfo (vi); free(mem); }
void adios_checkpoint_verify_variables(ADIOS_FILE* fp, const char* name, unsigned long origin) { ADIOS_VARINFO *vi; int count = 1; int size; vi = adios_inq_var(fp, name); if (vi->ndim > 0) { cout<<name<<" verification not passed, not a scalar"<<endl; return; } size = count*adios_type_size(vi->type, vi->value); unsigned long* mem= (unsigned long* )malloc(size); adios_schedule_read(fp, NULL, name, 0, 1, mem); adios_perform_reads(fp, 1); if(mem[0] == origin) { cout<<name<<" verification passed"<<mem[0]<<endl; } else { cout<<name<<" verification not passed, readin: "<<mem[0]<<" writeout: "<<origin<<endl; } adios_free_varinfo (vi); free(mem); }
/// Constructor. ADIOS_Var::ADIOS_Var(ADIOS_File* afile, const char* vn) : m_file(afile), m_name(vn), m_handle(0) { if (afile != 0 && afile->getHandle() != 0) m_handle = adios_inq_var(afile->getHandle(), vn); LOGGER(ibis::gVerbose > 8) << "adios_inq_var is called for " << m_name << ": m_handle=" << m_handle; }
std::string AdiosCheckpointInput::getScalar(const std::string& var_name) { //Scalars are all stored in the metadata so we can read them without disk access ADIOS_VARINFO* adios_inq = adios_inq_var(adios_file_handle, var_name.c_str()); int string_size = adios_type_size(adios_inq->type, adios_inq->value); std::string string_val = static_cast<char*>(*adios_inq->value); adios_free_varinfo(adios_inq); return string_val; }
void adios_checkpoint_verify_random_variables(ADIOS_FILE* fp, const char* name, uint_type* origin) { ADIOS_VARINFO *vi; int count_int = 1; int size; uint64_t *start; uint64_t *count; vi = adios_inq_var(fp, name); adios_inq_var_blockinfo (fp, vi); if (vi->ndim > 0) { start = (uint64_t *)malloc(vi->ndim * sizeof(uint64_t)); count = (uint64_t *)malloc(vi->ndim * sizeof(uint64_t)); } for (int j=0; j<vi->nblocks[0]; j++) { if(j == OHMMS::Controller->rank()) { for (int k=0; k<vi->ndim; k++) { start[k] = vi->blockinfo[j].start[k]; count[k] = vi->blockinfo[j].count[k]; count_int *= count[k]; //cout<<OHMMS::Controller->rank()<<" count "<<start[k]<<" "<<count[k]<<endl; } } } size = count_int*adios_type_size(vi->type, vi->value); uint_type *mem= (uint_type*)malloc(size); ADIOS_SELECTION *sel = adios_selection_boundingbox(vi->ndim, start, count); adios_schedule_read(fp, sel, name, 0, 1, mem); adios_perform_reads(fp, 1); int flag = 0; for(int i=0; i<count_int; i++) { if(mem[i] == origin[i]) { //cout<<name<<"["<<i<<"]verification passed "<<mem[i]<<endl; } else { flag = 1; cout<<name<<"["<<i<<"]verification not passed, readin: "<<mem[i]<<" writeout: "<<origin[i]<<endl; } } if (flag == 0) cout<<name<<" verification passed "<<endl; else cout<<name<<" verification not passed "<<endl; adios_free_varinfo (vi); adios_selection_delete(sel); free(start); free(count); free(mem); }
void ADIOS1CommonRead::ScheduleReadCommon(const std::string &name, const Dims &offs, const Dims &ldims, const int fromStep, const int nSteps, const bool readAsLocalValue, const bool readAsJoinedArray, void *data) { if (readAsLocalValue) { /* Get all the requested values from metadata now */ ADIOS_VARINFO *vi = adios_inq_var(m_fh, name.c_str()); if (vi) { adios_inq_var_stat(m_fh, vi, 0, 1); int elemsize = adios_type_size(vi->type, nullptr); long long blockidx = 0; for (int i = 0; i < fromStep; i++) { blockidx += vi->nblocks[i]; } char *dest = (char *)data; for (int i = fromStep; i < fromStep + nSteps; i++) { for (int j = 0; j < vi->nblocks[i]; j++) { memcpy(dest, vi->statistics->blocks->mins[blockidx], elemsize); ++blockidx; dest += elemsize; } } adios_free_varinfo(vi); } } else { uint64_t start[32], count[32]; for (int i = 0; i < ldims.size(); i++) { start[i] = (uint64_t)offs[i]; count[i] = (uint64_t)ldims[i]; } ADIOS_SELECTION *sel = nullptr; if (ldims.size() > 0) { sel = adios_selection_boundingbox(ldims.size(), start, count); } adios_schedule_read(m_fh, sel, name.c_str(), (int)fromStep, (int)nSteps, data); adios_selection_delete(sel); } }
T AdiosCheckpointInput::getScalar(const std::string& var_name) { //Scalars are all stored in the metadata so we can read them without disk access ADIOS_VARINFO* adios_inq = adios_inq_var(adios_file_handle, var_name.c_str()); if (adios_type_size(adios_inq->type, adios_inq->value) != sizeof(T)) { qmcplusplus::app_error() << "Data type does not match data type found in file: " << std::endl; return ; } T scalar_value = static_cast<T>(*adios_inq->value); adios_free_varinfo(adios_inq); return scalar_value; }
void ToolsAdiosParallel::convertToText() { if(m_options.data.size() == 0) throw std::runtime_error("No datasets requested"); for (size_t i = 0; i < m_options.data.size(); ++i) { ADIOS_VARINFO *pVarInfo; //get name of dataset to print std::string nodeName = m_options.data[i]; uint8_t *P; int varElement = 1; int varTypeSize = 0; adios_read_init_method(ADIOS_READ_METHOD_BP, comm, nodeName.c_str()); pVarInfo = adios_inq_var(pFile, nodeName.c_str()); varTypeSize = adios_type_size(pVarInfo->type, NULL); // get number of elements combined in a dataset for(int j = 0; j < pVarInfo->ndim; j++) { varElement = varElement * pVarInfo->dims[j]; } // allocate memory P = (uint8_t*) malloc (sizeof(uint8_t) * varTypeSize * varElement); adios_schedule_read(pFile, NULL, nodeName.c_str(), 0, 1, P); adios_perform_reads(pFile, 1); if(pVarInfo->ndim > 0) { for(int k = 0; k < varElement; k++) { printValue(pVarInfo->type, &P[k*varTypeSize]); } } else { printValue(pVarInfo->type, pVarInfo->value); } adios_free_varinfo(pVarInfo); } }
/** Read the skalar field and optionally the attribute into the values referenced by the pointers */ void operator()(ThreadParams& params, const std::string& name, T_Scalar* value, const std::string& attrName = "", T_Attribute* attribute = nullptr) { log<picLog::INPUT_OUTPUT> ("ADIOS: read %1%D scalars: %2%") % simDim % name; std::string datasetName = params.adiosBasePath + name; ADIOS_VARINFO* varInfo; ADIOS_CMD_EXPECT_NONNULL( varInfo = adios_inq_var(params.fp, datasetName.c_str()) ); if(varInfo->ndim != simDim) throw std::runtime_error(std::string("Invalid dimensionality for ") + name); if(varInfo->type != traits::PICToAdios<T_Scalar>().type) throw std::runtime_error(std::string("Invalid type for ") + name); DataSpace<simDim> gridPos = Environment<simDim>::get().GridController().getPosition(); uint64_t start[varInfo->ndim]; uint64_t count[varInfo->ndim]; for(int d = 0; d < varInfo->ndim; ++d) { /* \see adios_define_var: z,y,x in C-order */ start[d] = gridPos.revert()[d]; count[d] = 1; } ADIOS_SELECTION* fSel = adios_selection_boundingbox(varInfo->ndim, start, count); // avoid deadlock between not finished pmacc tasks and mpi calls in adios __getTransactionEvent().waitForFinished(); /* specify what we want to read, but start reading at below at `adios_perform_reads` */ /* magic parameters (0, 1): `from_step` (not used in streams), `nsteps` to read (must be 1 for stream) */ log<picLog::INPUT_OUTPUT > ("ADIOS: Schedule read skalar %1%)") % datasetName; ADIOS_CMD( adios_schedule_read(params.fp, fSel, datasetName.c_str(), 0, 1, (void*)value) ); /* start a blocking read of all scheduled variables */ ADIOS_CMD( adios_perform_reads(params.fp, 1) ); adios_selection_delete(fSel); adios_free_varinfo(varInfo); if(!attrName.empty()) { log<picLog::INPUT_OUTPUT> ("ADIOS: read attribute %1% for scalars: %2%") % attrName % name; *attribute = readAttribute<T_Attribute>(params.fp, datasetName, attrName); } }
int print_varinfo (ADIOS_FILE *f, int start_step) { ADIOS_VARINFO * v; int i,j,k; v = adios_inq_var (f, "t"); adios_inq_var_blockinfo (f, v); printf ("ndim = %d\n", v->ndim); printf ("dims[%llu]", v->dims[0]); if (v->dims[0] != gdims[start_step]) { printf ("\tERROR: expected [%llu]", gdims[start_step]); nerrors++; } printf("\n"); printf ("nsteps = %d\n", v->nsteps); printf ("sum_nblocks = %d\n", v->sum_nblocks); k = 0; // blockinfo is a contigous 1D array of elements from 0 to v->sum_nblocks-1 for (i = 0; i < v->nsteps; i++) { printf (" nblocks[%d] = %d\n", i, v->nblocks[i]); for (j = 0; j < v->nblocks[i]; j++) { printf(" block %2d: [%llu:%llu]", j, v->blockinfo[k].start[0], v->blockinfo[k].start[0] + v->blockinfo[k].count[0]-1); if (v->blockinfo[k].start[0] != block_offset [(start_step+i)*nblocks_per_step*size + j] || v->blockinfo[k].count[0] != block_count [(start_step+i)*nblocks_per_step*size + j] ) { nerrors++; printf ("\tERROR: expected [%llu:%llu]", block_offset [(start_step+i)*nblocks_per_step*size + j], block_offset [(start_step+i)*nblocks_per_step*size + j] + block_count [(start_step+i)*nblocks_per_step*size + j] -1 ); } printf("\n"); k++; } } adios_free_varinfo (v); }
static void check_xform_var(ADIOS_GROUP *ag, double *arr) { int64_t readlen; double *readarr = malloc(N * sizeof(double)); memset(readarr, 0, N*sizeof(double)); // Check the transformed data ADIOS_VARINFO *vi_xform = adios_inq_var(ag, XFORM_VAR); assert(vi_xform); assert(vi_xform->type == adios_double); assert(vi_xform->ndim == 1); // Apparently the "dummy" dimension has gone away for timed arrays that are written only once assert(vi_xform->dims[0] == N); readlen = adios_read_var_byid(ag, vi_xform->varid, (uint64_t[]){0}, vi_xform->dims, readarr); assert(readlen == N * sizeof(double)); assert(memcmp(readarr, arr, N * sizeof(double)) == 0); // Cleanup adios_free_varinfo(vi_xform); free(readarr); }
void AdiosCheckpointInput::getVector(const std::string& var_name, vector<T>& buffer) { ADIOS_VARINFO* adios_inq = adios_inq_var(adios_file_handle, var_name.c_str()); //Check to make sure the T is the same size as the data type on the disk if (adios_type_size(adios_inq->type, NULL) != sizeof(T)) { qmcplusplus::app_error() << "Data type does not match data type found in file: " << std::endl; APP_ABORT("qmcapp"); return ; } uint64_t total_size = 1; for (int i = 0; i < adios_inq->ndim; i++) total_size *= adios_inq->dims[i]; //make sure we have enough space to copy all the data from the file buffer.reserve(total_size); //schedule the read adios_schedule_read(adios_file_handle, sel, var_name.c_str(), 0, 1, &(buffer[0])); //perform the read adios_perform_reads(adios_file_handle, 1); //Don't need the information about the variable anymore adios_free_varinfo(adios_inq); }
int64_t getByteEstimationOnFile(ADIOS_FILE* f, int rank) { // check all vars int numVars = f->nvars; uint64_t bytes = 0; int i=0; for (i=0; i<numVars; i++) { char* varName = f->var_namelist[i]; ADIOS_VARINFO* v = adios_inq_var(f, varName); uint64_t varSize = estimateBytesOnVar(f, v); if (rank == 0) { printf(" var: %s has size: %" PRId64 "\n", varName, varSize); } bytes += varSize; adios_free_varinfo(v); } return bytes; }
void AdiosCheckpointInput::performReads() { for (int i = 0; i < read_queue.size(); i++) { const char* var_name = read_queue[i].c_str(); //grab the information from the metadata about this variable ADIOS_VARINFO* adios_inq = adios_inq_var(adios_file_handle, var_name); //Calculate the total size of the vector based on the metadata uint64_t total_size = adios_type_size(adios_inq->type, NULL); for (int i = 0; i < adios_inq->ndim; i++) total_size *= adios_inq->dims[i]; //allocate and then store for when retrieve void* buff = malloc(total_size); buffers.push_back(buff); //schedule the read adios_schedule_read(adios_file_handle, sel, var_name, 0, 1, buff); //Don't need the information about the variable anymore adios_free_varinfo(adios_inq); } //perform the read adios_perform_reads(adios_file_handle, 1); }
void adios_trace_verify_local_variables(ADIOS_FILE* fp, const char* name, double* origin) { ADIOS_VARINFO *vi; int count = 1; unsigned long size = 1; vi = adios_inq_var(fp, name); adios_inq_var_blockinfo(fp, vi); for (int j=0; j<vi->nblocks[0]; j++) { if(OHMMS::Controller->rank() == j) { for (int k=0; k<vi->ndim; k++) { count *= vi->blockinfo[j].count[k]; } } } size = count * adios_type_size(vi->type, vi->value); double* mem = (double*)malloc(size); ADIOS_SELECTION *sel = adios_selection_writeblock(OHMMS::Controller->rank()); adios_schedule_read(fp, sel, name, 0, 1, mem); adios_perform_reads(fp, 1); int flag = 0; for(int i=0; i<count; i++) { if(mem[i] == origin[i]) { } else { flag = 1; cout<<OHMMS::Controller->rank()<<" "<<name<<"["<<i<<"]verification not passed, readin: "<<mem[i]<<" writeout: "<<origin[i]<<endl; } } if(flag == 0) cout<<OHMMS::Controller->rank()<<" "<<name<<" verification passed"<<endl; adios_selection_delete(sel); adios_free_varinfo (vi); free(mem); }
void buildIndexOnAllVar(ADIOS_FILE* f, int rank, int size) { int numVars = f->nvars; int i=0; for (i=0; i<numVars; i++) { char* varName = f->var_namelist[i]; ADIOS_VARINFO* v = adios_inq_var(f, varName); if (rank == 0) { printf("\n==> building fastbit index on %dth variable: %s, %s ", i, varName, gBinningOption); } if (v->ndim > 0) { buildIndex_mpi(f, v, rank, size); } else { if (rank == 0) { printf("\t ... skipping scalar ...\n"); } } adios_free_varinfo(v); } }
static void test_file_mode_reads_on_var(ADIOS_FILE *fp, const char *bp_filename, const char *varname) { int i; ADIOS_VARINFO *varinfo = adios_inq_var(fp, varname); MPI_Assert(COMM, varinfo); if (varinfo->value != NULL) { //if (rank == 0) fprintf(stderr, "(skipping scalar variable '%s')\n", varname); adios_free_varinfo(varinfo); return; } fprintf(stderr, "[rank %d/%d] Starting file-mode writeblock reads on %s:/%s\n", rank, size, bp_filename, varname); adios_inq_var_blockinfo(fp, varinfo); MPI_Assert(COMM, varinfo->blockinfo); const enum ADIOS_DATATYPES datatype = varinfo->type; const int datatypesize = adios_get_type_size(datatype, NULL); int timestep, timestep_blockidx, blockidx = 0; for (timestep = 0; timestep < varinfo->nsteps; ++timestep) { for (timestep_blockidx = 0; timestep_blockidx < varinfo->nblocks[timestep]; ++timestep_blockidx, ++blockidx) { if (blockidx % size != rank) continue; const ADIOS_VARBLOCK *vb = &varinfo->blockinfo[blockidx]; ADIOS_SELECTION *block_bb = adios_selection_boundingbox(varinfo->ndim, vb->start, vb->count); ADIOS_SELECTION *block_wb = adios_selection_writeblock(timestep_blockidx); ADIOS_SELECTION *block_abs_wb = adios_selection_writeblock(blockidx); block_abs_wb->u.block.is_absolute_index = 1; uint64_t blocksize = datatypesize; for (i = 0; i < varinfo->ndim; ++i) blocksize *= vb->count[i]; void *buf_bb = malloc(blocksize); void *buf_wb = malloc(blocksize); void *buf_abs_wb = malloc(blocksize); memset(buf_bb, 0, blocksize); memset(buf_wb, 1, blocksize); memset(buf_abs_wb, 2, blocksize); MPI_Assert(COMM, buf_bb && buf_wb && buf_abs_wb); adios_schedule_read(fp, block_bb, varname, timestep, 1, buf_bb ); adios_schedule_read(fp, block_wb, varname, timestep, 1, buf_wb ); adios_schedule_read(fp, block_abs_wb, varname, timestep, 1, buf_abs_wb); adios_perform_reads(fp, 1); fprintf(stderr, "[rank %d/%d] Checking file-mode blockidx %d BB vs. WB...\n", rank, size, blockidx); MPI_Assert(COMM, memcmp(buf_bb, buf_wb, blocksize) == 0); fprintf(stderr, "[rank %d/%d] Checking file-mode blockidx %d BB vs. abs-WB...\n", rank, size, blockidx); MPI_Assert(COMM, memcmp(buf_bb, buf_abs_wb, blocksize) == 0); free(buf_bb); free(buf_wb); free(buf_abs_wb); adios_selection_delete(block_bb); adios_selection_delete(block_wb); adios_selection_delete(block_abs_wb); } } adios_free_varinfo(varinfo); fprintf(stderr, "[rank %d/%d] Finished file-mode writeblock reads on %s:/%s\n", rank, size, bp_filename, varname); }
int main (int argc, char ** argv) { int rank, size, i, j; MPI_Comm comm = MPI_COMM_WORLD; enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_BP; MPI_Init (&argc, &argv); MPI_Comm_rank (comm, &rank); MPI_Comm_size (comm, &size); adios_read_init_method (method, comm, "verbose=3"); ADIOS_FILE * f = adios_read_open_file("adios_stat.bp", method, comm); if (f == NULL) { fprintf (stderr, "%s\n", adios_errmsg()); return -1; } ADIOS_VARINFO * v = adios_inq_var (f, "temperature"); if (v) { /* get statistics, for each individual time-step */ adios_inq_var_stat (f, v, 1, 0); printf("Global MIN of temperature: %lf\n", * (double *) v->statistics->min); printf("Global MAX of temperature: %lf\n", * (double *) v->statistics->max); printf("Global AVG of temperature: %lf\n", * (double *) v->statistics->avg); printf("Global STD DEV of temperature: %lf\n", * (double *) v->statistics->std_dev); printf("\n"); printf("---------------------------------------------------------------------------\n"); if (v->statistics->steps) { printf("MIN\t\tMAX\t\tAVG\t\tSTD_DEV\t\tHISTOGRAM\n"); for(i = 0; i < v->nsteps; i++) { if (v->statistics->steps->mins[i]) printf("%lf\t", * (double *) v->statistics->steps->mins[i]); else printf("--\t\t"); if (v->statistics->steps->maxs[i]) printf("%lf\t", * (double *) v->statistics->steps->maxs[i]); else printf("--\t\t"); if (v->statistics->steps->avgs[i]) printf("%lf\t", * (double *) v->statistics->steps->avgs[i]); else printf("--\t\t"); if (v->statistics->steps->std_devs[i]) printf("%lf\t", * (double *) v->statistics->steps->std_devs[i]); else printf("--\t\t"); if (v->statistics->histogram) { for(j = 0; j <= v->statistics->histogram->num_breaks; j++) { printf("%d ", v->statistics->histogram->frequencies[i][j]); } printf("\n"); } else { printf("--\t\t"); } printf("\n"); } } else { printf ("Per step statistics is missing\n"); } printf("---------------------------------------------------------------------------\n"); printf("\n"); if (v->statistics->histogram) { printf("Break points:\t\t\t"); for(j = 0; j < v->statistics->histogram->num_breaks; j++) printf("%6.2lf\t", v->statistics->histogram->breaks[j]); printf("\n"); printf("Frequencies:\t\t\t"); for(j = 0; j <= v->statistics->histogram->num_breaks; j++) printf("%6d\t", v->statistics->histogram->gfrequencies[j]); } printf("\n\n"); #if 0 printf ("Auto covariance of MIN values of temperature over time: %lf\n", adios_stat_cov (v, v, "min", 0, 12, 0)); printf ("Auto correlation of MAX values of temperature over time, with lag 2 units: %lf\n", adios_stat_cor (v, v, "max", 0, 8, 2)); printf("\n\n"); #endif adios_free_varinfo (v); } else { fprintf (stderr, "ERROR: Cannot inquire statistics of variable 'temperature': %s\n", adios_errmsg()); } v = adios_inq_var (f, "complex"); if (v) { adios_inq_var_stat (f, v, 1, 0); double *C = v->statistics->min; printf("Global Minimum of variable complex - Magnitude: %lf\n", C[0]); printf("Global Minimum of variable complex - Real part: %lf\n", C[1]); printf("Global Minimum of variable complex - Imaginary part: %lfi\n", C[2]); double ** Cmin; Cmin = (double **) v->statistics->steps->mins; printf("\nMagnitude\t\tReal\t\t\tImaginary\n"); for (j = 0; j < v->nsteps; j++) { printf ("%lf\t\t%lf\t\t%lf\n", Cmin[j][0], Cmin[j][1], Cmin[j][2]); } printf("\n"); adios_free_varinfo (v); } else { fprintf (stderr, "ERROR: Cannot inquire statistics of variable 'complex': %s\n", adios_errmsg()); } adios_read_close (f); MPI_Barrier (comm); adios_read_finalize_method (method); MPI_Finalize (); return 0; }
int main (int argc, char ** argv) { int rank, size, i, j; MPI_Comm comm = MPI_COMM_WORLD; ADIOS_FILE * f; ADIOS_VARINFO * v; ADIOS_SELECTION * sel; int steps = 0; int retval = 0; float timeout_sec = 1.0; void * data = NULL; uint64_t start[2], count[2]; MPI_Init (&argc, &argv); MPI_Comm_rank (comm, &rank); MPI_Comm_size (comm, &size); adios_read_init_method (ADIOS_READ_METHOD_BP, comm, "verbose=3"); f = adios_read_open ("adios_globaltime.bp", ADIOS_READ_METHOD_BP, comm, ADIOS_LOCKMODE_NONE, timeout_sec); if (adios_errno == err_file_not_found) { printf ("rank %d: Stream not found after waiting %f seconds: %s\n", rank, timeout_sec, adios_errmsg()); retval = adios_errno; } else if (adios_errno == err_end_of_stream) { printf ("rank %d: Stream terminated before open. %s\n", rank, adios_errmsg()); retval = adios_errno; } else if (f == NULL) { printf ("rank %d: Error at opening stream: %s\n", rank, adios_errmsg()); retval = adios_errno; } else { /* process file here... */ v = adios_inq_var (f, "temperature"); adios_inq_var_blockinfo (f, v); printf ("ndim = %d\n", v->ndim); //printf ("nsteps = %d\n", v->nsteps); printf ("dims[%llu][%llu]\n", v->dims[0], v->dims[1]); uint64_t slice_size = v->dims[0]/size; if (rank == size-1) slice_size = slice_size + v->dims[0]%size; start[0] = rank * slice_size; count[0] = slice_size; start[1] = 0; count[1] = v->dims[1]; data = malloc (slice_size * v->dims[1] * 8); /* Processing loop over the steps (we are already in the first one) */ while (adios_errno != err_end_of_stream) { steps++; // steps start counting from 1 sel = adios_selection_boundingbox (v->ndim, start, count); adios_schedule_read (f, sel, "temperature", 0, 1, data); adios_perform_reads (f, 1); if (rank == 0) printf ("--------- Step: %d --------------------------------\n", f->current_step); printf("rank=%d: [0:%lld,0:%lld] = [", rank, v->dims[0], v->dims[1]); for (i = 0; i < slice_size; i++) { printf (" ["); for (j = 0; j < v->dims[1]; j++) { printf ("%g ", *((double *)data + i * v->dims[1] + j)); } printf ("]"); } printf (" ]\n\n"); // advance to 1) next available step with 2) blocking wait adios_advance_step (f, 0, timeout_sec); if (adios_errno == err_step_notready) { printf ("rank %d: No new step arrived within the timeout. Quit. %s\n", rank, adios_errmsg()); break; // quit while loop } } adios_read_close (f); } if (rank==0) printf ("We have processed %d steps\n", steps); adios_read_finalize_method (ADIOS_READ_METHOD_BP); free (data); MPI_Finalize (); return retval; }
void eavlXGCParticleImporter::Initialize() { ephase.clear(); iphase.clear(); egid.clear(); igid.clear(); last_available_timestep = fp->last_step; if(readingRestartFile) //check for restart file format { if(fp->nvars <= 25) readingRestartFile = 0; } if(readingRestartFile) { int numVarsPerGroup = 25; nvars = fp->nvars/numVarsPerGroup; if(nvars <= mpiRank) { printf("Warning! :: Thread[%i] is wasting cycles :: too many processors for data\n", mpiRank); return; } //----Set indexes for each reader if there is more than one int endIndex; int startIndex = (nvars / numMPITasks) * mpiRank; if (nvars % numMPITasks > mpiRank) { startIndex += mpiRank; endIndex = startIndex + (nvars / numMPITasks) + 1; } else { startIndex += nvars % numMPITasks; endIndex = startIndex + (nvars / numMPITasks); } startIndex *= numVarsPerGroup; endIndex *= numVarsPerGroup; //-- for(int i = startIndex; i < endIndex; i++) { ADIOS_VARINFO *avi = adios_inq_var_byid(fp, i); cerr << __LINE__ << endl; string longvarNm(&fp->var_namelist[i][0]); //!!This changed to not remove first char cerr << __LINE__ << endl; string varNm = longvarNm.substr(longvarNm.find("/",1,1)+1,longvarNm.length()); if(varNm == "ephase") { ephase[longvarNm] = avi; ephaseAvail = 1; } else if(varNm == "egid") { egid[longvarNm] = avi; } else if(varNm == "iphase") { iphase[longvarNm] = avi; iphaseAvail = 1; } else if(varNm == "igid") { igid[longvarNm] = avi; } else if(varNm == "inum") { int ipart; AdiosGetValue (fp, i, ipart); totalIParticles += ipart; //totalIParticles += (int)(*(int *)avi->value); adios_free_varinfo(avi); } else if(varNm == "enum") { int epart; AdiosGetValue (fp, i, epart); totalEParticles += epart; //totalEParticles += (int)(*(int *)avi->value); adios_free_varinfo(avi); } else if(i < startIndex + numVarsPerGroup) { if (varNm == "timestep") { //timestep = (int)(*(int *)avi->value); AdiosGetValue (fp, i, timestep); } else if(varNm == "time") { AdiosGetValue (fp, i, time); //time = (double)(*(double *)avi->value); } else if(varNm == "maxnum") { AdiosGetValue (fp, i, maxnum); //maxnum = (int)(*(int *)avi->value); } else if (varNm == "inphase") { AdiosGetValue (fp, i, inphase); //inphase = (int)(*(int *)avi->value); } else if(varNm == "enphase") { AdiosGetValue (fp, i, enphase); //enphase = (int)(*(int *)avi->value); } else if(varNm == "emaxgid") { AdiosGetValue (fp, i, emaxgid); //emaxgid = (long long)(*(long long *)avi->value); } else if(varNm == "imaxgid") { AdiosGetValue (fp, i, imaxgid); //imaxgid = (long long)(*(long long *)avi->value); } adios_free_varinfo(avi); } else { adios_free_varinfo(avi); } } } else //reading particle file or aggregated restart file { for(int i = 0; i < fp->nvars; i++) { string varNm(&fp->var_namelist[i][0]); if(varNm == "ephase") { ephaseAvail = 1; //----Set indexes for each reader for the ELECTRONS ADIOS_VARINFO *avi = adios_inq_var(fp, "ephase"); nvars = avi->dims[0]; int endIndex; int startIndex = (nvars / numMPITasks) * mpiRank; if (nvars % numMPITasks > mpiRank) { startIndex += mpiRank; endIndex = startIndex + (nvars / numMPITasks) + 1; } else { startIndex += nvars % numMPITasks; endIndex = startIndex + (nvars / numMPITasks); } ELECTRONendIndex = endIndex; ELECTRONstartIndex = startIndex; adios_free_varinfo(avi); //-- } else if(varNm == "iphase") { iphaseAvail = 1; //----Set indexes for each reader for the IONS ADIOS_VARINFO *avi = adios_inq_var(fp, "iphase"); nvars = avi->dims[0]; int endIndex; int startIndex = (nvars / numMPITasks) * mpiRank; if (nvars % numMPITasks > mpiRank) { startIndex += mpiRank; endIndex = startIndex + (nvars / numMPITasks) + 1; } else { startIndex += nvars % numMPITasks; endIndex = startIndex + (nvars / numMPITasks); } IONendIndex = endIndex; IONstartIndex = startIndex; adios_free_varinfo(avi); //-- } } for(int i = 0; i < fp->nvars; i++) { ADIOS_VARINFO *avi = adios_inq_var_byid(fp, i); string varNm(&fp->var_namelist[i][0]); if(varNm == "ephase") { ephase[varNm] = avi; } else if(varNm == "egid") { egid[varNm] = avi; } else if(varNm == "iphase") { iphase[varNm] = avi; } else if(varNm == "igid") { igid[varNm] = avi; } else if(varNm == "inum") { totalIParticles = IONendIndex - IONstartIndex; adios_free_varinfo(avi); } else if(varNm == "enum") { totalEParticles = ELECTRONendIndex - ELECTRONstartIndex; adios_free_varinfo(avi); } else if (varNm == "timestep") { //timestep = (int)(*(int *)avi->value); AdiosGetValue (fp, i, timestep); adios_free_varinfo(avi); } else if(varNm == "time") { AdiosGetValue (fp, i, time); //time = (double)(*(double *)avi->value); adios_free_varinfo(avi); } else if (varNm == "inphase") { AdiosGetValue (fp, i, inphase); //inphase = (int)(*(int *)avi->value); adios_free_varinfo(avi); } else if(varNm == "enphase") { AdiosGetValue (fp, i, enphase); //enphase = (int)(*(int *)avi->value); adios_free_varinfo(avi); } else if(varNm == "enum_total") { AdiosGetValue (fp, i, emaxgid); //emaxgid = (long long)(*(long long *)avi->value); adios_free_varinfo(avi); } else if(varNm == "inum_total") { AdiosGetValue (fp, i, imaxgid); //imaxgid = (long long)(*(long long *)avi->value); adios_free_varinfo(avi); } else { adios_free_varinfo(avi); } } } }
int read_scalar_stepbystep () { ADIOS_FILE * f; float timeout_sec = 0.0; int steps = 0; int retval = 0; MPI_Comm comm = MPI_COMM_SELF; adios_read_init_method (ADIOS_READ_METHOD_BP, comm, "verbose=3"); printf ("\n--------- Read scalar in stream using varinfo->value ------------\n"); f = adios_read_open (fname, ADIOS_READ_METHOD_BP, comm, ADIOS_LOCKMODE_NONE, timeout_sec); if (adios_errno == err_file_not_found) { printf ("Stream not found after waiting %f seconds: %s\n", timeout_sec, adios_errmsg()); retval = adios_errno; } else if (adios_errno == err_end_of_stream) { printf ("Stream terminated before open. %s\n", adios_errmsg()); retval = adios_errno; } else if (f == NULL) { printf ("Error at opening stream: %s\n", adios_errmsg()); retval = adios_errno; } else { /* Processing loop over the steps (we are already in the first one) */ while (adios_errno != err_end_of_stream) { steps++; // steps start counting from 1 printf ("Step: %d\n", f->current_step); /* Check the scalar O with varinfo->value */ ADIOS_VARINFO * v = adios_inq_var (f, "NX"); int value = *(int*)v->value; printf ("Scalar NX = %d", value); if (value != block_count [f->current_step*nblocks_per_step*size]) { printf ("\tERROR expected = %llu", block_count [f->current_step*nblocks_per_step*size]); nerrors++; } printf ("\n"); // advance to 1) next available step with 2) blocking wait adios_advance_step (f, 0, timeout_sec); if (adios_errno == err_step_notready) { //printf ("No new step arrived within the timeout. Quit. %s\n", // adios_errmsg()); break; // quit while loop } } adios_read_close (f); } adios_read_finalize_method (ADIOS_READ_METHOD_BP); //printf ("We have processed %d steps\n", steps); return retval; }
int print_scalar (ADIOS_FILE *f, char * name) { ADIOS_VARINFO * v; int i,j,k; v = adios_inq_var (f, name); adios_inq_var_blockinfo (f, v); printf ("Scalar '%s':\n", name); printf ("nsteps = %d\n", v->nsteps); printf ("nblocks per step = %d\n", v->nblocks[0]); int err; /* Read one writeblock across all timesteps */ int *data = (int*) calloc (v->nsteps, sizeof(int)); ADIOS_SELECTION *s; printf ("Read same instance across all timesteps:\n"); for (i=0; i < v->nblocks[0]; i++) { s = adios_selection_writeblock(i); err = adios_schedule_read_byid(f, s, v->varid, 0, v->nsteps, data); if (!err) { err = adios_perform_reads(f, 1); if (!err) { err = 0; printf (" block %d = [", i); for (j=0; j < v->nsteps; j++) { printf ("%d", data[j]); if (data[j] != block_offset [j*nblocks_per_step*size + i]) { err = 1; } if (j < v->nsteps-1) printf(","); } printf("]"); if (err) { nerrors++; printf ("\tERROR expected = ["); for (j=0; j < v->nsteps; j++) { printf ("%llu", block_offset [j*nblocks_per_step*size + i]); if (j < v->nsteps-1) printf(","); } printf("]"); } printf("\n"); } else { printf ("ERROR at reading scalar '%s': %s\n", name, adios_errmsg()); } } else { printf ("ERROR at scheduling read for scalar '%s': %s\n", name, adios_errmsg()); } adios_selection_delete(s); } /* Now read piecewise, one writeblock at a time */ printf ("Read each instance individually:\n"); for (j=0; j < v->nsteps; j++) { printf (" step %d: \n", j); for (i=0; i < v->nblocks[j]; i++) { s = adios_selection_writeblock(i); err = adios_schedule_read_byid(f, s, v->varid, j, 1, data); if (!err) { err = adios_perform_reads(f, 1); if (!err) { printf (" block %d = %d", i, data[0]); if (data[0] != block_offset [j*nblocks_per_step*size + i]) { printf ("\tERROR expected = %llu", block_offset [j*nblocks_per_step*size + i]); nerrors++; } printf ("\n"); } else { printf ("ERROR at reading scalar '%s': %s\n", name, adios_errmsg()); } } else { printf ("ERROR at scheduling read for scalar '%s': %s\n", name, adios_errmsg()); } adios_selection_delete(s); } } /* Now get them piecewise, but not with reading but through statistics */ printf ("Get each instance individually from available statistics:\n"); adios_inq_var_stat (f, v, 0, 1); if (v->statistics && v->statistics->blocks) { ADIOS_VARSTAT *stat = v->statistics; int blockid = 0; for (j=0; j < v->nsteps; j++) { printf (" step %d: \n", j); for (i=0; i < v->nblocks[j]; i++) { printf (" block %d = %d", i, *(int*)stat->blocks->mins[blockid]); if (*(int*)stat->blocks->mins[blockid] != block_offset [j*nblocks_per_step*size + i]) { printf ("\tERROR expected = %llu", block_offset [j*nblocks_per_step*size + i]); nerrors++; } printf ("\n"); blockid++; } } } adios_free_varinfo (v); free(data); }
int main (int argc, char ** argv) { int i, j, datasize; MPI_Comm comm = MPI_COMM_WORLD; enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_BP; ADIOS_SELECTION * sel1; double * data = NULL; uint64_t start[2], count[2]; MPI_Init (&argc, &argv); #ifdef WITH_NCSU_TIMER timer_init(); #endif adios_read_init_method (method, comm, NULL); ADIOS_FILE * f = adios_read_open_file ("adios_global.bp", method, comm); ADIOS_VARINFO * varinfo = adios_inq_var (f, "temperature"); if (varinfo) { int nranks; assert(varinfo->ndim == 2); nranks = varinfo->dims[0]; assert(nranks % 4 == 0); assert(varinfo->dims[1] == 10); datasize = (nranks / 2) * varinfo->dims[1] * sizeof(double); data = malloc (datasize); start[0] = nranks / 4; start[1] = 2; count[0] = nranks / 2; count[1] = 6; sel1 = adios_selection_boundingbox (varinfo->ndim, start, count); adios_schedule_read (f, sel1, "temperature", 0, 1, data); adios_perform_reads (f, 1); printf("Subvolume at (%" PRIu64 ",%" PRIu64 ") of size (%" PRIu64 ",%" PRIu64 "):\n", start[0], start[1], count[0], count[1]); for (i = 0; i < count[0]; i++) { printf("[ "); for (j = 0; j < count[1]; j++) { printf("%.0lf ", data[i * count[1] + j]); } printf("]\n"); } adios_selection_delete (sel1); } adios_free_varinfo (varinfo); adios_read_close (f); adios_read_finalize_method (ADIOS_READ_METHOD_BP); #ifdef WITH_NCSU_TIMER printf("[TIMERS] "); timer_result_t *results = timer_get_results_sorted(); for (i = 0; i < timer_get_num_timers(); i++) { printf("%s: %0.4lf ", results[i].name, results[i].time); } printf("\n"); free(results); #endif #ifdef WITH_NCSU_TIMER timer_finalize(); #endif MPI_Finalize (); return 0; }
int main (int argc, char ** argv) { char filename [256] = "stream.bp"; int rank, size; int NX, NY; int len, off; double *t = NULL; MPI_Comm comm = MPI_COMM_WORLD; int64_t adios_handle; uint64_t adios_groupsize, adios_totalsize; uint64_t start[2], count[2]; ADIOS_SELECTION *sel; int steps = 0; MPI_Init (&argc, &argv); MPI_Comm_rank (comm, &rank); MPI_Comm_size (comm, &size); // ADIOS read init adios_read_init_method (ADIOS_READ_METHOD_BP, comm, "verbose=3"); ADIOS_FILE* fp = adios_read_open_file ("kstar.bp", ADIOS_READ_METHOD_BP, comm); assert(fp != NULL); ADIOS_VARINFO* nx_info = adios_inq_var( fp, "N"); ADIOS_VARINFO* ny_info = adios_inq_var( fp, "L"); NX = *((int *)nx_info->value); NY= *((int*)ny_info->value); len = NX / size; off = len * rank; if (rank == size-1) len = len + NX % size; printf("\trank=%d: NX,NY,len,off = %d\t%d\t%d\t%d\n", rank, NX, NY, len, off); assert(len > 0); t = (double *) malloc(sizeof(double) * len * NY); memset(t, '\0', sizeof(double) * len * NY); assert(t != NULL); start[0] = off; start[1] = 0; count[0] = len; count[1] = NY; sel = adios_selection_boundingbox (2, start, count); // ADIOS write init adios_init ("adios.xml", comm); remove (filename); //int ii; //for(ii = 0; ii<10; ii++){ // for (i = 0; i < len * NY; i++) // t[i] = ii*1000 + rank; while(adios_errno != err_end_of_stream && adios_errno != err_step_notready) { steps++; // Reading adios_schedule_read (fp, sel, "var", 0, 1, t); adios_perform_reads (fp, 1); // Debugging //for (i = 0; i < len*NY; i++) t[i] = off * NY + i; printf("step=%d\trank=%d\t[%d,%d]\n", steps, rank, len, NY); // Writing adios_open (&adios_handle, "writer", filename, "a", comm); adios_groupsize = 4*4 + 8*len*NY; adios_group_size (adios_handle, adios_groupsize, &adios_totalsize); adios_write (adios_handle, "NX", &NX); adios_write (adios_handle, "NY", &NY); adios_write (adios_handle, "len", &len); adios_write (adios_handle, "off", &off); adios_write (adios_handle, "var_2d_array", t); adios_close (adios_handle); // Advance MPI_Barrier (comm); adios_advance_step(fp, 0, TIMEOUT_SEC); } free(t); MPI_Barrier (comm); adios_read_close(fp); if (rank==0) printf ("We have processed %d steps\n", steps); MPI_Barrier (comm); adios_read_finalize_method(ADIOS_READ_METHOD_BP); adios_finalize (rank); MPI_Finalize (); return 0; }
int main (int argc, char ** argv) { int rank, size; int NX, NY; int len, off; double *t = NULL; MPI_Comm comm = MPI_COMM_WORLD; uint64_t start[2], count[2]; ADIOS_SELECTION *sel; int steps = 0; #ifdef _USE_GNUPLOT int i, j; double *tmp; FILE *pipe; #else // Variables for ADIOS write int64_t adios_handle; uint64_t adios_groupsize, adios_totalsize; char outfn[256]; #endif MPI_Init (&argc, &argv); MPI_Comm_rank (comm, &rank); MPI_Comm_size (comm, &size); adios_read_init_method(ADIOS_READ_METHOD_FLEXPATH, comm, ""); ADIOS_FILE* fp = adios_read_open("stream.bp", ADIOS_READ_METHOD_FLEXPATH, comm, ADIOS_LOCKMODE_NONE, 0.0); assert(fp != NULL); ADIOS_VARINFO* nx_info = adios_inq_var( fp, "NX"); ADIOS_VARINFO* ny_info = adios_inq_var( fp, "NY"); NX = *((int *)nx_info->value); NY= *((int*)ny_info->value); len = NX / size; off = len * rank; if (rank == size-1) len = len + NX % size; printf("\trank=%d: NX,NY,len,off = %d\t%d\t%d\t%d\n", rank, NX, NY, len, off); assert(len > 0); t = (double *) malloc(sizeof(double) * len * NY); memset(t, '\0', sizeof(double) * len * NY); assert(t != NULL); start[0] = off; start[1] = 0; count[0] = len; count[1] = NY; // Not working ... //sel = adios_selection_boundingbox (2, start, count); sel = malloc(sizeof(ADIOS_SELECTION)); sel->type=ADIOS_SELECTION_WRITEBLOCK; sel->u.block.index = rank; #ifdef _USE_GNUPLOT if ((NX % size) > 0) { fprintf(stderr, "Equal distribution is required\n"); return -1; } if (rank == 0) { pipe = popen("gnuplot", "w"); fprintf(pipe, "set view map\n"); fprintf(pipe, "set xrange [0:%d]\n", NX-1); tmp = (double *) malloc(sizeof(double) * NX * NY); assert(tmp != NULL); } #else // ADIOS write init adios_init ("adios.xml", comm); #endif //while(adios_errno != err_end_of_stream && adios_errno != err_step_notready) while(1) { steps++; // Reading adios_schedule_read (fp, sel, "var_2d_array", 0, 1, t); adios_perform_reads (fp, 1); printf("step=%d\trank=%d\tfp->current_step=%d\t[%d,%d]\n", steps, rank, fp->current_step, len, NY); /* // Debugging for (i=0; i<len; i++) { printf("%d: rank=%d: t[%d,0:4] = ", steps, rank, off+i); for (j=0; j<5; j++) { printf(", %g", t[i*NY + j]); } printf(" ...\n"); } */ // Do something #ifdef _USE_GNUPLOT // Option 1: plotting MPI_Gather(t, len * NY, MPI_DOUBLE, tmp, len * NY, MPI_DOUBLE, 0, comm); if (rank == 0) { fprintf(pipe, "set title 'Soft X-Rray Signal (shot #%d)'\n", steps); fprintf(pipe, "set xlabel 'Channel#'\n"); fprintf(pipe, "set ylabel 'Timesteps'\n"); fprintf(pipe, "set cblabel 'Voltage (eV)'\n"); # ifndef _GNUPLOT_INTERACTIVE fprintf(pipe, "set terminal png\n"); fprintf(pipe, "set output 'fig%03d.png'\n", steps); # endif fprintf(pipe, "splot '-' matrix with image\n"); //fprintf(pipe, "plot '-' with lines, '-' with lines, '-' with lines\n"); double *sum = calloc(NX, sizeof(double)); for (j = 0; j < NY; j++) { for (i = 0; i < NX; i++) { sum[i] += tmp[i * NY + j]; } } for (j = 0; j < NY; j++) { for (i = 0; i < NX; i++) { fprintf (pipe, "%g ", (-tmp[i * NY + j] + sum[i]/NY)/3276.8); } fprintf(pipe, "\n"); } fprintf(pipe, "e\n"); fprintf(pipe, "e\n"); fflush (pipe); # ifdef _GNUPLOT_INTERACTIVE printf ("Press [Enter] to continue . . ."); fflush (stdout); getchar (); # endif free(sum); } #else // Option 2: BP writing snprintf (outfn, sizeof(outfn), "reader_%3.3d.bp", steps); adios_open (&adios_handle, "reader", outfn, "w", comm); adios_groupsize = 4 * sizeof(int) + sizeof(double) * len * NY; adios_group_size (adios_handle, adios_groupsize, &adios_totalsize); adios_write (adios_handle, "NX", &NX); adios_write (adios_handle, "NY", &NY); adios_write (adios_handle, "len", &len); adios_write (adios_handle, "off", &off); adios_write (adios_handle, "var", t); adios_close (adios_handle); #endif // Advance MPI_Barrier (comm); adios_advance_step(fp, 0, TIMEOUT_SEC); if (adios_errno == err_end_of_stream) { printf("rank %d, Stream terminated. Quit\n", rank); break; // quit while loop } else if (adios_errno == err_step_notready) { printf ("rank %d: No new step arrived within the timeout. Quit.\n", rank); break; // quit while loop } else if (adios_errno != err_no_error) { printf("ADIOS returned code=%d, msg:%s\n", adios_errno, adios_get_last_errmsg()); break; // quit while loop } } // free(t); adios_read_close(fp); //printf("rank %d, Successfully closed stream\n", rank); adios_read_finalize_method(ADIOS_READ_METHOD_FLEXPATH); //printf("rank %d, Successfully finalized read method\n", rank); #ifndef _USE_GNUPLOT adios_finalize (rank); //printf("rank %d, Successfully finalized adios\n", rank); #else if (rank==0) { free(tmp); pclose(pipe); } #endif MPI_Finalize (); return 0; }
int main(int argc, char ** argv) { int rank, size, varid, numvars; int bins, step, mod; char *filename, *in_stream, *data_var_name; MPI_Comm comm = MPI_COMM_WORLD; enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_FLEXPATH; //enum ADIOS_READ_METHOD method = ADIOS_READ_METHOD_BP; ADIOS_SELECTION * global_range_select; double *data; uint64_t tstep, global_size, mysize, mystart, sz; MPI_Init (&argc, &argv); MPI_Comm_rank(comm, &rank); MPI_Comm_size(comm, &size); /* Command line parsing */ if (rank == 0 && argc < 4) { fprintf(stderr, "\nHistogram usage: <exec> input-stream-name num-bins" " arr1 [arr2] [arr3] [...]\n" "\t where arr1, arr2, arr3 ... are the names of the arrays to be analyzed.\n"); MPI_Abort(comm, -1); } MPI_Barrier(comm); in_stream = argv[1]; //Parse cmd line bins = atoi(argv[2]); numvars = argc - 3; const char *vars[numvars]; for (varid=0; varid < numvars; varid++) { vars[varid] = argv[varid + 3]; } /* Adios open and init */ adios_read_init_method (method, comm, "verbose=1"); ADIOS_FILE * f = adios_read_open (in_stream, method, comm, ADIOS_LOCKMODE_ALL, -1); step = 0; //not used now while (adios_errno != err_end_of_stream){ //resource monitor /*loop over different arrays inside stream*/ for (varid = 0; varid < numvars; varid++){ #ifdef ENABLE_MONITOR //double t1 = wfgettimeofday(); lib_mem_init(); ind_timer_start(0, "whole timestep"); #endif //Init variables.... global_size = 0; tstep = 0; mod = 0; mysize = 0; mystart = 0; adios_schedule_read (f, NULL, "ntimestep", 0, 1, &tstep); adios_perform_reads (f, 1); ADIOS_VARINFO * glob_info = adios_inq_var (f, vars[varid]); global_size = glob_info->dims[0]; //printf("[DEBUG] global_size = %" PRIu64 " ntimestep = %" PRIu64 "\n", // global_size, tstep); //printf("[HIST%d] received data for timestep %" PRIu64 " with ndim: %d and globalsize:%" // PRIu64 " \n", rank, tstep, ndim, global_size); //sleep(800); //debug //Array slice computation mod = global_size % size;//size = MPI size if (mod == 0){ mysize = global_size / size; mystart = mysize * rank; } else { mysize = global_size / (size); if (rank < mod){ mysize++; mystart = mysize * rank; } else { mystart = (mod * (mysize + 1)) + ((rank - mod) * mysize); } } #ifdef ENABLE_MONITOR nohandler_mem(rank); #endif //printf("[HISTO%d]: mysize = %" PRIu64" mystart = %" PRIu64 "\n", rank, mysize, mystart); //debug //if (step == 0) sleep(800); uint64_t starts[] = {mystart}; uint64_t counts[] = {mysize}; global_range_select = adios_selection_boundingbox (1, starts, counts); //Allocate space for arrays uint64_t msize = ((uint64_t) sizeof(double) * mysize); //printf("[DEBUG] mysize = %" PRIu64 " msize= %" PRIu64" \n", mysize, msize); //data = (double *) malloc(sizeof(double) * mysize); data = new double[mysize]; if (data == NULL){ //printf("DEBUG: malloc returned NULL, size was %d\n", msize); } else { if (rank == 0) printf("[HIST0] DEBUG: malloc successful, size was %d\n", mysize); } //memset (data, 0, sizeof(double) * mysize); //Read data adios_schedule_read (f, global_range_select, vars[varid], 0, 1, data); adios_perform_reads (f, 1); #ifdef ENABLE_MONITOR nohandler_mem(rank); #endif //printf("PERFORM_READS success of variable: %s\n", vars[varid]); /* Data check if (step == 4) { FILE *fp; char *log; asprintf(&log, "histo-input%d-%d.log", step, rank); fp = fopen(log, "w"); fprintf(fp, "timestep: %" PRIu64 " mysize: %"PRIu64 "\n", tstep, mysize); for (i=0; i<(int)mysize; i++){ fprintf(fp, "%lf\n", data[i]); } fclose(fp); sleep(800); } */ // find max and min sz = 0; sz = mysize; double min = data[0]; double max = data[0]; for (uint64_t i = 1; i < sz; ++i) { if (data[i] > max) max = data[i]; if (data[i] < min) min = data[i]; }//local max, min found. //local data should just use shared mem. double g_min, g_max; // Find the global max/min MPI_Allreduce (&min, &g_min, 1, MPI_DOUBLE, MPI_MIN, comm); MPI_Allreduce (&max, &g_max, 1, MPI_DOUBLE, MPI_MAX, comm); //printf("[HIST%d] glob-min: %f, glob-max: %f\n", rank, g_min, g_max); nohandler_mem(rank); double width = (g_max - g_min)/bins; std::vector<uint64_t> hist(bins); for (uint64_t i = 0; i < sz; ++i)//fill local bins { //printf("[HISTO%d] local filling adding index %" PRIu64 "\n", rank, i); int idx = int((data[i] - g_min)/width);//discover index if (idx == bins) // we hit the max --idx; //printf("[%d]: %f -> %d\n", rank, data[i], idx); ++hist[idx]; } delete[] data; // Global reduce histograms std::vector<uint64_t> g_hist(bins); MPI_Reduce(&hist[0], &g_hist[0], bins, MPI_UINT64_T, MPI_SUM, 0, comm); //debug //printf("[Completed histogram routine]\n"); if (rank == 0) //print histogram to file { FILE *fp; const char *log = "histograms.log"; fp = fopen(log, "a"); fprintf(fp, "Histogram for %s, timestep %" PRIu64"\n", vars[varid], tstep); for (int i = 0; i < bins; ++i) fprintf(fp, " %f-%f: %" PRIu64 "\n", g_min + i*width, g_min + (i+1)*width, g_hist[i]); fclose (fp); } #ifdef ENABLE_MONITOR nohandler_mem(rank); #endif if (rank == 0) //print histogram to terminal { printf("Histogram for %s, timestep %" PRIu64"\n", vars[varid], tstep); for (int i = 0; i < bins; ++i) printf(" %f-%f: %" PRIu64 "\n", g_min + i*width, g_min + (i+1)*width, g_hist[i]); } //resource monitor #ifdef ENABLE_MONITOR //double t2 = wfgettimeofday(); ind_timer_end(0); char monitor_title[40]; sprintf(monitor_title, "histogram-%s", vars[varid]); monitor_out (rank, size, tstep, msize, t1, t2, comm, monitor_title); #endif } //end of read + analysis for 3 variables adios_release_step(f); //delete[] data; if (rank == 0) printf("[HIST%d] read and wrote data for timestep %" PRIu64 "\n", rank, tstep); step++; adios_advance_step(f, 0, -1); /* if (step == 6){ double t1 = wfgettimeofday(); FILE *tfp; tfp = fopen("time.log", "a"); fprintf(tfp, "rank %d histogram end time: %f\n", rank, t1); fclose(tfp); } */ }//end of adios stream while loop if (rank == 0) printf("[HIST%d] out of read loop\n", rank); /* performance measurement */ /* if (rank == 0){ double t3 = wfgettimeofday(); FILE *tfp; tfp = fopen("time.log", "a"); fprintf(tfp, "master histogram end time: %f\n", t3); fclose(tfp); } */ #ifdef ENABLE_MONITOR outer_timer_end(rank, "histogram"); #endif adios_read_close(f); adios_read_finalize_method(method); MPI_Finalize(); return 0; }