int main(int argc, char **argv) { std::string varPathStr; std::string varNameStr1; std::string varNameStr2; std::string varNameStr3; parseArgs(argc, argv); std::vector<double> beginList; std::vector<double> endList; std::vector<double> strideList; beginList.resize(dimension); endList.resize(dimension); strideList.resize(dimension); if (datafile.empty() || condstring == 0 || dimension == 0 ) { std::cerr << "Usage:\n" << *argv << " -f data-file-name" << " -q query-conditions-in-a-single-string" << " -x histogram-dimension" << " -y begin" << " -e end" << " -s stride" << " [-i index-file-name]" << " [-g log-file-name]" << " [-n name-of-variable]" << " [-p path-of-variable]" << " [-m file model [HDF5(default), H5PART, NETCDF, PNETCDF]" << " [-b use-boundingbox-data-selection]" << " [-v verboseness]" << " [-l mpi-subarray-length]" //<< "\n e.g: ./histogram -f h5uc-data-index.h5 -q 'px < 0.3' -n y -p TimeStep2 -x 1\n" << "\n e.g: ./histogram -f h5uc-data.h5 -i indexfile -q 'px<0.3 && py>0' -x 2 -n py,pz;" << " -y '0,-0.5;' -s '0.1,0.02;' -e '1,0;' -p TimeStep2\n\n" << "\tFor More detailed usage description and examples, please see file GUIDE" << std::endl; return -1; } #ifndef FQ_NOMPI MPI_Init(&argc, &argv); int mpi_size, mpi_rank; MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); #endif ibis::gParameters().add(FQ_REPORT_STATISTIC, "true"); ibis::horometer totTimer; totTimer.start(); FQ::FileFormat model = FQ::FQ_HDF5; if (fileModel != 0) { std::string format = fileModel; if (format.compare("HDF5") == 0) { model = FQ::FQ_HDF5; } else if (format.compare("H5PART") == 0) { model = FQ::FQ_H5Part; } else if (format.compare("NETCDF") == 0) { model = FQ::FQ_NetCDF; } else if (format.compare("PNETCDF") == 0) { model = FQ::FQ_pnetCDF; } } if (! indexfile.empty()) { if (verboseness > 1) std::cout << "DEBUG: using indexfile \"" << indexfile.c_str() << "\" ... \n"; } if (varPath != 0) { if (verboseness > 1) std::cout << "Debug: use variable path \"" << varPath << "\"\n"; varPathStr = varPath; } // std::cout << "varName:" << varName << " begin:" << begin << " end:" << end << " stride:" << stride << std::endl; varName1 = strtok(varName, ",;"); if (dimension>1) varName2 = strtok(NULL, ",;"); if (dimension>2) varName3 = strtok(NULL, ",;"); begin1 = atof(strtok(begin, ",;")); if (dimension>1) begin2 = atof(strtok(NULL, ",;")); if (dimension>2) begin3 = atof(strtok(NULL, ",;")); end1 = atof(strtok(end, ",;")); if (dimension>1) end2 = atof(strtok(NULL, ",;")); if (dimension>2) end3 = atof(strtok(NULL, ",;")); stride1 = atof(strtok(stride, ",;")); if (dimension>1) stride2 = atof(strtok(NULL, ",;")); if (dimension>2) stride3 = atof(strtok(NULL, ",;")); //std::cout << "varName is " << varName1 << ", " << varName2 << ", " << varName3 << std::endl; //std::cout << "begin is " << begin1 << ", " << begin2 << ", " << begin3 << std::endl; //std::cout << "end is " << end1 << ", " << end2 << ", " << end3 << std::endl; //std::cout << "stride is " << stride1 << ", " << stride2 << ", " << stride3 << std::endl; if (varName1!=0) varNameStr1 = varName1; if (dimension>1 && varName2!=0) varNameStr2 = varName2; if (dimension>2 && varName3!=0) varNameStr3 = varName3; /* if (mpi_rank==0) { unsigned int dims1 = static_cast<uint32_t>(1+floor((end1-begin1)/stride1)); unsigned int dims2 = static_cast<uint32_t>(1+floor((end2-begin2)/stride2)); unsigned int dims3 = static_cast<uint32_t>(1+floor((end3-begin3)/stride3)); std::cout << "dims1 * dims2 * dims3 = " << dims1 << " * " << dims2 << " * " << dims3 << std::endl; } */ if (logfile.str().empty() != true) { #ifndef FQ_NOMPI logfile << mpi_rank << ".log"; #endif if (verboseness > 1) std::cout << "Debug: using logfile \"" << logfile.str().c_str() << "\"\n"; } if (verboseness >1) { std::cout << "open the file handler" << std::endl; } // open the named file QueryProcessor* queryProcessor = new QueryProcessor(datafile, model, indexfile, verboseness, "", logfile.str().c_str()); // the file handler if (queryProcessor->isValid() == false) { if (verboseness > 0) { std::cout << "ERROR: failed to initiate the QueryProcessor object for file \"" << datafile.c_str() << "\" ...\n"; std::cout << "REPORT: failed to complete processing query" << std::endl; } delete(queryProcessor); #ifndef FQ_NOMPI MPI_Finalize(); #endif return -1; } uint64_t hits = 0; // getNumHits ibis::horometer timer; timer.start(); hits = queryProcessor->getNumHits(condstring, varPathStr, mpi_dim, mpi_len); timer.stop(); if (verboseness > 1) std::cout << "Debug: conditions \"" << condstring << "\" number of hits " << hits << std::endl;; if (hits == 0) { if (verboseness > 1) { std::cout << "Warning -- No element is seleteced ==>" << " the rest of the test is skipped!" << std::endl; } if (verboseness > 0) { #ifndef FQ_NOMPI if (mpi_rank==0) { #endif std::cout << "REPORT: successfully completed processing query with " << hits << " hits" << std::endl; #ifndef FQ_NOMPI } #endif } delete(queryProcessor); #ifndef FQ_NOMPI MPI_Finalize(); #endif return hits; } // executeQuery std::vector<uint64_t> coords; std::vector<uint32_t> counts; bool herr = true; // if (mpi_rank==0) std::cout<<"histogram starting..."<<std::endl; if (varPath != 0) { //coords.reserve(hits*dims.size()); // hits1 = queryProcessor->executeQuery((char*)condstring, coords, varPathStr, FQ::POINTS_SELECTION, mpi_dim, mpi_len); if (dimension==1) { //counts.assign(static_cast<uint32_t>(1+floor(end1-begin1)/stride1), 0); herr = queryProcessor->get1DHistogram ((char*) condstring, varNameStr1, varPathStr, begin1, end1, stride1, counts, mpi_dim, mpi_len); } else if (dimension==2) { // if (mpi_rank==0) std::cout << "in 2Dhistogram" << std::endl; //counts.assign(static_cast<uint32_t>(1+floor(end1-begin1)/stride1)* // static_cast<uint32_t>(1+floor(end2-begin2)/stride2), 0); herr = queryProcessor->get2DHistogram ((char*) condstring, varPathStr, varNameStr1, begin1, end1, stride1, varNameStr2, begin2, end2, stride2, counts, mpi_dim, mpi_len); // if (mpi_rank==0) std::cout << "out 2Dhistogram" << std::endl; } else if (dimension==3) { herr = queryProcessor->get3DHistogram ((char*) condstring, varPathStr, varNameStr1, begin1, end1, stride1, varNameStr2, begin2, end2, stride2, varNameStr3, begin3, end3, stride3, counts, mpi_dim, mpi_len); } if (! herr) { LOGGER(ibis::gVerbose >= 0) << *argv << " failed to compute the histogram"; return -2; } /************************/ /* verify part */ /************************/ if (verification) { //verify the Histogram //if (mpi_rank==0) std::cout << "starting verify the histogram..." << std::endl; uint64_t len = 1; if (len) { //std::cout<<"Warning: May use too large memory. Can only check sum.\n"; } else { /* double data[len]; // std::vector<uint32_t> temp_counts; temp_counts.assign(static_cast<uint32_t>(1+floor(end-begin)/stride), 0); bool verr = true; //std::cout << "starting getData...." << std::endl; verr = queryProcessor->getData(varNameStr, &data[0], varPathStr); #ifndef FQ_NOMPI if (mpi_rank==0) { #endif if (len<=1000000) { //std::cout << "getData success" << std::endl; std::cout << "temp Histogram" << std::endl; // copy from fasbit parth.cpp get1DHistogram if (len != 0) { for (uint32_t i = 0; i < len; ++ i) { ++ temp_counts[static_cast<uint32_t>((data[i] - begin) / stride)]; } } // std::cout << "temp Histogram" << std::endl; std::cout << "temp_counts.size is "<< temp_counts.size() << std::endl; for (int i=0; i<temp_counts.size(); i++) { std::cout << "[" << begin+i*stride << ", " << begin+(i+1)*stride << "]:\t" << temp_counts[i] << std::endl; } std::cout << "test Histogram" << std::endl; std::cout << "counts.size is "<< counts.size() << std::endl; for (int i=0; i<counts.size(); i++) { std::cout << "[" << begin+i*stride << ", " << begin+(i+1)*stride << "]:\t" << counts[i] << std::endl; } // verify two histogram vectors if (counts!=temp_counts) { std::cout << "ERROR:Vector is not match.Histogram fail." << std::endl; } else { std::cout << "histogram success" << std::endl; } } #ifndef FQ_NOMPI } #endif */ } //unsigned int hits = 0 ; //hits = queryProcessor->getNumHits(condstring, varPathStr, mpi_dim, mpi_len); uint64_t hits1 = 0; for (int i=0; i<counts.size(); i++) { hits1 += counts[i]; } if (hits1 != hits) { std::cout<<"Error:\tcheck sum failed. Num of Hit is " << hits << ",and histogram number is " << hits1<<std::endl; } else std::cout<<"verification result is correct.\n"; } std::fstream histogramFile; #ifndef FQ_NOMPI if (mpi_rank==0) { #endif if (dimension==1) { std::fstream file; //char fileName[100]=""; //char path[]="/global/homes/v/vidcina/fq/example/"; //fileName<<dimension<<"D"<<"histogram["<<begin1<<":"<<stride1<<":"<<end1<<"].out"; //sprintf(fileName, "%s%d%s%d%s%d%s%d%s", path, dimension, "Dhistogram[", begin1, ":", stride1, ":", end1, "].out"); //std::string temp=""; //temp.push_back(fileName.str()); std::ostringstream fileName; fileName << hist_path << "_"<< dimension << "D" << "histogram[" << begin1 << ":" << stride1 << ":" <<end1 << "].out"; std::string str = fileName.str(); const char* chr = str.c_str(); file.open(chr, std::ios::out); if ( file.fail() ) { std::cout << str << std::endl; std::cout << "openFile fail" << std::endl; } else { for (int i=0; i<counts.size(); i++) { file << begin1+i*stride1 << "\t" << begin1+(i+1)*stride1 << "\t" << counts[i] << std::endl; } } //histogramFile.close(); } else if (dimension==2) { std::cout << "2DHistogram " << "Variable1 "<< varName1 << " begin " << begin1 << " to " << end1 <<" stride is " << stride1 << "Variable2 "<< varName2 << " begin " << begin2 << " to " << end2 <<" stride is " << stride2 << std::endl ; std::cout << "counts.size is "<< counts.size() << std::endl; unsigned int imax = static_cast<uint32_t>(1+floor((end1-begin1)/stride1)); unsigned int jmax = static_cast<uint32_t>(1+floor((end2-begin2)/stride2)); for (unsigned int i=0; i<imax; i++) { for (unsigned int j=0; j<jmax; j++) { std::cout << "[" << begin1+i*stride1 << ", " << begin1+(i+1)*stride1 << "), [" << begin2+j*stride2 << ", " << begin2+(j+1)*stride2 << "):\t" << counts[i*jmax+j] << std::endl; } } } else if (dimension==3) { std::cout << "3DHistogram " << "Variable1 "<< varName1 << " begin " << begin1 << " to " << end1 << " stride is " << stride1 << "Variable2 "<< varName2 << " begin " << begin2 << " to " << end2 << " stride is " << stride2 << "Variable3 "<< varName3 << " begin " << begin3 << " to " << end2 << " stride is " << stride3 << std::endl ; std::cout << "counts.size is "<< counts.size() << std::endl; unsigned int imax = static_cast<uint32_t> (1+floor((end1-begin1)/stride1)); unsigned int jmax = static_cast<uint32_t> (1+floor((end2-begin2)/stride2)); unsigned int kmax = static_cast<uint32_t> (1+floor((end3-begin3)/stride3)); #ifndef FQ_NOMPI if (mpi_rank==0 && imax*jmax*kmax!=counts.size()) { std::cout<<"ERROR: counts.size not match."<<std::endl; delete(queryProcessor); MPI_Finalize(); return 0; } #endif for (unsigned int i=0; i<imax; i++) { for (unsigned int j=0; j<jmax; j++) { for (unsigned int k=0; k<kmax; k++) { if (easyToShow) { if (counts[i*jmax*kmax + j*kmax + k]!=0) { std::cout << "[" << begin1+i*stride1 << ", " << begin1+(i+1)*stride1 << "), [" << begin2+j*stride2 << ", " << begin2+(j+1)*stride2 << "), [" << begin3+k*stride3 << ", " << begin3+(k+1)*stride3 << "):\t" << counts[i*jmax*kmax + j*kmax + k] << std::endl; } } else { std::cout << "[" << begin1+i*stride1 << ", " << begin1+(i+1)*stride1 << "), [" << begin2+j*stride2 << ", " << begin2+(j+1)*stride2 << "), [" << begin3+k*stride3 << ", " << begin3+(k+1)*stride3 << "):\t" << counts[i*jmax*kmax + j*kmax + k] << std::endl; } } } } std::cout << "successfuly printed histogram" << std::endl; } #ifndef FQ_NOMPI } #endif // }//end else }//end if(!varPath) // MPI_Barrier(MPI_COMM_WORLD); /* if (hits != hits1) { std::cout << "Error -- number of hits does not match!" << std::endl; std::cout << "REPORT: failed to complete processing query" << std::endl; delete(queryProcessor); #ifndef FQ_NOMPI MPI_Finalize(); #endif return -1; } */ if (verboseness > 0) { #ifndef FQ_NOMPI if (mpi_rank==0) { #endif std::cout << "REPORT: successfully completed get1DHistogram with " << counts.size() << " histogram size" << std::endl; #ifndef FQ_NOMPI } #endif } delete(queryProcessor); #ifndef FQ_NOMPI MPI_Finalize(); #endif totTimer.stop(); LOGGER(FastQuery::reportTiming()) << "Statistic\thistogram::totTimer\t" << totTimer.CPUTime() << "\t" << totTimer.realTime() << "\t"; return hits; } // main