void run() { std::mt19937 generator(randomSeed()); std::uniform_int_distribution<size_t> distribution(0, queries.size() - 1); for (size_t i = 0; i < concurrency; ++i) pool.schedule(std::bind(&Benchmark::thread, this, connections.get())); InterruptListener interrupt_listener; info_per_interval.watch.restart(); delay_watch.restart(); /// Push queries into queue for (size_t i = 0; !max_iterations || i < max_iterations; ++i) { size_t query_index = randomize ? distribution(generator) : i % queries.size(); if (!tryPushQueryInteractively(queries[query_index], interrupt_listener)) break; } shutdown = true; pool.wait(); info_total.watch.stop(); if (!json_path.empty()) reportJSON(info_total, json_path); printNumberOfQueriesExecuted(info_total.queries); report(info_total); }
void printNumberOfQueriesExecuted(size_t num) { std::cerr << "\nQueries executed: " << num; if (queries.size() > 1) std::cerr << " (" << (num * 100.0 / queries.size()) << "%)"; std::cerr << ".\n"; }
void pcl::gpu::Octree::approxNearestSearch(const Queries& queries, NeighborIndices& results) const { assert(queries.size() > 0); results.create(static_cast<int> (queries.size()), 1); const OctreeImpl::Queries& q = (const OctreeImpl::Queries&)queries; static_cast<OctreeImpl*>(impl)->approxNearestSearch(q, results); }
void pcl::gpu::Octree::radiusSearch(const Queries& queries, float radius, int max_results, NeighborIndices& results) const { assert(queries.size() > 0); results.create(static_cast<int> (queries.size()), max_results); results.sizes.create(queries.size()); const OctreeImpl::Queries& q = (const OctreeImpl::Queries&)queries; static_cast<OctreeImpl*>(impl)->radiusSearch(q, radius, results); }
string Relation::parse(Queries query){ Relation editedRelation = *this; editedRelation = rename(query.getTuple(), editedRelation); editedRelation = select(query.getTuple(), editedRelation); editedRelation = project(query.getTuple(), editedRelation); return editedRelation.toFinalString(query); }
void pcl::gpu::Octree::nearestKSearchBatch(const Queries& queries, int k, NeighborIndices& results) const { if (k != 1) throw pcl::PCLException("OctreeGPU::knnSearch is supported only for k == 1", __FILE__, "", __LINE__); assert(queries.size() > 0); results.create(static_cast<int> (queries.size()), k); const OctreeImpl::Queries& q = (const OctreeImpl::Queries&)queries; static_cast<OctreeImpl*>(impl)->nearestKSearchBatch(q, k, results); }
void readQueries() { ReadBufferFromFileDescriptor in(STDIN_FILENO); while (!in.eof()) { std::string query; readText(query, in); assertChar('\n', in); if (!query.empty()) queries.emplace_back(query); } if (queries.empty()) throw Exception("Empty list of queries."); std::cerr << "Loaded " << queries.size() << " queries.\n"; }
string Relation::toFinalString(Queries query){ stringstream result; if(rows.size()>0){ result << query.toString() << " Yes(" << rows.size() << ")\n"; for(int i=0; i<rows.size(); i++){ for(int j=0; j<schema.size(); j++){ if(j==0){ result << " " << schema[j].getTokensValue() << "=" << getRowAtIndex(i).elements[j].getTokensValue(); } else{ result << ", " << schema[j].getTokensValue() << "=" << getRowAtIndex(i).elements[j].getTokensValue(); } } if(schema.size()>0){ result << "\n"; } } } else{ result << query.toString() << " No\n"; } return result.str(); }
void generateFile(std::string const& filename, const unsigned wordsCount, Dict const& dict, Prng& prng, Queries& queries) { cout << "generating file " << filename << "..." << endl; std::ofstream os(filename); if( not os.is_open() ) throw std::runtime_error{"unable to open file for writing: " + filename}; for(unsigned i=0; i<wordsCount; ++i) { const auto addNumber = (i % numberedWordEveryIteration) == 0; auto word = dict[prng()] + ( addNumber ? std::to_string(prng()) : "" ); os << word << "\n"; if( (i % queryEveryIteration) == 0 ) queries.insert(std::move(word)); } }
int main(int argc, char **argv) { QTextStream qout(stdout, QIODevice::WriteOnly); QTextStream qerr(stderr, QIODevice::WriteOnly); QTextStream qin(stdin, QIODevice::ReadOnly); Args args; args.add(new Arg("h", "help", Arg::setTrue, QVariant(false))); args.add(new Arg("v", "verbose", Arg::setTrue, QVariant(false))); //args.add(new Arg("i", "input-file", Arg::readableFile, QVariant("/var/log/postgresql.log"))); args.add(new Arg("i", "input-file", Arg::readableFile, QVariant("/var/log/postgresql/postgresql-9.1-main.log"))); args.add(new Arg("o", "output-file", Arg::writableFile, QVariant("report.html"))); args.add(new Arg("u", "users", Arg::toString, QVariant())); args.add(new Arg("d", "databases", Arg::toString, QVariant())); args.add(new Arg("top", Arg::toInt, QVariant(20))); args.add(new Arg("t", "query-types", Arg::toString, QVariant("SELECT,UPDATE,INSERT,DELETE"))); if(!args.parse(argc, argv)) { args.help(); return -1; } QStringList users = args.getStringList("users"); QStringList databases = args.getStringList("databases"); QStringList query_types = args.getStringList("query-types"); int ret; QString line; int start_index; int stop_index; int old_query_id; int new_query_id; int old_line_id; int new_line_id; QString database; QString user; QString statement; uint duration; QFile input_file; ret = args.getFile(&input_file, "input-file", QIODevice::ReadOnly | QIODevice::Text); if(ret)return ret; QFile output_file; ret = args.getFile(&output_file, "output-file", QIODevice::WriteOnly | QIODevice::Text); if(ret)return ret; QTextStream output(&output_file); /* display the top N queries */ int top = args.getInt("top"); Queries queries; old_query_id = -1; old_line_id = -0; statement = ""; duration = 0; QTime timer; timer.start(); uint lines = 0; if(input_file.atEnd()) { qerr << "The input file (" << input_file.fileName(); qerr << ") seems to be empty" << endl; } while (!input_file.atEnd()) { line = input_file.readLine(4096); lines++; if(lines % 1000 == 0) { qout << "Read " << lines << " lines." << endl; qout.flush(); } if(line[0] == '\t') { statement.append(line); continue; } start_index = line.indexOf("[", 15); start_index = line.indexOf("[", start_index + 3) + 1; stop_index = line.indexOf("-", start_index); new_query_id = line.mid(start_index, stop_index - start_index).toInt(); start_index = stop_index + 1; stop_index = line.indexOf("]", start_index); new_line_id = line.mid(start_index, stop_index - start_index).toInt(); if(new_query_id != old_query_id || old_line_id < new_line_id) { old_query_id = new_query_id; QString hashStatement = Query::normalize(statement); statement = Query::format(statement); if(( hashStatement.startsWith("INSERT") || hashStatement.startsWith("DELETE") || hashStatement.startsWith("UPDATE") || hashStatement.startsWith("SELECT") ) && (!users.length() || users.contains(user)) && (!databases.length() || databases.contains(database))) { uint hash = qHash(hashStatement); if(queries.contains(hash)) { queries[hash]->addStatement(duration, statement); } else { queries.insert(hash, new Query(hashStatement, statement, user, database, duration)); } } user = ""; database = ""; duration = 0; statement = ""; start_index = line.indexOf("user="******",", start_index); user = line.mid(start_index, stop_index - start_index); start_index = line.indexOf("db=", stop_index) + 3; stop_index = line.indexOf(" ", start_index); if(start_index == -1 || stop_index == -1)continue; database = line.mid(start_index, stop_index - start_index); start_index = line.indexOf("duration: ", stop_index) + 10; stop_index = line.indexOf(" ", start_index); duration = line.mid(start_index, stop_index - start_index).toDouble() * 1000; start_index = line.indexOf("statement: ", stop_index) + 11; if(start_index < stop_index)continue; stop_index = line.length(); statement = line.mid(start_index, stop_index - start_index); } else { start_index = line.indexOf("] ", stop_index); if(start_index != -1) { start_index += 2; stop_index = line.length() - start_index; statement.append(line.mid(start_index, line.length() - start_index)); } } } QFile header(":templates/header.html"); if (!header.open(QIODevice::ReadOnly | QIODevice::Text)) { qout << "Unable to open templates/header.html" << endl; return -4; } else { output << header.readAll(); header.close(); } QList<Query*> queries_sorted; output << QString("<div class=\"information\">" "<ul>" "<li>Generated on %1</li>" "<li>Parsed %2 (%3 lines) in %4 ms</li>" "</ul>" "</div>") \ .arg(QDateTime::currentDateTime().toString("yyyy-MM-dd hh:mm:ss")) \ .arg(args.getString("input_file")) \ .arg(lines) \ .arg(timer.elapsed()); output << "<div class=\"reports\">"; output << "<h2 id=\"NormalizedQueriesMostTimeReport\">Queries that took up the most time (N) <a href=\"#top\" title=\"Back to top\">^</a></h2>"; output << print_queries(queries.sortedQueries(Queries::mostTotalDuration), top); output << "<h2 id=\"NormalizedQueriesMostFrequentReport\">Most frequent queries (N) <a href=\"#top\" title=\"Back to top\">^</a></h2>"; output << print_queries(queries.sortedQueries(Queries::mostExecutions), top); output << "<h2 id=\"NormalizedQueriesSlowestAverageReport\">Slowest queries (N) <a href=\"#top\" title=\"Back to top\">^</a></h2>"; output << print_queries(queries.sortedQueries(Queries::mostAverageDuration), top); QFile footer(":templates/footer.html"); if (!footer.open(QIODevice::ReadOnly | QIODevice::Text)) { qout << "Unable to open templates/footer.html" << endl; return -5; } else { output << footer.readAll(); footer.close(); } qout << "Wrote to file " << args.getString("output-file") << endl; }