const _PlanOperation *_PlanOperation::execute() { epoch_t startTime = get_epoch_nanoseconds(); refreshInput(); setupPlanOperation(); PapiTracer pt; pt.addEvent("PAPI_TOT_CYC"); pt.addEvent(getEvent()); pt.start(); executePlanOperation(); pt.stop(); teardownPlanOperation(); epoch_t endTime = get_epoch_nanoseconds(); std::string threadId = boost::lexical_cast<std::string>(std::this_thread::get_id()); if (_performance_attr != nullptr) *_performance_attr = (performance_attributes_t) { pt.value("PAPI_TOT_CYC"), pt.value(getEvent()), getEvent() , planOperationName(), _operatorId, startTime, endTime, threadId }; setState(OpSuccess); return this; }
const PlanOperation* PlanOperation::execute() { const bool recordPerformance = _performance_attr != nullptr; // Check if we really need this epoch_t startTime = 0; if (recordPerformance) startTime = get_epoch_nanoseconds(); PapiTracer pt; // Start the execution refreshInput(); setupPlanOperation(); if (recordPerformance) { pt.addEvent("PAPI_TOT_CYC"); pt.addEvent(getEvent()); pt.start(); } executePlanOperation(); if (recordPerformance) pt.stop(); teardownPlanOperation(); if (recordPerformance) { epoch_t endTime = get_epoch_nanoseconds(); std::string threadId = boost::lexical_cast<std::string>(std::this_thread::get_id()); size_t cardinality; if (getResultTable() != empty_result) cardinality = getResultTable()->size(); else // the cardinality is max(size_t) by convention if there is no return table cardinality = std::numeric_limits<size_t>::max(); *_performance_attr = (performance_attributes_t) {pt.value("PAPI_TOT_CYC"), pt.value(getEvent()), getEvent(), planOperationName(), _operatorId, startTime, endTime, threadId, cardinality}; } setState(OpSuccess); return this; }
void StartProfiling::executePlanOperation() { output = input; #ifdef HYRISE_USE_GOOGLE_PROFILER ProfilerStart((Settings::getInstance()->getProfilePath() + "/profile_" + std::to_string(get_epoch_nanoseconds()) + ".gprof").c_str()); #endif }
Json::Value ResponseTask::generateResponseJson() { Json::Value response; epoch_t responseStart = _recordPerformanceData ? get_epoch_nanoseconds() : 0; PapiTracer pt; pt.addEvent("PAPI_TOT_CYC"); if (_recordPerformanceData) pt.start(); auto predecessor = getResultTask(); const auto& result = predecessor->getResultTable(); if (getState() != OpFail) { if (!_isAutoCommit) { response["session_context"] = std::to_string(_txContext.tid).append(" ").append(std::to_string(_txContext.lastCid)); } if (result) { // Make header Json::Value json_header(Json::arrayValue); for (unsigned col = 0; col < result->columnCount(); ++col) { Json::Value colname(result->nameOfColumn(col)); json_header.append(colname); } // Copy the complete result response["real_size"] = result->size(); response["rows"] = generateRowsJson(result, _transmitLimit, _transmitOffset); response["header"] = json_header; } //////////////////////////////////////////////////////////////////////////////////////// // Copy Performance Data if (_recordPerformanceData) { Json::Value json_perf(Json::arrayValue); for (const auto& attr : performance_data) { Json::Value element; element["papi_event"] = Json::Value(attr->papiEvent); element["duration"] = Json::Value((Json::UInt64)attr->duration); element["data"] = Json::Value((Json::UInt64)attr->data); element["name"] = Json::Value(attr->name); element["id"] = Json::Value(attr->operatorId); element["startTime"] = Json::Value((double)(attr->startTime - queryStart) / 1000000); element["endTime"] = Json::Value((double)(attr->endTime - queryStart) / 1000000); element["executingThread"] = Json::Value(attr->executingThread); element["lastCore"] = Json::Value(attr->core); element["lastNode"] = Json::Value(attr->node); // Put null for in/outRows if -1 was set element["inRows"] = attr->in_rows ? Json::Value(*(attr->in_rows)) : Json::Value(); element["outRows"] = attr->out_rows ? Json::Value(*(attr->out_rows)) : Json::Value(); if (_getSubQueryPerformanceData) { element["subQueryPerformanceData"] = _scriptOperation->getSubQueryPerformanceData(); } json_perf.append(element); } pt.stop(); Json::Value responseElement; responseElement["duration"] = Json::Value((Json::UInt64)pt.value("PAPI_TOT_CYC")); responseElement["name"] = Json::Value("ResponseTask"); responseElement["id"] = Json::Value("respond"); responseElement["startTime"] = Json::Value((double)(responseStart - queryStart) / 1000000); responseElement["endTime"] = Json::Value((double)(get_epoch_nanoseconds() - queryStart) / 1000000); std::string threadId = boost::lexical_cast<std::string>(std::this_thread::get_id()); responseElement["executingThread"] = Json::Value(threadId); responseElement["lastCore"] = Json::Value(getCurrentCore()); responseElement["lastNode"] = Json::Value(getCurrentNode()); std::optional<size_t> result_size; if (result) { result_size = result->size(); } responseElement["inRows"] = result_size ? Json::Value(*result_size) : Json::Value(); responseElement["outRows"] = Json::Value(); json_perf.append(responseElement); response["performanceData"] = json_perf; } Json::Value jsonKeys(Json::arrayValue); for (const auto& x : _generatedKeyRefs) { for (const auto& key : *x) { Json::Value element(key); jsonKeys.append(element); } } response["generatedKeys"] = jsonKeys; response["affectedRows"] = Json::Value(_affectedRows); if (_getSubQueryPerformanceData) { response["subQueryDataflow"] = _scriptOperation->getSubQueryDataflow(); } } LOG4CXX_DEBUG(_logger, "Table Use Count: " << result.use_count()); return response; }
void ResponseTask::operator()() { epoch_t responseStart = get_epoch_nanoseconds(); Json::Value response; if (getDependencyCount() > 0) { PapiTracer pt; pt.addEvent("PAPI_TOT_CYC"); pt.start(); auto predecessor = getResultTask(); const auto& result = predecessor->getResultTable(); if (predecessor->getState() != OpFail) { if (result) { // Make header Json::Value json_header(Json::arrayValue); for (unsigned col = 0; col < result->columnCount(); ++col) { Json::Value colname(result->nameOfColumn(col)); json_header.append(colname); } // Copy the complete result response["real_size"] = result->size(); response["rows"] = generateRowsJson(result, _transmitLimit); response["header"] = json_header; } // Copy Performance Data Json::Value json_perf(Json::arrayValue); for (const auto & attr: performance_data) { Json::Value element; element["papi_event"] = Json::Value(attr->papiEvent); element["duration"] = Json::Value((Json::UInt64) attr->duration); element["data"] = Json::Value((Json::UInt64) attr->data); element["name"] = Json::Value(attr->name); element["id"] = Json::Value(attr->operatorId); element["startTime"] = Json::Value((double)(attr->startTime - queryStart) / 1000000); element["endTime"] = Json::Value((double)(attr->endTime - queryStart) / 1000000); element["executingThread"] = Json::Value(attr->executingThread); json_perf.append(element); } pt.stop(); Json::Value responseElement; responseElement["duration"] = Json::Value((Json::UInt64) pt.value("PAPI_TOT_CYC")); responseElement["name"] = Json::Value("ResponseTask"); responseElement["id"] = Json::Value("respond"); responseElement["startTime"] = Json::Value((double)(responseStart - queryStart) / 1000000); responseElement["endTime"] = Json::Value((double)(get_epoch_nanoseconds() - queryStart) / 1000000); std::string threadId = boost::lexical_cast<std::string>(std::this_thread::get_id()); responseElement["executingThread"] = Json::Value(threadId); json_perf.append(responseElement); response["performanceData"] = json_perf; } else { LOG4CXX_ERROR(_logger, "Error during plan execution: " << predecessor->getErrorMessage()); response["error"] = predecessor->getErrorMessage(); } LOG4CXX_DEBUG(_logger, "Table Use Count: " << result.use_count()); } else { response["error"] = "Query parsing failed, see server error log"; } connection->respond(response.toStyledString()); }
void RequestParseTask::operator()() { assert((_responseTask != nullptr) && "Response needs to be set"); const auto& scheduler = SharedScheduler::getInstance().getScheduler(); performance_vector_t& performance_data = _responseTask->getPerformanceData(); bool recordPerformance = false; std::vector<std::shared_ptr<Task> > tasks; int priority = Task::DEFAULT_PRIORITY; int sessionId = 0; if (_connection->hasBody()) { // The body is a wellformed HTTP Post body, with key value pairs std::string body(_connection->getBody()); std::map<std::string, std::string> body_data = parseHTTPFormData(body); boost::optional<tx::TXContext> ctx; auto ctx_it = body_data.find("session_context"); if (ctx_it != body_data.end()) { boost::optional<tx::transaction_id_t> tid; if ((tid = parseNumeric<tx::transaction_id_t>(ctx_it->second)) && (tx::TransactionManager::isRunningTransaction(*tid))) { LOG4CXX_DEBUG(_logger, "Picking up transaction id " << *tid); ctx = tx::TransactionManager::getContext(*tid); } else { LOG4CXX_ERROR(_logger, "Invalid transaction id " << *tid); _responseTask->addErrorMessage("Invalid transaction id set, aborting execution."); } } else { ctx = tx::TransactionManager::beginTransaction(); LOG4CXX_DEBUG(_logger, "Creating new transaction context " << (*ctx).tid); } Json::Value request_data; Json::Reader reader; const std::string& query_string = urldecode(body_data["query"]); if (ctx && reader.parse(query_string, request_data)) { _responseTask->setTxContext(*ctx); recordPerformance = getOrDefault(body_data, "performance", "false") == "true"; // the performance attribute for this operation (at [0]) if (recordPerformance) { performance_data.push_back(std::unique_ptr<performance_attributes_t>(new performance_attributes_t)); } LOG4CXX_DEBUG(_query_logger, request_data); const std::string& final_hash = hash(query_string); std::shared_ptr<Task> result = nullptr; if(request_data.isMember("priority")) priority = request_data["priority"].asInt(); if(request_data.isMember("sessionId")) sessionId = request_data["sessionId"].asInt(); _responseTask->setPriority(priority); _responseTask->setSessionId(sessionId); _responseTask->setRecordPerformanceData(recordPerformance); try { tasks = QueryParser::instance().deserialize( QueryTransformationEngine::getInstance()->transform(request_data), &result); } catch (const std::exception &ex) { // clean up, so we don't end up with a whole mess due to thrown exceptions LOG4CXX_ERROR(_logger, "Received\n:" << request_data); LOG4CXX_ERROR(_logger, "Exception thrown during query deserialization:\n" << ex.what()); _responseTask->addErrorMessage(std::string("RequestParseTask: ") + ex.what()); tasks.clear(); result = nullptr; } auto autocommit_it = body_data.find("autocommit"); if (autocommit_it != body_data.end() && (autocommit_it->second == "true")) { auto commit = std::make_shared<Commit>(); commit->setOperatorId("__autocommit"); commit->setPlanOperationName("Commit"); commit->addDependency(result); result = commit; tasks.push_back(commit); } if (result != nullptr) { _responseTask->addDependency(result); } else { LOG4CXX_ERROR(_logger, "Json did not yield tasks"); } for (const auto & func: tasks) { if (auto task = std::dynamic_pointer_cast<PlanOperation>(func)) { task->setPriority(priority); task->setSessionId(sessionId); task->setPlanId(final_hash); task->setTXContext(*ctx); task->setId((*ctx).tid); _responseTask->registerPlanOperation(task); if (!task->hasSuccessors()) { // The response has to depend on all tasks, ie. we don't // want to respond before all tasks finished running, even // if they don't contribute to the result. This prevents // dangling tasks _responseTask->addDependency(task); } } } } else { LOG4CXX_ERROR(_logger, "Failed to parse: " << urldecode(body_data["query"]) << "\n" << body_data["query"] << "\n" << reader.getFormatedErrorMessages()); } // Update the transmission limit for the response task if (atoi(body_data["limit"].c_str()) > 0) _responseTask->setTransmitLimit(atol(body_data["limit"].c_str())); if (atoi(body_data["offset"].c_str()) > 0) _responseTask->setTransmitOffset(atol(body_data["offset"].c_str())); } else { LOG4CXX_WARN(_logger, "no body received!"); } // high priority tasks are expected to be scheduled sequentially if(priority == Task::HIGH_PRIORITY){ if (recordPerformance) { *(performance_data.at(0)) = { 0, 0, "NO_PAPI", "RequestParseTask", "requestParse", _queryStart, get_epoch_nanoseconds(), boost::lexical_cast<std::string>(std::this_thread::get_id()) }; } int number_of_tasks = tasks.size(); std::vector<bool> isExecuted(number_of_tasks, false); int executedTasks = 0; while(executedTasks < number_of_tasks){ for(int i = 0; i < number_of_tasks; i++){ if(!isExecuted[i] && tasks[i]->isReady()){ (*tasks[i])(); tasks[i]->notifyDoneObservers(); executedTasks++; isExecuted[i] = true; } } } _responseTask->setQueryStart(_queryStart); (*_responseTask)(); _responseTask.reset(); // yield responsibility } else { scheduler->schedule(_responseTask); scheduler->scheduleQuery(tasks); if (recordPerformance) { *(performance_data.at(0)) = { 0, 0, "NO_PAPI", "RequestParseTask", "requestParse", _queryStart, get_epoch_nanoseconds(), boost::lexical_cast<std::string>(std::this_thread::get_id()) }; } _responseTask->setQueryStart(_queryStart); _responseTask.reset(); // yield responsibility } }
RequestParseTask::RequestParseTask(net::AbstractConnection* connection) : _connection(connection), _responseTask(std::make_shared<ResponseTask>(connection)), _queryStart(get_epoch_nanoseconds()){}