void SamplingUpdateHeader(ConnectionPtr connection, long pageID, int testServerID, const std::string& startTimeStr, int overallResultCode, long DNSTimeOffsetMs, long firstByteTimeOffsetMs, long totalLastByteTimeOffsetMs, long totalBytes, resultid_t* resultID, long firstDataTimeOffsetMs, const std::string& sampleDbName, const std::string& request, const std::string& header, const std::string& html, const std::string& extraInfo, SC_TEST_TYPE testType, long customerNo, SC_WEBPAGE_TYPE pageType, const std::string& stepName, long pageNumber, long currentColourCode, int localOffset, long searchID, const std::string& searchName, long runID) { // updates the ResultHeaderYYMMDD table with the results of this file / page for reloaded steps // it just adds the details to the current result. // passed in dailyDbName try { // Get the resultID. ResultPtr queryResult; char resultIdQuery[(EXTRA_SQL_MAX_LEN * 4) + 100]; #ifdef SHARD_AWARE sprintf(resultIdQuery, SQL_WTT_HEADER_SAMPLE_MAX_RESULTID_C3, sampleDbName.c_str(), pageID, getBoundedDateTimeString(KEY_TIME_COLUMN_RESULTHEADER).c_str()); #else sprintf(resultIdQuery, SQL_WTT_HEADER_SAMPLE_MAX_RESULTID, sampleDbName.c_str(), pageID); #endif Trace("Issuing query", resultIdQuery, 0); queryResult = connection->Query(resultIdQuery); queryResult->Next(); RowPtr row = queryResult->GetCurrentRow(); *resultID = strtoull(row->GetField(1), 0, 10); } catch (const std::exception &e) { LogError("SamplingUpdateHeader() - Caught Exception querying max sampling resultID", e.what(), pageID); } ResultPtr result; char strQuery[(EXTRA_SQL_MAX_LEN * 4) + 200]; try { // update the current result in the daily table // leave start, dns, connect and data start the same // change complete time // update result code, #ifdef SHARD_AWARE sprintf(strQuery, SQL_WTT_HEADER_SAMPLE_UPD_C3, sampleDbName.c_str(), overallResultCode, totalLastByteTimeOffsetMs, totalBytes, *resultID, getBoundedDateTimeString(KEY_TIME_COLUMN_RESULTHEADER).c_str() ); #else sprintf(strQuery, SQL_WTT_HEADER_SAMPLE_UPD, sampleDbName.c_str(), overallResultCode, totalLastByteTimeOffsetMs, totalBytes, *resultID ); #endif Trace("SamplingUpdateHeader(): Query for ResultHeaderYYMMDD", strQuery, 0); result = connection->Query(strQuery); } catch (const std::exception &e) { LogError("SamplingUpdateHeader() - table ResultHeaderYYMMDD Caught Exception", e.what(), pageID); } // DO NOT store the extra info for the first component against the header - ALREADY STORED }
// per page tests void runWTTT(scriptid_t scriptID, unsigned int shardNumber, const char* runNo, long batchID, bool wait, bool debug) { // only do a sleep if the first run - not for manual or retests if ((atoi(runNo) == 1) && wait) { TraceNoise("Sleeping for a bit", runNo, scriptID); // sleep for an offset of seconds sleep(scriptID % 60); } ConnectionPtr connection; std::string strQuery; bool download = true; try { #ifdef SHARD_AWARE TraceLogic("Connecting to db", CConfig::getShardDatabase(shardNumber).c_str(), shardNumber); connection.reset( new Connection( CConfig::getShardServer(shardNumber), CConfig::getShardDatabase(shardNumber), CConfig::getShardUser(shardNumber), CConfig::getShardPass(shardNumber), CConfig::getDBRetryTime())); #else TraceLogic("Connecting to db", CConfig::getDBDatabase(), 0); connection.reset( new Connection( CConfig::getDBServer(), CConfig::getDBDatabase(), CConfig::getDBUser(), CConfig::getDBPass(), CConfig::getDBRetryTime())); #endif } catch (const std::exception &e) { LogError("Exception Caught:", e.what(), scriptID); return; } // if the ScriptID == 0 - get it from the batch table if (scriptID == 0) { strQuery = Format(SQL_WTTT_BATCH_SEL, batchID); TraceSQL("Issuing", "", 0); if (ResultPtr result = connection->Query(strQuery)) { if (result->Next()) { RowPtr row = result->GetCurrentRow(); scriptID = row->GetFieldLong(1); TraceLogic("batchID", "", batchID); TraceLogic("scriptID", "", scriptID); } } } // script and archive names const std::string filename = Format("/home/sc/bin/monitor_scripts/%ld_%u", scriptID, shardNumber); const std::string gzfilename = filename + ".gz"; // Get script details long customerNo = 0; SC_SCRIPT_LANGUAGE scriptlanguage = SC_SCRIPT_LANGUAGE_UNKNOWN; std::string scriptversion; strQuery = Format(SQL_WTTT_SCRIPTINFO_SEL, scriptID); TraceSQL("Issuing", "", 0); if (ResultPtr result = connection->Query(strQuery)) { if (result->Next()) { RowPtr row = result->GetCurrentRow(); customerNo = row->GetFieldLong(1); scriptlanguage = static_cast<SC_SCRIPT_LANGUAGE>(row->GetFieldLong(2)); // Field can be NULL, so check before dereference if (const char* str = row->GetField(3)) { scriptversion = str; } TraceLogic("customerNo", "", customerNo); TraceLogic("scriptlanguage", to_string(scriptlanguage).c_str(), int(scriptlanguage)); TraceLogic("scriptversion", scriptversion.c_str(), scriptversion.size()); } } switch (scriptlanguage) { case SC_SCRIPT_LANGUAGE_UNKNOWN: // // No recognised script or script language returned // LogError( "Script not found", Format("ScriptID=%d Shard=%d DB.name=%s DB.host=%s", scriptID, shardNumber, CConfig::getShardDatabase(shardNumber).c_str(), CConfig::getShardServer(shardNumber).c_str()).c_str(), 0); exit(1); break; case SC_SCRIPT_LANGUAGE_PHP: // // Handle PHP script // strQuery = Format("select Code, length(Code) from Script where ScriptID=%ld", scriptID); TraceSQL("Issuing", "", 0); if (ResultPtr result = connection->Query(strQuery)) { if (result->Next()) { RowPtr row = result->GetCurrentRow(); int scriptLength = row->GetFieldInteger(2); std::string scriptString(row->GetField(1), scriptLength); TraceLogic("length(Code)", "", scriptLength); umask(SC_WTTT_UMASK); int filePtr = open(filename.c_str(), O_CREAT | O_TRUNC | O_WRONLY, S_IRWXU); if (filePtr != -1) { // output the body to the file int nbytes = write(filePtr, scriptString.c_str(), scriptLength); if (nbytes != scriptLength) { LogError("Incomplete write of PHP executeable to disc", to_string(nbytes).c_str(), scriptLength); } close(filePtr); } else { LogError("Unable to write PHP executable to disc", "", scriptID); } } } break; case SC_SCRIPT_LANGUAGE_CXX: // // Handle compiled C++ script // if (Exist(filename)) { // Is the file out of date? strQuery = Format(SQL_WTTT_SCRIPTAGE_SEL, scriptID); TraceSQL("Issuing", "", 0); if (ResultPtr result = connection->Query(strQuery)) { if (result->Next()) { RowPtr row = result->GetCurrentRow(); long dbage = row->GetFieldLong(1); TraceLogic("dbage", "", dbage); struct stat statbuf; bzero(&statbuf, 0); int ret = stat(filename.c_str(), &statbuf); TraceLogic("stat", "download", ret); download = ret == -1 || statbuf.st_mtime < dbage; TraceLogic("Overwrite the existing %s?", "download", int(download)); TraceLogic("statbuf.st_mtime", "", statbuf.st_mtime); } } } // // Extract script from database // if (download) { strQuery = Format(SQL_WTTT_SCRIPT_SEL, scriptID); TraceSQL("Issuing", "", 0); if (ResultPtr result = connection->Query(strQuery)) { if (result->Next()) { RowPtr row = result->GetCurrentRow(); int scriptLength = row->GetFieldInteger(2); std::string scriptString(row->GetField(1), scriptLength); // Field can be NULL, so check before dereference int agentVersion = 0; if (const char* str = row->GetField(3)) { agentVersion = atoi(str); } TraceLogic("scriptLength", "", scriptLength); TraceLogic("scriptString", scriptString.c_str(), scriptString.size()); TraceLogic("agentVersion", "", agentVersion); umask(SC_WTTT_UMASK); int filePtr = open(gzfilename.c_str(), O_CREAT | O_TRUNC | O_WRONLY, S_IRWXU); if (filePtr != -1) { // output the body to the file int nbytes = write(filePtr, scriptString.c_str(), scriptLength); close(filePtr); if (nbytes == scriptLength) { // unzip, set permissions and prelink doUnzip(gzfilename); chmod(filename.c_str(), 0775); bool prelink = false; if (!CConfig::getIMMode()) { TraceLogic("Connecting to passport db", CConfig::getPassportDbDatabase().c_str(), 0); ConnectionPtr connectionPassport( new Connection( CConfig::getPassportDbServer(), CConfig::getPassportDbDatabase(), CConfig::getPassportDbUser(), CConfig::getPassportDbPass(), CConfig::getDBRetryTime())); strQuery = Format("select Prelink from LibraryVersions where Version=%d", agentVersion); TraceSQL("Issuing", "", 0); if (ResultPtr result = connectionPassport->Query(strQuery)) { if (result->Next()) { RowPtr row = result->GetCurrentRow(); prelink = row->GetFieldInteger(1) != 0; TraceLogic("prelink", to_string(prelink).c_str(), int(prelink)); } } TraceLogic("prelink", "", int(prelink)); } else { TraceLogic("prelink", "Don't prelink on IM node", 0); } if (prelink) { int ret = doPrelink(filename, SC_AGENT_VER_NUM); if (ret != 0) { LogError("Prelink failed", filename.c_str(), ret); } } } else { LogError("Incomplete write of C++ executeable to disc", to_string(nbytes).c_str(), scriptLength); return; } } else { LogError("Unable to write C++ executable to disc", "", scriptID); return; } } else { LogError("C++ Executable not in database", gzfilename.c_str(), result->GetRowCount()); return; } } else { LogError("Query for C++ executable failed", strQuery.c_str(), result->GetRowCount()); return; } } break; default: LogError("Fatal: Unknown language", "", int(scriptlanguage)); return; } // close the db connection Clear(connection); // // Build command line // std::string systemCmd; #ifdef USE_SHARED // For version 7 if (scriptlanguage == SC_SCRIPT_LANGUAGE_CXX && Exist("/home/sc/lib/libsiteconfidence.so." + scriptversion) && Exist("/home/sc/lib/libscparser.so." + scriptversion) && Exist("/home/sc/lib/libsiteconfidenceinterface.so." + scriptversion)) { std::ostringstream os; os << "export LD_PRELOAD="; os << "/home/sc/lib/libsiteconfidence.so." << scriptversion << ":"; os << "/home/sc/lib/libscparser.so." << scriptversion << ":"; os << "/home/sc/lib/libsiteconfidenceinterface.so." << scriptversion << "; "; systemCmd += os.str(); } #endif #ifdef SHARD_AWARE // set environment variables for debug and shardNumber { std::ostringstream os; os << "export SHARD_NUMBER=" << shardNumber << "; " << "export SC_DEBUG_MANUAL_MODE=" << (debug != 0) << "; "; systemCmd += os.str(); } #endif // program command line { std::ostringstream os; os << filename << " " << runNo << " " << batchID << " " << scriptID << " " << customerNo; systemCmd += os.str(); } //Print(Format("starting script:%s\n", systemCmd.c_str())); // execute TraceLogic("Starting script", systemCmd.c_str(), scriptID); int sysVal = system(systemCmd.c_str()); int exitVal = WEXITSTATUS(sysVal); TraceLogic("System call returned", systemCmd.c_str(), exitVal); //Print(Format("script returned:%d\n", exitVal)); }
void SamplingStoreHeader(ConnectionPtr connection, long pageID, int testServerID, const std::string& startTimeStr, int overallResultCode, long DNSTimeOffsetMs, long firstByteTimeOffsetMs, long totalLastByteTimeOffsetMs, long totalBytes, resultid_t* resultID, long firstDataTimeOffsetMs, const std::string& sampleDbName, const std::string& request, const std::string& header, const std::string& html, const std::string& extraInfo, SC_TEST_TYPE testType, long customerNo, SC_WEBPAGE_TYPE pageType, const std::string& stepName, long pageNumber, long currentColourCode, int localOffset, long searchID, const std::string& searchName, long runID, long sslConnectTime, long requestSentTime, long requestHeaderSize, long requestContentSize, long responseHeaderSize) { // stores the results of this file / page into the result detail table ResultPtr result; char strQuery[(EXTRA_SQL_MAX_LEN * 4) + 200]; try { // remove the current one stored Trace("remove the current one stored", "", pageID); #ifdef SHARD_AWARE sprintf(strQuery, SQL_WTT_HEADER_SAMPLE_LIST_DEL_C3, sampleDbName.c_str(), pageID, getBoundedDateTimeString(KEY_TIME_COLUMN_RESULTHEADER).c_str()); #else sprintf(strQuery, SQL_WTT_HEADER_SAMPLE_LIST_DEL, sampleDbName.c_str(), pageID); #endif Trace("Issuing query", strQuery, 0); result = connection->Query(strQuery); bool addComma = false; std::string refList; while (result->Next()) { RowPtr row = result->GetCurrentRow(); if (addComma) { refList += ","; } refList += row->GetField(1); addComma = true; } if (addComma) { // there was at least one in the list to delete #ifdef SHARD_AWARE sprintf(strQuery, SQL_WTT_HEADER_SAMPLE_DEL_C3, sampleDbName.c_str(), refList.c_str(), getBoundedDateTimeString(KEY_TIME_COLUMN_RESULTHEADER).c_str()); #else sprintf(strQuery, SQL_WTT_HEADER_SAMPLE_DEL, sampleDbName.c_str(), refList.c_str()); #endif Trace("Issuing query", strQuery, 0); result = connection->Query(strQuery); #ifdef SHARD_AWARE sprintf(strQuery, SQL_WTT_HEADER_EXTRA_INFO_SAMPLE_DEL_C3, sampleDbName.c_str(), refList.c_str(), getBoundedDateTimeString(KEY_TIME_COLUMN_RESULTHEADEREXTRAINFO).c_str()); #else sprintf(strQuery, SQL_WTT_HEADER_EXTRA_INFO_SAMPLE_DEL, sampleDbName.c_str(), refList.c_str()); #endif Trace("Issuing query", strQuery, 0); result = connection->Query(strQuery); #ifdef SHARD_AWARE sprintf(strQuery, SQL_WTT_HEADER_DETAIL_SAMPLE_DEL_C3, sampleDbName.c_str(), refList.c_str(), getBoundedDateTimeString(KEY_TIME_COLUMN_RESULTDETAIL).c_str()); #else sprintf(strQuery, SQL_WTT_HEADER_DETAIL_SAMPLE_DEL, sampleDbName.c_str(), refList.c_str()); #endif Trace("Issuing query", strQuery, 0); result = connection->Query(strQuery); #ifdef SHARD_AWARE sprintf(strQuery, SQL_WTT_HEADER_REQUEST_SAMPLE_DEL_C3, sampleDbName.c_str(), refList.c_str(), getBoundedDateTimeString(KEY_TIME_COLUMN_REQUEST).c_str()); #else sprintf(strQuery, SQL_WTT_HEADER_REQUEST_SAMPLE_DEL, sampleDbName.c_str(), refList.c_str()); #endif Trace("Issuing query", strQuery, 0); result = connection->Query(strQuery); #ifdef SHARD_AWARE sprintf(strQuery, SQL_WTT_HEADER_HEADER_SAMPLE_DEL_C3, sampleDbName.c_str(), refList.c_str(), getBoundedDateTimeString(KEY_TIME_COLUMN_HEADER).c_str()); #else sprintf(strQuery, SQL_WTT_HEADER_HEADER_SAMPLE_DEL, sampleDbName.c_str(), refList.c_str()); #endif Trace("Issuing query", strQuery, 0); result = connection->Query(strQuery); #ifdef SHARD_AWARE sprintf(strQuery, SQL_WTT_HEADER_HTML_SAMPLE_DEL_C3, sampleDbName.c_str(), refList.c_str(), getBoundedDateTimeString(KEY_TIME_COLUMN_HTML).c_str()); #else sprintf(strQuery, SQL_WTT_HEADER_HTML_SAMPLE_DEL, sampleDbName.c_str(), refList.c_str()); #endif Trace("Issuing query", strQuery, 0); result = connection->Query(strQuery); #ifdef SHARD_AWARE sprintf(strQuery, SQL_WTT_HEADER_EXTRA_SAMPLE_DEL_C3, sampleDbName.c_str(), refList.c_str(), getBoundedDateTimeString(KEY_TIME_COLUMN_EXTRAINFO).c_str()); #else sprintf(strQuery, SQL_WTT_HEADER_EXTRA_SAMPLE_DEL, sampleDbName.c_str(), refList.c_str()); #endif Trace("Issuing query", strQuery, 0); result = connection->Query(strQuery); } sprintf(strQuery, SQL_WTT_HEADER_SAMPLE_INS, sampleDbName.c_str(), pageID, testServerID, startTimeStr.c_str(), overallResultCode, DNSTimeOffsetMs, firstByteTimeOffsetMs, totalLastByteTimeOffsetMs, totalBytes, runID, firstDataTimeOffsetMs, int(testType), startTimeStr.c_str(), localOffset, customerNo, pageType, stepName.c_str(), pageNumber, currentColourCode, searchID, searchName.c_str(), sslConnectTime, requestSentTime, requestHeaderSize, requestContentSize, responseHeaderSize); Trace("Query for SampleCCYYMM.ResultHeader", strQuery, 0); result = connection->Query(strQuery); snprintf(strQuery, HTML_SQL_MAX_LEN + 100, SQL_WTT_HEADER_SAMPLE_INS_ID, sampleDbName.c_str()); Trace("Query for SampleCCYYMM.ResultHeader", strQuery, 0); result = connection->Query(strQuery); result->Next(); RowPtr row = result->GetCurrentRow(); Trace("LastInsertID for SampleCCYYMM.ResultHeader", row->GetField(1), 0); *resultID = strtoull(row->GetField(1), 0, 10); } catch (const std::exception &e) { // set the error code to 0 so that we don't connect to the wrong header *resultID = 0; LogError("SamplingStoreHeader() - table ResultHeader Caught Exception", e.what(), pageID); } // store the extra info for the first component against the header try { if ((!request.empty()) || (!header.empty()) || (!html.empty()) || (!extraInfo.empty())) { int len = 0; char sqlStr[EXTRA_SQL_MAX_LEN + 1]; // convert the string for inserting len = mysql_escape_string(sqlStr, request.c_str(), (request.size() < (EXTRA_SQL_MAX_LEN / 2)) ? (request.size()) : (EXTRA_SQL_MAX_LEN / 2)); // terminate the string if ( len > (int)EXTRA_SQL_MAX_LEN) { sqlStr[EXTRA_SQL_MAX_LEN] = '\0'; } else { sqlStr[len] = '\0'; } // make sure last char not a \ - it would escape the final quote if ( sqlStr[EXTRA_SQL_MAX_LEN] == '\\') sqlStr[EXTRA_SQL_MAX_LEN] = ' '; char sqlStr2[EXTRA_SQL_MAX_LEN + 1]; // convert the string for inserting len = mysql_escape_string(sqlStr2, header.c_str(), (header.size() < (EXTRA_SQL_MAX_LEN / 2)) ? (header.size()) : (EXTRA_SQL_MAX_LEN / 2)); // terminate the string if ( len > (int)EXTRA_SQL_MAX_LEN) { sqlStr2[EXTRA_SQL_MAX_LEN] = '\0'; } else { sqlStr2[len] = '\0'; } // make sure last char not a \ - it would escape the final quote if ( sqlStr2[EXTRA_SQL_MAX_LEN] == '\\') sqlStr2[EXTRA_SQL_MAX_LEN] = ' '; char sqlStr3[EXTRA_SQL_MAX_LEN + 1]; // convert the string for inserting len = mysql_escape_string(sqlStr3, html.c_str(), (html.size() < (EXTRA_SQL_MAX_LEN / 2)) ? (html.size()) : (EXTRA_SQL_MAX_LEN / 2)); // terminate the string if ( len > (int)EXTRA_SQL_MAX_LEN) { sqlStr3[EXTRA_SQL_MAX_LEN] = '\0'; } else { sqlStr3[len] = '\0'; } // make sure last char not a \ - it would escape the final quote if ( sqlStr3[EXTRA_SQL_MAX_LEN] == '\\') sqlStr3[EXTRA_SQL_MAX_LEN] = ' '; char sqlStr4[EXTRA_SQL_MAX_LEN + 1]; // convert the string for inserting len = mysql_escape_string(sqlStr4, extraInfo.c_str(), (extraInfo.size() < (EXTRA_SQL_MAX_LEN / 2)) ? (extraInfo.size()) : (EXTRA_SQL_MAX_LEN / 2)); // terminate the string if ( len > (int)EXTRA_SQL_MAX_LEN) { sqlStr4[EXTRA_SQL_MAX_LEN] = '\0'; } else { sqlStr4[len] = '\0'; } // make sure last char not a \ - it would escape the final quote if ( sqlStr4[EXTRA_SQL_MAX_LEN] == '\\') sqlStr4[EXTRA_SQL_MAX_LEN] = ' '; #ifdef SHARD_AWARE snprintf(strQuery, 204800 - 1, SQL_WTT_HEADER_EXTRA_INFO_SAMPLE_INS_C3, sampleDbName.c_str(), *resultID, sqlStr, sqlStr2, sqlStr3, sqlStr4, startTimeStr.c_str() ); #else snprintf(strQuery, 204800 - 1, SQL_WTT_HEADER_EXTRA_INFO_SAMPLE_INS, sampleDbName.c_str(), *resultID, sqlStr, sqlStr2, sqlStr3, sqlStr4 ); #endif Trace("Query for ResultHeaderExtraInfo", strQuery, 0); result = connection->Query(strQuery); } } catch (const std::exception &e) { LogError("SamplingStoreHeader() - table ResultHeaderExtraInfo Caught Exception", e.what(), pageID); } }
void CPage::loadJavaScriptCheck(ConnectionPtr connection) { Trace("loadJavaScriptCheck()", "",0); std::string strQuery; // need to change all to lower case std::string lowerCaseBody; try { Mutex::Lock lock(m_javascriptListLock); strQuery = Format(SQL_WTT_JAVASCRIPT1); if (ResultPtr result = connection->Query(strQuery)) { Trace("RowPtr s:","",result->GetRowCount()); while(result->Next()) { RowPtr row = result->GetCurrentRow(); // add to the list Trace("loadJavaScriptCheck() - search", row->GetField(1),0); Trace("loadJavaScriptCheck() - replace", row->GetField(2),0); m_javascriptSearchList.push_back(row->GetFieldString(1)); m_javascriptReplaceList.push_back(row->GetFieldString(2)); } } strQuery = Format(SQL_WTT_JAVASCRIPT2, getPageID()); if (ResultPtr result = connection->Query(strQuery)) { Trace("RowPtr s:","",result->GetRowCount()); while(result->Next()) { RowPtr row = result->GetCurrentRow(); // add to the list Trace("loadJavaScriptCheck() - search", row->GetField(1),0); Trace("loadJavaScriptCheck() - replace", row->GetField(2),0); m_javascriptSearchList.push_back(row->GetFieldString(1)); m_javascriptReplaceList.push_back(row->GetFieldString(2)); } } } catch (const std::exception& e) { LogError("Exception Caught:", e.what(), 0); } try { Mutex::Lock lock(m_javascriptExtractListLock); strQuery = Format(SQL_WTT_JAVASCRIPT3, getPageID()); if (ResultPtr result = connection->Query(strQuery)) { Trace("RowPtr s:","",result->GetRowCount()); while(result->Next()) { RowPtr row = result->GetCurrentRow(); // add to the list Trace("loadJavaScriptCheck() - start", row->GetField(1),0); Trace("loadJavaScriptCheck() - end", row->GetField(2),0); m_javascriptStartList.push_back(row->GetFieldString(1)); m_javascriptEndList.push_back(row->GetFieldString(2)); } } } catch (const std::exception& e) { LogError("Exception Caught:", e.what(), 0); } }