void CPage::SaveCache(ConnectionPtr connection) { Trace("SaveCache", "", m_totalUrlList.size()); // save all the cookies in the current memory list to the database // in the WebPageCookies table ncc::safe_array<char> strQuery(102400); ncc::safe_array<char> sqlCache(40961); { // scope for lock // delete the existing cache in the table for this run snprintf(strQuery, strQuery.size(), SQL_WTT_DEL_CACHE, m_runID); Trace("Query for SaveCache", strQuery, 0); connection->Query(strQuery); try { for (URLList::const_iterator it = m_totalUrlList.begin(); it != m_totalUrlList.end(); ++it) { // convert the string ready to be inserted int len = 0; // convert the string for inserting len = mysql_escape_string(sqlCache, (*it).c_str(),((*it).size()<40960)?((*it).size()):40960); // terminate the string if( len > 40960) { sqlCache[40960] = '\0'; } else { sqlCache[len] = '\0'; } // make sure last char not a \ - it would escape the final quote if( sqlCache[40959] == '\\') { sqlCache[40959] = ' '; } else if( sqlCache[len-1] == '\\') { sqlCache[len-1] = ' '; } // save to the database snprintf(strQuery, strQuery.size(), SQL_WTT_SAVE_CACHE, m_runID, static_cast<const char*>(sqlCache)); Trace("Query for SaveCache", strQuery, 0); connection->Query(strQuery); } } catch (const std::exception& e) { LogError2("SaveCache() - Caught Exception", e.what(), 0, SC_ERR_GROUP_DB, SC_ERR_CODE_STORE_CACHE); } } }
void CPage::LoadCache(ConnectionPtr connection) { Trace("LoadCache", "",0); ResultPtr result; ncc::safe_array<char> strQuery(10240); try { // load from the database snprintf(strQuery, strQuery.size(), SQL_WTT_LOAD_CACHE, m_runID); Trace("Query for LoadCache", strQuery, 0); result = connection->Query(strQuery); while(result->Next()) { RowPtr row = result->GetCurrentRow(); // add it std::string urlStr; urlStr = row->GetFieldString(1); { // scope for lock Mutex::Lock lock(sm_urlListLock); m_totalUrlList.push_back(urlStr); } } } catch (const std::exception& e) { LogError2("LoadCache() - Caught Exception", e.what(), 0, SC_ERR_GROUP_DB, SC_ERR_CODE_LOAD_CACHE); } }
void CPage::loadContentError(ConnectionPtr connection) { Trace("loadContentError()", "", 0); std::string strQuery; try { Mutex::Lock lock(m_contentErrorListLock); // get all of the pages for this customer to test strQuery = Format(SQL_WTT_CONTENT_ERRORS); if (ResultPtr result = connection->Query(strQuery)) { while (result->Next()) { RowPtr row = result->GetCurrentRow(); StringIDList::value_type contentError(row->GetFieldString(1), row->GetFieldLong(2)); contentError.first = " " + contentError.first + " "; Trace("loadContentError() - loading string", contentError.first.c_str(), contentError.second); m_contentError.push_back(contentError); } } } catch (const std::exception& e) { LogError("Exception Caught:", e.what(), 0); } }
void CPage::loadPhraseCheck(ConnectionPtr connection) { Trace("loadPhraseCheck()", "", 0); std::string strQuery; try { Mutex::Lock lock(m_phraseListLock); strQuery = Format(SQL_WTT_PHRASE_WORDS, getPageID()); if (ResultPtr result = connection->Query(strQuery)) { Trace("RowPtr s:","",result->GetRowCount()); while (result->Next()) { RowPtr row = result->GetCurrentRow(); StringIDList::value_type phrase(row->GetFieldString(1), row->GetFieldLong(2)); Trace("loadPhraseCheck_LT(): pre", phrase.first.c_str(), phrase.second); std::transform(phrase.first.begin(), phrase.first.end(), phrase.first.begin(), tolower); Trace("loadPhraseCheck_LT(): post", phrase.first.c_str(), phrase.second); m_phrase.push_back(phrase); } } } catch (const std::exception& e) { LogError("Exception Caught:", e.what(), 0); } }
void CPage::checkErrorsAlertable(ConnectionPtr connection) { Trace("checkErrorsAlertable", getURL().c_str(), getOverallResultCode()); std::string strQuery; std::string colour; // if colour = red/black then go to downtime testing and send alert to alertable people try { // select Colour from Severity where ErrorCode = %d strQuery = Format(SQL_WTT_ERROR_COLOUR, (long int)getOverallResultCode()); if (ResultPtr result = connection->Query(strQuery)) { Trace("RowPtr s:","",result->GetRowCount()); if (result->Next()) { RowPtr row = result->GetCurrentRow(); colour += row->GetFieldString(1); } } } catch (const std::exception& e) { LogError("Exception Caught:", e.what(), 0); } Trace("checkErrorsAlertable", colour.c_str(), getOverallResultCode()); // only report an error if page failed and not a benchmark and IS a normal page if ((colour == SC_SEVERITY_BLACK) || (colour == SC_SEVERITY_RED)) { sm_errorAlertable = true; } }
void SamplingUpdateHeader(ConnectionPtr connection, long pageID, int testServerID, const std::string& startTimeStr, int overallResultCode, long DNSTimeOffsetMs, long firstByteTimeOffsetMs, long totalLastByteTimeOffsetMs, long totalBytes, resultid_t* resultID, long firstDataTimeOffsetMs, const std::string& sampleDbName, const std::string& request, const std::string& header, const std::string& html, const std::string& extraInfo, SC_TEST_TYPE testType, long customerNo, SC_WEBPAGE_TYPE pageType, const std::string& stepName, long pageNumber, long currentColourCode, int localOffset, long searchID, const std::string& searchName, long runID) { // updates the ResultHeaderYYMMDD table with the results of this file / page for reloaded steps // it just adds the details to the current result. // passed in dailyDbName try { // Get the resultID. ResultPtr queryResult; char resultIdQuery[(EXTRA_SQL_MAX_LEN * 4) + 100]; #ifdef SHARD_AWARE sprintf(resultIdQuery, SQL_WTT_HEADER_SAMPLE_MAX_RESULTID_C3, sampleDbName.c_str(), pageID, getBoundedDateTimeString(KEY_TIME_COLUMN_RESULTHEADER).c_str()); #else sprintf(resultIdQuery, SQL_WTT_HEADER_SAMPLE_MAX_RESULTID, sampleDbName.c_str(), pageID); #endif Trace("Issuing query", resultIdQuery, 0); queryResult = connection->Query(resultIdQuery); queryResult->Next(); RowPtr row = queryResult->GetCurrentRow(); *resultID = strtoull(row->GetField(1), 0, 10); } catch (const std::exception &e) { LogError("SamplingUpdateHeader() - Caught Exception querying max sampling resultID", e.what(), pageID); } ResultPtr result; char strQuery[(EXTRA_SQL_MAX_LEN * 4) + 200]; try { // update the current result in the daily table // leave start, dns, connect and data start the same // change complete time // update result code, #ifdef SHARD_AWARE sprintf(strQuery, SQL_WTT_HEADER_SAMPLE_UPD_C3, sampleDbName.c_str(), overallResultCode, totalLastByteTimeOffsetMs, totalBytes, *resultID, getBoundedDateTimeString(KEY_TIME_COLUMN_RESULTHEADER).c_str() ); #else sprintf(strQuery, SQL_WTT_HEADER_SAMPLE_UPD, sampleDbName.c_str(), overallResultCode, totalLastByteTimeOffsetMs, totalBytes, *resultID ); #endif Trace("SamplingUpdateHeader(): Query for ResultHeaderYYMMDD", strQuery, 0); result = connection->Query(strQuery); } catch (const std::exception &e) { LogError("SamplingUpdateHeader() - table ResultHeaderYYMMDD Caught Exception", e.what(), pageID); } // DO NOT store the extra info for the first component against the header - ALREADY STORED }
long SamplingStoreResult(ConnectionPtr connection, const CResultDetail* pDetail, resultid_t resultID, const std::string& sampleDbName, int ComponentNo, long reloadTimeOffset) { // stores the results of this file / page into the result detail table ResultPtr result; char strQuery[HTML_SQL_MAX_LEN + 100]; // make max of the stuff inserted plus 100 for the SQL try { sprintf(strQuery, SQL_WTT_DETAIL_SAMPLE_INS, sampleDbName.c_str(), resultID, pDetail->getStartTimeStr().c_str(), reloadTimeOffset + pDetail->getStartTimeOffsetMs(), pDetail->getDNSTimeOffsetMs(), pDetail->getConnectTimeOffsetMs(), pDetail->getFirstDataTimeOffsetMs(), pDetail->getLastByteTimeOffsetMs(), pDetail->getResultCode(), pDetail->getContentLength(), pDetail->getURLsql().c_str(), // url with 's escaped and only 255 in length ComponentNo, pDetail->getSslConnectTimeOffsetMs(), pDetail->getRequestSentTimeOffsetMs(), pDetail->getRequestHeaderSize(), pDetail->getRequestContentSize(), pDetail->getResponseHeaderSize() ); Trace("Query for ResultDetail", strQuery, 0); result = connection->Query(strQuery); } catch (const std::exception &e) { std::string errorBuffer = Format("ResultID: %llu", resultID); LogError("SamplingStoreResult() - table ResultDetailYYMMDD Caught Exception", e.what(), 0); LogError("SamplingStoreResult() - table ResultDetailYYMMDD Caught Exception:", errorBuffer.c_str(), 0); } try { // convert the string ready to be inserted int len = 0; char sqlStr[REQUEST_SQL_MAX_LEN + 1]; // convert the string for inserting len = mysql_escape_string(sqlStr, pDetail->getRequest().c_str(), (pDetail->getRequest().size() < (REQUEST_SQL_MAX_LEN / 2)) ? (pDetail->getRequest().size()) : (REQUEST_SQL_MAX_LEN / 2)); // terminate the string if ( len > (int)REQUEST_SQL_MAX_LEN) { sqlStr[REQUEST_SQL_MAX_LEN] = '\0'; } else { sqlStr[len] = '\0'; } // make sure last char not a \ - it would escape the final quote if ( sqlStr[REQUEST_SQL_MAX_LEN] == '\\') sqlStr[REQUEST_SQL_MAX_LEN] = ' '; #ifdef SHARD_AWARE // insert the request sprintf(strQuery, SQL_WTT_REQUEST_SAMPLE_INS_C3, sampleDbName.c_str(), resultID, ComponentNo, sqlStr, // with 's escaped and only 255 in length pDetail->getStartTimeStr().c_str()); #else // insert the request sprintf(strQuery, SQL_WTT_REQUEST_SAMPLE_INS, sampleDbName.c_str(), resultID, ComponentNo, sqlStr // with 's escaped and only 255 in length ); #endif Trace("Query for Request", strQuery, 0); result = connection->Query(strQuery); } catch (const std::exception &e) { std::string errorBuffer = Format("ResultID: %llu", resultID); LogError("SamplingStoreResult() - table RequestYYMMDD Caught Exception", e.what(), 0); LogError("SamplingStoreResult() - table RequestYYMMDD Caught Exception:", errorBuffer.c_str(), 0); } try { int len = 0; char sqlStr[HEADER_SQL_MAX_LEN + 1]; // convert the string for inserting len = mysql_escape_string(sqlStr, pDetail->getHeaders().c_str(), (pDetail->getHeaders().size() < (HEADER_SQL_MAX_LEN / 2)) ? (pDetail->getHeaders().size()) : (HEADER_SQL_MAX_LEN / 2)); // terminate the string if ( len > (int)HEADER_SQL_MAX_LEN) { sqlStr[HEADER_SQL_MAX_LEN] = '\0'; } else { sqlStr[len] = '\0'; } // make sure last char not a \ - it would escape the final quote if ( sqlStr[HEADER_SQL_MAX_LEN] == '\\') sqlStr[HEADER_SQL_MAX_LEN] = ' '; #ifdef SHARD_AWARE // insert the header sprintf(strQuery, SQL_WTT_HEADERS_SAMPLE_INS_C3, sampleDbName.c_str(), resultID, ComponentNo, sqlStr, // with 's escaped and only 255 in length pDetail->getStartTimeStr().c_str()); #else // insert the header sprintf(strQuery, SQL_WTT_HEADERS_SAMPLE_INS, sampleDbName.c_str(), resultID, ComponentNo, sqlStr // with 's escaped and only 255 in length ); #endif Trace("Query for Headers", strQuery, 0); result = connection->Query(strQuery); } catch (const std::exception &e) { std::string errorBuffer = Format("ResultID: %llu", resultID); LogError("SamplingStoreResult() - table HeaderYYMMDD Caught Exception", e.what(), 0); LogError("SamplingStoreResult() - table HeaderYYMMDD Caught Exception:", errorBuffer.c_str(), 0); } try { int len = 0; char sqlStr[HTML_SQL_MAX_LEN + 1]; // convert the string for inserting len = mysql_escape_string(sqlStr, pDetail->getPageText().c_str(), (pDetail->getPageText().size() < (HTML_SQL_MAX_LEN / 2)) ? (pDetail->getPageText().size()) : (HTML_SQL_MAX_LEN / 2)); // terminate the string if ( len > (int)HTML_SQL_MAX_LEN) { sqlStr[HTML_SQL_MAX_LEN] = '\0'; } else { sqlStr[len] = '\0'; } // make sure last char not a \ - it would escape the final quote if ( sqlStr[HTML_SQL_MAX_LEN] == '\\') sqlStr[HTML_SQL_MAX_LEN] = ' '; #ifdef SHARD_AWARE // insert the html sprintf(strQuery, SQL_WTT_HTML_SAMPLE_INS_C3, sampleDbName.c_str(), resultID, ComponentNo, sqlStr, // with 's escaped and only 255 in length pDetail->getStartTimeStr().c_str()); #else // insert the html sprintf(strQuery, SQL_WTT_HTML_SAMPLE_INS, sampleDbName.c_str(), resultID, ComponentNo, sqlStr // with 's escaped and only 255 in length ); #endif Trace("Query for HTML", strQuery, 0); result = connection->Query(strQuery); } catch (const std::exception &e) { std::string errorBuffer = Format("ResultID: %llu", resultID); LogError("SamplingStoreResult() - table HTMLYYMMDD Caught Exception", e.what(), 0); LogError("SamplingStoreResult() - table HTMLYYMMDD Caught Exception:", errorBuffer.c_str(), 0); } // store any extra info collected...(if there is any) try { Trace("EXTRA", pDetail->getExtraInfo().c_str(), pDetail->getExtraInfo().size()); // decide if to store it or not if (!pDetail->getExtraInfo().empty()) { int len = 0; char sqlStr[EXTRA_SQL_MAX_LEN + 100]; // convert the string for inserting len = mysql_escape_string(sqlStr, pDetail->getExtraInfo().c_str(), (pDetail->getExtraInfo().size() < (EXTRA_SQL_MAX_LEN / 2)) ? (pDetail->getExtraInfo().size()) : (EXTRA_SQL_MAX_LEN / 2)); // terminate the string if ( len > (int)EXTRA_SQL_MAX_LEN) { sqlStr[EXTRA_SQL_MAX_LEN] = '\0'; } else { sqlStr[len] = '\0'; } // make sure last char not a \ - it would escape the final quote if ( sqlStr[EXTRA_SQL_MAX_LEN] == '\\') sqlStr[EXTRA_SQL_MAX_LEN] = ' '; #ifdef SHARD_AWARE // insert the html sprintf(strQuery, SQL_WTT_EXTRA_SAMPLE_INS_C3, sampleDbName.c_str(), resultID, ComponentNo, sqlStr, // with 's escaped and only 255 in length pDetail->getStartTimeStr().c_str()); #else // insert the html sprintf(strQuery, SQL_WTT_EXTRA_SAMPLE_INS, sampleDbName.c_str(), resultID, ComponentNo, sqlStr // with 's escaped and only 255 in length ); #endif Trace("Query for EXTRA", strQuery, 0); result = connection->Query(strQuery); } } catch (const std::exception &e) { std::string errorBuffer = Format("ResultID: %llu", resultID); LogError("SamplingStoreResult() - table ExtraInfoYYMMDD Caught Exception", e.what(), 0); LogError("SamplingStoreResult() - table ExtraInfoYYMMDD Caught Exception:", errorBuffer.c_str(), 0); } return (0); }
void SamplingStoreHeader(ConnectionPtr connection, long pageID, int testServerID, const std::string& startTimeStr, int overallResultCode, long DNSTimeOffsetMs, long firstByteTimeOffsetMs, long totalLastByteTimeOffsetMs, long totalBytes, resultid_t* resultID, long firstDataTimeOffsetMs, const std::string& sampleDbName, const std::string& request, const std::string& header, const std::string& html, const std::string& extraInfo, SC_TEST_TYPE testType, long customerNo, SC_WEBPAGE_TYPE pageType, const std::string& stepName, long pageNumber, long currentColourCode, int localOffset, long searchID, const std::string& searchName, long runID, long sslConnectTime, long requestSentTime, long requestHeaderSize, long requestContentSize, long responseHeaderSize) { // stores the results of this file / page into the result detail table ResultPtr result; char strQuery[(EXTRA_SQL_MAX_LEN * 4) + 200]; try { // remove the current one stored Trace("remove the current one stored", "", pageID); #ifdef SHARD_AWARE sprintf(strQuery, SQL_WTT_HEADER_SAMPLE_LIST_DEL_C3, sampleDbName.c_str(), pageID, getBoundedDateTimeString(KEY_TIME_COLUMN_RESULTHEADER).c_str()); #else sprintf(strQuery, SQL_WTT_HEADER_SAMPLE_LIST_DEL, sampleDbName.c_str(), pageID); #endif Trace("Issuing query", strQuery, 0); result = connection->Query(strQuery); bool addComma = false; std::string refList; while (result->Next()) { RowPtr row = result->GetCurrentRow(); if (addComma) { refList += ","; } refList += row->GetField(1); addComma = true; } if (addComma) { // there was at least one in the list to delete #ifdef SHARD_AWARE sprintf(strQuery, SQL_WTT_HEADER_SAMPLE_DEL_C3, sampleDbName.c_str(), refList.c_str(), getBoundedDateTimeString(KEY_TIME_COLUMN_RESULTHEADER).c_str()); #else sprintf(strQuery, SQL_WTT_HEADER_SAMPLE_DEL, sampleDbName.c_str(), refList.c_str()); #endif Trace("Issuing query", strQuery, 0); result = connection->Query(strQuery); #ifdef SHARD_AWARE sprintf(strQuery, SQL_WTT_HEADER_EXTRA_INFO_SAMPLE_DEL_C3, sampleDbName.c_str(), refList.c_str(), getBoundedDateTimeString(KEY_TIME_COLUMN_RESULTHEADEREXTRAINFO).c_str()); #else sprintf(strQuery, SQL_WTT_HEADER_EXTRA_INFO_SAMPLE_DEL, sampleDbName.c_str(), refList.c_str()); #endif Trace("Issuing query", strQuery, 0); result = connection->Query(strQuery); #ifdef SHARD_AWARE sprintf(strQuery, SQL_WTT_HEADER_DETAIL_SAMPLE_DEL_C3, sampleDbName.c_str(), refList.c_str(), getBoundedDateTimeString(KEY_TIME_COLUMN_RESULTDETAIL).c_str()); #else sprintf(strQuery, SQL_WTT_HEADER_DETAIL_SAMPLE_DEL, sampleDbName.c_str(), refList.c_str()); #endif Trace("Issuing query", strQuery, 0); result = connection->Query(strQuery); #ifdef SHARD_AWARE sprintf(strQuery, SQL_WTT_HEADER_REQUEST_SAMPLE_DEL_C3, sampleDbName.c_str(), refList.c_str(), getBoundedDateTimeString(KEY_TIME_COLUMN_REQUEST).c_str()); #else sprintf(strQuery, SQL_WTT_HEADER_REQUEST_SAMPLE_DEL, sampleDbName.c_str(), refList.c_str()); #endif Trace("Issuing query", strQuery, 0); result = connection->Query(strQuery); #ifdef SHARD_AWARE sprintf(strQuery, SQL_WTT_HEADER_HEADER_SAMPLE_DEL_C3, sampleDbName.c_str(), refList.c_str(), getBoundedDateTimeString(KEY_TIME_COLUMN_HEADER).c_str()); #else sprintf(strQuery, SQL_WTT_HEADER_HEADER_SAMPLE_DEL, sampleDbName.c_str(), refList.c_str()); #endif Trace("Issuing query", strQuery, 0); result = connection->Query(strQuery); #ifdef SHARD_AWARE sprintf(strQuery, SQL_WTT_HEADER_HTML_SAMPLE_DEL_C3, sampleDbName.c_str(), refList.c_str(), getBoundedDateTimeString(KEY_TIME_COLUMN_HTML).c_str()); #else sprintf(strQuery, SQL_WTT_HEADER_HTML_SAMPLE_DEL, sampleDbName.c_str(), refList.c_str()); #endif Trace("Issuing query", strQuery, 0); result = connection->Query(strQuery); #ifdef SHARD_AWARE sprintf(strQuery, SQL_WTT_HEADER_EXTRA_SAMPLE_DEL_C3, sampleDbName.c_str(), refList.c_str(), getBoundedDateTimeString(KEY_TIME_COLUMN_EXTRAINFO).c_str()); #else sprintf(strQuery, SQL_WTT_HEADER_EXTRA_SAMPLE_DEL, sampleDbName.c_str(), refList.c_str()); #endif Trace("Issuing query", strQuery, 0); result = connection->Query(strQuery); } sprintf(strQuery, SQL_WTT_HEADER_SAMPLE_INS, sampleDbName.c_str(), pageID, testServerID, startTimeStr.c_str(), overallResultCode, DNSTimeOffsetMs, firstByteTimeOffsetMs, totalLastByteTimeOffsetMs, totalBytes, runID, firstDataTimeOffsetMs, int(testType), startTimeStr.c_str(), localOffset, customerNo, pageType, stepName.c_str(), pageNumber, currentColourCode, searchID, searchName.c_str(), sslConnectTime, requestSentTime, requestHeaderSize, requestContentSize, responseHeaderSize); Trace("Query for SampleCCYYMM.ResultHeader", strQuery, 0); result = connection->Query(strQuery); snprintf(strQuery, HTML_SQL_MAX_LEN + 100, SQL_WTT_HEADER_SAMPLE_INS_ID, sampleDbName.c_str()); Trace("Query for SampleCCYYMM.ResultHeader", strQuery, 0); result = connection->Query(strQuery); result->Next(); RowPtr row = result->GetCurrentRow(); Trace("LastInsertID for SampleCCYYMM.ResultHeader", row->GetField(1), 0); *resultID = strtoull(row->GetField(1), 0, 10); } catch (const std::exception &e) { // set the error code to 0 so that we don't connect to the wrong header *resultID = 0; LogError("SamplingStoreHeader() - table ResultHeader Caught Exception", e.what(), pageID); } // store the extra info for the first component against the header try { if ((!request.empty()) || (!header.empty()) || (!html.empty()) || (!extraInfo.empty())) { int len = 0; char sqlStr[EXTRA_SQL_MAX_LEN + 1]; // convert the string for inserting len = mysql_escape_string(sqlStr, request.c_str(), (request.size() < (EXTRA_SQL_MAX_LEN / 2)) ? (request.size()) : (EXTRA_SQL_MAX_LEN / 2)); // terminate the string if ( len > (int)EXTRA_SQL_MAX_LEN) { sqlStr[EXTRA_SQL_MAX_LEN] = '\0'; } else { sqlStr[len] = '\0'; } // make sure last char not a \ - it would escape the final quote if ( sqlStr[EXTRA_SQL_MAX_LEN] == '\\') sqlStr[EXTRA_SQL_MAX_LEN] = ' '; char sqlStr2[EXTRA_SQL_MAX_LEN + 1]; // convert the string for inserting len = mysql_escape_string(sqlStr2, header.c_str(), (header.size() < (EXTRA_SQL_MAX_LEN / 2)) ? (header.size()) : (EXTRA_SQL_MAX_LEN / 2)); // terminate the string if ( len > (int)EXTRA_SQL_MAX_LEN) { sqlStr2[EXTRA_SQL_MAX_LEN] = '\0'; } else { sqlStr2[len] = '\0'; } // make sure last char not a \ - it would escape the final quote if ( sqlStr2[EXTRA_SQL_MAX_LEN] == '\\') sqlStr2[EXTRA_SQL_MAX_LEN] = ' '; char sqlStr3[EXTRA_SQL_MAX_LEN + 1]; // convert the string for inserting len = mysql_escape_string(sqlStr3, html.c_str(), (html.size() < (EXTRA_SQL_MAX_LEN / 2)) ? (html.size()) : (EXTRA_SQL_MAX_LEN / 2)); // terminate the string if ( len > (int)EXTRA_SQL_MAX_LEN) { sqlStr3[EXTRA_SQL_MAX_LEN] = '\0'; } else { sqlStr3[len] = '\0'; } // make sure last char not a \ - it would escape the final quote if ( sqlStr3[EXTRA_SQL_MAX_LEN] == '\\') sqlStr3[EXTRA_SQL_MAX_LEN] = ' '; char sqlStr4[EXTRA_SQL_MAX_LEN + 1]; // convert the string for inserting len = mysql_escape_string(sqlStr4, extraInfo.c_str(), (extraInfo.size() < (EXTRA_SQL_MAX_LEN / 2)) ? (extraInfo.size()) : (EXTRA_SQL_MAX_LEN / 2)); // terminate the string if ( len > (int)EXTRA_SQL_MAX_LEN) { sqlStr4[EXTRA_SQL_MAX_LEN] = '\0'; } else { sqlStr4[len] = '\0'; } // make sure last char not a \ - it would escape the final quote if ( sqlStr4[EXTRA_SQL_MAX_LEN] == '\\') sqlStr4[EXTRA_SQL_MAX_LEN] = ' '; #ifdef SHARD_AWARE snprintf(strQuery, 204800 - 1, SQL_WTT_HEADER_EXTRA_INFO_SAMPLE_INS_C3, sampleDbName.c_str(), *resultID, sqlStr, sqlStr2, sqlStr3, sqlStr4, startTimeStr.c_str() ); #else snprintf(strQuery, 204800 - 1, SQL_WTT_HEADER_EXTRA_INFO_SAMPLE_INS, sampleDbName.c_str(), *resultID, sqlStr, sqlStr2, sqlStr3, sqlStr4 ); #endif Trace("Query for ResultHeaderExtraInfo", strQuery, 0); result = connection->Query(strQuery); } } catch (const std::exception &e) { LogError("SamplingStoreHeader() - table ResultHeaderExtraInfo Caught Exception", e.what(), pageID); } }
bool checkConnectivity(ConnectionPtr connection) { // checks to see if this server has connectivity // it does so by finding two servers (from the TestServer table) in other groups, and hence on other carriers // if both of these pings fail then we can assume that it is our connectivity // we can then re-schedule the job // now also used to see if we are down before the end of the check std::string testHost; std::string externalTestHost; std::string externalTestHost1; std::string externalTestHost2; int ourGroup = 0; int testGroup = 0; ResultPtr result; ncc::safe_array<char> selectQuery(1024); bool retVal = true; std::string site; std::string clusterGroup; Trace("checkConnectivity", "CConfig::getServerID()", CConfig::getServerID()); try { #ifdef SHARD_AWARE ConnectionPtr passportShardConnection( new Connection( CConfig::getPassportDbServer(), CConfig::getPassportDbDatabase(), CConfig::getPassportDbUser(), CConfig::getPassportDbPass(), CConfig::getDBRetryTime())); // get the group of this test server snprintf(selectQuery, selectQuery.size(), SQL_SCS_TEST_GROUP_CG3, CConfig::getServerID()); Trace("Issuing query",selectQuery,0); result = passportShardConnection->Query(selectQuery); Trace("Rows:","",result->GetRowCount()); if (result->GetRowCount() > 0) { result->Next(); RowPtr row = result->GetCurrentRow(); ourGroup = row->GetFieldInteger(1); } snprintf(selectQuery, selectQuery.size(), SQL_SCS_RETEST_SERVERS_GROUP_RAND_CG3, SC_SERVER_STATUS_LIVE, ourGroup, 0); Trace("Issuing query",selectQuery,0); result = passportShardConnection->Query(selectQuery); Trace("Rows:","",result->GetRowCount()); #else // get the group of this test server snprintf(selectQuery, selectQuery.size(), SQL_SCS_TEST_GROUP, CConfig::getServerID()); Trace("Issuing query",selectQuery,0); result = connection->Query(selectQuery); Trace("Rows:","",result->GetRowCount()); if (result->GetRowCount() > 0) { result->Next(); RowPtr row = result->GetCurrentRow(); ourGroup = row->GetFieldInteger(1); site = row->GetFieldString(2); clusterGroup = row->GetFieldString(3); } snprintf(selectQuery, selectQuery.size(), SQL_SCS_RETEST_SERVERS_GROUP_RAND, SC_SERVER_STATUS_LIVE, ourGroup, site.c_str(), clusterGroup.c_str()); Trace("Issuing query",selectQuery,0); result = connection->Query(selectQuery); Trace("Rows:","",result->GetRowCount()); #endif int testGroup1 = 0; int testGroup2 = 0; std::string externalTestHost1; std::string externalTestHost2; if (result->GetRowCount() > 0) { while(result->Next()) { RowPtr row = result->GetCurrentRow(); testHost = row->GetFieldString(2); testGroup = row->GetFieldInteger(3); externalTestHost = row->GetFieldString(4); Trace("externalHost",externalTestHost.c_str(),testGroup); if (testGroup1 == 0) { // set this to be the first group to test testGroup1 = testGroup; externalTestHost1 = externalTestHost; } else if (testGroup1 != testGroup) { // different group - use for the second test testGroup2 = testGroup; externalTestHost2 = externalTestHost; // all done now break; } // else must be same group as testGroup1 - ignore it } bool ping1OK = true; bool ping2OK = true; if ((testGroup1 != 0) && (externalTestHost1 != "")) { Trace("Tesing against group 1",externalTestHost1.c_str(),testGroup1); ping1OK = pingSiteAvailable(externalTestHost1); } else { // nothing to test - assume it worked ? Trace("Tesing against group 1","n/a",testGroup1); } if ((testGroup2 != 0) && (externalTestHost2 != "")) { Trace("Tesing against group 2",externalTestHost2.c_str(),testGroup2); ping2OK = pingSiteAvailable(externalTestHost2); } else if ((testGroup1 != 0) && (externalTestHost1 != "")) { // we tested a 'real' #1 - test the one from sc.opt Trace("Tesing against group 2 - sc.opt",CConfig::getPing1Name().c_str(),testGroup2); ping2OK = pingSiteAvailable(CConfig::getPing1Name()); } else { // else nothing to test - assume it worked ? Trace("Tesing against group 2","n/a",testGroup2); } if (!ping1OK && !ping2OK) { // first two have failed // check yahoo if (!pingSiteAvailable(CConfig::getPing1Name())) { retVal = false; } } } } catch (const std::exception& e) { Trace("Exception caught", e.what(), 0); printf("Error: %s\n", e.what()); } return retVal; }
void schedSecondTest(ConnectionPtr connection, long idNo, int portNo, SC_TEST_TYPE testType, unsigned int shardNumber) { Trace("schedSecondTest", "", idNo); std::string strQuery; // find a server to send the test to try { #ifdef SHARD_AWARE ConnectionPtr passportShardConnection( new Connection( CConfig::getPassportDbServer(), CConfig::getPassportDbDatabase(), CConfig::getPassportDbUser(), CConfig::getPassportDbPass(), CConfig::getDBRetryTime())); // get the group of this test server strQuery = Format(SQL_SCS_TEST_GROUP_CG3, CConfig::getServerID()); TraceSQL("Issuing query", "", 0); int testGroup = 0; if (ResultPtr result = passportShardConnection->Query(strQuery)) { TraceSQL("Rows:","",result->GetRowCount()); if (result->GetRowCount() > 0) { result->Next(); RowPtr row = result->GetCurrentRow(); testGroup = row->GetFieldInteger(1); } } // get the carrier to exclude if (portNo == TCP_PORT_WTTGD) { strQuery = Format(SQL_SCS_TEST_GROUP_EXCLUDE_SCRIPT, idNo); } else if (portNo == TCP_PORT_ETTD) { strQuery = Format(SQL_SCS_TEST_GROUP_EXCLUDE_ISP, idNo); } else { strQuery = Format(SQL_SCS_TEST_GROUP_EXCLUDE_WEBPAGE, idNo); } TraceSQL("Issuing query", "", 0); unsigned int carrierToExclude = 0; if (ResultPtr result = connection->Query(strQuery)) { TraceSQL("Rows:","",result->GetRowCount()); if (result->GetRowCount() > 0) { result->Next(); RowPtr row = result->GetCurrentRow(); // make it the customer one carrierToExclude = row->GetFieldInteger(1); // override with Script or WebPage one if (row->GetFieldInteger(2) != 0) carrierToExclude = row->GetFieldInteger(2); } } #ifdef SC_NON_WEBKIT strQuery = Format(SQL_SCS_RETEST_SERVERS_GROUP_RAND_CG3, SC_SERVER_STATUS_LIVE, testGroup, carrierToExclude); #else #ifdef SC_WEBKIT_V89 strQuery = Format(SQL_SCS_RETEST_SERVERS_GROUP_RAND_CG3, SC_SERVER_JACK_STATUS_LIVE_89, testGroup, carrierToExclude); #else strQuery = Format(SQL_SCS_RETEST_SERVERS_GROUP_RAND_CG3, SC_SERVER_JACK_STATUS_LIVE, testGroup, carrierToExclude); #endif // SC_WEBKIT_V89 #endif // SC_NON_WEBKIT TraceSQL("Issuing query", "", 0); if (ResultPtr result = passportShardConnection->Query(strQuery)) { TraceSQL("Rows:","",result->GetRowCount()); #else // get the group of this test server strQuery = Format(SQL_SCS_TEST_GROUP, CConfig::getServerID()); TraceSQL("Issuing query", "", 0); if (ResultPtr result = connection->Query(strQuery)) { std::string site; std::string clusterGroup; TraceSQL("Rows:","",result->GetRowCount()); if (result->GetRowCount() > 0) { result->Next(); RowPtr row = result->GetCurrentRow(); testGroup = row->GetFieldInteger(1); site = row->GetFieldString(2); clusterGroup = row->GetFieldString(3); } // first work out the minute //time_t timeNow = time(NULL); //struct tm* timeStruct; //timeStruct = localtime( &timeNow ); strQuery = Format(SQL_SCS_RETEST_SERVERS_GROUP_RAND, SC_SERVER_STATUS_LIVE, testGroup, site.c_str(), clusterGroup.c_str()); } TraceSQL("Issuing query", "", 0); if (ResultPtr result = connection->Query(strQuery)) { TraceSQL("Rows:","",result->GetRowCount()); #endif if (result->GetRowCount() > 0) { std::string testHost; while (result->Next()) { RowPtr row = result->GetCurrentRow(); testHost = row->GetFieldString(2); // try to send the job // send the test request CTcp tcpConnection(testHost, portNo); int fd = tcpConnection.tcp_connect(WTT_WTTD_TIMEOUT_SECS); if (fd > 0) { #ifdef SHARD_AWARE // send the info to the testing tool if (testType == SC_TEST_TYPE_UNKNOWN) { strQuery = Format("%ld %d", idNo, shardNumber); } else { strQuery = Format("%ld %d %d", idNo, shardNumber, int(testType)); } #else // send the info to the testing tool if (testType == SC_TEST_TYPE_UNKNOWN) { strQuery = Format("%ld", idNo); } else { strQuery = Format("%ld %d", idNo, int(testType)); } #endif TraceLogic("tcpConnection.writen", strQuery.c_str(), strQuery.size()) if (tcpConnection.writen(strQuery.c_str(), strQuery.size()) > 0) { Trace("Job sent to test server", strQuery.c_str(), fd); return; } else { LogError("Error sending job to test server", strQuery.c_str(), idNo); } } else { LogError("Timed out sending job to test server", "", idNo); } } }
// per page tests void runWTTT(scriptid_t scriptID, unsigned int shardNumber, const char* runNo, long batchID, bool wait, bool debug) { // only do a sleep if the first run - not for manual or retests if ((atoi(runNo) == 1) && wait) { TraceNoise("Sleeping for a bit", runNo, scriptID); // sleep for an offset of seconds sleep(scriptID % 60); } ConnectionPtr connection; std::string strQuery; bool download = true; try { #ifdef SHARD_AWARE TraceLogic("Connecting to db", CConfig::getShardDatabase(shardNumber).c_str(), shardNumber); connection.reset( new Connection( CConfig::getShardServer(shardNumber), CConfig::getShardDatabase(shardNumber), CConfig::getShardUser(shardNumber), CConfig::getShardPass(shardNumber), CConfig::getDBRetryTime())); #else TraceLogic("Connecting to db", CConfig::getDBDatabase(), 0); connection.reset( new Connection( CConfig::getDBServer(), CConfig::getDBDatabase(), CConfig::getDBUser(), CConfig::getDBPass(), CConfig::getDBRetryTime())); #endif } catch (const std::exception &e) { LogError("Exception Caught:", e.what(), scriptID); return; } // if the ScriptID == 0 - get it from the batch table if (scriptID == 0) { strQuery = Format(SQL_WTTT_BATCH_SEL, batchID); TraceSQL("Issuing", "", 0); if (ResultPtr result = connection->Query(strQuery)) { if (result->Next()) { RowPtr row = result->GetCurrentRow(); scriptID = row->GetFieldLong(1); TraceLogic("batchID", "", batchID); TraceLogic("scriptID", "", scriptID); } } } // script and archive names const std::string filename = Format("/home/sc/bin/monitor_scripts/%ld_%u", scriptID, shardNumber); const std::string gzfilename = filename + ".gz"; // Get script details long customerNo = 0; SC_SCRIPT_LANGUAGE scriptlanguage = SC_SCRIPT_LANGUAGE_UNKNOWN; std::string scriptversion; strQuery = Format(SQL_WTTT_SCRIPTINFO_SEL, scriptID); TraceSQL("Issuing", "", 0); if (ResultPtr result = connection->Query(strQuery)) { if (result->Next()) { RowPtr row = result->GetCurrentRow(); customerNo = row->GetFieldLong(1); scriptlanguage = static_cast<SC_SCRIPT_LANGUAGE>(row->GetFieldLong(2)); // Field can be NULL, so check before dereference if (const char* str = row->GetField(3)) { scriptversion = str; } TraceLogic("customerNo", "", customerNo); TraceLogic("scriptlanguage", to_string(scriptlanguage).c_str(), int(scriptlanguage)); TraceLogic("scriptversion", scriptversion.c_str(), scriptversion.size()); } } switch (scriptlanguage) { case SC_SCRIPT_LANGUAGE_UNKNOWN: // // No recognised script or script language returned // LogError( "Script not found", Format("ScriptID=%d Shard=%d DB.name=%s DB.host=%s", scriptID, shardNumber, CConfig::getShardDatabase(shardNumber).c_str(), CConfig::getShardServer(shardNumber).c_str()).c_str(), 0); exit(1); break; case SC_SCRIPT_LANGUAGE_PHP: // // Handle PHP script // strQuery = Format("select Code, length(Code) from Script where ScriptID=%ld", scriptID); TraceSQL("Issuing", "", 0); if (ResultPtr result = connection->Query(strQuery)) { if (result->Next()) { RowPtr row = result->GetCurrentRow(); int scriptLength = row->GetFieldInteger(2); std::string scriptString(row->GetField(1), scriptLength); TraceLogic("length(Code)", "", scriptLength); umask(SC_WTTT_UMASK); int filePtr = open(filename.c_str(), O_CREAT | O_TRUNC | O_WRONLY, S_IRWXU); if (filePtr != -1) { // output the body to the file int nbytes = write(filePtr, scriptString.c_str(), scriptLength); if (nbytes != scriptLength) { LogError("Incomplete write of PHP executeable to disc", to_string(nbytes).c_str(), scriptLength); } close(filePtr); } else { LogError("Unable to write PHP executable to disc", "", scriptID); } } } break; case SC_SCRIPT_LANGUAGE_CXX: // // Handle compiled C++ script // if (Exist(filename)) { // Is the file out of date? strQuery = Format(SQL_WTTT_SCRIPTAGE_SEL, scriptID); TraceSQL("Issuing", "", 0); if (ResultPtr result = connection->Query(strQuery)) { if (result->Next()) { RowPtr row = result->GetCurrentRow(); long dbage = row->GetFieldLong(1); TraceLogic("dbage", "", dbage); struct stat statbuf; bzero(&statbuf, 0); int ret = stat(filename.c_str(), &statbuf); TraceLogic("stat", "download", ret); download = ret == -1 || statbuf.st_mtime < dbage; TraceLogic("Overwrite the existing %s?", "download", int(download)); TraceLogic("statbuf.st_mtime", "", statbuf.st_mtime); } } } // // Extract script from database // if (download) { strQuery = Format(SQL_WTTT_SCRIPT_SEL, scriptID); TraceSQL("Issuing", "", 0); if (ResultPtr result = connection->Query(strQuery)) { if (result->Next()) { RowPtr row = result->GetCurrentRow(); int scriptLength = row->GetFieldInteger(2); std::string scriptString(row->GetField(1), scriptLength); // Field can be NULL, so check before dereference int agentVersion = 0; if (const char* str = row->GetField(3)) { agentVersion = atoi(str); } TraceLogic("scriptLength", "", scriptLength); TraceLogic("scriptString", scriptString.c_str(), scriptString.size()); TraceLogic("agentVersion", "", agentVersion); umask(SC_WTTT_UMASK); int filePtr = open(gzfilename.c_str(), O_CREAT | O_TRUNC | O_WRONLY, S_IRWXU); if (filePtr != -1) { // output the body to the file int nbytes = write(filePtr, scriptString.c_str(), scriptLength); close(filePtr); if (nbytes == scriptLength) { // unzip, set permissions and prelink doUnzip(gzfilename); chmod(filename.c_str(), 0775); bool prelink = false; if (!CConfig::getIMMode()) { TraceLogic("Connecting to passport db", CConfig::getPassportDbDatabase().c_str(), 0); ConnectionPtr connectionPassport( new Connection( CConfig::getPassportDbServer(), CConfig::getPassportDbDatabase(), CConfig::getPassportDbUser(), CConfig::getPassportDbPass(), CConfig::getDBRetryTime())); strQuery = Format("select Prelink from LibraryVersions where Version=%d", agentVersion); TraceSQL("Issuing", "", 0); if (ResultPtr result = connectionPassport->Query(strQuery)) { if (result->Next()) { RowPtr row = result->GetCurrentRow(); prelink = row->GetFieldInteger(1) != 0; TraceLogic("prelink", to_string(prelink).c_str(), int(prelink)); } } TraceLogic("prelink", "", int(prelink)); } else { TraceLogic("prelink", "Don't prelink on IM node", 0); } if (prelink) { int ret = doPrelink(filename, SC_AGENT_VER_NUM); if (ret != 0) { LogError("Prelink failed", filename.c_str(), ret); } } } else { LogError("Incomplete write of C++ executeable to disc", to_string(nbytes).c_str(), scriptLength); return; } } else { LogError("Unable to write C++ executable to disc", "", scriptID); return; } } else { LogError("C++ Executable not in database", gzfilename.c_str(), result->GetRowCount()); return; } } else { LogError("Query for C++ executable failed", strQuery.c_str(), result->GetRowCount()); return; } } break; default: LogError("Fatal: Unknown language", "", int(scriptlanguage)); return; } // close the db connection Clear(connection); // // Build command line // std::string systemCmd; #ifdef USE_SHARED // For version 7 if (scriptlanguage == SC_SCRIPT_LANGUAGE_CXX && Exist("/home/sc/lib/libsiteconfidence.so." + scriptversion) && Exist("/home/sc/lib/libscparser.so." + scriptversion) && Exist("/home/sc/lib/libsiteconfidenceinterface.so." + scriptversion)) { std::ostringstream os; os << "export LD_PRELOAD="; os << "/home/sc/lib/libsiteconfidence.so." << scriptversion << ":"; os << "/home/sc/lib/libscparser.so." << scriptversion << ":"; os << "/home/sc/lib/libsiteconfidenceinterface.so." << scriptversion << "; "; systemCmd += os.str(); } #endif #ifdef SHARD_AWARE // set environment variables for debug and shardNumber { std::ostringstream os; os << "export SHARD_NUMBER=" << shardNumber << "; " << "export SC_DEBUG_MANUAL_MODE=" << (debug != 0) << "; "; systemCmd += os.str(); } #endif // program command line { std::ostringstream os; os << filename << " " << runNo << " " << batchID << " " << scriptID << " " << customerNo; systemCmd += os.str(); } //Print(Format("starting script:%s\n", systemCmd.c_str())); // execute TraceLogic("Starting script", systemCmd.c_str(), scriptID); int sysVal = system(systemCmd.c_str()); int exitVal = WEXITSTATUS(sysVal); TraceLogic("System call returned", systemCmd.c_str(), exitVal); //Print(Format("script returned:%d\n", exitVal)); }
void CPage::loadJavaScriptCheck(ConnectionPtr connection) { Trace("loadJavaScriptCheck()", "",0); std::string strQuery; // need to change all to lower case std::string lowerCaseBody; try { Mutex::Lock lock(m_javascriptListLock); strQuery = Format(SQL_WTT_JAVASCRIPT1); if (ResultPtr result = connection->Query(strQuery)) { Trace("RowPtr s:","",result->GetRowCount()); while(result->Next()) { RowPtr row = result->GetCurrentRow(); // add to the list Trace("loadJavaScriptCheck() - search", row->GetField(1),0); Trace("loadJavaScriptCheck() - replace", row->GetField(2),0); m_javascriptSearchList.push_back(row->GetFieldString(1)); m_javascriptReplaceList.push_back(row->GetFieldString(2)); } } strQuery = Format(SQL_WTT_JAVASCRIPT2, getPageID()); if (ResultPtr result = connection->Query(strQuery)) { Trace("RowPtr s:","",result->GetRowCount()); while(result->Next()) { RowPtr row = result->GetCurrentRow(); // add to the list Trace("loadJavaScriptCheck() - search", row->GetField(1),0); Trace("loadJavaScriptCheck() - replace", row->GetField(2),0); m_javascriptSearchList.push_back(row->GetFieldString(1)); m_javascriptReplaceList.push_back(row->GetFieldString(2)); } } } catch (const std::exception& e) { LogError("Exception Caught:", e.what(), 0); } try { Mutex::Lock lock(m_javascriptExtractListLock); strQuery = Format(SQL_WTT_JAVASCRIPT3, getPageID()); if (ResultPtr result = connection->Query(strQuery)) { Trace("RowPtr s:","",result->GetRowCount()); while(result->Next()) { RowPtr row = result->GetCurrentRow(); // add to the list Trace("loadJavaScriptCheck() - start", row->GetField(1),0); Trace("loadJavaScriptCheck() - end", row->GetField(2),0); m_javascriptStartList.push_back(row->GetFieldString(1)); m_javascriptEndList.push_back(row->GetFieldString(2)); } } } catch (const std::exception& e) { LogError("Exception Caught:", e.what(), 0); } }