bool DataSetValues::SetDataSet(DataSet *ds)
{
if (ds && !ds->GetReadable())
{
LogError2(GetType()->GetLogCategory(), GetName(),tr("Data Set Values Display can't be set to a data set that is not readable."));
return false;
}
if (m_dataSet)
{
disconnect(m_dataSet,SIGNAL(BeforeDeletion(DataClass*)),this,SLOT(OnBeforeDataSetRemovedSlot(DataClass*)));
disconnect(m_dataSet,SIGNAL(InputSourceAssigned(DataSet*)),this,SLOT(OnDataSetSourceSet(DataSet*)));
disconnect(m_dataSet,SIGNAL(NewData(DataClass*)),this,SLOT(OnNewData(DataClass*)));
disconnect(m_dataSet,SIGNAL(StructureChanged(DataSource*)),this,SLOT(OnDataSetSourceStructureChanged(DataSource*)));
disconnect(m_dataSet,SIGNAL(NameChanged(DataClass*)),this,SLOT(OnDataSetNameChanged(DataClass*)));
}
m_dataSet = ds;
if (m_dataSet)
{
connect(m_dataSet,SIGNAL(BeforeDeletion(DataClass*)),this,SLOT(OnBeforeDataSetRemovedSlot(DataClass*)));
connect(m_dataSet,SIGNAL(InputSourceAssigned(DataSet*)),this,SLOT(OnDataSetSourceSet(DataSet*)));
connect(m_dataSet,SIGNAL(NewData(DataClass*)),this,SLOT(OnNewData(DataClass*)));
connect(m_dataSet,SIGNAL(StructureChanged(DataSource*)),this,SLOT(OnDataSetSourceStructureChanged(DataSource*)));
connect(m_dataSet,SIGNAL(NameChanged(DataClass*)),this,SLOT(OnDataSetNameChanged(DataClass*)));
m_dataType = ds->GetDataType();
m_firstVisibleElement = ds->GetFirstIndex();
OnDataSetNameChanged(m_dataSet);
}
ParametersUpdated();
return true;
}
void CPage::LoadCache(ConnectionPtr connection)
{
Trace("LoadCache", "",0);
ResultPtr result;
ncc::safe_array<char> strQuery(10240);
try
{
// load from the database
snprintf(strQuery, strQuery.size(), SQL_WTT_LOAD_CACHE, m_runID);
Trace("Query for LoadCache", strQuery, 0);
result = connection->Query(strQuery);
while(result->Next())
{
RowPtr row = result->GetCurrentRow();
// add it
std::string urlStr;
urlStr = row->GetFieldString(1);
{ // scope for lock
Mutex::Lock lock(sm_urlListLock);
m_totalUrlList.push_back(urlStr);
}
}
}
catch (const std::exception& e)
{
LogError2("LoadCache() - Caught Exception", e.what(), 0, SC_ERR_GROUP_DB, SC_ERR_CODE_LOAD_CACHE);
}
}
void CPage::SaveCache(ConnectionPtr connection)
{
Trace("SaveCache", "", m_totalUrlList.size());
// save all the cookies in the current memory list to the database
// in the WebPageCookies table
ncc::safe_array<char> strQuery(102400);
ncc::safe_array<char> sqlCache(40961);
{ // scope for lock
// delete the existing cache in the table for this run
snprintf(strQuery, strQuery.size(), SQL_WTT_DEL_CACHE, m_runID);
Trace("Query for SaveCache", strQuery, 0);
connection->Query(strQuery);
try
{
for (URLList::const_iterator it = m_totalUrlList.begin(); it != m_totalUrlList.end(); ++it)
{
// convert the string ready to be inserted
int len = 0;
// convert the string for inserting
len = mysql_escape_string(sqlCache, (*it).c_str(),((*it).size()<40960)?((*it).size()):40960);
// terminate the string
if( len > 40960)
{
sqlCache[40960] = '\0';
}
else
{
sqlCache[len] = '\0';
}
// make sure last char not a \ - it would escape the final quote
if( sqlCache[40959] == '\\')
{
sqlCache[40959] = ' ';
}
else if( sqlCache[len-1] == '\\')
{
sqlCache[len-1] = ' ';
}
// save to the database
snprintf(strQuery, strQuery.size(), SQL_WTT_SAVE_CACHE, m_runID, static_cast<const char*>(sqlCache));
Trace("Query for SaveCache", strQuery, 0);
connection->Query(strQuery);
}
}
catch (const std::exception& e)
{
LogError2("SaveCache() - Caught Exception", e.what(), 0, SC_ERR_GROUP_DB, SC_ERR_CODE_STORE_CACHE);
}
}
}
ConnectionPtr CPage::getDbConnection()
{
// if not already connected - create one
if (sm_pDbConnection.get() == NULL)
{
sm_pDbConnection.reset(
new Connection(
CConfig::getDBServer(),
CConfig::getDBDatabase(),
CConfig::getDBUser(),
CConfig::getDBPass(),
CConfig::getDBRetryTime()));
}
if (sm_pDbConnection.get() == NULL)
{
LogError2("CPage::getDbConnection()", "Unable to create new Connection", 0,
SC_ERR_GROUP_AGENT, SC_ERR_CODE_UNABLE_TO_CREATE_TCP_CONN);
}
return sm_pDbConnection;
}
resultid_t runWTTI(const char* url, std::string& dbName, unsigned int wttiRunID, unsigned int shardNumber, int webkitTimeout)
{
std::string strQuery;
sem_init( &(CWebPage::Instance()->sm_hostSemaphore), 0, 0 );
sem_init( &(CWebPage::Instance()->sm_threadControlSemaphore), 0, 0 );
#ifdef SHARD_AWARE
ConnectionPtr connection(
new Connection(
CConfig::getShardServer(shardNumber),
CConfig::getShardDatabase(shardNumber),
CConfig::getShardUser(shardNumber),
CConfig::getShardPass(shardNumber),
CConfig::getDBRetryTime()));
#else
ConnectionPtr connection(
new Connection(
CConfig::getDBServer(),
CConfig::getDBDatabase(),
CConfig::getDBUser(),
CConfig::getDBPass(),
CConfig::getDBRetryTime()));
#endif
std::string urlToTest;
// Get URL from the WttiTests table.
ResultPtr resultUrl;
strQuery = Format(SQL_WTTI_SELECT_URL, wttiRunID);
resultUrl = connection->Query(strQuery.c_str());
if ( resultUrl->Next() )
{
RowPtr row = resultUrl->GetCurrentRow();
urlToTest = row->GetFieldString(1);
Trace("Rows:", "", resultUrl->GetRowCount());
}
resultid_t dailyResultID = 0;
PackDiagStorage store;
int pageID = 0;
CResultSet resultSet;
CTcp* pTcpConnection = new CTcp("", 0);
// Turn CSS Parsing on permanently.
CWebPage::Instance()->sm_doCssParsing = 1;
//Turn SSL validation on permanently
CWebPage::Instance()->sm_doSslValidation = 1;
CTcp::setDoSslValidation(1);
// create the new web page object
CWebPage* pPage = new CWebPage(
resultSet,
pTcpConnection,
0,
SC_SERVICE_BENCHMARK,
urlToTest, // url
pageID, // page id
SC_WEBPAGE_TYPE_IMED, // page type
true, // reset cookies
std::string(), // auth user
std::string(), // auth pass
DOWNLOAD_THRESHOLD_SECS, // download threshold
true, // reset cache
CConfig::getServerID(),
SC_WEBPAGE_STATUS_OK, // bodge to be status live page
HTTP_METHOD_GET, // bodge to get GET by default
std::string(),
std::vector<std::string>(),
std::vector<char>(), // emptyVector, // bodge - no form variables
std::string() // fileStr // bodge - no file variables
);
// reset the counters
pPage->resetPage();
// record when we started the test
//pPage->setStartTime();
// make sure we don't report a hacking / login failure
// pPage->phraseWasFound();
// Set that a connection exists to this host.
//pPage->isHostConnectionsExceeded(static_cast<std::string>(url));
// do the load of this page
//pPage->load();
// add the page just loaded to the list of those done so it doesn't get done again
// via a redirect
//CWebPage::AddURLDone(url);
// reset the first byte count for the page
//pPage->resetSpeedCounters();
MultiThreadSetup();
// ----------------------
// --- THREAD MANAGER ---
// ----------------------
std::string protoAndHost;
protoAndHost = urlToTest;
// Calculate protocol type.
if (strncasecmp (protoAndHost.c_str(), "http://", 7) == 0)
{
std::string::size_type hostLength = protoAndHost.find("/", 7);
if (hostLength != std::string::npos)
protoAndHost = protoAndHost.substr(0, hostLength);
}
else if (strncasecmp (protoAndHost.c_str(), "https://", 8) == 0)
{
std::string::size_type hostLength = protoAndHost.find("/", 8);
if (hostLength != std::string::npos)
protoAndHost = protoAndHost.substr(0, hostLength);
}
else
{
std::string::size_type hostLength = protoAndHost.find("/");
if (hostLength != std::string::npos)
protoAndHost = "http://" + protoAndHost.substr(0, hostLength);
else
protoAndHost = protoAndHost.insert(0, "http://");
}
// Add the host that we're about to connect to so it does not get added again.
{
Mutex::Lock lock(CWebPage::Instance()->sm_connectedHostsLock);
CWebPage::Instance()->sm_connectedHosts.push_back(std::make_pair(true, protoAndHost));
}
std::vector<ThreadArgs*> threadArgsVector;
// JYLS WEBKIT START
#ifndef SC_NON_WEBKIT
#ifdef SC_WEBKIT_V89
curlwrapper_init(pPage, startThread);
//Use the default value from the opt file for these
ww_setMaxNumberOfThreadsPerHost(CConfig::getDefaultMaxThreadsPerHost());
ww_setMaxTotalNumberOfThreads(CConfig::getDefaultMaxTotalThreads());
ww_setScreenXDimension(CConfig::getScreenXDimension());
ww_setScreenYDimension(CConfig::getScreenYDimension());
ww_setJavascriptTimeout(CConfig::getJsExecutionTimeout());
ww_loadPageByWebkit(pPage->getURL().c_str(), true, DOWNLOAD_THRESHOLD_SECS, webkitTimeout);
#else
bool isJavascriptEnabled = true;
soupwrapper_init(pPage, startThread);
ww_setPageId(pageID);
ww_setWebkitTimeout(webkitTimeout);
ww_setWebkitTimeoutHard(CConfig::getWebkitHardTimeout());
load_page_by_webkit(pPage->getURL(), isJavascriptEnabled);
#endif // SC_WEBKIT_V89
#else
std::vector<pthread_t> threadIds;
pthread_t threadId;
int res = 0;
unsigned int maxThreadsPerHost = 0;
if (maxThreadsPerHost == 0)
maxThreadsPerHost = CConfig::getDefaultMaxThreadsPerHost();
//Limit the value of maxThreadsPerHost
if (maxThreadsPerHost > CConfig::getMaxThreadsPerHostLimit())
maxThreadsPerHost = CConfig::getMaxThreadsPerHostLimit();
unsigned int numThreads = maxThreadsPerHost;
for (unsigned int a = 0; a < numThreads; ++a)
{
ThreadArgs* args1 = new ThreadArgs;
args1->isFirstPage = (a == 0);
args1->pPage = pPage;
args1->host = protoAndHost;
res = pthread_create(&(threadId), NULL, startThread, (void*)args1);
if ( res != 0 )
{
LogError("Failed to start thread for", protoAndHost.c_str(), 0);
}
else
{
threadIds.push_back(threadId);
threadArgsVector.push_back(args1);
// Wait for the second thread to signal it has started.
sem_wait(&(CWebPage::Instance()->sm_threadControlSemaphore));
}
}
std::vector<CWebPage::HostPair>::const_iterator hostIt;
Trace("before starting threads", "", 0);
while ((CWebPage::getUrlListSize()) > 0 || (CWebPage::Instance()->sm_numWorkingThreads > 0))
{
{
Mutex::Lock lock(CWebPage::Instance()->sm_connectedHostsLock);
for ( hostIt = CWebPage::Instance()->sm_connectedHosts.begin(); hostIt != CWebPage::Instance()->sm_connectedHosts.end(); ++hostIt )
{
if ( !(hostIt->first) )
{
unsigned int numThreads = maxThreadsPerHost;
for (unsigned int a = 0; a < numThreads; ++a)
{
bool errorOccured = false;
std::string hostToConnect;
hostToConnect.assign(hostIt->second);
// Start threads per new host encountered.
Trace("Starting threads for", hostIt->second.c_str(), 0);
ThreadArgs* args2 = new ThreadArgs;
args2->isFirstPage = false;
args2->pPage = pPage;
args2->host = hostToConnect;
res = pthread_create(&(threadId), NULL, startThread, (void*)args2);
if ( res != 0 )
{
LogError("Failed to start thread for", hostIt->second.c_str(), 2);
errorOccured = true;
}
else
{
threadIds.push_back(threadId);
threadArgsVector.push_back(args2);
// Wait for thread to start.
sem_wait(&(CWebPage::Instance()->sm_threadControlSemaphore));
}
// Connected ok.
hostIt->first = true;
}
}
}
}
// Wait for more hosts to be encountered.
sem_wait(&(CWebPage::Instance()->sm_hostSemaphore));
}
#endif // SC_NON_WEBKIT
// JYLS WEBKIT END
Trace("Deleting semaphore", "", 0);
sem_destroy(&(CWebPage::Instance()->sm_hostSemaphore));
sem_destroy(&(CWebPage::Instance()->sm_threadControlSemaphore));
#ifdef SC_NON_WEBKIT
// Write debug information if components remain.
if (CWebPage::getUrlListSize() > 0)
{
CWebPage::writeThreadsDebug();
int count = 0;
std::vector<ThreadArgs*>::const_iterator threadsIt;
for (threadsIt = threadArgsVector.begin(); threadsIt != threadArgsVector.end(); ++threadsIt)
{
LogInfo("Threads started=", (*threadsIt)->host.c_str(), count);
++count;
}
}
#endif // SC_NON_WEBKIT
#ifndef SC_NON_WEBKIT
// Sleep to allow for extra components from eg javascript
usleep(1000000);
//Actually check how many threads are running, go ahead if 0
while (CWebPage::Instance()->sm_numWorkingThreads)
{
usleep(10000);
}
#else
void* threadResult;
// Join started threads.
for (unsigned int i = 0; i < threadIds.size(); ++i)
{
res = pthread_join(threadIds[i], &threadResult);
if (res != 0)
{
LogError("Unable to join thread", "", res);
}
else
{
Trace("Joined thread", "", res);
}
}
#endif // SC_NON_WEBKIT
// Delete all ThreadArgs used.
while ( !threadArgsVector.empty() )
{
delete threadArgsVector.back();
threadArgsVector.pop_back();
}
setResultCode(pPage, CWebPage::Instance()->sm_firstPageResultCode);
try
{
Trace("Connecting to db", "", 0);
std::string requestStr;
std::string headerStr;
std::string htmlStr;
std::string extraInfoStr;
// extra info for ResultHeader
if (!resultSet.getResultDetails().empty())
{
const CResultDetail* pResultDetail = resultSet.getFirstResultDetail();
requestStr = pResultDetail->getRequest();
headerStr = pResultDetail->getHeaders();
htmlStr = pResultDetail->getPageText();
extraInfoStr = pResultDetail->getExtraInfo();
}
const bool pageInError = pPage->getOverallResultCode() != SC_HEADER_RESULT_SUCCESS && pPage->getOverallResultCode() != SC_HEADER_RESULT_DUMMY_TRANSACTION;
// 3.0.0 Daily ResultHeader tables
StoreHeaderDaily(
connection,
resultSet,
pPage->getPageID(),
CConfig::getServerID(),
pPage->getStartTimeStr(),
pPage->getOverallResultCode(),
pPage->getDNSTimeOffsetMs(),
pPage->getConnectTimeOffsetMs(),
pPage->getTotalLastByteTimeOffsetMs(),
pPage->getTotalBytes(),
&dailyResultID,
pPage->getFirstDataTimeOffsetMs(),
pPage->getDailyTableName(),
PackDiagOverride(),
store,
pageInError,
false,
SC_TEST_TYPE_WTTI,
0,
0,
0,
CSCGlobal::gAgentVersion,
pPage->getTotalGzipBytes(),
false, //diagstored
pPage->getSslConnectTimeOffsetMs(),
pPage->getRequestSentTimeOffsetMs(),
pPage->getRequestHeaderSize(),
pPage->getRequestContentSize(),
pPage->getResponseHeaderSize());
// now store all the detail - always do this for wtti
int ComponentNo = 1;
for (CResultSet::ResultList::const_iterator p = resultSet.getResultDetails().begin(); p != resultSet.getResultDetails().end(); ++p, ++ComponentNo)
{
// now go and get the rest of the files - with threads
if (const CResultDetail* pResultDetail = p->pResultDetail)
{
pResultDetail->setComponentNo(ComponentNo);
// 3.0.0 Daily ResultDetail tables
StoreResultDaily(
connection,
resultSet,
ComponentNo,
pResultDetail,
dailyResultID,
pPage->getDailyTableName());
}
}
}
catch (const std::exception &e)
{
Trace("Exception caught", e.what(), 0);
LogError2("Exception Caught:", e.what(), 0, SC_ERR_GROUP_AGENT, SC_ERR_CODE_WTTI);
return 0;
}
#ifdef SHARD_AWARE
// Write the result ID back to the transient table so the website can pick it up.
strQuery = Format(SQL_WTTI_INSERT_RESULTID, dailyResultID, wttiRunID);
Trace("Issuing query", strQuery.c_str(), dailyResultID);
connection->Query(strQuery);
#endif
//
dbName = pPage->getDailyTableName();
// complete it
delete pPage;
pPage = NULL;
MultiThreadCleanup();
// Remove /tmp/session_id.<pid>p.host files
long int uniqueId = getpid();
std::string identifier = WTTP_SESSION_ID_IDENTIFIER;
strQuery = Format("rm -f /tmp/session_id.%ld%s.*", uniqueId, identifier.c_str());
int ret = system(strQuery.c_str());
Trace("System call returned", strQuery.c_str(), ret);
return dailyResultID;
}
