bool CPage::hasPageTimedOut()
{
bool result = false;
// this checks if the whole download has passed the 'download' threshold
Trace("Download Threshold","",getDownloadThreshold());
Trace("Overall Time","",getOverallTimeMs());
if (getDownloadThreshold() < (getOverallTimeMs() / 1000))
{
result = true;
// set timeout code for this page
setResultCode(SC_RESULT_CODE_TIMEDOUT);
// set timeout time
setLastByteTime();
}
return result;
}
/**
* Marks the whole task as complete and sets the result code.
*
* If the result code indicates a failure (|FAILED(@a aResultCode)|) then this
* method will import the error info from the current thread and assign it to
* the errorInfo attribute (it will return an error if no info is available in
* such case).
*
* If the result code indicates a success (|SUCCEEDED(@a aResultCode)|) then
* the current operation is set to the last.
*
* Note that this method may be called only once for the given Progress object.
* Subsequent calls will assert.
*
* @param aResultCode Operation result code.
*/
HRESULT Progress::notifyComplete(HRESULT aResultCode)
{
AutoCaller autoCaller(this);
AssertComRCReturnRC(autoCaller.rc());
AutoWriteLock alock(this COMMA_LOCKVAL_SRC_POS);
AssertReturn(mCompleted == FALSE, E_FAIL);
LogFunc(("aResultCode=%d\n", aResultCode));
if (mCanceled && SUCCEEDED(aResultCode))
aResultCode = E_FAIL;
HRESULT rc = setResultCode(aResultCode);
mCompleted = TRUE;
if (!FAILED(aResultCode))
{
m_ulCurrentOperation = m_cOperations - 1; /* last operation */
m_ulOperationPercent = 100;
}
#if !defined VBOX_COM_INPROC
/* remove from the global collection of pending progress operations */
if (mParent)
mParent->removeProgress(mId.ref());
#endif
/* wake up all waiting threads */
if (mWaitersCount > 0)
RTSemEventMultiSignal(mCompletedSem);
return rc;
}
resultid_t runWTTI(const char* url, std::string& dbName, unsigned int wttiRunID, unsigned int shardNumber, int webkitTimeout)
{
std::string strQuery;
sem_init( &(CWebPage::Instance()->sm_hostSemaphore), 0, 0 );
sem_init( &(CWebPage::Instance()->sm_threadControlSemaphore), 0, 0 );
#ifdef SHARD_AWARE
ConnectionPtr connection(
new Connection(
CConfig::getShardServer(shardNumber),
CConfig::getShardDatabase(shardNumber),
CConfig::getShardUser(shardNumber),
CConfig::getShardPass(shardNumber),
CConfig::getDBRetryTime()));
#else
ConnectionPtr connection(
new Connection(
CConfig::getDBServer(),
CConfig::getDBDatabase(),
CConfig::getDBUser(),
CConfig::getDBPass(),
CConfig::getDBRetryTime()));
#endif
std::string urlToTest;
// Get URL from the WttiTests table.
ResultPtr resultUrl;
strQuery = Format(SQL_WTTI_SELECT_URL, wttiRunID);
resultUrl = connection->Query(strQuery.c_str());
if ( resultUrl->Next() )
{
RowPtr row = resultUrl->GetCurrentRow();
urlToTest = row->GetFieldString(1);
Trace("Rows:", "", resultUrl->GetRowCount());
}
resultid_t dailyResultID = 0;
PackDiagStorage store;
int pageID = 0;
CResultSet resultSet;
CTcp* pTcpConnection = new CTcp("", 0);
// Turn CSS Parsing on permanently.
CWebPage::Instance()->sm_doCssParsing = 1;
//Turn SSL validation on permanently
CWebPage::Instance()->sm_doSslValidation = 1;
CTcp::setDoSslValidation(1);
// create the new web page object
CWebPage* pPage = new CWebPage(
resultSet,
pTcpConnection,
0,
SC_SERVICE_BENCHMARK,
urlToTest, // url
pageID, // page id
SC_WEBPAGE_TYPE_IMED, // page type
true, // reset cookies
std::string(), // auth user
std::string(), // auth pass
DOWNLOAD_THRESHOLD_SECS, // download threshold
true, // reset cache
CConfig::getServerID(),
SC_WEBPAGE_STATUS_OK, // bodge to be status live page
HTTP_METHOD_GET, // bodge to get GET by default
std::string(),
std::vector<std::string>(),
std::vector<char>(), // emptyVector, // bodge - no form variables
std::string() // fileStr // bodge - no file variables
);
// reset the counters
pPage->resetPage();
// record when we started the test
//pPage->setStartTime();
// make sure we don't report a hacking / login failure
// pPage->phraseWasFound();
// Set that a connection exists to this host.
//pPage->isHostConnectionsExceeded(static_cast<std::string>(url));
// do the load of this page
//pPage->load();
// add the page just loaded to the list of those done so it doesn't get done again
// via a redirect
//CWebPage::AddURLDone(url);
// reset the first byte count for the page
//pPage->resetSpeedCounters();
MultiThreadSetup();
// ----------------------
// --- THREAD MANAGER ---
// ----------------------
std::string protoAndHost;
protoAndHost = urlToTest;
// Calculate protocol type.
if (strncasecmp (protoAndHost.c_str(), "http://", 7) == 0)
{
std::string::size_type hostLength = protoAndHost.find("/", 7);
if (hostLength != std::string::npos)
protoAndHost = protoAndHost.substr(0, hostLength);
}
else if (strncasecmp (protoAndHost.c_str(), "https://", 8) == 0)
{
std::string::size_type hostLength = protoAndHost.find("/", 8);
if (hostLength != std::string::npos)
protoAndHost = protoAndHost.substr(0, hostLength);
}
else
{
std::string::size_type hostLength = protoAndHost.find("/");
if (hostLength != std::string::npos)
protoAndHost = "http://" + protoAndHost.substr(0, hostLength);
else
protoAndHost = protoAndHost.insert(0, "http://");
}
// Add the host that we're about to connect to so it does not get added again.
{
Mutex::Lock lock(CWebPage::Instance()->sm_connectedHostsLock);
CWebPage::Instance()->sm_connectedHosts.push_back(std::make_pair(true, protoAndHost));
}
std::vector<ThreadArgs*> threadArgsVector;
// JYLS WEBKIT START
#ifndef SC_NON_WEBKIT
#ifdef SC_WEBKIT_V89
curlwrapper_init(pPage, startThread);
//Use the default value from the opt file for these
ww_setMaxNumberOfThreadsPerHost(CConfig::getDefaultMaxThreadsPerHost());
ww_setMaxTotalNumberOfThreads(CConfig::getDefaultMaxTotalThreads());
ww_setScreenXDimension(CConfig::getScreenXDimension());
ww_setScreenYDimension(CConfig::getScreenYDimension());
ww_setJavascriptTimeout(CConfig::getJsExecutionTimeout());
ww_loadPageByWebkit(pPage->getURL().c_str(), true, DOWNLOAD_THRESHOLD_SECS, webkitTimeout);
#else
bool isJavascriptEnabled = true;
soupwrapper_init(pPage, startThread);
ww_setPageId(pageID);
ww_setWebkitTimeout(webkitTimeout);
ww_setWebkitTimeoutHard(CConfig::getWebkitHardTimeout());
load_page_by_webkit(pPage->getURL(), isJavascriptEnabled);
#endif // SC_WEBKIT_V89
#else
std::vector<pthread_t> threadIds;
pthread_t threadId;
int res = 0;
unsigned int maxThreadsPerHost = 0;
if (maxThreadsPerHost == 0)
maxThreadsPerHost = CConfig::getDefaultMaxThreadsPerHost();
//Limit the value of maxThreadsPerHost
if (maxThreadsPerHost > CConfig::getMaxThreadsPerHostLimit())
maxThreadsPerHost = CConfig::getMaxThreadsPerHostLimit();
unsigned int numThreads = maxThreadsPerHost;
for (unsigned int a = 0; a < numThreads; ++a)
{
ThreadArgs* args1 = new ThreadArgs;
args1->isFirstPage = (a == 0);
args1->pPage = pPage;
args1->host = protoAndHost;
res = pthread_create(&(threadId), NULL, startThread, (void*)args1);
if ( res != 0 )
{
LogError("Failed to start thread for", protoAndHost.c_str(), 0);
}
else
{
threadIds.push_back(threadId);
threadArgsVector.push_back(args1);
// Wait for the second thread to signal it has started.
sem_wait(&(CWebPage::Instance()->sm_threadControlSemaphore));
}
}
std::vector<CWebPage::HostPair>::const_iterator hostIt;
Trace("before starting threads", "", 0);
while ((CWebPage::getUrlListSize()) > 0 || (CWebPage::Instance()->sm_numWorkingThreads > 0))
{
{
Mutex::Lock lock(CWebPage::Instance()->sm_connectedHostsLock);
for ( hostIt = CWebPage::Instance()->sm_connectedHosts.begin(); hostIt != CWebPage::Instance()->sm_connectedHosts.end(); ++hostIt )
{
if ( !(hostIt->first) )
{
unsigned int numThreads = maxThreadsPerHost;
for (unsigned int a = 0; a < numThreads; ++a)
{
bool errorOccured = false;
std::string hostToConnect;
hostToConnect.assign(hostIt->second);
// Start threads per new host encountered.
Trace("Starting threads for", hostIt->second.c_str(), 0);
ThreadArgs* args2 = new ThreadArgs;
args2->isFirstPage = false;
args2->pPage = pPage;
args2->host = hostToConnect;
res = pthread_create(&(threadId), NULL, startThread, (void*)args2);
if ( res != 0 )
{
LogError("Failed to start thread for", hostIt->second.c_str(), 2);
errorOccured = true;
}
else
{
threadIds.push_back(threadId);
threadArgsVector.push_back(args2);
// Wait for thread to start.
sem_wait(&(CWebPage::Instance()->sm_threadControlSemaphore));
}
// Connected ok.
hostIt->first = true;
}
}
}
}
// Wait for more hosts to be encountered.
sem_wait(&(CWebPage::Instance()->sm_hostSemaphore));
}
#endif // SC_NON_WEBKIT
// JYLS WEBKIT END
Trace("Deleting semaphore", "", 0);
sem_destroy(&(CWebPage::Instance()->sm_hostSemaphore));
sem_destroy(&(CWebPage::Instance()->sm_threadControlSemaphore));
#ifdef SC_NON_WEBKIT
// Write debug information if components remain.
if (CWebPage::getUrlListSize() > 0)
{
CWebPage::writeThreadsDebug();
int count = 0;
std::vector<ThreadArgs*>::const_iterator threadsIt;
for (threadsIt = threadArgsVector.begin(); threadsIt != threadArgsVector.end(); ++threadsIt)
{
LogInfo("Threads started=", (*threadsIt)->host.c_str(), count);
++count;
}
}
#endif // SC_NON_WEBKIT
#ifndef SC_NON_WEBKIT
// Sleep to allow for extra components from eg javascript
usleep(1000000);
//Actually check how many threads are running, go ahead if 0
while (CWebPage::Instance()->sm_numWorkingThreads)
{
usleep(10000);
}
#else
void* threadResult;
// Join started threads.
for (unsigned int i = 0; i < threadIds.size(); ++i)
{
res = pthread_join(threadIds[i], &threadResult);
if (res != 0)
{
LogError("Unable to join thread", "", res);
}
else
{
Trace("Joined thread", "", res);
}
}
#endif // SC_NON_WEBKIT
// Delete all ThreadArgs used.
while ( !threadArgsVector.empty() )
{
delete threadArgsVector.back();
threadArgsVector.pop_back();
}
setResultCode(pPage, CWebPage::Instance()->sm_firstPageResultCode);
try
{
Trace("Connecting to db", "", 0);
std::string requestStr;
std::string headerStr;
std::string htmlStr;
std::string extraInfoStr;
// extra info for ResultHeader
if (!resultSet.getResultDetails().empty())
{
const CResultDetail* pResultDetail = resultSet.getFirstResultDetail();
requestStr = pResultDetail->getRequest();
headerStr = pResultDetail->getHeaders();
htmlStr = pResultDetail->getPageText();
extraInfoStr = pResultDetail->getExtraInfo();
}
const bool pageInError = pPage->getOverallResultCode() != SC_HEADER_RESULT_SUCCESS && pPage->getOverallResultCode() != SC_HEADER_RESULT_DUMMY_TRANSACTION;
// 3.0.0 Daily ResultHeader tables
StoreHeaderDaily(
connection,
resultSet,
pPage->getPageID(),
CConfig::getServerID(),
pPage->getStartTimeStr(),
pPage->getOverallResultCode(),
pPage->getDNSTimeOffsetMs(),
pPage->getConnectTimeOffsetMs(),
pPage->getTotalLastByteTimeOffsetMs(),
pPage->getTotalBytes(),
&dailyResultID,
pPage->getFirstDataTimeOffsetMs(),
pPage->getDailyTableName(),
PackDiagOverride(),
store,
pageInError,
false,
SC_TEST_TYPE_WTTI,
0,
0,
0,
CSCGlobal::gAgentVersion,
pPage->getTotalGzipBytes(),
false, //diagstored
pPage->getSslConnectTimeOffsetMs(),
pPage->getRequestSentTimeOffsetMs(),
pPage->getRequestHeaderSize(),
pPage->getRequestContentSize(),
pPage->getResponseHeaderSize());
// now store all the detail - always do this for wtti
int ComponentNo = 1;
for (CResultSet::ResultList::const_iterator p = resultSet.getResultDetails().begin(); p != resultSet.getResultDetails().end(); ++p, ++ComponentNo)
{
// now go and get the rest of the files - with threads
if (const CResultDetail* pResultDetail = p->pResultDetail)
{
pResultDetail->setComponentNo(ComponentNo);
// 3.0.0 Daily ResultDetail tables
StoreResultDaily(
connection,
resultSet,
ComponentNo,
pResultDetail,
dailyResultID,
pPage->getDailyTableName());
}
}
}
catch (const std::exception &e)
{
Trace("Exception caught", e.what(), 0);
LogError2("Exception Caught:", e.what(), 0, SC_ERR_GROUP_AGENT, SC_ERR_CODE_WTTI);
return 0;
}
#ifdef SHARD_AWARE
// Write the result ID back to the transient table so the website can pick it up.
strQuery = Format(SQL_WTTI_INSERT_RESULTID, dailyResultID, wttiRunID);
Trace("Issuing query", strQuery.c_str(), dailyResultID);
connection->Query(strQuery);
#endif
//
dbName = pPage->getDailyTableName();
// complete it
delete pPage;
pPage = NULL;
MultiThreadCleanup();
// Remove /tmp/session_id.<pid>p.host files
long int uniqueId = getpid();
std::string identifier = WTTP_SESSION_ID_IDENTIFIER;
strQuery = Format("rm -f /tmp/session_id.%ld%s.*", uniqueId, identifier.c_str());
int ret = system(strQuery.c_str());
Trace("System call returned", strQuery.c_str(), ret);
return dailyResultID;
}
virtual void doWork()
{
try
{
StringBuffer userId;
StringBuffer password;
bool bLinux;
if (m_sConfigAddress.length() < 1)
{
m_pService->getAccountAndPlatformInfo(m_sAddress.str(), userId, password, bLinux);
}
else
{
m_pService->getAccountAndPlatformInfo(m_sConfigAddress.str(), userId, password, bLinux);
}
if (!m_userId.length() || !m_password.length())
{
//BUG: 9825 - remote execution on linux needs to use individual accounts
//use userid/password in ESP context for remote execution...
if (bLinux)
{
userId.clear();
password.clear();
m_context.getUserID(userId);
m_context.getPassword(password);
}
}
else
{
userId.clear().append(m_userId);
password.clear().append(m_password);
}
if (!m_sCommand.length())
setResultCode(-1);
else
{
IFRunSSH * connection = createFRunSSH();
connection->init(m_sCommand.str(),NULL,NULL,NULL,5,0);
connection->exec(m_sAddress.str(),NULL,true);
}
}
// CFRunSSH uses a MakeStringExceptionDirect throw to pass code and result string to caller
catch(IException* e)
{
// errorCode == -1 on successful CFRunSSH execution
if(e->errorCode() == -1)
setResultCode(0);
else
setResultCode(e->errorCode());
StringBuffer buf;
e->errorMessage(buf);
if (buf.length() && buf.charAt(buf.length() - 1) == '\n') // strip newline
buf.setLength(buf.length() - 1);
// set regardless of return
setResponse(buf.str());
e->Release();
}
#ifndef NO_CATCHALL
catch(...)
{
setResponse("An unknown exception occurred!");
setResultCode(-1);
}
#endif
}//doWork()
virtual void doWork()
{
try
{
StringBuffer cmdLine;
StringBuffer userId;
StringBuffer password;
bool bLinux;
int exitCode = -1;
if (m_sConfigAddress.length() < 1)
{
m_pService->getAccountAndPlatformInfo(m_sAddress.str(), userId, password, bLinux);
}
else
{
m_pService->getAccountAndPlatformInfo(m_sConfigAddress.str(), userId, password, bLinux);
}
if (m_userId.length() < 1 || m_userId.length() < 1)
{
//BUG: 9825 - remote execution on linux needs to use individual accounts
//use userid/password in ESP context for remote execution...
if (bLinux)
{
userId.clear();
password.clear();
m_context.getUserID(userId);
m_context.getPassword(password);
}
}
else
{
userId.clear().append(m_userId);
password.clear().append(m_password);
}
#ifdef _WIN32
///#define CHECK_LINUX_COMMAND
#ifndef CHECK_LINUX_COMMAND
#define popen _popen
#define pclose _pclose
// Use psexec as default remote control program
if (bLinux)
{
if (!checkFileExists(".\\plink.exe"))
throw MakeStringException(ECLWATCH_PLINK_NOT_INSTALLED, "Invalid ESP installation: missing plink.exe to execute the remote program!");
m_sCommand.replace('\\', '/');//replace all '\\' by '/'
/*
note that if we use plink (cmd line ssh client) for the first time with a computer,
it generates the following message:
The server's host key is not cached in the registry. You have no guarantee that the
server is the computer you think it is. The server's key fingerprint is:
1024 aa:bb:cc:dd:ee:ff:gg:hh:ii:jj:kk:ll:mm:nn:oo:pp
If you trust this host, enter "y" to add the key to
PuTTY's cache and carry on connecting. If you want to carry on connecting just once,
without adding the key to the cache, enter "n".If you do not trust this host, press
Return to abandon the connection.
To get around this, we pipe "n" to plink without using its -batch parameter. We need
help from cmd.exe to do this though...
*/
if (!m_useDefaultSSHUserID)
{
cmdLine.appendf("cmd /c \"echo y | .\\plink.exe -ssh -l espuser -i id_rsa.ppk %s bash -c '%s' 2>&1\"",
m_sAddress.str(), m_sCommand.str());
}
else
{
cmdLine.appendf("cmd /c \"echo y | .\\plink.exe -ssh -l %s -pw %s %s sudo bash -c '%s' 2>&1\"",
userId.str(), password.str(), m_sAddress.str(), m_sCommand.str());
}
}
else
{
if (!checkFileExists(".\\psexec.exe"))
throw MakeStringException(ECLWATCH_PSEXEC_NOT_INSTALLED, "Invalid ESP installation: missing psexec.exe to execute the remote program!");
cmdLine.appendf(".\\psexec \\\\%s -u %s -p %s %s cmd /c %s 2>&1",
m_sAddress.str(), userId.str(), password.str(),
m_bWait ? "" : "-d", m_sCommand.str());
}
#else
if (bLinux)
{
if (!m_useDefaultSSHUserID)
{
m_sCommand.replace('\\', '/');//replace all '\\' by '/'
cmdLine.appendf("ssh -o StrictHostKeyChecking=no -i /home/espuser/.ssh/id_rsa espuser@%s '%s'", m_sAddress.str(), m_sCommand.str());
}
else
{
m_sCommand.replace('\\', '/');//replace all '\\' by '/'
cmdLine.appendf("ssh -o StrictHostKeyChecking=no %s '%s'", m_sAddress.str(), m_sCommand.str());
}
}
else
{
setResponse("Remote execution from Linux to Windows is not supported!");
exitCode = 1;
}
#endif
#else
if (bLinux)
{
if (!m_useDefaultSSHUserID)
{
m_sCommand.replace('\\', '/');//replace all '\\' by '/'
cmdLine.appendf("ssh -o StrictHostKeyChecking=no -i /home/espuser/.ssh/id_rsa espuser@%s '%s'", m_sAddress.str(), m_sCommand.str());
}
else
{
m_sCommand.replace('\\', '/');//replace all '\\' by '/'
cmdLine.appendf("ssh -o StrictHostKeyChecking=no %s '%s'", m_sAddress.str(), m_sCommand.str());
}
}
else
{
setResponse("Remote execution from Linux to Windows is not supported!");
exitCode = 1;
}
#endif
if (*cmdLine.str())
{
if (m_bWait)
{
StringBuffer response, response1;
exitCode = invoke_program(cmdLine, response);
if (exitCode < 0)
response1.append("Failed in executing a system command.\n");
else
response1.append("System command(s) has been executed.\n");
//remove \n at the end
int len = response.length();
if (len > 0 && response.charAt(--len) == '\n')
response.setLength(len);
setResponse(response1.str());
}
else
{
DWORD runCode;
::invoke_program(cmdLine, runCode, false);
exitCode = (int) runCode;
}
}
setResultCode(exitCode);
}
catch(IException* e)
{
StringBuffer buf;
e->errorMessage(buf);
setResponse(buf.str());
setResultCode(e->errorCode());
}
#ifndef NO_CATCHALL
catch(...)
{
setResponse("An unknown exception occurred!");
setResultCode(-1);
}
#endif
}//doWork()
