bool CPage::hasPageTimedOut() { bool result = false; // this checks if the whole download has passed the 'download' threshold Trace("Download Threshold","",getDownloadThreshold()); Trace("Overall Time","",getOverallTimeMs()); if (getDownloadThreshold() < (getOverallTimeMs() / 1000)) { result = true; // set timeout code for this page setResultCode(SC_RESULT_CODE_TIMEDOUT); // set timeout time setLastByteTime(); } return result; }
/** * Marks the whole task as complete and sets the result code. * * If the result code indicates a failure (|FAILED(@a aResultCode)|) then this * method will import the error info from the current thread and assign it to * the errorInfo attribute (it will return an error if no info is available in * such case). * * If the result code indicates a success (|SUCCEEDED(@a aResultCode)|) then * the current operation is set to the last. * * Note that this method may be called only once for the given Progress object. * Subsequent calls will assert. * * @param aResultCode Operation result code. */ HRESULT Progress::notifyComplete(HRESULT aResultCode) { AutoCaller autoCaller(this); AssertComRCReturnRC(autoCaller.rc()); AutoWriteLock alock(this COMMA_LOCKVAL_SRC_POS); AssertReturn(mCompleted == FALSE, E_FAIL); LogFunc(("aResultCode=%d\n", aResultCode)); if (mCanceled && SUCCEEDED(aResultCode)) aResultCode = E_FAIL; HRESULT rc = setResultCode(aResultCode); mCompleted = TRUE; if (!FAILED(aResultCode)) { m_ulCurrentOperation = m_cOperations - 1; /* last operation */ m_ulOperationPercent = 100; } #if !defined VBOX_COM_INPROC /* remove from the global collection of pending progress operations */ if (mParent) mParent->removeProgress(mId.ref()); #endif /* wake up all waiting threads */ if (mWaitersCount > 0) RTSemEventMultiSignal(mCompletedSem); return rc; }
resultid_t runWTTI(const char* url, std::string& dbName, unsigned int wttiRunID, unsigned int shardNumber, int webkitTimeout) { std::string strQuery; sem_init( &(CWebPage::Instance()->sm_hostSemaphore), 0, 0 ); sem_init( &(CWebPage::Instance()->sm_threadControlSemaphore), 0, 0 ); #ifdef SHARD_AWARE ConnectionPtr connection( new Connection( CConfig::getShardServer(shardNumber), CConfig::getShardDatabase(shardNumber), CConfig::getShardUser(shardNumber), CConfig::getShardPass(shardNumber), CConfig::getDBRetryTime())); #else ConnectionPtr connection( new Connection( CConfig::getDBServer(), CConfig::getDBDatabase(), CConfig::getDBUser(), CConfig::getDBPass(), CConfig::getDBRetryTime())); #endif std::string urlToTest; // Get URL from the WttiTests table. ResultPtr resultUrl; strQuery = Format(SQL_WTTI_SELECT_URL, wttiRunID); resultUrl = connection->Query(strQuery.c_str()); if ( resultUrl->Next() ) { RowPtr row = resultUrl->GetCurrentRow(); urlToTest = row->GetFieldString(1); Trace("Rows:", "", resultUrl->GetRowCount()); } resultid_t dailyResultID = 0; PackDiagStorage store; int pageID = 0; CResultSet resultSet; CTcp* pTcpConnection = new CTcp("", 0); // Turn CSS Parsing on permanently. CWebPage::Instance()->sm_doCssParsing = 1; //Turn SSL validation on permanently CWebPage::Instance()->sm_doSslValidation = 1; CTcp::setDoSslValidation(1); // create the new web page object CWebPage* pPage = new CWebPage( resultSet, pTcpConnection, 0, SC_SERVICE_BENCHMARK, urlToTest, // url pageID, // page id SC_WEBPAGE_TYPE_IMED, // page type true, // reset cookies std::string(), // auth user std::string(), // auth pass DOWNLOAD_THRESHOLD_SECS, // download threshold true, // reset cache CConfig::getServerID(), SC_WEBPAGE_STATUS_OK, // bodge to be status live page HTTP_METHOD_GET, // bodge to get GET by default std::string(), std::vector<std::string>(), std::vector<char>(), // emptyVector, // bodge - no form variables std::string() // fileStr // bodge - no file variables ); // reset the counters pPage->resetPage(); // record when we started the test //pPage->setStartTime(); // make sure we don't report a hacking / login failure // pPage->phraseWasFound(); // Set that a connection exists to this host. //pPage->isHostConnectionsExceeded(static_cast<std::string>(url)); // do the load of this page //pPage->load(); // add the page just loaded to the list of those done so it doesn't get done again // via a redirect //CWebPage::AddURLDone(url); // reset the first byte count for the page //pPage->resetSpeedCounters(); MultiThreadSetup(); // ---------------------- // --- THREAD MANAGER --- // ---------------------- std::string protoAndHost; protoAndHost = urlToTest; // Calculate protocol type. if (strncasecmp (protoAndHost.c_str(), "http://", 7) == 0) { std::string::size_type hostLength = protoAndHost.find("/", 7); if (hostLength != std::string::npos) protoAndHost = protoAndHost.substr(0, hostLength); } else if (strncasecmp (protoAndHost.c_str(), "https://", 8) == 0) { std::string::size_type hostLength = protoAndHost.find("/", 8); if (hostLength != std::string::npos) protoAndHost = protoAndHost.substr(0, hostLength); } else { std::string::size_type hostLength = protoAndHost.find("/"); if (hostLength != std::string::npos) protoAndHost = "http://" + protoAndHost.substr(0, hostLength); else protoAndHost = protoAndHost.insert(0, "http://"); } // Add the host that we're about to connect to so it does not get added again. { Mutex::Lock lock(CWebPage::Instance()->sm_connectedHostsLock); CWebPage::Instance()->sm_connectedHosts.push_back(std::make_pair(true, protoAndHost)); } std::vector<ThreadArgs*> threadArgsVector; // JYLS WEBKIT START #ifndef SC_NON_WEBKIT #ifdef SC_WEBKIT_V89 curlwrapper_init(pPage, startThread); //Use the default value from the opt file for these ww_setMaxNumberOfThreadsPerHost(CConfig::getDefaultMaxThreadsPerHost()); ww_setMaxTotalNumberOfThreads(CConfig::getDefaultMaxTotalThreads()); ww_setScreenXDimension(CConfig::getScreenXDimension()); ww_setScreenYDimension(CConfig::getScreenYDimension()); ww_setJavascriptTimeout(CConfig::getJsExecutionTimeout()); ww_loadPageByWebkit(pPage->getURL().c_str(), true, DOWNLOAD_THRESHOLD_SECS, webkitTimeout); #else bool isJavascriptEnabled = true; soupwrapper_init(pPage, startThread); ww_setPageId(pageID); ww_setWebkitTimeout(webkitTimeout); ww_setWebkitTimeoutHard(CConfig::getWebkitHardTimeout()); load_page_by_webkit(pPage->getURL(), isJavascriptEnabled); #endif // SC_WEBKIT_V89 #else std::vector<pthread_t> threadIds; pthread_t threadId; int res = 0; unsigned int maxThreadsPerHost = 0; if (maxThreadsPerHost == 0) maxThreadsPerHost = CConfig::getDefaultMaxThreadsPerHost(); //Limit the value of maxThreadsPerHost if (maxThreadsPerHost > CConfig::getMaxThreadsPerHostLimit()) maxThreadsPerHost = CConfig::getMaxThreadsPerHostLimit(); unsigned int numThreads = maxThreadsPerHost; for (unsigned int a = 0; a < numThreads; ++a) { ThreadArgs* args1 = new ThreadArgs; args1->isFirstPage = (a == 0); args1->pPage = pPage; args1->host = protoAndHost; res = pthread_create(&(threadId), NULL, startThread, (void*)args1); if ( res != 0 ) { LogError("Failed to start thread for", protoAndHost.c_str(), 0); } else { threadIds.push_back(threadId); threadArgsVector.push_back(args1); // Wait for the second thread to signal it has started. sem_wait(&(CWebPage::Instance()->sm_threadControlSemaphore)); } } std::vector<CWebPage::HostPair>::const_iterator hostIt; Trace("before starting threads", "", 0); while ((CWebPage::getUrlListSize()) > 0 || (CWebPage::Instance()->sm_numWorkingThreads > 0)) { { Mutex::Lock lock(CWebPage::Instance()->sm_connectedHostsLock); for ( hostIt = CWebPage::Instance()->sm_connectedHosts.begin(); hostIt != CWebPage::Instance()->sm_connectedHosts.end(); ++hostIt ) { if ( !(hostIt->first) ) { unsigned int numThreads = maxThreadsPerHost; for (unsigned int a = 0; a < numThreads; ++a) { bool errorOccured = false; std::string hostToConnect; hostToConnect.assign(hostIt->second); // Start threads per new host encountered. Trace("Starting threads for", hostIt->second.c_str(), 0); ThreadArgs* args2 = new ThreadArgs; args2->isFirstPage = false; args2->pPage = pPage; args2->host = hostToConnect; res = pthread_create(&(threadId), NULL, startThread, (void*)args2); if ( res != 0 ) { LogError("Failed to start thread for", hostIt->second.c_str(), 2); errorOccured = true; } else { threadIds.push_back(threadId); threadArgsVector.push_back(args2); // Wait for thread to start. sem_wait(&(CWebPage::Instance()->sm_threadControlSemaphore)); } // Connected ok. hostIt->first = true; } } } } // Wait for more hosts to be encountered. sem_wait(&(CWebPage::Instance()->sm_hostSemaphore)); } #endif // SC_NON_WEBKIT // JYLS WEBKIT END Trace("Deleting semaphore", "", 0); sem_destroy(&(CWebPage::Instance()->sm_hostSemaphore)); sem_destroy(&(CWebPage::Instance()->sm_threadControlSemaphore)); #ifdef SC_NON_WEBKIT // Write debug information if components remain. if (CWebPage::getUrlListSize() > 0) { CWebPage::writeThreadsDebug(); int count = 0; std::vector<ThreadArgs*>::const_iterator threadsIt; for (threadsIt = threadArgsVector.begin(); threadsIt != threadArgsVector.end(); ++threadsIt) { LogInfo("Threads started=", (*threadsIt)->host.c_str(), count); ++count; } } #endif // SC_NON_WEBKIT #ifndef SC_NON_WEBKIT // Sleep to allow for extra components from eg javascript usleep(1000000); //Actually check how many threads are running, go ahead if 0 while (CWebPage::Instance()->sm_numWorkingThreads) { usleep(10000); } #else void* threadResult; // Join started threads. for (unsigned int i = 0; i < threadIds.size(); ++i) { res = pthread_join(threadIds[i], &threadResult); if (res != 0) { LogError("Unable to join thread", "", res); } else { Trace("Joined thread", "", res); } } #endif // SC_NON_WEBKIT // Delete all ThreadArgs used. while ( !threadArgsVector.empty() ) { delete threadArgsVector.back(); threadArgsVector.pop_back(); } setResultCode(pPage, CWebPage::Instance()->sm_firstPageResultCode); try { Trace("Connecting to db", "", 0); std::string requestStr; std::string headerStr; std::string htmlStr; std::string extraInfoStr; // extra info for ResultHeader if (!resultSet.getResultDetails().empty()) { const CResultDetail* pResultDetail = resultSet.getFirstResultDetail(); requestStr = pResultDetail->getRequest(); headerStr = pResultDetail->getHeaders(); htmlStr = pResultDetail->getPageText(); extraInfoStr = pResultDetail->getExtraInfo(); } const bool pageInError = pPage->getOverallResultCode() != SC_HEADER_RESULT_SUCCESS && pPage->getOverallResultCode() != SC_HEADER_RESULT_DUMMY_TRANSACTION; // 3.0.0 Daily ResultHeader tables StoreHeaderDaily( connection, resultSet, pPage->getPageID(), CConfig::getServerID(), pPage->getStartTimeStr(), pPage->getOverallResultCode(), pPage->getDNSTimeOffsetMs(), pPage->getConnectTimeOffsetMs(), pPage->getTotalLastByteTimeOffsetMs(), pPage->getTotalBytes(), &dailyResultID, pPage->getFirstDataTimeOffsetMs(), pPage->getDailyTableName(), PackDiagOverride(), store, pageInError, false, SC_TEST_TYPE_WTTI, 0, 0, 0, CSCGlobal::gAgentVersion, pPage->getTotalGzipBytes(), false, //diagstored pPage->getSslConnectTimeOffsetMs(), pPage->getRequestSentTimeOffsetMs(), pPage->getRequestHeaderSize(), pPage->getRequestContentSize(), pPage->getResponseHeaderSize()); // now store all the detail - always do this for wtti int ComponentNo = 1; for (CResultSet::ResultList::const_iterator p = resultSet.getResultDetails().begin(); p != resultSet.getResultDetails().end(); ++p, ++ComponentNo) { // now go and get the rest of the files - with threads if (const CResultDetail* pResultDetail = p->pResultDetail) { pResultDetail->setComponentNo(ComponentNo); // 3.0.0 Daily ResultDetail tables StoreResultDaily( connection, resultSet, ComponentNo, pResultDetail, dailyResultID, pPage->getDailyTableName()); } } } catch (const std::exception &e) { Trace("Exception caught", e.what(), 0); LogError2("Exception Caught:", e.what(), 0, SC_ERR_GROUP_AGENT, SC_ERR_CODE_WTTI); return 0; } #ifdef SHARD_AWARE // Write the result ID back to the transient table so the website can pick it up. strQuery = Format(SQL_WTTI_INSERT_RESULTID, dailyResultID, wttiRunID); Trace("Issuing query", strQuery.c_str(), dailyResultID); connection->Query(strQuery); #endif // dbName = pPage->getDailyTableName(); // complete it delete pPage; pPage = NULL; MultiThreadCleanup(); // Remove /tmp/session_id.<pid>p.host files long int uniqueId = getpid(); std::string identifier = WTTP_SESSION_ID_IDENTIFIER; strQuery = Format("rm -f /tmp/session_id.%ld%s.*", uniqueId, identifier.c_str()); int ret = system(strQuery.c_str()); Trace("System call returned", strQuery.c_str(), ret); return dailyResultID; }
virtual void doWork() { try { StringBuffer userId; StringBuffer password; bool bLinux; if (m_sConfigAddress.length() < 1) { m_pService->getAccountAndPlatformInfo(m_sAddress.str(), userId, password, bLinux); } else { m_pService->getAccountAndPlatformInfo(m_sConfigAddress.str(), userId, password, bLinux); } if (!m_userId.length() || !m_password.length()) { //BUG: 9825 - remote execution on linux needs to use individual accounts //use userid/password in ESP context for remote execution... if (bLinux) { userId.clear(); password.clear(); m_context.getUserID(userId); m_context.getPassword(password); } } else { userId.clear().append(m_userId); password.clear().append(m_password); } if (!m_sCommand.length()) setResultCode(-1); else { IFRunSSH * connection = createFRunSSH(); connection->init(m_sCommand.str(),NULL,NULL,NULL,5,0); connection->exec(m_sAddress.str(),NULL,true); } } // CFRunSSH uses a MakeStringExceptionDirect throw to pass code and result string to caller catch(IException* e) { // errorCode == -1 on successful CFRunSSH execution if(e->errorCode() == -1) setResultCode(0); else setResultCode(e->errorCode()); StringBuffer buf; e->errorMessage(buf); if (buf.length() && buf.charAt(buf.length() - 1) == '\n') // strip newline buf.setLength(buf.length() - 1); // set regardless of return setResponse(buf.str()); e->Release(); } #ifndef NO_CATCHALL catch(...) { setResponse("An unknown exception occurred!"); setResultCode(-1); } #endif }//doWork()
virtual void doWork() { try { StringBuffer cmdLine; StringBuffer userId; StringBuffer password; bool bLinux; int exitCode = -1; if (m_sConfigAddress.length() < 1) { m_pService->getAccountAndPlatformInfo(m_sAddress.str(), userId, password, bLinux); } else { m_pService->getAccountAndPlatformInfo(m_sConfigAddress.str(), userId, password, bLinux); } if (m_userId.length() < 1 || m_userId.length() < 1) { //BUG: 9825 - remote execution on linux needs to use individual accounts //use userid/password in ESP context for remote execution... if (bLinux) { userId.clear(); password.clear(); m_context.getUserID(userId); m_context.getPassword(password); } } else { userId.clear().append(m_userId); password.clear().append(m_password); } #ifdef _WIN32 ///#define CHECK_LINUX_COMMAND #ifndef CHECK_LINUX_COMMAND #define popen _popen #define pclose _pclose // Use psexec as default remote control program if (bLinux) { if (!checkFileExists(".\\plink.exe")) throw MakeStringException(ECLWATCH_PLINK_NOT_INSTALLED, "Invalid ESP installation: missing plink.exe to execute the remote program!"); m_sCommand.replace('\\', '/');//replace all '\\' by '/' /* note that if we use plink (cmd line ssh client) for the first time with a computer, it generates the following message: The server's host key is not cached in the registry. You have no guarantee that the server is the computer you think it is. The server's key fingerprint is: 1024 aa:bb:cc:dd:ee:ff:gg:hh:ii:jj:kk:ll:mm:nn:oo:pp If you trust this host, enter "y" to add the key to PuTTY's cache and carry on connecting. If you want to carry on connecting just once, without adding the key to the cache, enter "n".If you do not trust this host, press Return to abandon the connection. To get around this, we pipe "n" to plink without using its -batch parameter. We need help from cmd.exe to do this though... */ if (!m_useDefaultSSHUserID) { cmdLine.appendf("cmd /c \"echo y | .\\plink.exe -ssh -l espuser -i id_rsa.ppk %s bash -c '%s' 2>&1\"", m_sAddress.str(), m_sCommand.str()); } else { cmdLine.appendf("cmd /c \"echo y | .\\plink.exe -ssh -l %s -pw %s %s sudo bash -c '%s' 2>&1\"", userId.str(), password.str(), m_sAddress.str(), m_sCommand.str()); } } else { if (!checkFileExists(".\\psexec.exe")) throw MakeStringException(ECLWATCH_PSEXEC_NOT_INSTALLED, "Invalid ESP installation: missing psexec.exe to execute the remote program!"); cmdLine.appendf(".\\psexec \\\\%s -u %s -p %s %s cmd /c %s 2>&1", m_sAddress.str(), userId.str(), password.str(), m_bWait ? "" : "-d", m_sCommand.str()); } #else if (bLinux) { if (!m_useDefaultSSHUserID) { m_sCommand.replace('\\', '/');//replace all '\\' by '/' cmdLine.appendf("ssh -o StrictHostKeyChecking=no -i /home/espuser/.ssh/id_rsa espuser@%s '%s'", m_sAddress.str(), m_sCommand.str()); } else { m_sCommand.replace('\\', '/');//replace all '\\' by '/' cmdLine.appendf("ssh -o StrictHostKeyChecking=no %s '%s'", m_sAddress.str(), m_sCommand.str()); } } else { setResponse("Remote execution from Linux to Windows is not supported!"); exitCode = 1; } #endif #else if (bLinux) { if (!m_useDefaultSSHUserID) { m_sCommand.replace('\\', '/');//replace all '\\' by '/' cmdLine.appendf("ssh -o StrictHostKeyChecking=no -i /home/espuser/.ssh/id_rsa espuser@%s '%s'", m_sAddress.str(), m_sCommand.str()); } else { m_sCommand.replace('\\', '/');//replace all '\\' by '/' cmdLine.appendf("ssh -o StrictHostKeyChecking=no %s '%s'", m_sAddress.str(), m_sCommand.str()); } } else { setResponse("Remote execution from Linux to Windows is not supported!"); exitCode = 1; } #endif if (*cmdLine.str()) { if (m_bWait) { StringBuffer response, response1; exitCode = invoke_program(cmdLine, response); if (exitCode < 0) response1.append("Failed in executing a system command.\n"); else response1.append("System command(s) has been executed.\n"); //remove \n at the end int len = response.length(); if (len > 0 && response.charAt(--len) == '\n') response.setLength(len); setResponse(response1.str()); } else { DWORD runCode; ::invoke_program(cmdLine, runCode, false); exitCode = (int) runCode; } } setResultCode(exitCode); } catch(IException* e) { StringBuffer buf; e->errorMessage(buf); setResponse(buf.str()); setResultCode(e->errorCode()); } #ifndef NO_CATCHALL catch(...) { setResponse("An unknown exception occurred!"); setResultCode(-1); } #endif }//doWork()