void schedSecondTest(ConnectionPtr connection, long idNo, int portNo, SC_TEST_TYPE testType, unsigned int shardNumber)
{
Trace("schedSecondTest", "", idNo);
std::string strQuery;
// find a server to send the test to
try
{
#ifdef SHARD_AWARE
ConnectionPtr passportShardConnection(
new Connection(
CConfig::getPassportDbServer(),
CConfig::getPassportDbDatabase(),
CConfig::getPassportDbUser(),
CConfig::getPassportDbPass(),
CConfig::getDBRetryTime()));
// get the group of this test server
strQuery = Format(SQL_SCS_TEST_GROUP_CG3, CConfig::getServerID());
TraceSQL("Issuing query", "", 0);
int testGroup = 0;
if (ResultPtr result = passportShardConnection->Query(strQuery))
{
TraceSQL("Rows:","",result->GetRowCount());
if (result->GetRowCount() > 0)
{
result->Next();
RowPtr row = result->GetCurrentRow();
testGroup = row->GetFieldInteger(1);
}
}
// get the carrier to exclude
if (portNo == TCP_PORT_WTTGD)
{
strQuery = Format(SQL_SCS_TEST_GROUP_EXCLUDE_SCRIPT, idNo);
}
else if (portNo == TCP_PORT_ETTD)
{
strQuery = Format(SQL_SCS_TEST_GROUP_EXCLUDE_ISP, idNo);
}
else
{
strQuery = Format(SQL_SCS_TEST_GROUP_EXCLUDE_WEBPAGE, idNo);
}
TraceSQL("Issuing query", "", 0);
unsigned int carrierToExclude = 0;
if (ResultPtr result = connection->Query(strQuery))
{
TraceSQL("Rows:","",result->GetRowCount());
if (result->GetRowCount() > 0)
{
result->Next();
RowPtr row = result->GetCurrentRow();
// make it the customer one
carrierToExclude = row->GetFieldInteger(1);
// override with Script or WebPage one
if (row->GetFieldInteger(2) != 0)
carrierToExclude = row->GetFieldInteger(2);
}
}
#ifdef SC_NON_WEBKIT
strQuery = Format(SQL_SCS_RETEST_SERVERS_GROUP_RAND_CG3, SC_SERVER_STATUS_LIVE, testGroup, carrierToExclude);
#else
#ifdef SC_WEBKIT_V89
strQuery = Format(SQL_SCS_RETEST_SERVERS_GROUP_RAND_CG3, SC_SERVER_JACK_STATUS_LIVE_89, testGroup, carrierToExclude);
#else
strQuery = Format(SQL_SCS_RETEST_SERVERS_GROUP_RAND_CG3, SC_SERVER_JACK_STATUS_LIVE, testGroup, carrierToExclude);
#endif // SC_WEBKIT_V89
#endif // SC_NON_WEBKIT
TraceSQL("Issuing query", "", 0);
if (ResultPtr result = passportShardConnection->Query(strQuery))
{
TraceSQL("Rows:","",result->GetRowCount());
#else
// get the group of this test server
strQuery = Format(SQL_SCS_TEST_GROUP, CConfig::getServerID());
TraceSQL("Issuing query", "", 0);
if (ResultPtr result = connection->Query(strQuery))
{
std::string site;
std::string clusterGroup;
TraceSQL("Rows:","",result->GetRowCount());
if (result->GetRowCount() > 0)
{
result->Next();
RowPtr row = result->GetCurrentRow();
testGroup = row->GetFieldInteger(1);
site = row->GetFieldString(2);
clusterGroup = row->GetFieldString(3);
}
// first work out the minute
//time_t timeNow = time(NULL);
//struct tm* timeStruct;
//timeStruct = localtime( &timeNow );
strQuery = Format(SQL_SCS_RETEST_SERVERS_GROUP_RAND, SC_SERVER_STATUS_LIVE, testGroup, site.c_str(), clusterGroup.c_str());
}
TraceSQL("Issuing query", "", 0);
if (ResultPtr result = connection->Query(strQuery))
{
TraceSQL("Rows:","",result->GetRowCount());
#endif
if (result->GetRowCount() > 0)
{
std::string testHost;
while (result->Next())
{
RowPtr row = result->GetCurrentRow();
testHost = row->GetFieldString(2);
// try to send the job
// send the test request
CTcp tcpConnection(testHost, portNo);
int fd = tcpConnection.tcp_connect(WTT_WTTD_TIMEOUT_SECS);
if (fd > 0)
{
#ifdef SHARD_AWARE
// send the info to the testing tool
if (testType == SC_TEST_TYPE_UNKNOWN)
{
strQuery = Format("%ld %d", idNo, shardNumber);
}
else
{
strQuery = Format("%ld %d %d", idNo, shardNumber, int(testType));
}
#else
// send the info to the testing tool
if (testType == SC_TEST_TYPE_UNKNOWN)
{
strQuery = Format("%ld", idNo);
}
else
{
strQuery = Format("%ld %d", idNo, int(testType));
}
#endif
TraceLogic("tcpConnection.writen", strQuery.c_str(), strQuery.size())
if (tcpConnection.writen(strQuery.c_str(), strQuery.size()) > 0)
{
Trace("Job sent to test server", strQuery.c_str(), fd);
return;
}
else
{
LogError("Error sending job to test server", strQuery.c_str(), idNo);
}
}
else
{
LogError("Timed out sending job to test server", "", idNo);
}
}
}
bool checkConnectivity(ConnectionPtr connection)
{
// checks to see if this server has connectivity
// it does so by finding two servers (from the TestServer table) in other groups, and hence on other carriers
// if both of these pings fail then we can assume that it is our connectivity
// we can then re-schedule the job
// now also used to see if we are down before the end of the check
std::string testHost;
std::string externalTestHost;
std::string externalTestHost1;
std::string externalTestHost2;
int ourGroup = 0;
int testGroup = 0;
ResultPtr result;
ncc::safe_array<char> selectQuery(1024);
bool retVal = true;
std::string site;
std::string clusterGroup;
Trace("checkConnectivity", "CConfig::getServerID()", CConfig::getServerID());
try
{
#ifdef SHARD_AWARE
ConnectionPtr passportShardConnection(
new Connection(
CConfig::getPassportDbServer(),
CConfig::getPassportDbDatabase(),
CConfig::getPassportDbUser(),
CConfig::getPassportDbPass(),
CConfig::getDBRetryTime()));
// get the group of this test server
snprintf(selectQuery, selectQuery.size(), SQL_SCS_TEST_GROUP_CG3, CConfig::getServerID());
Trace("Issuing query",selectQuery,0);
result = passportShardConnection->Query(selectQuery);
Trace("Rows:","",result->GetRowCount());
if (result->GetRowCount() > 0)
{
result->Next();
RowPtr row = result->GetCurrentRow();
ourGroup = row->GetFieldInteger(1);
}
snprintf(selectQuery, selectQuery.size(), SQL_SCS_RETEST_SERVERS_GROUP_RAND_CG3,
SC_SERVER_STATUS_LIVE, ourGroup, 0);
Trace("Issuing query",selectQuery,0);
result = passportShardConnection->Query(selectQuery);
Trace("Rows:","",result->GetRowCount());
#else
// get the group of this test server
snprintf(selectQuery, selectQuery.size(), SQL_SCS_TEST_GROUP, CConfig::getServerID());
Trace("Issuing query",selectQuery,0);
result = connection->Query(selectQuery);
Trace("Rows:","",result->GetRowCount());
if (result->GetRowCount() > 0)
{
result->Next();
RowPtr row = result->GetCurrentRow();
ourGroup = row->GetFieldInteger(1);
site = row->GetFieldString(2);
clusterGroup = row->GetFieldString(3);
}
snprintf(selectQuery, selectQuery.size(), SQL_SCS_RETEST_SERVERS_GROUP_RAND, SC_SERVER_STATUS_LIVE, ourGroup, site.c_str(), clusterGroup.c_str());
Trace("Issuing query",selectQuery,0);
result = connection->Query(selectQuery);
Trace("Rows:","",result->GetRowCount());
#endif
int testGroup1 = 0;
int testGroup2 = 0;
std::string externalTestHost1;
std::string externalTestHost2;
if (result->GetRowCount() > 0)
{
while(result->Next())
{
RowPtr row = result->GetCurrentRow();
testHost = row->GetFieldString(2);
testGroup = row->GetFieldInteger(3);
externalTestHost = row->GetFieldString(4);
Trace("externalHost",externalTestHost.c_str(),testGroup);
if (testGroup1 == 0)
{
// set this to be the first group to test
testGroup1 = testGroup;
externalTestHost1 = externalTestHost;
}
else if (testGroup1 != testGroup)
{
// different group - use for the second test
testGroup2 = testGroup;
externalTestHost2 = externalTestHost;
// all done now
break;
}
// else must be same group as testGroup1 - ignore it
}
bool ping1OK = true;
bool ping2OK = true;
if ((testGroup1 != 0) && (externalTestHost1 != ""))
{
Trace("Tesing against group 1",externalTestHost1.c_str(),testGroup1);
ping1OK = pingSiteAvailable(externalTestHost1);
}
else
{
// nothing to test - assume it worked ?
Trace("Tesing against group 1","n/a",testGroup1);
}
if ((testGroup2 != 0) && (externalTestHost2 != ""))
{
Trace("Tesing against group 2",externalTestHost2.c_str(),testGroup2);
ping2OK = pingSiteAvailable(externalTestHost2);
}
else if ((testGroup1 != 0) && (externalTestHost1 != ""))
{
// we tested a 'real' #1 - test the one from sc.opt
Trace("Tesing against group 2 - sc.opt",CConfig::getPing1Name().c_str(),testGroup2);
ping2OK = pingSiteAvailable(CConfig::getPing1Name());
}
else
{
// else nothing to test - assume it worked ?
Trace("Tesing against group 2","n/a",testGroup2);
}
if (!ping1OK && !ping2OK)
{
// first two have failed
// check yahoo
if (!pingSiteAvailable(CConfig::getPing1Name()))
{
retVal = false;
}
}
}
}
catch (const std::exception& e)
{
Trace("Exception caught", e.what(), 0);
printf("Error: %s\n", e.what());
}
return retVal;
}
// per page tests
void runWTTT(scriptid_t scriptID, unsigned int shardNumber, const char* runNo, long batchID, bool wait, bool debug)
{
// only do a sleep if the first run - not for manual or retests
if ((atoi(runNo) == 1) && wait)
{
TraceNoise("Sleeping for a bit", runNo, scriptID);
// sleep for an offset of seconds
sleep(scriptID % 60);
}
ConnectionPtr connection;
std::string strQuery;
bool download = true;
try
{
#ifdef SHARD_AWARE
TraceLogic("Connecting to db", CConfig::getShardDatabase(shardNumber).c_str(), shardNumber);
connection.reset(
new Connection(
CConfig::getShardServer(shardNumber),
CConfig::getShardDatabase(shardNumber),
CConfig::getShardUser(shardNumber),
CConfig::getShardPass(shardNumber),
CConfig::getDBRetryTime()));
#else
TraceLogic("Connecting to db", CConfig::getDBDatabase(), 0);
connection.reset(
new Connection(
CConfig::getDBServer(),
CConfig::getDBDatabase(),
CConfig::getDBUser(),
CConfig::getDBPass(),
CConfig::getDBRetryTime()));
#endif
}
catch (const std::exception &e)
{
LogError("Exception Caught:", e.what(), scriptID);
return;
}
// if the ScriptID == 0 - get it from the batch table
if (scriptID == 0)
{
strQuery = Format(SQL_WTTT_BATCH_SEL, batchID);
TraceSQL("Issuing", "", 0);
if (ResultPtr result = connection->Query(strQuery))
{
if (result->Next())
{
RowPtr row = result->GetCurrentRow();
scriptID = row->GetFieldLong(1);
TraceLogic("batchID", "", batchID);
TraceLogic("scriptID", "", scriptID);
}
}
}
// script and archive names
const std::string filename = Format("/home/sc/bin/monitor_scripts/%ld_%u", scriptID, shardNumber);
const std::string gzfilename = filename + ".gz";
// Get script details
long customerNo = 0;
SC_SCRIPT_LANGUAGE scriptlanguage = SC_SCRIPT_LANGUAGE_UNKNOWN;
std::string scriptversion;
strQuery = Format(SQL_WTTT_SCRIPTINFO_SEL, scriptID);
TraceSQL("Issuing", "", 0);
if (ResultPtr result = connection->Query(strQuery))
{
if (result->Next())
{
RowPtr row = result->GetCurrentRow();
customerNo = row->GetFieldLong(1);
scriptlanguage = static_cast<SC_SCRIPT_LANGUAGE>(row->GetFieldLong(2));
// Field can be NULL, so check before dereference
if (const char* str = row->GetField(3))
{
scriptversion = str;
}
TraceLogic("customerNo", "", customerNo);
TraceLogic("scriptlanguage", to_string(scriptlanguage).c_str(), int(scriptlanguage));
TraceLogic("scriptversion", scriptversion.c_str(), scriptversion.size());
}
}
switch (scriptlanguage)
{
case SC_SCRIPT_LANGUAGE_UNKNOWN:
//
// No recognised script or script language returned
//
LogError(
"Script not found",
Format("ScriptID=%d Shard=%d DB.name=%s DB.host=%s",
scriptID,
shardNumber,
CConfig::getShardDatabase(shardNumber).c_str(),
CConfig::getShardServer(shardNumber).c_str()).c_str(),
0);
exit(1);
break;
case SC_SCRIPT_LANGUAGE_PHP:
//
// Handle PHP script
//
strQuery = Format("select Code, length(Code) from Script where ScriptID=%ld", scriptID);
TraceSQL("Issuing", "", 0);
if (ResultPtr result = connection->Query(strQuery))
{
if (result->Next())
{
RowPtr row = result->GetCurrentRow();
int scriptLength = row->GetFieldInteger(2);
std::string scriptString(row->GetField(1), scriptLength);
TraceLogic("length(Code)", "", scriptLength);
umask(SC_WTTT_UMASK);
int filePtr = open(filename.c_str(), O_CREAT | O_TRUNC | O_WRONLY, S_IRWXU);
if (filePtr != -1)
{
// output the body to the file
int nbytes = write(filePtr, scriptString.c_str(), scriptLength);
if (nbytes != scriptLength)
{
LogError("Incomplete write of PHP executeable to disc", to_string(nbytes).c_str(), scriptLength);
}
close(filePtr);
}
else
{
LogError("Unable to write PHP executable to disc", "", scriptID);
}
}
}
break;
case SC_SCRIPT_LANGUAGE_CXX:
//
// Handle compiled C++ script
//
if (Exist(filename))
{
// Is the file out of date?
strQuery = Format(SQL_WTTT_SCRIPTAGE_SEL, scriptID);
TraceSQL("Issuing", "", 0);
if (ResultPtr result = connection->Query(strQuery))
{
if (result->Next())
{
RowPtr row = result->GetCurrentRow();
long dbage = row->GetFieldLong(1);
TraceLogic("dbage", "", dbage);
struct stat statbuf;
bzero(&statbuf, 0);
int ret = stat(filename.c_str(), &statbuf);
TraceLogic("stat", "download", ret);
download = ret == -1 || statbuf.st_mtime < dbage;
TraceLogic("Overwrite the existing %s?", "download", int(download));
TraceLogic("statbuf.st_mtime", "", statbuf.st_mtime);
}
}
}
//
// Extract script from database
//
if (download)
{
strQuery = Format(SQL_WTTT_SCRIPT_SEL, scriptID);
TraceSQL("Issuing", "", 0);
if (ResultPtr result = connection->Query(strQuery))
{
if (result->Next())
{
RowPtr row = result->GetCurrentRow();
int scriptLength = row->GetFieldInteger(2);
std::string scriptString(row->GetField(1), scriptLength);
// Field can be NULL, so check before dereference
int agentVersion = 0;
if (const char* str = row->GetField(3))
{
agentVersion = atoi(str);
}
TraceLogic("scriptLength", "", scriptLength);
TraceLogic("scriptString", scriptString.c_str(), scriptString.size());
TraceLogic("agentVersion", "", agentVersion);
umask(SC_WTTT_UMASK);
int filePtr = open(gzfilename.c_str(), O_CREAT | O_TRUNC | O_WRONLY, S_IRWXU);
if (filePtr != -1)
{
// output the body to the file
int nbytes = write(filePtr, scriptString.c_str(), scriptLength);
close(filePtr);
if (nbytes == scriptLength)
{
// unzip, set permissions and prelink
doUnzip(gzfilename);
chmod(filename.c_str(), 0775);
bool prelink = false;
if (!CConfig::getIMMode())
{
TraceLogic("Connecting to passport db", CConfig::getPassportDbDatabase().c_str(), 0);
ConnectionPtr connectionPassport(
new Connection(
CConfig::getPassportDbServer(),
CConfig::getPassportDbDatabase(),
CConfig::getPassportDbUser(),
CConfig::getPassportDbPass(),
CConfig::getDBRetryTime()));
strQuery = Format("select Prelink from LibraryVersions where Version=%d", agentVersion);
TraceSQL("Issuing", "", 0);
if (ResultPtr result = connectionPassport->Query(strQuery))
{
if (result->Next())
{
RowPtr row = result->GetCurrentRow();
prelink = row->GetFieldInteger(1) != 0;
TraceLogic("prelink", to_string(prelink).c_str(), int(prelink));
}
}
TraceLogic("prelink", "", int(prelink));
}
else
{
TraceLogic("prelink", "Don't prelink on IM node", 0);
}
if (prelink)
{
int ret = doPrelink(filename, SC_AGENT_VER_NUM);
if (ret != 0)
{
LogError("Prelink failed", filename.c_str(), ret);
}
}
}
else
{
LogError("Incomplete write of C++ executeable to disc", to_string(nbytes).c_str(), scriptLength);
return;
}
}
else
{
LogError("Unable to write C++ executable to disc", "", scriptID);
return;
}
}
else
{
LogError("C++ Executable not in database", gzfilename.c_str(), result->GetRowCount());
return;
}
}
else
{
LogError("Query for C++ executable failed", strQuery.c_str(), result->GetRowCount());
return;
}
}
break;
default:
LogError("Fatal: Unknown language", "", int(scriptlanguage));
return;
}
// close the db connection
Clear(connection);
//
// Build command line
//
std::string systemCmd;
#ifdef USE_SHARED
// For version 7
if (scriptlanguage == SC_SCRIPT_LANGUAGE_CXX &&
Exist("/home/sc/lib/libsiteconfidence.so." + scriptversion) &&
Exist("/home/sc/lib/libscparser.so." + scriptversion) &&
Exist("/home/sc/lib/libsiteconfidenceinterface.so." + scriptversion))
{
std::ostringstream os;
os << "export LD_PRELOAD=";
os << "/home/sc/lib/libsiteconfidence.so." << scriptversion << ":";
os << "/home/sc/lib/libscparser.so." << scriptversion << ":";
os << "/home/sc/lib/libsiteconfidenceinterface.so." << scriptversion << "; ";
systemCmd += os.str();
}
#endif
#ifdef SHARD_AWARE
// set environment variables for debug and shardNumber
{
std::ostringstream os;
os << "export SHARD_NUMBER=" << shardNumber << "; "
<< "export SC_DEBUG_MANUAL_MODE=" << (debug != 0) << "; ";
systemCmd += os.str();
}
#endif
// program command line
{
std::ostringstream os;
os << filename << " " << runNo << " " << batchID << " " << scriptID << " " << customerNo;
systemCmd += os.str();
}
//Print(Format("starting script:%s\n", systemCmd.c_str()));
// execute
TraceLogic("Starting script", systemCmd.c_str(), scriptID);
int sysVal = system(systemCmd.c_str());
int exitVal = WEXITSTATUS(sysVal);
TraceLogic("System call returned", systemCmd.c_str(), exitVal);
//Print(Format("script returned:%d\n", exitVal));
}