void CDirectory::InitStats(int num)
{
if (num) {
NumVariations = num;
if (Mode == RM_EXE) {
stat = RLS_EXE;
::NumVariationsTotal[RLS_EXE] += num;
::NumVariationsTotal[RLS_TOTAL] += num;
}
else {
if (FBaseDiff || FBaseline) {
stat = RLS_BASELINES; // initial stat
::NumVariationsTotal[RLS_BASELINES] += num;
::NumVariationsTotal[RLS_TOTAL] += num;
}
if (FBaseDiff || !FBaseline) {
if (!FBaseDiff)
stat = RLS_DIFFS; // initial stat
::NumVariationsTotal[RLS_DIFFS] += num;
::NumVariationsTotal[RLS_TOTAL] += num;
}
}
}
else {
ASSERTNR(FBaseDiff && !IsBaseline());
ASSERTNR(stat == RLS_BASELINES);
ASSERTNR(NumVariationsRun == NumVariations);
stat = RLS_DIFFS;
}
NumVariationsRun = NumFailures = NumDiffs = 0;
}
void
RLFEAddLog(
CDirectory *pDir,
RLFE_STATUS status,
const char *testName,
const char *subTestName,
const char *logText
)
{
int pathLen, logLen, num;
BYTE buf[4];
char pathBuf[BUFFER_SIZE], *path;
ASSERTNR(testName || subTestName);
ASSERTNR(logText);
path = pathBuf;
if (testName)
path += sprintf_s(path, REMAININGARRAYLEN(pathBuf, path), "%c%s", PIPE_SEP_CHAR, testName);
if (subTestName)
path += sprintf_s(path, REMAININGARRAYLEN(pathBuf, path), "%c%s", PIPE_SEP_CHAR, subTestName);
// String has a leading PIPE_SEP_CHAR, so chop that off.
path = pathBuf + 1;
pathLen = (int)strlen(path);
ASSERTNR(pathLen < BUFFER_SIZE - 1);
logLen = (int)strlen(logText);
num = pDir->GetDirectoryNumber();
ASSERTNR((SHORT)num == num);
*(SHORT *)buf = (SHORT)num;
buf[2] = (BYTE)pDir->stat;
buf[3] = (BYTE)status;
#ifdef TRACE
printf("RLFEAddLog: %d, %d, %s, %s\n", pDir->stat, num, path, logText);
#endif
if (RLFEEnterCritSec()) {
SendCommand(RLFE_ADD_LOG, 2 + pathLen + 4 + logLen);
SendDataWithLen((BYTE *)path, pathLen);
SendData(buf, 4);
SendData((BYTE *)logText, logLen);
RLFELeaveCritSec();
}
}
void COutputBuffer::Flush()
{
FILE* fp;
if (_type == OUT_FILE) {
Flush(_pfile);
}
else {
ASSERTNR(_type == OUT_FILENAME);
if (_filename != NULL) {
fp = fopen_unsafe(_filename, "a");
if (fp == NULL) {
// We might not be able to open the log or full log output
// files because someone is grepping or otherwise looking
// through them while rl is active. In that case, we don't
// want to just kill rl, so just keep accumulating output
// and try again next time. Output a warning to the log so
// they know it happened (but they won't see it unless the
// output is flushed). We could consider having a maximum,
// after which we "turn off" this output buffer, but we're
// unlikely to ever have so much output that it causes a
// problem.
Warning("Cannot open '%s' for appending with error '%s'", _filename, strerror_unsafe(errno));
}
else {
Flush(fp);
fclose(fp);
}
}
}
}
void
RLFEThreadDir(
CDirectory *pDir,
BYTE threadNum
)
{
BYTE buf[4];
int num;
if (pDir) {
num = pDir->GetDirectoryNumber();
ASSERTNR((SHORT)num == num);
}
else {
num = 0;
}
--threadNum; // RL is 1-based, RLFE is 0-based
*(SHORT *)buf = (SHORT)num;
buf[2] = threadNum;
if (pDir)
buf[3] = (BYTE)pDir->stat;
#ifdef TRACE
printf("RLFEThreadDir: %d, %d, %d\n", num, threadNum, buf[3]);
#endif
if (RLFEEnterCritSec()) {
SendCommand(RLFE_THREAD_DIR, 4);
SendData(buf, 4);
RLFELeaveCritSec();
}
}
void
RLFEAddTest(
RL_STATS stat,
CDirectory *pDir
)
{
char *path;
int len, num;
path = pDir->GetDirectoryName();
num = pDir->GetDirectoryNumber();
ASSERTNR((SHORT)num == num);
#ifdef TRACE
printf("RLFEAddTest: %d, %s, %d, %d\n", stat,
path, num, pDir->NumVariations);
#endif
len = (int)strlen(path) + 7;
if (RLFEEnterCritSec()) {
SendCommand(RLFE_ADD_TEST, len);
SendData((BYTE *)path, len - 7);
SendData((BYTE *)&pDir->NumVariations, 4);
SendData((BYTE *)&stat, 1);
SendData((BYTE *)&num, 2);
RLFELeaveCritSec();
}
}
void
RLFETestStatus(
CDirectory *pDir
)
{
DWORD *d;
BYTE buf[3];
BYTE statBuf[3 * 4];
DWORD dataLen;
int num;
num = pDir->GetDirectoryNumber();
ASSERTNR((SHORT)num == num);
if (pDir->stat == RLS_DIFFS)
dataLen = 3 * 4;
else
dataLen = 2 * 4;
d = (DWORD *)statBuf;
*d++ = (int) pDir->NumVariationsRun;
*d++ = (int) pDir->NumFailures;
if (pDir->stat == RLS_DIFFS)
*d++ = (int) pDir->NumDiffs;
*(SHORT *)buf = (SHORT)num;
buf[2] = (BYTE)pDir->stat;
#ifdef TRACE
printf("RLFETestStatus: %d, %d, %d, %d, %d\n",
pDir->stat, num, pDir->NumVariationsRun, pDir->NumFailures,
pDir->NumDiffs);
#endif
if (RLFEEnterCritSec()) {
SendCommand(RLFE_TEST_STATUS, 2 + 1 + dataLen);
SendData((BYTE *)&buf, 3);
SendData(statBuf, dataLen);
RLFELeaveCritSec();
}
}
// Add without doing varargs formatting (avoids local buffer size problems)
void COutputBuffer::AddDirect(char* string)
{
ASSERTNR(!_textGrabbed);
size_t len = strlen(string);
while ((_end - _start) + len >= _bufSize) {
char* pNew = new char[_bufSize * 2];
memcpy(pNew, _start, _end - _start + 1); // copy null
_end = pNew + (_end - _start);
delete[] _start;
_start = pNew;
_bufSize *= 2;
}
memcpy(_end, string, len + 1); // copy null
_end += len;
if (!_buffered || (!FRLFE && (NumberOfThreads == 1))) {
// no need to synchronize; flush immediately to get faster interaction
Flush();
}
}
void
RLFEInit(
BYTE numThreads,
int numDirs
)
{
BYTE buf[4];
ASSERTNR((SHORT)numDirs == numDirs);
InitializeCriticalSection(&CS);
#ifdef TRACE
printf("RLFEAddInit: %d, %d\n", numDirs, numThreads);
#endif
*(SHORT *)buf = (SHORT)numDirs;
buf[2] = numThreads;
buf[3] = (BYTE)RLFE_VERSION;
SendCommand(RLFE_INIT, 4);
SendData(buf, 4);
}
void
RunInit()
{
char *opts; // value of EXEC_TESTS_FLAGS environment variable
int numOptions;
int numPogoOptions;
int i;
atexit(RunCleanUp);
// Break EXEC_TESTS up into different sets of flags. The sets should
// be separated by semi-colons. Options don't apply to Pogo testing
// unless prefixed with POGO_TEST_PREFIX. Those options _only_ apply
// to Pogo tests.
opts = EXEC_TESTS_FLAGS;
ASSERTNR(opts);
numOptions = numPogoOptions = 0;
while (opts) {
while (isspace(*opts))
opts++;
if (*opts == '\0')
break;
if (!_strnicmp(opts, POGO_TEST_PREFIX, strlen(POGO_TEST_PREFIX))) {
opts += strlen(POGO_TEST_PREFIX);
PogoOptFlags[numPogoOptions] = opts;
++numPogoOptions;
if (numPogoOptions == MAXOPTIONS)
Fatal("Too many options in EXEC_TESTS_FLAGS");
}
else {
OptFlags[numOptions] = opts;
++numOptions;
if (numOptions == MAXOPTIONS)
Fatal("Too many options in EXEC_TESTS_FLAGS");
}
opts = strchr(opts, ';');
if (opts)
*opts++ = '\0';
}
for (i = 0; i < numPogoOptions; i++) {
if (strstr(PogoOptFlags[i], "GL") == NULL) {
Fatal("Pogo without LTCG is not supported");
}
}
OptFlags[numOptions] = NULL;
PogoOptFlags[numPogoOptions] = NULL;
if (FVerbose) {
printf("(Normal) Exec flags:");
for (i = 0; i < numOptions; i++) {
printf(" '%s'", OptFlags[i]);
}
printf("\nPogo Exec flags:");
for (i = 0; i < numPogoOptions; i++) {
printf(" '%s'", PogoOptFlags[i]);
}
printf("\n");
}
}
// Handle external test scripts. We support three kinds, makefiles (rl.mak),
// command shell (dotest.cmd), and JScript (*.js).
//
// Standardized makefiles have the following targets:
// clean: delete all generated files
// build: build the test (OPT=compile options)
// run: run the test
// copy: copy the generated files to a subdirectory (COPYDIR=subdir)
//
int
DoOneExternalTest(
CDirectory* pDir,
TestVariant *pTestVariant,
const char *optFlags,
const char *inCCFlags,
const char *inLinkFlags,
const char *testCmd,
ExternalTestKind kind,
BOOL fSyncVariationWhenFinished,
BOOL fCleanBefore,
BOOL fCleanAfter,
BOOL fSuppressNoGPF,
void *envFlags,
DWORD millisecTimeout
)
{
#define NMAKE "nmake -nologo -R -f "
char full[MAX_PATH];
char cmdbuf[BUFFER_SIZE];
char buf[BUFFER_SIZE];
char ccFlags[BUFFER_SIZE];
char linkFlags[BUFFER_SIZE];
char nogpfFlags[BUFFER_SIZE];
char optReportBuf[BUFFER_SIZE];
char nonZeroReturnBuf[BUFFER_SIZE];
const char *reason = NULL;
time_t start_variation;
UINT elapsed_variation;
time_t start_build_variation;
UINT elapsed_build_variation;
LARGE_INTEGER start_run, end_run, frequency;
UINT elapsed_run;
BOOL fFailed = FALSE;
BOOL fDumpOutputFile = FVerbose;
BOOL fFileToDelete = FALSE;
int cmdResult;
static unsigned int testCount = 0;
unsigned int localTestCount = InterlockedIncrement(&testCount);
// Avoid conditionals by copying/creating ccFlags appropriately.
if (inCCFlags)
{
if (pDir->HasTestInfoData(TIK_SOURCE_PATH))
{
sprintf_s(ccFlags, " %s -baselinePath:%s", inCCFlags, pDir->GetDirectoryPath());
}
else
{
sprintf_s(ccFlags, " %s", inCCFlags);
}
}
else
{
ccFlags[0] = '\0';
}
switch (TargetMachine) {
case TM_WVM:
case TM_WVMX86:
case TM_WVM64:
strcat_s(ccFlags, " /BC ");
break;
}
if (inLinkFlags)
strcpy_s(linkFlags, inLinkFlags);
else
linkFlags[0] = '\0';
sprintf_s(optReportBuf, "%s%s%s", optFlags, *linkFlags ? ";" : "", linkFlags);
// Update the status.
sprintf_s(buf, " (%s)", optReportBuf);
ThreadInfo[ThreadId].SetCurrentTest(pDir->GetDirectoryName(),
buf, pDir->IsBaseline());
UpdateTitleStatus();
// Make sure the file that will say pass or fail is not present.
sprintf_s(full, "%s\\testout%d", pDir->GetDirectoryPath(), localTestCount);
DeleteFileIfFound(full);
start_variation = time(NULL);
if (kind == TK_MAKEFILE) {
Message(""); // newline
Message("Processing %s with '%s' flags",
testCmd, optReportBuf);
Message(""); // newline
if (FTest)
{
return 0;
}
if (fCleanBefore) {
// Clean the directory.
sprintf_s(cmdbuf, NMAKE"%s clean", testCmd);
Message(cmdbuf);
ExecuteCommand(pDir->GetDirectoryPath(), cmdbuf);
}
FillNoGPFFlags(nogpfFlags, fSuppressNoGPF);
// Build the test.
start_build_variation = time(NULL);
sprintf_s(cmdbuf, NMAKE"%s build OPT=\"%s %s%s\" LINKFLAGS=\"%s %s %s\"",
testCmd,
optFlags, EXTRA_CC_FLAGS, ccFlags,
LINKFLAGS, linkFlags, nogpfFlags);
if (strlen(cmdbuf) > BUFFER_SIZE - 1)
Fatal("Buffer overrun");
Message(cmdbuf);
fFailed = ExecuteCommand(pDir->GetDirectoryPath(), cmdbuf);
elapsed_build_variation = (int)(time(NULL) - start_build_variation);
if (Timing & TIME_VARIATION) {
Message("RL: Variation elapsed time (build) (%s, %s, %s): %02d:%02d",
pDir->GetDirectoryName(),
"rl.mak",
optReportBuf,
elapsed_build_variation / 60, elapsed_build_variation % 60);
}
if (fFailed) {
reason = "build failure";
goto logFailure;
}
// Run the test.
QueryPerformanceCounter(&start_run);
sprintf_s(cmdbuf, NMAKE"%s run", testCmd);
Message(cmdbuf);
cmdResult = ExecuteCommand(pDir->GetDirectoryPath(), cmdbuf, millisecTimeout);
QueryPerformanceCounter(&end_run);
QueryPerformanceFrequency(&frequency);
elapsed_run = (int) (((end_run.QuadPart - start_run.QuadPart) * 1000UI64) / frequency.QuadPart);
if (Timing & TIME_VARIATION) {
Message("RL: Variation elapsed time (run) (%s, %s, %s): %02d:%02d.%03d",
pDir->GetDirectoryName(),
"rl.mak",
optReportBuf,
elapsed_run / 60000, (elapsed_run % 60000)/1000, elapsed_run % 1000);
}
}
else if (kind == TK_CMDSCRIPT)
{
// Build up the test command string
sprintf_s(cmdbuf, "%s %s %s%s >testout%d", testCmd, optFlags, EXTRA_CC_FLAGS, ccFlags, localTestCount);
Message("Running '%s'", cmdbuf);
if (FTest)
{
return 0;
}
cmdResult = ExecuteCommand(pDir->GetDirectoryPath(), cmdbuf, millisecTimeout, envFlags);
}
else if (kind == TK_JSCRIPT || kind==TK_HTML || kind == TK_COMMAND)
{
char tempExtraCCFlags[MAX_PATH*2] = {0};
// Only append when EXTRA_CC_FLAGS isn't empty.
if (EXTRA_CC_FLAGS[0])
{
// Append test case unique identifier to the end of EXTRA_CC_FLAGS.
if (FAppendTestNameToExtraCCFlags)
{
sprintf_s(tempExtraCCFlags, "%s.%s", EXTRA_CC_FLAGS, pTestVariant->testInfo.data[TIK_FILES]);
}
else
{
strcpy_s(tempExtraCCFlags, EXTRA_CC_FLAGS);
}
}
const char* cmd = JCBinary;
if (kind != TK_JSCRIPT && kind != TK_HTML)
{
cmd = pTestVariant->testInfo.data[TIK_COMMAND];
}
sprintf_s(cmdbuf, "%s %s %s %s %s >%s 2>&1", cmd, optFlags, tempExtraCCFlags, ccFlags, testCmd, full);
Message("Running '%s'", cmdbuf);
if(FTest)
{
DeleteFileIfFound(full);
return 0;
}
cmdResult = ExecuteCommand(pDir->GetFullPathFromSourceOrDirectory(), cmdbuf, millisecTimeout, envFlags);
if (cmdResult && cmdResult != WAIT_TIMEOUT && !pTestVariant->testInfo.data[TIK_BASELINE]) // failure code, not baseline diffing
{
fFailed = TRUE;
sprintf_s(nonZeroReturnBuf, "non-zero (%08X) return value from test command", cmdResult);
reason = nonZeroReturnBuf;
goto logFailure;
}
}
else
{
ASSERTNR(UNREACHED);
cmdResult = NOERROR; // calm compiler warning about uninitialized variable usage
}
// Check for timeout.
if (cmdResult == WAIT_TIMEOUT) {
ASSERT(millisecTimeout != INFINITE);
sprintf_s(nonZeroReturnBuf, "timed out after %u second%s", millisecTimeout / 1000, millisecTimeout == 1000 ? "" : "s");
reason = nonZeroReturnBuf;
fFailed = TRUE;
goto logFailure;
}
// If we have a baseline test, we need to check the baseline file.
if (pTestVariant->testInfo.data[TIK_BASELINE]) {
char baseline_file[_MAX_PATH];
sprintf_s(baseline_file, "%s\\%s", pDir->GetFullPathFromSourceOrDirectory(),
pTestVariant->testInfo.data[TIK_BASELINE]);
if (DoCompare(baseline_file, full, pTestVariant->testInfo.hasData[TIK_EOL_NORMALIZATION])) {
reason = "diffs from baseline";
sprintf_s(optReportBuf, "%s", baseline_file);
fFailed = TRUE;
CopyRebaseFile(full, baseline_file);
}
}
else if ((kind == TK_JSCRIPT || kind == TK_HTML || kind == TK_COMMAND) && !pTestVariant->testInfo.hasData[TIK_BASELINE]) {
if (!CheckForPass(full, optReportBuf, cmdbuf, fDumpOutputFile)) {
fFailed = TRUE;
goto SkipLogFailure;
}
}
logFailure:
if (fFailed) {
LogOut("ERROR: Test failed to run correctly: %s (%s):",
reason, optReportBuf);
LogOut(" %s", cmdbuf);
if (fDumpOutputFile) {
DumpFileToLog(full);
}
}
SkipLogFailure:
if (fFileToDelete && !FNoDelete) {
DeleteFileRetryMsg(full);
}
elapsed_variation = (int)(time(NULL) - start_variation);
if (Timing & TIME_VARIATION) {
Message("RL: Variation elapsed time (%s, %s, %s): %02d:%02d",
pDir->GetDirectoryName(),
kind == TK_MAKEFILE ? "rl.mak" : "dotest.cmd",
optReportBuf,
elapsed_variation / 60, elapsed_variation % 60);
}
if (kind == TK_MAKEFILE) {
// If the test failed and we are asked to copy the failures, do so.
if (fFailed && FCopyOnFail) {
sprintf_s(cmdbuf, NMAKE"%s copy COPYDIR=\"fail.%s.%s\"",
testCmd, optFlags, linkFlags);
Message(cmdbuf);
ExecuteCommand(pDir->GetDirectoryPath(), cmdbuf);
}
// Clean up after ourselves.
if (!FNoDelete && (fFailed || fCleanAfter)) {
sprintf_s(cmdbuf, NMAKE"%s clean", testCmd);
Message(cmdbuf);
ExecuteCommand(pDir->GetDirectoryPath(), cmdbuf);
}
}
if (FSyncVariation) {
if (FRLFE && fFailed) {
RLFEAddLog(pDir, RLFES_FAILED, testCmd,
optReportBuf, ThreadOut->GetText());
}
if (fSyncVariationWhenFinished)
FlushOutput();
}
DeleteFileIfFound(full);
return fFailed ? -1 : 0;
}
int
ExecTest
(
CDirectory* pDir,
Test * pTest,
TestVariant * pTestVariant
)
{
char *p = NULL;
char full[MAX_PATH];
DWORD millisecTimeout = DEFAULT_TEST_TIMEOUT;
const char *strTimeout = pTestVariant->testInfo.data[TIK_TIMEOUT];
if (strTimeout) {
char *end;
_set_errno(0);
uint32 secTimeout = strtoul(strTimeout, &end, 10);
millisecTimeout = 1000 * secTimeout;
// Validation has already occurred so this string should
// parse fine and the value shouldn't overflow the DWORD.
ASSERT(errno == 0 && *end == 0);
ASSERT(millisecTimeout > secTimeout);
}
// Check to see if all of the files exist.
for (StringList * pFile = pTest->files; pFile != NULL; pFile = pFile->next)
{
// Get a pointer to the filename sans path, if present.
p = GetFilenamePtr(pFile->string);
// If we have no pathname, use the current directory.
if (p == pFile->string) {
sprintf_s(full, "%s\\", pDir->GetFullPathFromSourceOrDirectory());
}
else {
// Look for %REGRESS% specifier.
if (!_strnicmp(pFile->string, "%REGRESS%",
strlen("%REGRESS%"))) {
// Temporarily truncate the filename.
ASSERT(p[-1] == '\\');
p[-1] = '\0';
sprintf_s(full, "%s%s\\",
REGRESS, pFile->string + strlen("%REGRESS%"));
p[-1] = '\\';
}
else {
*p = '\0';
}
}
strcat_s(full, p);
if (GetFileAttributes(full) == INVALID_FILE_ATTRIBUTES) {
LogError("ERROR: '%s' does not exist", full);
return -1;
}
}
const char* ext = GetFilenameExt(p);
// Special case dotest.cmd
if (!_stricmp(p, "dotest.cmd")) {
// We don't handle these yet.
ASSERTNR(pTestVariant->testInfo.data[TIK_LINK_FLAGS] == NULL);
if (IsPogoTest(pTest))
return DoPogoExternalTest(pDir, pTestVariant, full, TK_CMDSCRIPT, TRUE, millisecTimeout);
else
return DoExternalTest(pDir, pTestVariant, full, TK_CMDSCRIPT, TRUE, millisecTimeout);
}
// Special case for standardized RL makefiles.
else if (!_stricmp(p, "rl.mak")) {
// We don't handle these yet.
ASSERTNR(pTestVariant->testInfo.data[TIK_LINK_FLAGS] == NULL);
if (IsPogoTest(pTest))
return DoPogoExternalTest(pDir, pTestVariant, full, TK_MAKEFILE, FALSE, millisecTimeout);
else
return DoExternalTest(pDir, pTestVariant, full, TK_MAKEFILE, SuppressNoGPF(pTest), millisecTimeout);
}
// Special case for files ending with ".js", ".html", ".htm" (<command> dealt with separately)
else if (pTestVariant->testInfo.data[TIK_COMMAND] == NULL
&& !_stricmp(ext, ".js"))
{
return DoExternalTest(pDir, pTestVariant, full, TK_JSCRIPT, FALSE, millisecTimeout);
}
else if (pTestVariant->testInfo.data[TIK_COMMAND] == NULL
&& (!_stricmp(ext, ".html") || !_stricmp(ext, ".htm")))
{
return DoExternalTest(pDir, pTestVariant, full, TK_HTML, FALSE, millisecTimeout);
}
// Special case for tests with a <command> argument
else if (pTestVariant->testInfo.data[TIK_COMMAND] != NULL)
{
return DoExternalTest(pDir, pTestVariant, full, TK_COMMAND, FALSE, millisecTimeout);
}
// Non-scripted test.
else {
if (IsPogoTest(pTest)) {
// We don't handle these yet.
ASSERTNR(pTestVariant->testInfo.data[TIK_LINK_FLAGS] == NULL);
return DoPogoSimpleTest(pDir, pTest, pTestVariant, FALSE, millisecTimeout);
}
else
{
return DoSimpleTest(pDir, pTest, pTestVariant, SuppressNoGPF(pTest), millisecTimeout);
}
}
}
