/*
* Checks if RunawayCleaner_RunawayCleanupDoneForProcess prevents multiple cleanup
* for a single runaway event by properly updating beginCleanupRunawayVersion and
* endCleanupRunawayVersion
*/
void
test__RunawayCleaner_RunawayCleanupDoneForProcess__PreventsDuplicateCleanup(void **state)
{
InitFakeSessionState(2 /* activeProcessCount */,
2 /* cleanupCountdown */,
RunawayStatus_PrimaryRunawaySession /* runawayStatus */, CLEANUP_COUNTDOWN /* pinCount */, 12345 /* vmem */);
static fakeLatestRunawayVersion = 10;
latestRunawayVersion = &fakeLatestRunawayVersion;
*latestRunawayVersion = 10;
/*
* Some imaginary cleanup begin/end event version. The idea is to ensure
* that once the RunawayCleaner_RunawayCleanupDoneForProcess call returns
* we will have both set to latestRunawayVersion
*/
beginCleanupRunawayVersion = *latestRunawayVersion;
endCleanupRunawayVersion = 0;
/* Make sure the cleanup goes through */
vmemTrackerInited = true;
isProcessActive = true;
RunawayCleaner_RunawayCleanupDoneForProcess(false /* ignoredCleanup */);
/* cleanupCountdown should be adjusted */
assert_true(MySessionState->cleanupCountdown == CLEANUP_COUNTDOWN - 1);
assert_true(beginCleanupRunawayVersion == endCleanupRunawayVersion);
assert_true(beginCleanupRunawayVersion == *latestRunawayVersion);
/* Second call shouldn't change anything */
RunawayCleaner_RunawayCleanupDoneForProcess(false /* ignoredCleanup */);
/* cleanupCountdown is unchanged */
assert_true(MySessionState->cleanupCountdown == CLEANUP_COUNTDOWN - 1);
}
/*
* Checks if RunawayCleaner_RunawayCleanupDoneForProcess reactivates the runaway detector
* once all the processes of the runaway session are done cleaning
*/
void
test__RunawayCleaner_RunawayCleanupDoneForProcess__ReactivatesRunawayDetection(void **state)
{
InitFakeSessionState(2 /* activeProcessCount */,
2 /* cleanupCountdown */,
RunawayStatus_PrimaryRunawaySession /* runawayStatus */, 2 /* pinCount */, 12345 /* vmem */);
static EventVersion fakeLatestRunawayVersion = 10;
latestRunawayVersion = &fakeLatestRunawayVersion;
/*
* Set beginCleanupRunawayVersion to latestRunawayVersion and endCleanupRunawayVersion
* to a smaller value to simulate an ongoing cleanup
*/
beginCleanupRunawayVersion = *latestRunawayVersion;
endCleanupRunawayVersion = 1;
/* Valid isRunawayDetector is necessary for Assert */
static uint32 fakeIsRunawayDetector = 1;
isRunawayDetector = &fakeIsRunawayDetector;
/* Just an active process that never became idle */
activationVersion = 0;
deactivationVersion = 0;
isProcessActive = true;
/* Make sure the cleanup goes through */
vmemTrackerInited = true;
RunawayCleaner_RunawayCleanupDoneForProcess(false /* ignoredCleanup */);
/* The cleanupCountdown must be decremented as we cleaned up */
assert_true(MySessionState->cleanupCountdown == 1);
/* We updated the endCleanupRunawayVersion to indicate that we finished cleanup */
assert_true(endCleanupRunawayVersion == beginCleanupRunawayVersion);
/* The runaway detector promotion should be disabled as we still have 1 QE unclean */
assert_true(*isRunawayDetector == 1);
/*
* Fake a ongoing cleanup by making endCleanupRunawayVersion < beginCleanupRunawayVersion
* so that we can execute cleanup one more time, marking all QEs clean
*/
endCleanupRunawayVersion = 1;
/*
* cleanupCountdown should reach 0, and immediately afterwards should be set to
* CLEANUP_COUNTDOWN_BEFORE_RUNAWAY
*/
RunawayCleaner_RunawayCleanupDoneForProcess(false /* ignoredCleanup */);
assert_true(MySessionState->cleanupCountdown == CLEANUP_COUNTDOWN_BEFORE_RUNAWAY);
/* Runaway detector should be re-enabled */
assert_true(*isRunawayDetector == 0);
}
/*
* Starts a runaway cleanup by triggering an ERROR if the VMEM tracker is active
* and a commit is not already in progress. Otherwise, it marks the process as clean
*/
void
RunawayCleaner_StartCleanup()
{
/*
* Cleanup can be attempted from multiple places, such as before deactivating
* a process (if a pending runaway event) or periodically from CHECK_FOR_INTERRUPTS
* (indirectly via RedZoneHandler_DetectRunaway). We don't carry multiple cleanup
* for a single runaway event. Every time we *start* a cleanup process, we set the
* beginCleanupRunawayVersion to the runaway version for which we started cleaning
* up. Later on, if we reenter this method (e.g., another CHECK_FOR_INTERRUPTS()
* during cleanup), we can observe that the cleanup already started from this runaway
* event, and therefore we skip duplicate cleanup
*/
if (RunawayCleaner_ShouldStartRunawayCleanup())
{
Assert(beginCleanupRunawayVersion < *latestRunawayVersion);
Assert(endCleanupRunawayVersion < *latestRunawayVersion);
/* We don't want to cleanup multiple times for same runaway event */
beginCleanupRunawayVersion = *latestRunawayVersion;
if (CritSectionCount == 0 && InterruptHoldoffCount == 0 && vmemTrackerInited &&
gp_command_count > 0 /* Cleaning up QEs that are not executing a valid command
may cause the QD to get stuck [MPP-24950] */ &&
/* Super user is terminated only when it's the primary runaway consumer (i.e., the top consumer) */
(!superuser() || MySessionState->runawayStatus == RunawayStatus_PrimaryRunawaySession))
{
#ifdef FAULT_INJECTOR
FaultInjector_InjectFaultIfSet(
RunawayCleanup,
DDLNotSpecified,
"", // databaseName
""); // tableName
#endif
ereport(ERROR, (errmsg("Canceling query because of high VMEM usage. Used: %dMB, available %dMB, red zone: %dMB",
VmemTracker_ConvertVmemChunksToMB(MySessionState->sessionVmem), VmemTracker_GetAvailableVmemMB(),
RedZoneHandler_GetRedZoneLimitMB()), errprintstack(true)));
}
/*
* If we cannot error out because of a critical section or because we are a super user
* or for some other reason (such as the QE is not running any valid command, i.e.,
* gp_command_count is not positive) simply declare this process as clean
*/
RunawayCleaner_RunawayCleanupDoneForProcess(true /* ignoredCleanup */);
}
}
/*
* Checks if RunawayCleaner_RunawayCleanupDoneForProcess reactivates a process
* if the deactivation process triggers cleanup for a pending runaway event
*/
void
test__RunawayCleaner_RunawayCleanupDoneForProcess__UndoDeactivation(void **state)
{
InitFakeSessionState(2 /* activeProcessCount */,
2 /* cleanupCountdown */,
RunawayStatus_PrimaryRunawaySession /* runawayStatus */, 2 /* pinCount */, 12345 /* vmem */);
static fakeLatestRunawayVersion = 10;
latestRunawayVersion = &fakeLatestRunawayVersion;
/*
* Set beginCleanupRunawayVersion to latestRunawayVersion and endCleanupRunawayVersion
* to a smaller value to simulate an ongoing cleanup
*/
beginCleanupRunawayVersion = *latestRunawayVersion;
endCleanupRunawayVersion = 1;
/* Valid isRunawayDetector is necessary for Assert */
static uint32 fakeIsRunawayDetector = 1;
isRunawayDetector = &fakeIsRunawayDetector;
/* Make sure we became idle after a pending runaway event */
activationVersion = 1;
deactivationVersion = *latestRunawayVersion + 1;
/* Make sure the cleanup goes through */
vmemTrackerInited = true;
isProcessActive = false;
/* We must undo the idle state */
will_be_called(IdleTracker_ActivateProcess);
RunawayCleaner_RunawayCleanupDoneForProcess(false /* ignoredCleanup */);
/* The cleanupCountdown must be decremented as we cleaned up */
assert_true(MySessionState->cleanupCountdown == 1);
/* We updated the endCleanupRunawayVersion to indicate that we finished cleanup */
assert_true(endCleanupRunawayVersion == beginCleanupRunawayVersion);
}
/*
* Checks if RunawayCleaner_RunawayCleanupDoneForProcess ignores duplicate cleanup
* for a single runaway event
*/
void
test__RunawayCleaner_RunawayCleanupDoneForProcess__IgnoresDuplicateCalls(void **state)
{
InitFakeSessionState(2 /* activeProcessCount */,
2 /* cleanupCountdown */,
RunawayStatus_PrimaryRunawaySession /* runawayStatus */, 2 /* pinCount */, 12345 /* vmem */);
static fakeLatestRunawayVersion = 10;
latestRunawayVersion = &fakeLatestRunawayVersion;
/*
* Set beginCleanupRunawayVersion and endCleanupRunawayVersion to
* latestRunawayVersion which should make the function call a no-op
*/
beginCleanupRunawayVersion = *latestRunawayVersion;
endCleanupRunawayVersion = *latestRunawayVersion;
/* Make sure the cleanup goes through */
vmemTrackerInited = true;
isProcessActive = true;
RunawayCleaner_RunawayCleanupDoneForProcess(false /* ignoredCleanup */);
/* Nothing got cleaned */
assert_true(MySessionState->cleanupCountdown == 2);
}
