/*
* Starts a runaway cleanup by triggering an ERROR if the VMEM tracker is active
* and a commit is not already in progress. Otherwise, it marks the process as clean
*/
void
RunawayCleaner_StartCleanup()
{
/*
* Cleanup can be attempted from multiple places, such as before deactivating
* a process (if a pending runaway event) or periodically from CHECK_FOR_INTERRUPTS
* (indirectly via RedZoneHandler_DetectRunaway). We don't carry multiple cleanup
* for a single runaway event. Every time we *start* a cleanup process, we set the
* beginCleanupRunawayVersion to the runaway version for which we started cleaning
* up. Later on, if we reenter this method (e.g., another CHECK_FOR_INTERRUPTS()
* during cleanup), we can observe that the cleanup already started from this runaway
* event, and therefore we skip duplicate cleanup
*/
if (RunawayCleaner_ShouldStartRunawayCleanup())
{
Assert(beginCleanupRunawayVersion < *latestRunawayVersion);
Assert(endCleanupRunawayVersion < *latestRunawayVersion);
/* We don't want to cleanup multiple times for same runaway event */
beginCleanupRunawayVersion = *latestRunawayVersion;
if (CritSectionCount == 0 && InterruptHoldoffCount == 0 && vmemTrackerInited &&
gp_command_count > 0 /* Cleaning up QEs that are not executing a valid command
may cause the QD to get stuck [MPP-24950] */ &&
/* Super user is terminated only when it's the primary runaway consumer (i.e., the top consumer) */
(!superuser() || MySessionState->runawayStatus == RunawayStatus_PrimaryRunawaySession))
{
#ifdef FAULT_INJECTOR
FaultInjector_InjectFaultIfSet(
RunawayCleanup,
DDLNotSpecified,
"", // databaseName
""); // tableName
#endif
ereport(ERROR, (errmsg("Canceling query because of high VMEM usage. Used: %dMB, available %dMB, red zone: %dMB",
VmemTracker_ConvertVmemChunksToMB(MySessionState->sessionVmem), VmemTracker_GetAvailableVmemMB(),
RedZoneHandler_GetRedZoneLimitMB()), errprintstack(true)));
}
/*
* If we cannot error out because of a critical section or because we are a super user
* or for some other reason (such as the QE is not running any valid command, i.e.,
* gp_command_count is not positive) simply declare this process as clean
*/
RunawayCleaner_RunawayCleanupDoneForProcess(true /* ignoredCleanup */);
}
}
/*
* gp_failed_to_alloc is called upon an OOM. We can have either a VMEM
* limited OOM (i.e., the system still has memory, but we ran out of either
* per-query VMEM limit or segment VMEM limit) or a true OOM, where the
* malloc returns a NULL pointer.
*
* This function logs OOM details, such as memory allocation/deallocation/peak.
* It also updates segment OOM time by calling UpdateTimeAtomically().
*
* Parameters:
*
* ec: error code; indicates what type of OOM event happend (system, VMEM, per-query VMEM)
* en: the last seen error number as retrieved by calling __error() or similar function
* sz: the requested allocation size for which we reached OOM
*/
static void gp_failed_to_alloc(MemoryAllocationStatus ec, int en, int sz)
{
/*
* A per-query vmem overflow shouldn't trigger a segment-wide
* OOM reporting.
*/
if (MemoryFailure_QueryMemoryExhausted != ec)
{
UpdateTimeAtomically(segmentOOMTime);
}
UpdateTimeAtomically(&alreadyReportedOOMTime);
/* Request 1 MB of waiver for processing error */
VmemTracker_RequestWaiver(1024 * 1024);
Insist(MemoryProtection_IsOwnerThread());
if (ec == MemoryFailure_QueryMemoryExhausted)
{
elog(LOG, "Logging memory usage for reaching per-query memory limit");
}
else if (ec == MemoryFailure_VmemExhausted)
{
elog(LOG, "Logging memory usage for reaching Vmem limit");
}
else if (ec == MemoryFailure_SystemMemoryExhausted)
{
/*
* The system memory is exhausted and malloc returned a null pointer.
* Although elog switches to ErrorContext, which already
* has pre-allocated space, we are not risking any new allocation until
* we dump the memory context and memory accounting tree. We are therefore
* printing the log message header using write_stderr.
*/
write_stderr("Logging memory usage for reaching system memory limit");
}
else
{
Assert(!"Unknown memory failure error code");
}
RedZoneHandler_LogVmemUsageOfAllSessions();
MemoryAccounting_SaveToLog();
MemoryContextStats(TopMemoryContext);
if(coredump_on_memerror)
{
/*
* Generate a core dump by writing to NULL pointer
*/
*(int *) NULL = ec;
}
if (ec == MemoryFailure_VmemExhausted)
{
/* Hit MOP limit */
ereport(ERROR, (errcode(ERRCODE_GP_MEMPROT_KILL),
errmsg("Out of memory"),
errdetail("VM Protect failed to allocate %d bytes, %d MB available",
sz, VmemTracker_GetAvailableVmemMB()
)
));
}
else if (ec == MemoryFailure_QueryMemoryExhausted)
{
/* Hit MOP limit */
ereport(ERROR, (errcode(ERRCODE_GP_MEMPROT_KILL),
errmsg("Out of memory"),
errdetail("Per-query VM protect limit reached: current limit is %d kB, requested %d bytes, available %d MB",
gp_vmem_limit_per_query, sz, VmemTracker_GetAvailableQueryVmemMB()
)
));
}
else if (ec == MemoryFailure_SystemMemoryExhausted)
{
ereport(ERROR, (errcode(ERRCODE_GP_MEMPROT_KILL),
errmsg("Out of memory"),
errdetail("VM protect failed to allocate %d bytes from system, VM Protect %d MB available",
sz, VmemTracker_GetAvailableVmemMB()
)
));
}
else
{
/* SemOp error. */
ereport(ERROR, (errcode(ERRCODE_GP_MEMPROT_KILL),
errmsg("Failed to allocate memory under virtual memory protection"),
errdetail("Error %d, errno %d, %s", ec, en, strerror(en))
));
}
}
