/*
* gp_failed_to_alloc is called upon an OOM. We can have either a VMEM
* limited OOM (i.e., the system still has memory, but we ran out of either
* per-query VMEM limit or segment VMEM limit) or a true OOM, where the
* malloc returns a NULL pointer.
*
* This function logs OOM details, such as memory allocation/deallocation/peak.
* It also updates segment OOM time by calling UpdateTimeAtomically().
*
* Parameters:
*
* ec: error code; indicates what type of OOM event happend (system, VMEM, per-query VMEM)
* en: the last seen error number as retrieved by calling __error() or similar function
* sz: the requested allocation size for which we reached OOM
*/
static void gp_failed_to_alloc(MemoryAllocationStatus ec, int en, int sz)
{
/*
* A per-query vmem overflow shouldn't trigger a segment-wide
* OOM reporting.
*/
if (MemoryFailure_QueryMemoryExhausted != ec)
{
UpdateTimeAtomically(segmentOOMTime);
}
UpdateTimeAtomically(&alreadyReportedOOMTime);
/* Request 1 MB of waiver for processing error */
VmemTracker_RequestWaiver(1024 * 1024);
Insist(MemoryProtection_IsOwnerThread());
if (ec == MemoryFailure_QueryMemoryExhausted)
{
elog(LOG, "Logging memory usage for reaching per-query memory limit");
}
else if (ec == MemoryFailure_VmemExhausted)
{
elog(LOG, "Logging memory usage for reaching Vmem limit");
}
else if (ec == MemoryFailure_SystemMemoryExhausted)
{
/*
* The system memory is exhausted and malloc returned a null pointer.
* Although elog switches to ErrorContext, which already
* has pre-allocated space, we are not risking any new allocation until
* we dump the memory context and memory accounting tree. We are therefore
* printing the log message header using write_stderr.
*/
write_stderr("Logging memory usage for reaching system memory limit");
}
else
{
Assert(!"Unknown memory failure error code");
}
RedZoneHandler_LogVmemUsageOfAllSessions();
MemoryAccounting_SaveToLog();
MemoryContextStats(TopMemoryContext);
if(coredump_on_memerror)
{
/*
* Generate a core dump by writing to NULL pointer
*/
*(int *) NULL = ec;
}
if (ec == MemoryFailure_VmemExhausted)
{
/* Hit MOP limit */
ereport(ERROR, (errcode(ERRCODE_GP_MEMPROT_KILL),
errmsg("Out of memory"),
errdetail("VM Protect failed to allocate %d bytes, %d MB available",
sz, VmemTracker_GetAvailableVmemMB()
)
));
}
else if (ec == MemoryFailure_QueryMemoryExhausted)
{
/* Hit MOP limit */
ereport(ERROR, (errcode(ERRCODE_GP_MEMPROT_KILL),
errmsg("Out of memory"),
errdetail("Per-query VM protect limit reached: current limit is %d kB, requested %d bytes, available %d MB",
gp_vmem_limit_per_query, sz, VmemTracker_GetAvailableQueryVmemMB()
)
));
}
else if (ec == MemoryFailure_SystemMemoryExhausted)
{
ereport(ERROR, (errcode(ERRCODE_GP_MEMPROT_KILL),
errmsg("Out of memory"),
errdetail("VM protect failed to allocate %d bytes from system, VM Protect %d MB available",
sz, VmemTracker_GetAvailableVmemMB()
)
));
}
else
{
/* SemOp error. */
ereport(ERROR, (errcode(ERRCODE_GP_MEMPROT_KILL),
errmsg("Failed to allocate memory under virtual memory protection"),
errdetail("Error %d, errno %d, %s", ec, en, strerror(en))
));
}
}
/*
* MemoryContextError
* Report failure of a memory context operation. Does not return.
*/
void
MemoryContextError(int errorcode, MemoryContext context,
const char *sfile, int sline,
const char *fmt, ...)
{
va_list args;
char buf[200];
/*
* Don't use elog, as we might have a malloc problem.
* Also, don't use write_log, as this method might be
* called from syslogger, which does not support
* write_log calls
*/
write_stderr("Logging memory usage for memory context error");
MemoryAccounting_SaveToLog();
MemoryContextStats(TopMemoryContext);
if(coredump_on_memerror)
{
/*
* Turn memory context into a SIGSEGV, so will generate
* a core dump.
*
* XXX What is the right way of doing this?
*/
*(int *) NULL = errorcode;
}
if(errorcode != ERRCODE_OUT_OF_MEMORY)
{
Assert(!"Memory context error!");
}
/* Format caller's message. */
va_start(args, fmt);
vsnprintf(buf, sizeof(buf)-32, fmt, args);
va_end(args);
/*
* This might fail if we run out of memory at the system level
* (i.e., malloc returned null), and the system is running so
* low in memory that ereport cannot format its parameter.
* However, we already dumped our usage information using
* write_stderr, so we are gonna take a chance by calling ereport.
* If we fail, we at least have OOM message in the log. If we succeed,
* we will also have the detail error code and location of the error.
* Note, ereport should switch to ErrorContext which should have
* some preallocated memory to handle this message. Therefore,
* our chance of success is quite high
*/
ereport(ERROR, (errcode(errorcode),
errmsg("%s (context '%s') (%s:%d)",
buf,
context->name,
sfile ? sfile : "",
sline)
));
/* not reached */
abort();
} /* MemoryContextError */
/*
* Marks the current process as clean. If all the processes are marked
* as clean for this session (i.e., cleanupCountdown == 0 in the
* MySessionState) then we reset session's runaway status as well as
* the runaway detector flag (i.e., a new runaway detector can run).
*
* Parameters:
* ignoredCleanup: whether the cleanup was ignored, i.e., no elog(ERROR, ...)
* was thrown. In such case a deactivated process is not reactivated as the
* deactivation didn't get interrupted.
*/
void
RunawayCleaner_RunawayCleanupDoneForProcess(bool ignoredCleanup)
{
/*
* We don't do anything if we don't have an ongoing cleanup, or we already finished
* cleanup once for the current runaway event
*/
if (beginCleanupRunawayVersion != *latestRunawayVersion ||
endCleanupRunawayVersion == beginCleanupRunawayVersion)
{
/* Either we never started cleanup, or we already finished */
return;
}
/* Disable repeating call */
endCleanupRunawayVersion = beginCleanupRunawayVersion;
Assert(NULL != MySessionState);
/*
* As the current cleanup holds leverage on the cleanupCountdown,
* the session must stay as runaway at least until the current
* process marks itself clean
*/
Assert(MySessionState->runawayStatus != RunawayStatus_NotRunaway);
/* We only cleanup if we were active when the runaway event happened */
Assert((!isProcessActive && *latestRunawayVersion < deactivationVersion &&
*latestRunawayVersion > activationVersion) ||
(*latestRunawayVersion > activationVersion &&
(activationVersion >= deactivationVersion && isProcessActive)));
/*
* We don't reactivate if the process is already active or a deactivated
* process never errored out during deactivation (i.e., failed to complete
* deactivation)
*/
if (!isProcessActive && !ignoredCleanup)
{
Assert(1 == *isRunawayDetector);
Assert(0 < MySessionState->cleanupCountdown);
/*
* As the process threw ERROR instead of going into ReadCommand() blocking
* state, we have to reactivate the process from its current Deactivated
* state
*/
IdleTracker_ActivateProcess();
}
Assert(0 < MySessionState->cleanupCountdown);
#if USE_ASSERT_CHECKING
int cleanProgress =
#endif
gp_atomic_add_32(&MySessionState->cleanupCountdown, -1);
Assert(0 <= cleanProgress);
bool finalCleaner = compare_and_swap_32((uint32*) &MySessionState->cleanupCountdown,
0, CLEANUP_COUNTDOWN_BEFORE_RUNAWAY);
if (finalCleaner)
{
/*
* The final cleaner is responsible to reset the runaway flag,
* and enable the runaway detection process.
*/
RunawayCleaner_RunawayCleanupDoneForSession();
}
/*
* Finally we are done with all critical cleanup, which includes releasing all our memory and
* releasing our cleanup counter so that another session can be marked as runaway, if needed.
* Now, we have some head room to actually record our usage.
*/
write_stderr("Logging memory usage because of runaway cleanup. Note, this is a post-cleanup logging and may be incomplete.");
MemoryAccounting_SaveToLog();
MemoryContextStats(TopMemoryContext);
}
