/* * gp_failed_to_alloc is called upon an OOM. We can have either a VMEM * limited OOM (i.e., the system still has memory, but we ran out of either * per-query VMEM limit or segment VMEM limit) or a true OOM, where the * malloc returns a NULL pointer. * * This function logs OOM details, such as memory allocation/deallocation/peak. * It also updates segment OOM time by calling UpdateTimeAtomically(). * * Parameters: * * ec: error code; indicates what type of OOM event happend (system, VMEM, per-query VMEM) * en: the last seen error number as retrieved by calling __error() or similar function * sz: the requested allocation size for which we reached OOM */ static void gp_failed_to_alloc(MemoryAllocationStatus ec, int en, int sz) { /* * A per-query vmem overflow shouldn't trigger a segment-wide * OOM reporting. */ if (MemoryFailure_QueryMemoryExhausted != ec) { UpdateTimeAtomically(segmentOOMTime); } UpdateTimeAtomically(&alreadyReportedOOMTime); /* Request 1 MB of waiver for processing error */ VmemTracker_RequestWaiver(1024 * 1024); Insist(MemoryProtection_IsOwnerThread()); if (ec == MemoryFailure_QueryMemoryExhausted) { elog(LOG, "Logging memory usage for reaching per-query memory limit"); } else if (ec == MemoryFailure_VmemExhausted) { elog(LOG, "Logging memory usage for reaching Vmem limit"); } else if (ec == MemoryFailure_SystemMemoryExhausted) { /* * The system memory is exhausted and malloc returned a null pointer. * Although elog switches to ErrorContext, which already * has pre-allocated space, we are not risking any new allocation until * we dump the memory context and memory accounting tree. We are therefore * printing the log message header using write_stderr. */ write_stderr("Logging memory usage for reaching system memory limit"); } else { Assert(!"Unknown memory failure error code"); } RedZoneHandler_LogVmemUsageOfAllSessions(); MemoryAccounting_SaveToLog(); MemoryContextStats(TopMemoryContext); if(coredump_on_memerror) { /* * Generate a core dump by writing to NULL pointer */ *(int *) NULL = ec; } if (ec == MemoryFailure_VmemExhausted) { /* Hit MOP limit */ ereport(ERROR, (errcode(ERRCODE_GP_MEMPROT_KILL), errmsg("Out of memory"), errdetail("VM Protect failed to allocate %d bytes, %d MB available", sz, VmemTracker_GetAvailableVmemMB() ) )); } else if (ec == MemoryFailure_QueryMemoryExhausted) { /* Hit MOP limit */ ereport(ERROR, (errcode(ERRCODE_GP_MEMPROT_KILL), errmsg("Out of memory"), errdetail("Per-query VM protect limit reached: current limit is %d kB, requested %d bytes, available %d MB", gp_vmem_limit_per_query, sz, VmemTracker_GetAvailableQueryVmemMB() ) )); } else if (ec == MemoryFailure_SystemMemoryExhausted) { ereport(ERROR, (errcode(ERRCODE_GP_MEMPROT_KILL), errmsg("Out of memory"), errdetail("VM protect failed to allocate %d bytes from system, VM Protect %d MB available", sz, VmemTracker_GetAvailableVmemMB() ) )); } else { /* SemOp error. */ ereport(ERROR, (errcode(ERRCODE_GP_MEMPROT_KILL), errmsg("Failed to allocate memory under virtual memory protection"), errdetail("Error %d, errno %d, %s", ec, en, strerror(en)) )); } }
/* * MemoryContextError * Report failure of a memory context operation. Does not return. */ void MemoryContextError(int errorcode, MemoryContext context, const char *sfile, int sline, const char *fmt, ...) { va_list args; char buf[200]; /* * Don't use elog, as we might have a malloc problem. * Also, don't use write_log, as this method might be * called from syslogger, which does not support * write_log calls */ write_stderr("Logging memory usage for memory context error"); MemoryAccounting_SaveToLog(); MemoryContextStats(TopMemoryContext); if(coredump_on_memerror) { /* * Turn memory context into a SIGSEGV, so will generate * a core dump. * * XXX What is the right way of doing this? */ *(int *) NULL = errorcode; } if(errorcode != ERRCODE_OUT_OF_MEMORY) { Assert(!"Memory context error!"); } /* Format caller's message. */ va_start(args, fmt); vsnprintf(buf, sizeof(buf)-32, fmt, args); va_end(args); /* * This might fail if we run out of memory at the system level * (i.e., malloc returned null), and the system is running so * low in memory that ereport cannot format its parameter. * However, we already dumped our usage information using * write_stderr, so we are gonna take a chance by calling ereport. * If we fail, we at least have OOM message in the log. If we succeed, * we will also have the detail error code and location of the error. * Note, ereport should switch to ErrorContext which should have * some preallocated memory to handle this message. Therefore, * our chance of success is quite high */ ereport(ERROR, (errcode(errorcode), errmsg("%s (context '%s') (%s:%d)", buf, context->name, sfile ? sfile : "", sline) )); /* not reached */ abort(); } /* MemoryContextError */
/* * Marks the current process as clean. If all the processes are marked * as clean for this session (i.e., cleanupCountdown == 0 in the * MySessionState) then we reset session's runaway status as well as * the runaway detector flag (i.e., a new runaway detector can run). * * Parameters: * ignoredCleanup: whether the cleanup was ignored, i.e., no elog(ERROR, ...) * was thrown. In such case a deactivated process is not reactivated as the * deactivation didn't get interrupted. */ void RunawayCleaner_RunawayCleanupDoneForProcess(bool ignoredCleanup) { /* * We don't do anything if we don't have an ongoing cleanup, or we already finished * cleanup once for the current runaway event */ if (beginCleanupRunawayVersion != *latestRunawayVersion || endCleanupRunawayVersion == beginCleanupRunawayVersion) { /* Either we never started cleanup, or we already finished */ return; } /* Disable repeating call */ endCleanupRunawayVersion = beginCleanupRunawayVersion; Assert(NULL != MySessionState); /* * As the current cleanup holds leverage on the cleanupCountdown, * the session must stay as runaway at least until the current * process marks itself clean */ Assert(MySessionState->runawayStatus != RunawayStatus_NotRunaway); /* We only cleanup if we were active when the runaway event happened */ Assert((!isProcessActive && *latestRunawayVersion < deactivationVersion && *latestRunawayVersion > activationVersion) || (*latestRunawayVersion > activationVersion && (activationVersion >= deactivationVersion && isProcessActive))); /* * We don't reactivate if the process is already active or a deactivated * process never errored out during deactivation (i.e., failed to complete * deactivation) */ if (!isProcessActive && !ignoredCleanup) { Assert(1 == *isRunawayDetector); Assert(0 < MySessionState->cleanupCountdown); /* * As the process threw ERROR instead of going into ReadCommand() blocking * state, we have to reactivate the process from its current Deactivated * state */ IdleTracker_ActivateProcess(); } Assert(0 < MySessionState->cleanupCountdown); #if USE_ASSERT_CHECKING int cleanProgress = #endif gp_atomic_add_32(&MySessionState->cleanupCountdown, -1); Assert(0 <= cleanProgress); bool finalCleaner = compare_and_swap_32((uint32*) &MySessionState->cleanupCountdown, 0, CLEANUP_COUNTDOWN_BEFORE_RUNAWAY); if (finalCleaner) { /* * The final cleaner is responsible to reset the runaway flag, * and enable the runaway detection process. */ RunawayCleaner_RunawayCleanupDoneForSession(); } /* * Finally we are done with all critical cleanup, which includes releasing all our memory and * releasing our cleanup counter so that another session can be marked as runaway, if needed. * Now, we have some head room to actually record our usage. */ write_stderr("Logging memory usage because of runaway cleanup. Note, this is a post-cleanup logging and may be incomplete."); MemoryAccounting_SaveToLog(); MemoryContextStats(TopMemoryContext); }