/* * Starts a runaway cleanup by triggering an ERROR if the VMEM tracker is active * and a commit is not already in progress. Otherwise, it marks the process as clean */ void RunawayCleaner_StartCleanup() { /* * Cleanup can be attempted from multiple places, such as before deactivating * a process (if a pending runaway event) or periodically from CHECK_FOR_INTERRUPTS * (indirectly via RedZoneHandler_DetectRunaway). We don't carry multiple cleanup * for a single runaway event. Every time we *start* a cleanup process, we set the * beginCleanupRunawayVersion to the runaway version for which we started cleaning * up. Later on, if we reenter this method (e.g., another CHECK_FOR_INTERRUPTS() * during cleanup), we can observe that the cleanup already started from this runaway * event, and therefore we skip duplicate cleanup */ if (RunawayCleaner_ShouldStartRunawayCleanup()) { Assert(beginCleanupRunawayVersion < *latestRunawayVersion); Assert(endCleanupRunawayVersion < *latestRunawayVersion); /* We don't want to cleanup multiple times for same runaway event */ beginCleanupRunawayVersion = *latestRunawayVersion; if (CritSectionCount == 0 && InterruptHoldoffCount == 0 && vmemTrackerInited && gp_command_count > 0 /* Cleaning up QEs that are not executing a valid command may cause the QD to get stuck [MPP-24950] */ && /* Super user is terminated only when it's the primary runaway consumer (i.e., the top consumer) */ (!superuser() || MySessionState->runawayStatus == RunawayStatus_PrimaryRunawaySession)) { #ifdef FAULT_INJECTOR FaultInjector_InjectFaultIfSet( RunawayCleanup, DDLNotSpecified, "", // databaseName ""); // tableName #endif ereport(ERROR, (errmsg("Canceling query because of high VMEM usage. Used: %dMB, available %dMB, red zone: %dMB", VmemTracker_ConvertVmemChunksToMB(MySessionState->sessionVmem), VmemTracker_GetAvailableVmemMB(), RedZoneHandler_GetRedZoneLimitMB()), errprintstack(true))); } /* * If we cannot error out because of a critical section or because we are a super user * or for some other reason (such as the QE is not running any valid command, i.e., * gp_command_count is not positive) simply declare this process as clean */ RunawayCleaner_RunawayCleanupDoneForProcess(true /* ignoredCleanup */); } }
/* * gp_failed_to_alloc is called upon an OOM. We can have either a VMEM * limited OOM (i.e., the system still has memory, but we ran out of either * per-query VMEM limit or segment VMEM limit) or a true OOM, where the * malloc returns a NULL pointer. * * This function logs OOM details, such as memory allocation/deallocation/peak. * It also updates segment OOM time by calling UpdateTimeAtomically(). * * Parameters: * * ec: error code; indicates what type of OOM event happend (system, VMEM, per-query VMEM) * en: the last seen error number as retrieved by calling __error() or similar function * sz: the requested allocation size for which we reached OOM */ static void gp_failed_to_alloc(MemoryAllocationStatus ec, int en, int sz) { /* * A per-query vmem overflow shouldn't trigger a segment-wide * OOM reporting. */ if (MemoryFailure_QueryMemoryExhausted != ec) { UpdateTimeAtomically(segmentOOMTime); } UpdateTimeAtomically(&alreadyReportedOOMTime); /* Request 1 MB of waiver for processing error */ VmemTracker_RequestWaiver(1024 * 1024); Insist(MemoryProtection_IsOwnerThread()); if (ec == MemoryFailure_QueryMemoryExhausted) { elog(LOG, "Logging memory usage for reaching per-query memory limit"); } else if (ec == MemoryFailure_VmemExhausted) { elog(LOG, "Logging memory usage for reaching Vmem limit"); } else if (ec == MemoryFailure_SystemMemoryExhausted) { /* * The system memory is exhausted and malloc returned a null pointer. * Although elog switches to ErrorContext, which already * has pre-allocated space, we are not risking any new allocation until * we dump the memory context and memory accounting tree. We are therefore * printing the log message header using write_stderr. */ write_stderr("Logging memory usage for reaching system memory limit"); } else { Assert(!"Unknown memory failure error code"); } RedZoneHandler_LogVmemUsageOfAllSessions(); MemoryAccounting_SaveToLog(); MemoryContextStats(TopMemoryContext); if(coredump_on_memerror) { /* * Generate a core dump by writing to NULL pointer */ *(int *) NULL = ec; } if (ec == MemoryFailure_VmemExhausted) { /* Hit MOP limit */ ereport(ERROR, (errcode(ERRCODE_GP_MEMPROT_KILL), errmsg("Out of memory"), errdetail("VM Protect failed to allocate %d bytes, %d MB available", sz, VmemTracker_GetAvailableVmemMB() ) )); } else if (ec == MemoryFailure_QueryMemoryExhausted) { /* Hit MOP limit */ ereport(ERROR, (errcode(ERRCODE_GP_MEMPROT_KILL), errmsg("Out of memory"), errdetail("Per-query VM protect limit reached: current limit is %d kB, requested %d bytes, available %d MB", gp_vmem_limit_per_query, sz, VmemTracker_GetAvailableQueryVmemMB() ) )); } else if (ec == MemoryFailure_SystemMemoryExhausted) { ereport(ERROR, (errcode(ERRCODE_GP_MEMPROT_KILL), errmsg("Out of memory"), errdetail("VM protect failed to allocate %d bytes from system, VM Protect %d MB available", sz, VmemTracker_GetAvailableVmemMB() ) )); } else { /* SemOp error. */ ereport(ERROR, (errcode(ERRCODE_GP_MEMPROT_KILL), errmsg("Failed to allocate memory under virtual memory protection"), errdetail("Error %d, errno %d, %s", ec, en, strerror(en)) )); } }