/* * Fills in the relation statistics for an append-only relation. * * This information is used to update the reltuples and relpages information * in pg_class. reltuples is the same as "pg_aoseg_<oid>:tupcount" * column and we simulate relpages by subdividing the eof value * ("pg_aoseg_<oid>:eof") over the defined page size. */ void vacuum_appendonly_fill_stats(Relation aorel, Snapshot snapshot, BlockNumber *rel_pages, double *rel_tuples, bool *relhasindex) { FileSegTotals *fstotal; BlockNumber nblocks; char *relname; double num_tuples; double totalbytes; double eof; int64 hidden_tupcount; AppendOnlyVisimap visimap; Assert(RelationIsAoRows(aorel) || RelationIsAoCols(aorel)); relname = RelationGetRelationName(aorel); /* get updated statistics from the pg_aoseg table */ if (RelationIsAoRows(aorel)) { fstotal = GetSegFilesTotals(aorel, snapshot); } else { Assert(RelationIsAoCols(aorel)); fstotal = GetAOCSSSegFilesTotals(aorel, snapshot); } /* calculate the values we care about */ eof = (double)fstotal->totalbytes; num_tuples = (double)fstotal->totaltuples; totalbytes = eof; nblocks = (uint32)RelationGuessNumberOfBlocks(totalbytes); AppendOnlyVisimap_Init(&visimap, aorel->rd_appendonly->visimaprelid, aorel->rd_appendonly->visimapidxid, AccessShareLock, snapshot); hidden_tupcount = AppendOnlyVisimap_GetRelationHiddenTupleCount(&visimap); num_tuples -= hidden_tupcount; Assert(num_tuples > -1.0); AppendOnlyVisimap_Finish(&visimap, AccessShareLock); elogif (Debug_appendonly_print_compaction, LOG, "Gather statistics after vacuum for append-only relation %s: " "page count %d, tuple count %f", relname, nblocks, num_tuples); *rel_pages = nblocks; *rel_tuples = num_tuples; *relhasindex = aorel->rd_rel->relhasindex; ereport(elevel, (errmsg("\"%s\": found %.0f rows in %u pages.", relname, num_tuples, nblocks))); pfree(fstotal); }
static void gp_statistics_estimate_reltuples_relpages_ao_rows(Relation rel, float4 *reltuples, float4 *relpages) { FileSegTotals *fstotal; AppendOnlyEntry *aoEntry; AppendOnlyVisimap visimap; int64 hidden_tupcount = 0; /** * Ensure that the right kind of relation with the right type of storage is passed to us. */ Assert(rel->rd_rel->relkind == RELKIND_RELATION); Assert(RelationIsAoRows(rel)); fstotal = GetSegFilesTotals(rel, SnapshotNow); Assert(fstotal); /** * The planner doesn't understand AO's blocks, so need this method to try to fudge up a number for * the planner. */ *relpages = RelationGuessNumberOfBlocks((double)fstotal->totalbytes); aoEntry = GetAppendOnlyEntry(RelationGetRelid(rel), SnapshotNow); AppendOnlyVisimap_Init(&visimap, aoEntry->visimaprelid, aoEntry->visimapidxid, AccessShareLock, SnapshotNow); hidden_tupcount = AppendOnlyVisimap_GetRelationHiddenTupleCount(&visimap); AppendOnlyVisimap_Finish(&visimap, AccessShareLock); /** * The number of tuples in AO table is known accurately. Therefore, we just utilize this value. */ *reltuples = (double)(fstotal->totaltuples - hidden_tupcount); pfree(fstotal); pfree(aoEntry); return; }
static void gp_statistics_estimate_reltuples_relpages_ao_cs(Relation rel, float4 *reltuples, float4 *relpages) { AOCSFileSegInfo **aocsInfo = NULL; int nsegs = 0; double totalBytes = 0; AppendOnlyEntry *aoEntry; int64 hidden_tupcount; AppendOnlyVisimap visimap; /** * Ensure that the right kind of relation with the right type of storage is passed to us. */ Assert(rel->rd_rel->relkind == RELKIND_RELATION); Assert(RelationIsAoCols(rel)); *reltuples = 0.0; *relpages = 0.0; /* get table level statistics from the pg_aoseg table */ aoEntry = GetAppendOnlyEntry(RelationGetRelid(rel), SnapshotNow); aocsInfo = GetAllAOCSFileSegInfo(rel, aoEntry, SnapshotNow, &nsegs); if (aocsInfo) { int i = 0; int j = 0; for(i = 0; i < nsegs; i++) { for(j = 0; j < RelationGetNumberOfAttributes(rel); j++) { AOCSVPInfoEntry *e = getAOCSVPEntry(aocsInfo[i], j); Assert(e); totalBytes += e->eof_uncompressed; } /* Do not include tuples from an awaiting drop segment file */ if (aocsInfo[i]->state != AOSEG_STATE_AWAITING_DROP) { *reltuples += aocsInfo[i]->total_tupcount; } } /** * The planner doesn't understand AO's blocks, so need this method to try to fudge up a number for * the planner. */ *relpages = RelationGuessNumberOfBlocks(totalBytes); } AppendOnlyVisimap_Init(&visimap, aoEntry->visimaprelid, aoEntry->visimapidxid, AccessShareLock, SnapshotNow); hidden_tupcount = AppendOnlyVisimap_GetRelationHiddenTupleCount(&visimap); AppendOnlyVisimap_Finish(&visimap, AccessShareLock); (*reltuples) -= hidden_tupcount; pfree(aoEntry); return; }
/* * Assumes that the segment file lock is already held. * Assumes that the segment file should be compacted. */ static bool AOCSSegmentFileFullCompaction(Relation aorel, AOCSInsertDesc insertDesc, AOCSFileSegInfo *fsinfo, Snapshot snapshot) { const char *relname; AppendOnlyVisimap visiMap; AOCSScanDesc scanDesc; TupleDesc tupDesc; TupleTableSlot *slot; int compact_segno; int64 movedTupleCount = 0; ResultRelInfo *resultRelInfo; MemTupleBinding *mt_bind; EState *estate; bool *proj; int i; AOTupleId *aoTupleId; int64 tupleCount = 0; int64 tuplePerPage = INT_MAX; Assert(Gp_role == GP_ROLE_EXECUTE || Gp_role == GP_ROLE_UTILITY); Assert(RelationIsAoCols(aorel)); Assert(insertDesc); compact_segno = fsinfo->segno; if (fsinfo->varblockcount > 0) { tuplePerPage = fsinfo->total_tupcount / fsinfo->varblockcount; } relname = RelationGetRelationName(aorel); AppendOnlyVisimap_Init(&visiMap, aorel->rd_appendonly->visimaprelid, aorel->rd_appendonly->visimapidxid, ShareLock, snapshot); elogif(Debug_appendonly_print_compaction, LOG, "Compact AO segfile %d, relation %sd", compact_segno, relname); proj = palloc0(sizeof(bool) * RelationGetNumberOfAttributes(aorel)); for (i = 0; i < RelationGetNumberOfAttributes(aorel); ++i) { proj[i] = true; } scanDesc = aocs_beginrangescan(aorel, snapshot, snapshot, &compact_segno, 1, NULL, proj); tupDesc = RelationGetDescr(aorel); slot = MakeSingleTupleTableSlot(tupDesc); mt_bind = create_memtuple_binding(tupDesc); /* * We need a ResultRelInfo and an EState so we can use the regular * executor's index-entry-making machinery. */ estate = CreateExecutorState(); resultRelInfo = makeNode(ResultRelInfo); resultRelInfo->ri_RangeTableIndex = 1; /* dummy */ resultRelInfo->ri_RelationDesc = aorel; resultRelInfo->ri_TrigDesc = NULL; /* we don't fire triggers */ ExecOpenIndices(resultRelInfo); estate->es_result_relations = resultRelInfo; estate->es_num_result_relations = 1; estate->es_result_relation_info = resultRelInfo; while (aocs_getnext(scanDesc, ForwardScanDirection, slot)) { CHECK_FOR_INTERRUPTS(); aoTupleId = (AOTupleId *) slot_get_ctid(slot); if (AppendOnlyVisimap_IsVisible(&scanDesc->visibilityMap, aoTupleId)) { AOCSMoveTuple(slot, insertDesc, resultRelInfo, estate); movedTupleCount++; } else { /* Tuple is invisible and needs to be dropped */ AppendOnlyThrowAwayTuple(aorel, slot, mt_bind); } /* * Check for vacuum delay point after approximatly a var block */ tupleCount++; if (VacuumCostActive && tupleCount % tuplePerPage == 0) { vacuum_delay_point(); } } SetAOCSFileSegInfoState(aorel, compact_segno, AOSEG_STATE_AWAITING_DROP); AppendOnlyVisimap_DeleteSegmentFile(&visiMap, compact_segno); /* Delete all mini pages of the segment files if block directory exists */ if (OidIsValid(aorel->rd_appendonly->blkdirrelid)) { AppendOnlyBlockDirectory_DeleteSegmentFile(aorel, snapshot, compact_segno, 0); } elogif(Debug_appendonly_print_compaction, LOG, "Finished compaction: " "AO segfile %d, relation %s, moved tuple count " INT64_FORMAT, compact_segno, relname, movedTupleCount); AppendOnlyVisimap_Finish(&visiMap, NoLock); ExecCloseIndices(resultRelInfo); FreeExecutorState(estate); ExecDropSingleTupleTableSlot(slot); destroy_memtuple_binding(mt_bind); aocs_endscan(scanDesc); pfree(proj); return true; }
/* * Returns true iff the given segment file should be compacted. */ bool AppendOnlyCompaction_ShouldCompact( Relation aoRelation, AppendOnlyEntry *aoEntry, int segno, int64 segmentTotalTupcount, bool isFull) { bool result; AppendOnlyVisimap visiMap; int64 hiddenTupcount; int hideRatio; Assert(RelationIsAoRows(aoRelation) || RelationIsAoCols(aoRelation)); if (!gp_appendonly_compaction) { ereport(LOG, (errmsg("Append-only compaction skipped on relation %s, segment file num %d", RelationGetRelationName(aoRelation), segno), errdetail("Compaction is disabled"))); /* Disable compaction by global guc. */ return false; } AppendOnlyVisimap_Init(&visiMap, aoEntry->visimaprelid, aoEntry->visimapidxid, ShareLock, SnapshotNow); hiddenTupcount = AppendOnlyVisimap_GetSegmentFileHiddenTupleCount( &visiMap, segno); result = true; if (isFull && hiddenTupcount > 0) { /* * if it is a full vacuum and there is any obsolete data, do a compaction */ result = true; } else { hideRatio = AppendOnlyCompaction_GetHideRatio(hiddenTupcount, segmentTotalTupcount); if (hideRatio <= gp_appendonly_compaction_threshold || gp_appendonly_compaction_threshold == 0) { if (hiddenTupcount > 0) { ereportif(Debug_appendonly_print_compaction, LOG, (errmsg("Append-only compaction skipped on relation %s, segment file num %d, " "hidden tupcount " INT64_FORMAT ", total tupcount " INT64_FORMAT ", " "hide ratio %d%%, threshold %d%%", RelationGetRelationName(aoRelation), segno, hiddenTupcount, segmentTotalTupcount, hideRatio, gp_appendonly_compaction_threshold))); ereport(LOG, (errmsg("Append-only compaction skipped on relation %s, segment file num %d", RelationGetRelationName(aoRelation), segno), errdetail("Ratio of obsolete tuples below threshold (%d%% vs %d%%)", hideRatio, gp_appendonly_compaction_threshold))); } else { ereportif(Debug_appendonly_print_compaction, LOG, (errmsg("Append-only compaction skipped on relation %s, segment file num %d, " "hidden tupcount " INT64_FORMAT ", total tupcount " INT64_FORMAT ", " "hide ratio %d%%, threshold %d%%", RelationGetRelationName(aoRelation), segno, hiddenTupcount, segmentTotalTupcount, hideRatio, gp_appendonly_compaction_threshold))); } result = false; } elogif(Debug_appendonly_print_compaction, LOG, "Schedule compaction: " "segno %d, " "hidden tupcount " INT64_FORMAT ", total tupcount " INT64_FORMAT ", " "hide ratio %d%%, threshold %d%%", segno, hiddenTupcount, segmentTotalTupcount, hideRatio, gp_appendonly_compaction_threshold); } AppendOnlyVisimap_Finish(&visiMap, ShareLock); return result; }
/* * Assumes that the segment file lock is already held. * Assumes that the segment file should be compacted. * */ static void AppendOnlySegmentFileFullCompaction(Relation aorel, AppendOnlyEntry *aoEntry, AppendOnlyInsertDesc insertDesc, FileSegInfo* fsinfo) { const char* relname; AppendOnlyVisimap visiMap; AppendOnlyScanDesc scanDesc; TupleDesc tupDesc; MemTuple tuple; TupleTableSlot *slot; MemTupleBinding *mt_bind; int compact_segno; int64 movedTupleCount = 0; ResultRelInfo *resultRelInfo; EState *estate; AOTupleId *aoTupleId; int64 tupleCount = 0; int64 tuplePerPage = INT_MAX; Assert(Gp_role == GP_ROLE_EXECUTE || Gp_role == GP_ROLE_UTILITY); Assert(RelationIsAoRows(aorel)); Assert(insertDesc); compact_segno = fsinfo->segno; if (fsinfo->varblockcount > 0) { tuplePerPage = fsinfo->total_tupcount / fsinfo->varblockcount; } relname = RelationGetRelationName(aorel); AppendOnlyVisimap_Init(&visiMap, aoEntry->visimaprelid, aoEntry->visimapidxid, ShareUpdateExclusiveLock, SnapshotNow); elogif(Debug_appendonly_print_compaction, LOG, "Compact AO segno %d, relation %s, insert segno %d", compact_segno, relname, insertDesc->storageWrite.segmentFileNum); /* * Todo: We need to limit the scan to one file and we need to avoid to * lock the file again. * * We use SnapshotAny to get visible and invisible tuples. */ scanDesc = appendonly_beginrangescan(aorel, SnapshotAny, SnapshotNow, &compact_segno, 1, 0, NULL); tupDesc = RelationGetDescr(aorel); slot = MakeSingleTupleTableSlot(tupDesc); mt_bind = create_memtuple_binding(tupDesc); /* * We need a ResultRelInfo and an EState so we can use the regular * executor's index-entry-making machinery. */ estate = CreateExecutorState(); resultRelInfo = makeNode(ResultRelInfo); resultRelInfo->ri_RangeTableIndex = 1; /* dummy */ resultRelInfo->ri_RelationDesc = aorel; resultRelInfo->ri_TrigDesc = NULL; /* we don't fire triggers */ ExecOpenIndices(resultRelInfo); estate->es_result_relations = resultRelInfo; estate->es_num_result_relations = 1; estate->es_result_relation_info = resultRelInfo; /* * Go through all visible tuples and move them to a new segfile. */ while ((tuple = appendonly_getnext(scanDesc, ForwardScanDirection, slot)) != NULL) { /* Check interrupts as this may take time. */ CHECK_FOR_INTERRUPTS(); aoTupleId = (AOTupleId*)slot_get_ctid(slot); if (AppendOnlyVisimap_IsVisible(&scanDesc->visibilityMap, aoTupleId)) { AppendOnlyMoveTuple(tuple, slot, mt_bind, insertDesc, resultRelInfo, estate); movedTupleCount++; } else { /* Tuple is invisible and needs to be dropped */ AppendOnlyThrowAwayTuple(aorel, tuple, slot, mt_bind); } /* * Check for vacuum delay point after approximatly a var block */ tupleCount++; if (VacuumCostActive && tupleCount % tuplePerPage == 0) { vacuum_delay_point(); } } SetFileSegInfoState(aorel, aoEntry, compact_segno, AOSEG_STATE_AWAITING_DROP); AppendOnlyVisimap_DeleteSegmentFile(&visiMap, compact_segno); /* Delete all mini pages of the segment files if block directory exists */ if (OidIsValid(aoEntry->blkdirrelid)) { AppendOnlyBlockDirectory_DeleteSegmentFile( aoEntry, SnapshotNow, compact_segno, 0); } elogif(Debug_appendonly_print_compaction, LOG, "Finished compaction: " "AO segfile %d, relation %s, moved tuple count " INT64_FORMAT, compact_segno, relname, movedTupleCount); AppendOnlyVisimap_Finish(&visiMap, NoLock); ExecCloseIndices(resultRelInfo); FreeExecutorState(estate); ExecDropSingleTupleTableSlot(slot); destroy_memtuple_binding(mt_bind); appendonly_endscan(scanDesc); }
static Datum gp_aovisimap_entry_internal(PG_FUNCTION_ARGS, Oid aoRelOid) { Datum values[4]; bool nulls[4]; HeapTuple tuple; Datum result; typedef struct Context { AppendOnlyVisimap visiMap; Relation parentRelation; IndexScanDesc indexScan; text *bitmapBuffer; } Context; FuncCallContext *funcctx; Context *context; if (SRF_IS_FIRSTCALL()) { TupleDesc tupdesc; MemoryContext oldcontext; /* create a function context for cross-call persistence */ funcctx = SRF_FIRSTCALL_INIT(); /* * switch to memory context appropriate for multiple function * calls */ oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); /* build tupdesc for result tuples */ tupdesc = CreateTemplateTupleDesc(4, false); TupleDescInitEntry(tupdesc, (AttrNumber) 1, "segno", INT4OID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "first_row_num", INT8OID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 3, "hidden_tupcount", INT4OID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 4, "bitmap", TEXTOID, -1, 0); funcctx->tuple_desc = BlessTupleDesc(tupdesc); /* * Collect all the locking information that we will format and send * out as a result set. */ context = (Context *) palloc0(sizeof(Context)); context->parentRelation = heap_open(aoRelOid, AccessShareLock); if (!(RelationIsAoRows(context->parentRelation) || RelationIsAoCols(context->parentRelation))) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("Function not supported on relation"))); } AppendOnlyVisimap_Init(&context->visiMap, context->parentRelation->rd_appendonly->visimaprelid, context->parentRelation->rd_appendonly->visimapidxid, AccessShareLock, SnapshotNow); context->indexScan = AppendOnlyVisimapStore_BeginScan(& context->visiMap.visimapStore, 0, NULL); context->bitmapBuffer = palloc0(VARHDRSZ + APPENDONLY_VISIMAP_MAX_RANGE + 1); funcctx->user_fctx = (void *) context; MemoryContextSwitchTo(oldcontext); } funcctx = SRF_PERCALL_SETUP(); context = (Context *) funcctx->user_fctx; if (AppendOnlyVisimapStore_GetNext(&context->visiMap.visimapStore, context->indexScan, ForwardScanDirection, &context->visiMap.visimapEntry, NULL)) { AppendOnlyVisimapEntry *visimapEntry = &context->visiMap.visimapEntry; MemSet(values, 0, sizeof(values)); MemSet(nulls, false, sizeof(nulls)); values[0] = Int32GetDatum(visimapEntry->segmentFileNum); values[1] = Int64GetDatum(visimapEntry->firstRowNum); values[2] = Int32GetDatum( (int32)AppendOnlyVisimapEntry_GetHiddenTupleCount(visimapEntry)); gp_aovisimap_encode_bitmap(VARDATA(context->bitmapBuffer), visimapEntry->bitmap); SET_VARSIZE(context->bitmapBuffer, APPENDONLY_VISIMAP_MAX_RANGE); values[3] = PointerGetDatum(context->bitmapBuffer); tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls); result = HeapTupleGetDatum(tuple); SRF_RETURN_NEXT(funcctx, result); } AppendOnlyVisimapStore_EndScan(&context->visiMap.visimapStore, context->indexScan); AppendOnlyVisimap_Finish(&context->visiMap, AccessShareLock); heap_close(context->parentRelation, AccessShareLock); pfree(context->bitmapBuffer); pfree(context); funcctx->user_fctx = NULL; SRF_RETURN_DONE(funcctx); }
static Datum gp_aovisimap_hidden_info_internal(PG_FUNCTION_ARGS, Oid aoRelOid) { Datum values[3]; bool nulls[3]; HeapTuple tuple; Datum result; typedef struct Context { AppendOnlyVisimap visiMap; Relation parentRelation; FileSegInfo **appendonlySegfileInfo; AOCSFileSegInfo **aocsSegfileInfo; int segfile_info_total; int i; } Context; FuncCallContext *funcctx; Context *context; if (SRF_IS_FIRSTCALL()) { TupleDesc tupdesc; MemoryContext oldcontext; /* create a function context for cross-call persistence */ funcctx = SRF_FIRSTCALL_INIT(); /* * switch to memory context appropriate for multiple function * calls */ oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); /* build tupdesc for result tuples */ tupdesc = CreateTemplateTupleDesc(3, false); TupleDescInitEntry(tupdesc, (AttrNumber) 1, "segno", INT4OID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 2, "hidden_tupcount", INT8OID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber) 3, "total_tupcount", INT8OID, -1, 0); funcctx->tuple_desc = BlessTupleDesc(tupdesc); /* * Collect all the locking information that we will format and send * out as a result set. */ context = (Context *) palloc0(sizeof(Context)); context->parentRelation = heap_open(aoRelOid, AccessShareLock); if (!(RelationIsAoRows(context->parentRelation) || RelationIsAoCols(context->parentRelation))) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("Function not supported on relation"))); } if (RelationIsAoRows(context->parentRelation)) { context->appendonlySegfileInfo = GetAllFileSegInfo( context->parentRelation, SnapshotNow, &context->segfile_info_total); } else { Assert(RelationIsAoCols(context->parentRelation)); context->aocsSegfileInfo = GetAllAOCSFileSegInfo(context->parentRelation, SnapshotNow, &context->segfile_info_total); } context->i = 0; AppendOnlyVisimap_Init(&context->visiMap, context->parentRelation->rd_appendonly->visimaprelid, context->parentRelation->rd_appendonly->visimapidxid, AccessShareLock, SnapshotNow); funcctx->user_fctx = (void *) context; MemoryContextSwitchTo(oldcontext); } funcctx = SRF_PERCALL_SETUP(); context = (Context *) funcctx->user_fctx; while (context->i < context->segfile_info_total) { int64 tupcount; int segno; if (context->appendonlySegfileInfo) { FileSegInfo *fsinfo = context->appendonlySegfileInfo[context->i]; tupcount = fsinfo->total_tupcount; segno = fsinfo->segno; } else if (context->aocsSegfileInfo) { AOCSFileSegInfo *fsinfo = context->aocsSegfileInfo[context->i]; tupcount = fsinfo->total_tupcount; segno = fsinfo->segno; } else { Insist(false); } MemSet(values, 0, sizeof(values)); MemSet(nulls, false, sizeof(nulls)); values[0] = Int32GetDatum(segno); values[1] = Int64GetDatum(AppendOnlyVisimap_GetSegmentFileHiddenTupleCount( &context->visiMap, segno)); values[2] = Int64GetDatum(tupcount); tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls); result = HeapTupleGetDatum(tuple); context->i++; SRF_RETURN_NEXT(funcctx, result); } AppendOnlyVisimap_Finish(&context->visiMap, AccessShareLock); if (context->appendonlySegfileInfo) { FreeAllSegFileInfo(context->appendonlySegfileInfo, context->segfile_info_total); pfree(context->appendonlySegfileInfo); context->appendonlySegfileInfo = NULL; } if (context->aocsSegfileInfo) { FreeAllAOCSSegFileInfo(context->aocsSegfileInfo, context->segfile_info_total); pfree(context->aocsSegfileInfo); context->aocsSegfileInfo = NULL; } heap_close(context->parentRelation, AccessShareLock); pfree(context); funcctx->user_fctx = NULL; SRF_RETURN_DONE(funcctx); }