static void LockSegfilesOnMasterForSingleRel(Relation rel, int32 segno) {
Insist(Gp_role == GP_ROLE_DISPATCH);
/*
* do not lock segfile with content id = -1
*/
/*
for (i = 1; i < rel->rd_segfile0_count; ++i)
{
if (RelationIsAoRows(rel) || RelationIsParquet(rel))
{
LockRelationAppendOnlySegmentFile(&rel->rd_node, segno,
AccessExclusiveLock, false, i - 1);
}
}
*/
{
if (RelationIsAoRows(rel) || RelationIsParquet(rel)) {
LockRelationAppendOnlySegmentFile(&rel->rd_node, segno,
AccessExclusiveLock,
false);
}
}
}
/*
* Update the eof and filetupcount of a parquet table.
*/
void
UpdateParquetFileSegInfo(Relation parentrel,
AppendOnlyEntry *aoEntry,
int segno,
int64 eof,
int64 eof_uncompressed,
int64 tuples_added)
{
LockAcquireResult acquireResult;
Relation pg_parquetseg_rel;
TupleDesc pg_parquetseg_dsc;
ScanKeyData key[1];
SysScanDesc parquetscan;
HeapTuple tuple, new_tuple;
Datum filetupcount;
Datum new_tuple_count;
Datum *new_record;
bool *new_record_nulls;
bool *new_record_repl;
bool isNull;
/* overflow sanity checks. don't check the same for tuples_added,
* it may be coming as a negative diff from gp_update_ao_master_stats */
Assert(eof >= 0);
Insist(Gp_role != GP_ROLE_EXECUTE);
elog(DEBUG3, "UpdateParquetFileSegInfo called. segno = %d", segno);
if (Gp_role != GP_ROLE_DISPATCH)
{
/*
* Verify we already have the write-lock!
*/
acquireResult = LockRelationAppendOnlySegmentFile(
&parentrel->rd_node,
segno,
AccessExclusiveLock,
/* dontWait */ false);
if (acquireResult != LOCKACQUIRE_ALREADY_HELD)
{
elog(ERROR, "Should already have the (transaction-scope) write-lock on Parquet segment file #%d, "
"relation %s", segno, RelationGetRelationName(parentrel));
}
}
/*
* Open the aoseg relation and its index.
*/
pg_parquetseg_rel = heap_open(aoEntry->segrelid, RowExclusiveLock);
pg_parquetseg_dsc = pg_parquetseg_rel->rd_att;
/*
* Setup a scan key to fetch from the index by segno.
*/
ScanKeyInit(&key[0], (AttrNumber) Anum_pg_parquetseg_segno,
BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(segno));
parquetscan = systable_beginscan(pg_parquetseg_rel, aoEntry->segidxid, TRUE,
SnapshotNow, 1, &key[0]);
tuple = systable_getnext(parquetscan);
if (!HeapTupleIsValid(tuple))
ereport(ERROR,
(errcode(ERRCODE_UNDEFINED_OBJECT),
errmsg("parquet table \"%s\" file segment \"%d\" entry "
"does not exist", RelationGetRelationName(parentrel),
segno)));
new_record = palloc0(sizeof(Datum) * pg_parquetseg_dsc->natts);
new_record_nulls = palloc0(sizeof(bool) * pg_parquetseg_dsc->natts);
new_record_repl = palloc0(sizeof(bool) * pg_parquetseg_dsc->natts);
/* get the current tuple count so we can add to it */
filetupcount = fastgetattr(tuple,
Anum_pg_parquetseg_tupcount,
pg_parquetseg_dsc,
&isNull);
if(isNull)
ereport(ERROR,
(errcode(ERRCODE_UNDEFINED_OBJECT),
errmsg("got invalid pg_aoseg filetupcount value: NULL")));
/* calculate the new tuple count */
new_tuple_count = DirectFunctionCall2(float8pl,
filetupcount,
Float8GetDatum((float8)tuples_added));
/*
* Build a tuple to update
*/
new_record[Anum_pg_parquetseg_eof - 1] = Float8GetDatum((float8)eof);
new_record_repl[Anum_pg_parquetseg_eof - 1] = true;
new_record[Anum_pg_parquetseg_tupcount - 1] = new_tuple_count;
new_record_repl[Anum_pg_parquetseg_tupcount - 1] = true;
new_record[Anum_pg_parquetseg_eofuncompressed - 1] = Float8GetDatum((float8)eof_uncompressed);
new_record_repl[Anum_pg_parquetseg_eofuncompressed - 1] = true;
/*
* update the tuple in the pg_aoseg table
*/
new_tuple = heap_modify_tuple(tuple, pg_parquetseg_dsc, new_record,
new_record_nulls, new_record_repl);
simple_heap_update(pg_parquetseg_rel, &tuple->t_self, new_tuple);
CatalogUpdateIndexes(pg_parquetseg_rel, new_tuple);
heap_freetuple(new_tuple);
/* Finish up scan */
systable_endscan(parquetscan);
heap_close(pg_parquetseg_rel, RowExclusiveLock);
pfree(new_record);
pfree(new_record_nulls);
pfree(new_record_repl);
}
/*
* Open the next file segment for write.
*/
static void SetCurrentFileSegForWrite(ParquetInsertDesc parquetInsertDesc, ResultRelSegFileInfo *segfileinfo) {
ParquetFileSegInfo *fsinfo;
int32 fileSegNo;
/* Make the 'segment' file name */
MakeAOSegmentFileName(parquetInsertDesc->parquet_rel,
parquetInsertDesc->cur_segno, -1, &fileSegNo,
parquetInsertDesc->parquetFilePathName);
Assert(
strlen(parquetInsertDesc->parquetFilePathName) + 1 <=
parquetInsertDesc->parquetFilePathNameMaxLen);
/*
* In order to append to this file segment entry we must first
* acquire the relation parquet segment file (transaction-scope) lock (tag
* LOCKTAG_RELATION_APPENDONLY_SEGMENT_FILE) in order to guarantee
* stability of the pg_aoseg information on this segment file and exclusive right
* to append data to the segment file.
*
* NOTE: This is a transaction scope lock that must be held until commit / abort.
*/
LockRelationAppendOnlySegmentFile(&parquetInsertDesc->parquet_rel->rd_node,
parquetInsertDesc->cur_segno, AccessExclusiveLock,
/* dontWait */false);
/* Now, get the information for the file segment we are going to append to. */
parquetInsertDesc->fsInfo = (ParquetFileSegInfo *) palloc0(sizeof(ParquetFileSegInfo));
/*
* in hawq, we cannot insert a new catalog entry and then update,
* since we cannot get the tid of added tuple.
* we should add the new catalog entry on master and then dispatch it to segments for update.
*/
Assert(parquetInsertDesc->fsInfo != NULL);
Assert(segfileinfo->numfiles == 1);
fsinfo = parquetInsertDesc->fsInfo;
fsinfo->segno = segfileinfo->segno;
fsinfo->tupcount = segfileinfo->tupcount;
fsinfo->eof = segfileinfo->eof[0];
fsinfo->eof_uncompressed = segfileinfo->uncompressed_eof[0];
parquetInsertDesc->fileLen = (int64)fsinfo->eof;
parquetInsertDesc->fileLen_uncompressed = (int64)fsinfo->eof_uncompressed;
parquetInsertDesc->rowCount = fsinfo->tupcount;
/* Open the existing file for write.*/
OpenSegmentFile(
parquetInsertDesc->mirroredOpen,
parquetInsertDesc->parquetFilePathName, fsinfo->eof,
&parquetInsertDesc->parquet_rel->rd_node,
parquetInsertDesc->cur_segno, parquetInsertDesc->relname,
&parquetInsertDesc->parquet_file,
&parquetInsertDesc->file_previousmetadata,
&parquetInsertDesc->protocol_read,
parquetInsertDesc->parquet_rel->rd_att,
&parquetInsertDesc->parquetMetadata,
&parquetInsertDesc->fileLen,
&parquetInsertDesc->fileLen_uncompressed,
&parquetInsertDesc->previous_rowgroupcnt);
initSerializeFooter(&(parquetInsertDesc->footerProtocol), parquetInsertDesc->parquetFilePathName);
}
/*
* Performs a compaction of an append-only relation in column-orientation.
*
* In non-utility mode, all compaction segment files should be
* marked as in-use/in-compaction in the appendonlywriter.c code. If
* set, the insert_segno should also be marked as in-use.
* When the insert segno is negative, only truncate to eof operations
* can be executed.
*
* The caller is required to hold either an AccessExclusiveLock (vacuum full)
* or a ShareLock on the relation.
*/
void
AOCSCompact(Relation aorel,
List *compaction_segno,
int insert_segno,
bool isFull)
{
const char *relname;
int total_segfiles;
AOCSFileSegInfo **segfile_array;
AOCSInsertDesc insertDesc = NULL;
int i,
segno;
LockAcquireResult acquireResult;
AOCSFileSegInfo *fsinfo;
Snapshot appendOnlyMetaDataSnapshot = RegisterSnapshot(GetCatalogSnapshot(InvalidOid));
Assert(RelationIsAoCols(aorel));
Assert(Gp_role == GP_ROLE_EXECUTE || Gp_role == GP_ROLE_UTILITY);
Assert(insert_segno >= 0);
relname = RelationGetRelationName(aorel);
elogif(Debug_appendonly_print_compaction, LOG,
"Compact AO relation %s", relname);
/* Get information about all the file segments we need to scan */
segfile_array = GetAllAOCSFileSegInfo(aorel, appendOnlyMetaDataSnapshot, &total_segfiles);
if (insert_segno >= 0)
{
insertDesc = aocs_insert_init(aorel, insert_segno, false);
}
for (i = 0; i < total_segfiles; i++)
{
segno = segfile_array[i]->segno;
if (!list_member_int(compaction_segno, segno))
{
continue;
}
if (segno == insert_segno)
{
/* We cannot compact the segment file we are inserting to. */
continue;
}
/*
* Try to get the transaction write-lock for the Append-Only segment
* file.
*
* NOTE: This is a transaction scope lock that must be held until
* commit / abort.
*/
acquireResult = LockRelationAppendOnlySegmentFile(
&aorel->rd_node,
segfile_array[i]->segno,
AccessExclusiveLock,
/* dontWait */ true);
if (acquireResult == LOCKACQUIRE_NOT_AVAIL)
{
elog(DEBUG5, "compaction skips AOCS segfile %d, "
"relation %s", segfile_array[i]->segno, relname);
continue;
}
/* Re-fetch under the write lock to get latest committed eof. */
fsinfo = GetAOCSFileSegInfo(aorel, appendOnlyMetaDataSnapshot, segno);
/*
* This should not occur since this segfile info was found by the
* "all" method, but better to catch for trouble shooting (possibly
* index corruption?)
*/
if (fsinfo == NULL)
elog(ERROR, "file seginfo for AOCS relation %s %u/%u/%u (segno=%u) is missing",
relname,
aorel->rd_node.spcNode,
aorel->rd_node.dbNode,
aorel->rd_node.relNode,
segno);
if (AppendOnlyCompaction_ShouldCompact(aorel,
fsinfo->segno, fsinfo->total_tupcount, isFull,
appendOnlyMetaDataSnapshot))
{
AOCSSegmentFileFullCompaction(aorel, insertDesc, fsinfo,
appendOnlyMetaDataSnapshot);
}
pfree(fsinfo);
}
if (insertDesc != NULL)
aocs_insert_finish(insertDesc);
if (segfile_array)
{
FreeAllAOCSSegFileInfo(segfile_array, total_segfiles);
pfree(segfile_array);
}
UnregisterSnapshot(appendOnlyMetaDataSnapshot);
}
/*
* Performs a compaction of an append-only AOCS relation.
*
* In non-utility mode, all compaction segment files should be
* marked as in-use/in-compaction in the appendonlywriter.c code.
*
*/
void
AOCSDrop(Relation aorel,
List *compaction_segno)
{
const char *relname;
int total_segfiles;
AOCSFileSegInfo **segfile_array;
int i,
segno;
LockAcquireResult acquireResult;
AOCSFileSegInfo *fsinfo;
Snapshot appendOnlyMetaDataSnapshot = RegisterSnapshot(GetCatalogSnapshot(InvalidOid));
Assert(Gp_role == GP_ROLE_EXECUTE || Gp_role == GP_ROLE_UTILITY);
Assert(RelationIsAoCols(aorel));
relname = RelationGetRelationName(aorel);
elogif(Debug_appendonly_print_compaction, LOG,
"Drop AOCS relation %s", relname);
/* Get information about all the file segments we need to scan */
segfile_array = GetAllAOCSFileSegInfo(aorel,
appendOnlyMetaDataSnapshot, &total_segfiles);
for (i = 0; i < total_segfiles; i++)
{
segno = segfile_array[i]->segno;
if (!list_member_int(compaction_segno, segno))
{
continue;
}
/*
* Try to get the transaction write-lock for the Append-Only segment
* file.
*
* NOTE: This is a transaction scope lock that must be held until
* commit / abort.
*/
acquireResult = LockRelationAppendOnlySegmentFile(
&aorel->rd_node,
segfile_array[i]->segno,
AccessExclusiveLock,
/* dontWait */ true);
if (acquireResult == LOCKACQUIRE_NOT_AVAIL)
{
elog(DEBUG5, "drop skips AOCS segfile %d, "
"relation %s", segfile_array[i]->segno, relname);
continue;
}
/* Re-fetch under the write lock to get latest committed eof. */
fsinfo = GetAOCSFileSegInfo(aorel, appendOnlyMetaDataSnapshot, segno);
if (fsinfo->state == AOSEG_STATE_AWAITING_DROP)
{
Assert(HasLockForSegmentFileDrop(aorel));
AOCSCompaction_DropSegmentFile(aorel, segno);
ClearAOCSFileSegInfo(aorel, segno, AOSEG_STATE_DEFAULT);
}
pfree(fsinfo);
}
if (segfile_array)
{
FreeAllAOCSSegFileInfo(segfile_array, total_segfiles);
pfree(segfile_array);
}
UnregisterSnapshot(appendOnlyMetaDataSnapshot);
}
/*
* Truncates each segment file to the AOCS relation to its EOF.
* If we cannot get a lock on the segment file (because e.g. a concurrent insert)
* the segment file is skipped.
*/
void
AOCSTruncateToEOF(Relation aorel)
{
const char *relname;
int total_segfiles;
AOCSFileSegInfo **segfile_array;
int i,
segno;
LockAcquireResult acquireResult;
AOCSFileSegInfo *fsinfo;
Snapshot appendOnlyMetaDataSnapshot = RegisterSnapshot(GetCatalogSnapshot(InvalidOid));
Assert(RelationIsAoCols(aorel));
relname = RelationGetRelationName(aorel);
elogif(Debug_appendonly_print_compaction, LOG,
"Compact AO relation %s", relname);
/* Get information about all the file segments we need to scan */
segfile_array = GetAllAOCSFileSegInfo(aorel, appendOnlyMetaDataSnapshot, &total_segfiles);
for (i = 0; i < total_segfiles; i++)
{
segno = segfile_array[i]->segno;
/*
* Try to get the transaction write-lock for the Append-Only segment
* file.
*
* NOTE: This is a transaction scope lock that must be held until
* commit / abort.
*/
acquireResult = LockRelationAppendOnlySegmentFile(
&aorel->rd_node,
segfile_array[i]->segno,
AccessExclusiveLock,
/* dontWait */ true);
if (acquireResult == LOCKACQUIRE_NOT_AVAIL)
{
elog(DEBUG5, "truncate skips AO segfile %d, "
"relation %s", segfile_array[i]->segno, relname);
continue;
}
/* Re-fetch under the write lock to get latest committed eof. */
fsinfo = GetAOCSFileSegInfo(aorel, appendOnlyMetaDataSnapshot, segno);
/*
* This should not occur since this segfile info was found by the
* "all" method, but better to catch for trouble shooting (possibly
* index corruption?)
*/
if (fsinfo == NULL)
elog(ERROR, "file seginfo for AOCS relation %s %u/%u/%u (segno=%u) is missing",
relname,
aorel->rd_node.spcNode,
aorel->rd_node.dbNode,
aorel->rd_node.relNode,
segno);
AOCSSegmentFileTruncateToEOF(aorel, fsinfo);
pfree(fsinfo);
}
if (segfile_array)
{
FreeAllAOCSSegFileInfo(segfile_array, total_segfiles);
pfree(segfile_array);
}
UnregisterSnapshot(appendOnlyMetaDataSnapshot);
}
/*
* Performs a compaction of an append-only relation.
*
* In non-utility mode, all compaction segment files should be
* marked as in-use/in-compaction in the appendonlywriter.c code. If
* set, the insert_segno should also be marked as in-use.
* When the insert segno is negative, only truncate to eof operations
* can be executed.
*
* The caller is required to hold either an AccessExclusiveLock (vacuum full)
* or a ShareLock on the relation.
*/
void
AppendOnlyCompact(Relation aorel,
List* compaction_segno,
int insert_segno,
bool isFull)
{
const char* relname;
int total_segfiles;
FileSegInfo** segfile_array;
AppendOnlyInsertDesc insertDesc = NULL;
int i, segno;
FileSegInfo* fsinfo;
Assert (Gp_role == GP_ROLE_EXECUTE || Gp_role == GP_ROLE_UTILITY);
Assert(insert_segno >= 0);
relname = RelationGetRelationName(aorel);
AppendOnlyEntry *aoEntry = GetAppendOnlyEntry(RelationGetRelid(aorel), SnapshotNow);
elogif (Debug_appendonly_print_compaction, LOG,
"Compact AO relation %s", relname);
/* Get information about all the file segments we need to scan */
segfile_array = GetAllFileSegInfo(aorel, aoEntry, SnapshotNow, &total_segfiles);
insertDesc = appendonly_insert_init(aorel, SnapshotNow,
insert_segno, false);
for(i = 0 ; i < total_segfiles ; i++)
{
segno = segfile_array[i]->segno;
if (list_find_int(compaction_segno, segno) < 0)
{
continue;
}
if (segno == insert_segno)
{
/* We cannot compact the segment file we are inserting to. */
continue;
}
/*
* Try to get the transaction write-lock for the Append-Only segment file.
*
* NOTE: This is a transaction scope lock that must be held until commit / abort.
*/
LockRelationAppendOnlySegmentFile(
&aorel->rd_node,
segfile_array[i]->segno,
AccessExclusiveLock,
false);
/* Re-fetch under the write lock to get latest committed eof. */
fsinfo = GetFileSegInfo(aorel, aoEntry, SnapshotNow, segno);
/*
* This should not occur since this segfile info was found by the
* "all" method, but better to catch for trouble shooting
* (possibly index corruption?)
*/
if (fsinfo == NULL)
elog(ERROR, "file seginfo for AO relation %s %u/%u/%u (segno=%u) is missing",
relname,
aorel->rd_node.spcNode,
aorel->rd_node.dbNode,
aorel->rd_node.relNode,
segno);
if (AppendOnlyCompaction_ShouldCompact(aorel, aoEntry,
fsinfo->segno, fsinfo->total_tupcount, isFull))
{
AppendOnlySegmentFileFullCompaction(aorel, aoEntry,
insertDesc,
fsinfo);
}
pfree(fsinfo);
}
appendonly_insert_finish(insertDesc);
pfree(aoEntry);
if (segfile_array)
{
FreeAllSegFileInfo(segfile_array, total_segfiles);
pfree(segfile_array);
}
}
/*
* Performs a compaction of an append-only relation.
*
* In non-utility mode, all compaction segment files should be
* marked as in-use/in-compaction in the appendonlywriter.c code.
*
*/
void
AppendOnlyDrop(Relation aorel,
List* compaction_segno)
{
const char* relname;
int total_segfiles;
FileSegInfo** segfile_array;
int i, segno;
FileSegInfo* fsinfo;
Assert (Gp_role == GP_ROLE_EXECUTE || Gp_role == GP_ROLE_UTILITY);
Assert (RelationIsAoRows(aorel));
relname = RelationGetRelationName(aorel);
AppendOnlyEntry *aoEntry = GetAppendOnlyEntry(RelationGetRelid(aorel), SnapshotNow);
elogif (Debug_appendonly_print_compaction, LOG,
"Drop AO relation %s", relname);
/* Get information about all the file segments we need to scan */
segfile_array = GetAllFileSegInfo(aorel, aoEntry, SnapshotNow, &total_segfiles);
for(i = 0 ; i < total_segfiles ; i++)
{
segno = segfile_array[i]->segno;
if (list_find_int(compaction_segno, segno) < 0)
{
continue;
}
/*
* Try to get the transaction write-lock for the Append-Only segment file.
*
* NOTE: This is a transaction scope lock that must be held until commit / abort.
*/
LockRelationAppendOnlySegmentFile(
&aorel->rd_node,
segfile_array[i]->segno,
AccessExclusiveLock,
false);
/* Re-fetch under the write lock to get latest committed eof. */
fsinfo = GetFileSegInfo(aorel, aoEntry, SnapshotNow, segno);
if (fsinfo->state == AOSEG_STATE_AWAITING_DROP)
{
Assert(HasLockForSegmentFileDrop(aorel));
Assert(!HasSerializableBackends(false));
AppendOnlyCompaction_DropSegmentFile(aorel, segno);
ClearFileSegInfo(aorel, aoEntry, segno,
AOSEG_STATE_DEFAULT);
}
pfree(fsinfo);
}
pfree(aoEntry);
if (segfile_array)
{
FreeAllSegFileInfo(segfile_array, total_segfiles);
pfree(segfile_array);
}
}