/*
* Performs a compaction of an append-only relation.
*
* In non-utility mode, all compaction segment files should be
* marked as in-use/in-compaction in the appendonlywriter.c code. If
* set, the insert_segno should also be marked as in-use.
* When the insert segno is negative, only truncate to eof operations
* can be executed.
*
* The caller is required to hold either an AccessExclusiveLock (vacuum full)
* or a ShareLock on the relation.
*/
void
AppendOnlyCompact(Relation aorel,
List* compaction_segno,
int insert_segno,
bool isFull)
{
const char* relname;
int total_segfiles;
FileSegInfo** segfile_array;
AppendOnlyInsertDesc insertDesc = NULL;
int i, segno;
FileSegInfo* fsinfo;
Assert (Gp_role == GP_ROLE_EXECUTE || Gp_role == GP_ROLE_UTILITY);
Assert(insert_segno >= 0);
relname = RelationGetRelationName(aorel);
AppendOnlyEntry *aoEntry = GetAppendOnlyEntry(RelationGetRelid(aorel), SnapshotNow);
elogif (Debug_appendonly_print_compaction, LOG,
"Compact AO relation %s", relname);
/* Get information about all the file segments we need to scan */
segfile_array = GetAllFileSegInfo(aorel, aoEntry, SnapshotNow, &total_segfiles);
insertDesc = appendonly_insert_init(aorel, SnapshotNow,
insert_segno, false);
for(i = 0 ; i < total_segfiles ; i++)
{
segno = segfile_array[i]->segno;
if (list_find_int(compaction_segno, segno) < 0)
{
continue;
}
if (segno == insert_segno)
{
/* We cannot compact the segment file we are inserting to. */
continue;
}
/*
* Try to get the transaction write-lock for the Append-Only segment file.
*
* NOTE: This is a transaction scope lock that must be held until commit / abort.
*/
LockRelationAppendOnlySegmentFile(
&aorel->rd_node,
segfile_array[i]->segno,
AccessExclusiveLock,
false);
/* Re-fetch under the write lock to get latest committed eof. */
fsinfo = GetFileSegInfo(aorel, aoEntry, SnapshotNow, segno);
/*
* This should not occur since this segfile info was found by the
* "all" method, but better to catch for trouble shooting
* (possibly index corruption?)
*/
if (fsinfo == NULL)
elog(ERROR, "file seginfo for AO relation %s %u/%u/%u (segno=%u) is missing",
relname,
aorel->rd_node.spcNode,
aorel->rd_node.dbNode,
aorel->rd_node.relNode,
segno);
if (AppendOnlyCompaction_ShouldCompact(aorel, aoEntry,
fsinfo->segno, fsinfo->total_tupcount, isFull))
{
AppendOnlySegmentFileFullCompaction(aorel, aoEntry,
insertDesc,
fsinfo);
}
pfree(fsinfo);
}
appendonly_insert_finish(insertDesc);
pfree(aoEntry);
if (segfile_array)
{
FreeAllSegFileInfo(segfile_array, total_segfiles);
pfree(segfile_array);
}
}
/* ----------------------------------------------------------------
* ExecInsert
*
* INSERTs have to add the tuple into
* the base relation and insert appropriate tuples into the
* index relations.
* Insert can be part of an update operation when
* there is a preceding SplitUpdate node.
* ----------------------------------------------------------------
*/
void
ExecInsert(TupleTableSlot *slot,
DestReceiver *dest,
EState *estate,
PlanGenerator planGen,
bool isUpdate)
{
void *tuple = NULL;
ResultRelInfo *resultRelInfo = NULL;
Relation resultRelationDesc = NULL;
Oid newId = InvalidOid;
TupleTableSlot *partslot = NULL;
AOTupleId aoTupleId = AOTUPLEID_INIT;
bool rel_is_heap = false;
bool rel_is_aorows = false;
bool rel_is_external = false;
bool rel_is_parquet = false;
/*
* get information on the (current) result relation
*/
if (estate->es_result_partitions)
{
resultRelInfo = slot_get_partition(slot, estate);
estate->es_result_relation_info = resultRelInfo;
if (NULL != resultRelInfo->ri_parquetSendBack)
{
/*
* The Parquet part we are about to insert into
* has sendBack information. This means we're inserting into the
* part twice, which is not supported. Error out (GPSQL-2291)
*/
Assert(gp_parquet_insert_sort);
ereport(ERROR, (errcode(ERRCODE_CDB_FEATURE_NOT_YET),
errmsg("Cannot insert out-of-order tuples in parquet partitions"),
errhint("Sort the data on the partitioning key(s) before inserting"),
errOmitLocation(true)));
}
/*
* Check if we need to close the last parquet partition we
* inserted into (GPSQL-2291).
*/
Oid new_part_oid = resultRelInfo->ri_RelationDesc->rd_id;
if (gp_parquet_insert_sort &&
PLANGEN_OPTIMIZER == planGen &&
InvalidOid != estate->es_last_parq_part &&
new_part_oid != estate->es_last_parq_part)
{
Assert(NULL != estate->es_partition_state->result_partition_hash);
ResultPartHashEntry *entry = hash_search(estate->es_partition_state->result_partition_hash,
&estate->es_last_parq_part,
HASH_FIND,
NULL /* found */);
Assert(NULL != entry);
Assert(entry->offset < estate->es_num_result_relations);
ResultRelInfo *oldResultRelInfo = & estate->es_result_relations[entry->offset];
elog(DEBUG1, "Switching from old part oid=%d name=[%s] to new part oid=%d name=[%s]",
estate->es_last_parq_part,
oldResultRelInfo->ri_RelationDesc->rd_rel->relname.data,
new_part_oid,
resultRelInfo->ri_RelationDesc->rd_rel->relname.data);
/*
* We are opening a new partition, and the last partition we
* inserted into was a Parquet part. Let's close the old
* parquet insert descriptor to free the memory before
* opening the new one.
*/
ParquetInsertDescData *oldInsertDesc = oldResultRelInfo->ri_parquetInsertDesc;
/*
* We need to preserve the "sendback" information that needs to be
* sent back to the QD process from this part.
* Compute it here, and store it for later use.
*/
QueryContextDispatchingSendBack sendback =
CreateQueryContextDispatchingSendBack(1);
sendback->relid = RelationGetRelid(oldResultRelInfo->ri_RelationDesc);
oldInsertDesc->sendback = sendback;
parquet_insert_finish(oldInsertDesc);
/* Store the sendback information in the resultRelInfo for this part */
oldResultRelInfo->ri_parquetSendBack = sendback;
/* Record in the resultRelInfo that we closed the parquet insert descriptor */
oldResultRelInfo->ri_parquetInsertDesc = NULL;
/* Reset the last parquet part Oid, it's now closed */
estate->es_last_parq_part = InvalidOid;
}
}
else
{
resultRelInfo = estate->es_result_relation_info;
}
Assert (!resultRelInfo->ri_projectReturning);
resultRelationDesc = resultRelInfo->ri_RelationDesc;
rel_is_heap = RelationIsHeap(resultRelationDesc);
rel_is_aorows = RelationIsAoRows(resultRelationDesc);
rel_is_external = RelationIsExternal(resultRelationDesc);
rel_is_parquet = RelationIsParquet(resultRelationDesc);
/* Validate that insert is not part of an non-allowed update operation. */
if (isUpdate && (rel_is_aorows || rel_is_parquet))
{
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("Append-only tables are not updatable. Operation not permitted."),
errOmitLocation(true)));
}
partslot = reconstructMatchingTupleSlot(slot, resultRelInfo);
if (rel_is_heap || rel_is_external)
{
tuple = ExecFetchSlotHeapTuple(partslot);
}
else if (rel_is_aorows)
{
tuple = ExecFetchSlotMemTuple(partslot, false);
}
else if (rel_is_parquet)
{
tuple = NULL;
}
Assert( partslot != NULL );
Assert( rel_is_parquet || (tuple != NULL));
/* Execute triggers in Planner-generated plans */
if (planGen == PLANGEN_PLANNER)
{
/* BEFORE ROW INSERT Triggers */
if (resultRelInfo->ri_TrigDesc &&
resultRelInfo->ri_TrigDesc->n_before_row[TRIGGER_EVENT_INSERT] > 0)
{
HeapTuple newtuple;
/* NYI */
if(rel_is_parquet)
elog(ERROR, "triggers are not supported on tables that use column-oriented storage");
newtuple = ExecBRInsertTriggers(estate, resultRelInfo, tuple);
if (newtuple == NULL) /* "do nothing" */
{
return;
}
if (newtuple != tuple) /* modified by Trigger(s) */
{
/*
* Put the modified tuple into a slot for convenience of routines
* below. We assume the tuple was allocated in per-tuple memory
* context, and therefore will go away by itself. The tuple table
* slot should not try to clear it.
*/
TupleTableSlot *newslot = estate->es_trig_tuple_slot;
if (newslot->tts_tupleDescriptor != partslot->tts_tupleDescriptor)
ExecSetSlotDescriptor(newslot, partslot->tts_tupleDescriptor);
ExecStoreGenericTuple(newtuple, newslot, false);
newslot->tts_tableOid = partslot->tts_tableOid; /* for constraints */
tuple = newtuple;
partslot = newslot;
}
}
}
/*
* Check the constraints of the tuple
*/
if (resultRelationDesc->rd_att->constr &&
planGen == PLANGEN_PLANNER)
{
ExecConstraints(resultRelInfo, partslot, estate);
}
/*
* insert the tuple
*
* Note: heap_insert returns the tid (location) of the new tuple in the
* t_self field.
*
* NOTE: for append-only relations we use the append-only access methods.
*/
if (rel_is_aorows)
{
if (resultRelInfo->ri_aoInsertDesc == NULL)
{
ResultRelSegFileInfo *segfileinfo = NULL;
/* Set the pre-assigned fileseg number to insert into */
ResultRelInfoSetSegFileInfo(resultRelInfo, estate->es_result_segfileinfos);
segfileinfo = (ResultRelSegFileInfo *)list_nth(resultRelInfo->ri_aosegfileinfos, GetQEIndex());
resultRelInfo->ri_aoInsertDesc =
appendonly_insert_init(resultRelationDesc,
segfileinfo);
}
appendonly_insert(resultRelInfo->ri_aoInsertDesc, tuple, &newId, &aoTupleId);
}
else if (rel_is_external)
{
/* Writable external table */
if (resultRelInfo->ri_extInsertDesc == NULL)
resultRelInfo->ri_extInsertDesc = external_insert_init(
resultRelationDesc, 0);
newId = external_insert(resultRelInfo->ri_extInsertDesc, tuple);
}
else if(rel_is_parquet)
{
/* If there is no parquet insert descriptor, create it now. */
if (resultRelInfo->ri_parquetInsertDesc == NULL)
{
ResultRelSegFileInfo *segfileinfo = NULL;
ResultRelInfoSetSegFileInfo(resultRelInfo, estate->es_result_segfileinfos);
segfileinfo = (ResultRelSegFileInfo *)list_nth(resultRelInfo->ri_aosegfileinfos, GetQEIndex());
resultRelInfo->ri_parquetInsertDesc = parquet_insert_init(resultRelationDesc, segfileinfo);
/*
* Just opened a new parquet partition for insert. Save the Oid
* in estate, so that we can close it when switching to a
* new partition (GPSQL-2291)
*/
elog(DEBUG1, "Saving es_last_parq_part. Old=%d, new=%d", estate->es_last_parq_part, resultRelationDesc->rd_id);
estate->es_last_parq_part = resultRelationDesc->rd_id;
}
newId = parquet_insert(resultRelInfo->ri_parquetInsertDesc, partslot);
}
else
{
Insist(rel_is_heap);
newId = heap_insert(resultRelationDesc,
tuple,
estate->es_snapshot->curcid,
true, true, GetCurrentTransactionId());
}
IncrAppended();
(estate->es_processed)++;
(resultRelInfo->ri_aoprocessed)++;
estate->es_lastoid = newId;
partslot->tts_tableOid = RelationGetRelid(resultRelationDesc);
if (rel_is_aorows || rel_is_parquet)
{
/* NOTE: Current version does not support index upon parquet table. */
/*
* insert index entries for AO Row-Store tuple
*/
if (resultRelInfo->ri_NumIndices > 0 && !rel_is_parquet)
ExecInsertIndexTuples(partslot, (ItemPointer)&aoTupleId, estate, false);
}
else
{
/* Use parttuple for index update in case this is an indexed heap table. */
TupleTableSlot *xslot = partslot;
void *xtuple = tuple;
setLastTid(&(((HeapTuple) xtuple)->t_self));
/*
* insert index entries for tuple
*/
if (resultRelInfo->ri_NumIndices > 0)
ExecInsertIndexTuples(xslot, &(((HeapTuple) xtuple)->t_self), estate, false);
}
if (planGen == PLANGEN_PLANNER)
{
/* AFTER ROW INSERT Triggers */
ExecARInsertTriggers(estate, resultRelInfo, tuple);
}
}
/* ----------------------------------------------------------------
* ExecInsert
*
* INSERTs have to add the tuple into
* the base relation and insert appropriate tuples into the
* index relations.
* Insert can be part of an update operation when
* there is a preceding SplitUpdate node.
* ----------------------------------------------------------------
*/
void
ExecInsert(TupleTableSlot *slot,
DestReceiver *dest,
EState *estate,
PlanGenerator planGen,
bool isUpdate)
{
void *tuple = NULL;
ResultRelInfo *resultRelInfo = NULL;
Relation resultRelationDesc = NULL;
Oid newId = InvalidOid;
TupleTableSlot *partslot = NULL;
AOTupleId aoTupleId = AOTUPLEID_INIT;
bool rel_is_heap = false;
bool rel_is_aorows = false;
bool rel_is_aocols = false;
bool rel_is_external = false;
/*
* get information on the (current) result relation
*/
if (estate->es_result_partitions)
{
resultRelInfo = slot_get_partition(slot, estate);
/* Check whether the user provided the correct leaf part only if required */
if (!dml_ignore_target_partition_check)
{
Assert(NULL != estate->es_result_partitions->part &&
NULL != resultRelInfo->ri_RelationDesc);
List *resultRelations = estate->es_plannedstmt->resultRelations;
/*
* Only inheritance can generate multiple result relations and inheritance
* is not compatible with partitions. As we are in inserting in partitioned
* table, we should not have more than one resultRelation
*/
Assert(list_length(resultRelations) == 1);
/* We only have one resultRelations entry where the user originally intended to insert */
int rteIdxForUserRel = linitial_int(resultRelations);
Assert (rteIdxForUserRel > 0);
Oid userProvidedRel = InvalidOid;
if (1 == rteIdxForUserRel)
{
/* Optimization for typical case */
userProvidedRel = ((RangeTblEntry *) estate->es_plannedstmt->rtable->head->data.ptr_value)->relid;
}
else
{
userProvidedRel = getrelid(rteIdxForUserRel, estate->es_plannedstmt->rtable);
}
/* Error out if user provides a leaf partition that does not match with our calculated partition */
if (userProvidedRel != estate->es_result_partitions->part->parrelid &&
userProvidedRel != resultRelInfo->ri_RelationDesc->rd_id)
{
ereport(ERROR,
(errcode(ERRCODE_CHECK_VIOLATION),
errmsg("Trying to insert row into wrong partition"),
errdetail("Expected partition: %s, provided partition: %s",
resultRelInfo->ri_RelationDesc->rd_rel->relname.data,
estate->es_result_relation_info->ri_RelationDesc->rd_rel->relname.data)));
}
}
estate->es_result_relation_info = resultRelInfo;
}
else
{
resultRelInfo = estate->es_result_relation_info;
}
Assert (!resultRelInfo->ri_projectReturning);
resultRelationDesc = resultRelInfo->ri_RelationDesc;
rel_is_heap = RelationIsHeap(resultRelationDesc);
rel_is_aocols = RelationIsAoCols(resultRelationDesc);
rel_is_aorows = RelationIsAoRows(resultRelationDesc);
rel_is_external = RelationIsExternal(resultRelationDesc);
partslot = reconstructMatchingTupleSlot(slot, resultRelInfo);
if (rel_is_heap)
{
tuple = ExecFetchSlotHeapTuple(partslot);
}
else if (rel_is_aorows)
{
tuple = ExecFetchSlotMemTuple(partslot, false);
}
else if (rel_is_external)
{
if (estate->es_result_partitions &&
estate->es_result_partitions->part->parrelid != 0)
{
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("Insert into external partitions not supported.")));
return;
}
else
{
tuple = ExecFetchSlotHeapTuple(partslot);
}
}
else
{
Assert(rel_is_aocols);
tuple = ExecFetchSlotMemTuple(partslot, true);
}
Assert(partslot != NULL && tuple != NULL);
/* Execute triggers in Planner-generated plans */
if (planGen == PLANGEN_PLANNER)
{
/* BEFORE ROW INSERT Triggers */
if (resultRelInfo->ri_TrigDesc &&
resultRelInfo->ri_TrigDesc->n_before_row[TRIGGER_EVENT_INSERT] > 0)
{
HeapTuple newtuple;
/* NYI */
if(rel_is_aocols)
elog(ERROR, "triggers are not supported on tables that use column-oriented storage");
newtuple = ExecBRInsertTriggers(estate, resultRelInfo, tuple);
if (newtuple == NULL) /* "do nothing" */
{
return;
}
if (newtuple != tuple) /* modified by Trigger(s) */
{
/*
* Put the modified tuple into a slot for convenience of routines
* below. We assume the tuple was allocated in per-tuple memory
* context, and therefore will go away by itself. The tuple table
* slot should not try to clear it.
*/
TupleTableSlot *newslot = estate->es_trig_tuple_slot;
if (newslot->tts_tupleDescriptor != partslot->tts_tupleDescriptor)
ExecSetSlotDescriptor(newslot, partslot->tts_tupleDescriptor);
ExecStoreGenericTuple(newtuple, newslot, false);
newslot->tts_tableOid = partslot->tts_tableOid; /* for constraints */
tuple = newtuple;
partslot = newslot;
}
}
}
/*
* Check the constraints of the tuple
*/
if (resultRelationDesc->rd_att->constr &&
planGen == PLANGEN_PLANNER)
{
ExecConstraints(resultRelInfo, partslot, estate);
}
/*
* insert the tuple
*
* Note: heap_insert returns the tid (location) of the new tuple in the
* t_self field.
*
* NOTE: for append-only relations we use the append-only access methods.
*/
if (rel_is_aorows)
{
if (resultRelInfo->ri_aoInsertDesc == NULL)
{
/* Set the pre-assigned fileseg number to insert into */
ResultRelInfoSetSegno(resultRelInfo, estate->es_result_aosegnos);
resultRelInfo->ri_aoInsertDesc =
appendonly_insert_init(resultRelationDesc,
ActiveSnapshot,
resultRelInfo->ri_aosegno,
false);
}
appendonly_insert(resultRelInfo->ri_aoInsertDesc, tuple, &newId, &aoTupleId);
}
else if (rel_is_aocols)
{
if (resultRelInfo->ri_aocsInsertDesc == NULL)
{
ResultRelInfoSetSegno(resultRelInfo, estate->es_result_aosegnos);
resultRelInfo->ri_aocsInsertDesc = aocs_insert_init(resultRelationDesc,
resultRelInfo->ri_aosegno, false);
}
newId = aocs_insert(resultRelInfo->ri_aocsInsertDesc, partslot);
aoTupleId = *((AOTupleId*)slot_get_ctid(partslot));
}
else if (rel_is_external)
{
/* Writable external table */
if (resultRelInfo->ri_extInsertDesc == NULL)
resultRelInfo->ri_extInsertDesc = external_insert_init(resultRelationDesc);
newId = external_insert(resultRelInfo->ri_extInsertDesc, tuple);
}
else
{
Insist(rel_is_heap);
newId = heap_insert(resultRelationDesc,
tuple,
estate->es_snapshot->curcid,
true, true, GetCurrentTransactionId());
}
IncrAppended();
(estate->es_processed)++;
(resultRelInfo->ri_aoprocessed)++;
estate->es_lastoid = newId;
partslot->tts_tableOid = RelationGetRelid(resultRelationDesc);
if (rel_is_aorows || rel_is_aocols)
{
/*
* insert index entries for AO Row-Store tuple
*/
if (resultRelInfo->ri_NumIndices > 0)
ExecInsertIndexTuples(partslot, (ItemPointer)&aoTupleId, estate, false);
}
else
{
/* Use parttuple for index update in case this is an indexed heap table. */
TupleTableSlot *xslot = partslot;
void *xtuple = tuple;
setLastTid(&(((HeapTuple) xtuple)->t_self));
/*
* insert index entries for tuple
*/
if (resultRelInfo->ri_NumIndices > 0)
ExecInsertIndexTuples(xslot, &(((HeapTuple) xtuple)->t_self), estate, false);
}
if (planGen == PLANGEN_PLANNER)
{
/* AFTER ROW INSERT Triggers */
ExecARInsertTriggers(estate, resultRelInfo, tuple);
}
}
