/*
* get_all_brokers
*
* Return a list of all brokers in pipeline_kafka_brokers
*/
static List *
get_all_brokers(void)
{
HeapTuple tup = NULL;
HeapScanDesc scan;
Relation brokers = open_pipeline_kafka_brokers();
TupleTableSlot *slot = MakeSingleTupleTableSlot(RelationGetDescr(brokers));
List *result = NIL;
scan = heap_beginscan(brokers, GetTransactionSnapshot(), 0, NULL);
while ((tup = heap_getnext(scan, ForwardScanDirection)) != NULL)
{
char *host;
Datum d;
bool isnull;
ExecStoreTuple(tup, slot, InvalidBuffer, false);
d = slot_getattr(slot, BROKER_ATTR_HOST, &isnull);
host = TextDatumGetCString(d);
result = lappend(result, host);
}
ExecDropSingleTupleTableSlot(slot);
heap_endscan(scan);
heap_close(brokers, NoLock);
return result;
}
void
CheckerInit(Checker *checker, Relation rel, TupleChecker *tchecker)
{
TupleDesc desc;
checker->tchecker = tchecker;
/*
* When specify ENCODING, we check the input data encoding.
* Convert encoding if the client and server encodings are different.
*/
checker->db_encoding = GetDatabaseEncoding();
if (checker->encoding != -1 &&
(checker->encoding != PG_SQL_ASCII ||
checker->db_encoding != PG_SQL_ASCII))
checker->check_encoding = true;
if (!rel)
return;
/* When specify CHECK_CONSTRAINTS, we check the constraints */
desc = RelationGetDescr(rel);
if (desc->constr &&
(checker->check_constraints || desc->constr->has_not_null))
{
if (checker->check_constraints)
checker->has_constraints = true;
if (desc->constr->has_not_null)
checker->has_not_null = true;
checker->resultRelInfo = makeNode(ResultRelInfo);
checker->resultRelInfo->ri_RangeTableIndex = 1; /* dummy */
checker->resultRelInfo->ri_RelationDesc = rel;
checker->resultRelInfo->ri_TrigDesc = NULL; /* TRIGGER is not supported */
checker->resultRelInfo->ri_TrigInstrument = NULL;
}
if (checker->has_constraints)
{
checker->estate = CreateExecutorState();
checker->estate->es_result_relations = checker->resultRelInfo;
checker->estate->es_num_result_relations = 1;
checker->estate->es_result_relation_info = checker->resultRelInfo;
/* Set up a tuple slot too */
checker->slot = MakeSingleTupleTableSlot(desc);
}
if (!checker->has_constraints && checker->has_not_null)
{
int i;
checker->desc = CreateTupleDescCopy(desc);
for (i = 0; i < desc->natts; i++)
checker->desc->attrs[i]->attnotnull = desc->attrs[i]->attnotnull;
}
}
/*
* load_consumer_offsets
*
* Load all offsets for all of this consumer's partitions
*/
static void
load_consumer_offsets(KafkaConsumer *consumer, struct rd_kafka_metadata_topic *meta, int64_t offset)
{
MemoryContext old;
ScanKeyData skey[1];
HeapTuple tup = NULL;
HeapScanDesc scan;
Relation offsets = open_pipeline_kafka_offsets();
TupleTableSlot *slot = MakeSingleTupleTableSlot(RelationGetDescr(offsets));
int i;
ScanKeyInit(&skey[0], OFFSETS_ATTR_CONSUMER, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(consumer->id));
scan = heap_beginscan(offsets, GetTransactionSnapshot(), 1, skey);
old = MemoryContextSwitchTo(CacheMemoryContext);
consumer->offsets = palloc0(meta->partition_cnt * sizeof(int64_t));
MemoryContextSwitchTo(old);
/* by default, begin consuming from the end of a stream */
for (i = 0; i < meta->partition_cnt; i++)
consumer->offsets[i] = offset;
consumer->num_partitions = meta->partition_cnt;
while ((tup = heap_getnext(scan, ForwardScanDirection)) != NULL)
{
Datum d;
bool isnull;
int partition;
ExecStoreTuple(tup, slot, InvalidBuffer, false);
d = slot_getattr(slot, OFFSETS_ATTR_PARTITION, &isnull);
partition = DatumGetInt32(d);
if(partition > consumer->num_partitions)
elog(ERROR, "invalid partition id: %d", partition);
if (offset == RD_KAFKA_OFFSET_NULL)
{
d = slot_getattr(slot, OFFSETS_ATTR_OFFSET, &isnull);
if (isnull)
offset = RD_KAFKA_OFFSET_END;
else
offset = DatumGetInt64(d);
}
consumer->offsets[partition] = DatumGetInt64(offset);
}
ExecDropSingleTupleTableSlot(slot);
heap_endscan(scan);
heap_close(offsets, RowExclusiveLock);
}
TupleTableSlot *
table_slot_create(Relation relation, List **reglist)
{
const TupleTableSlotOps *tts_cb;
TupleTableSlot *slot;
tts_cb = table_slot_callbacks(relation);
slot = MakeSingleTupleTableSlot(RelationGetDescr(relation), tts_cb);
if (reglist)
*reglist = lappend(*reglist, slot);
return slot;
}
/*
* ExecInitJunkFilter
*
* Initialize the Junk filter.
*
* The source targetlist is passed in. The output tuple descriptor is
* built from the non-junk tlist entries, plus the passed specification
* of whether to include room for an OID or not.
* An optional resultSlot can be passed as well.
*/
JunkFilter *
ExecInitJunkFilter(List *targetList, bool hasoid, TupleTableSlot *slot)
{
JunkFilter *junkfilter;
TupleDesc cleanTupType;
int cleanLength;
AttrNumber *cleanMap;
ListCell *t;
AttrNumber cleanResno;
/*
* Compute the tuple descriptor for the cleaned tuple.
*/
cleanTupType = ExecCleanTypeFromTL(targetList, hasoid);
/*
* Use the given slot, or make a new slot if we weren't given one.
*/
if (slot)
ExecSetSlotDescriptor(slot, cleanTupType);
else
slot = MakeSingleTupleTableSlot(cleanTupType);
/*
* Now calculate the mapping between the original tuple's attributes and
* the "clean" tuple's attributes.
*
* The "map" is an array of "cleanLength" attribute numbers, i.e. one
* entry for every attribute of the "clean" tuple. The value of this entry
* is the attribute number of the corresponding attribute of the
* "original" tuple. (Zero indicates a NULL output attribute, but we do
* not use that feature in this routine.)
*/
cleanLength = cleanTupType->natts;
if (cleanLength > 0)
{
cleanMap = (AttrNumber *) palloc(cleanLength * sizeof(AttrNumber));
cleanResno = 1;
foreach(t, targetList)
{
TargetEntry *tle = lfirst(t);
if (!tle->resjunk)
{
cleanMap[cleanResno - 1] = tle->resno;
cleanResno++;
}
}
/* ----------------------------------------------------------------
* ExecInitPartitionSelector
*
* Create the run-time state information for PartitionSelector node
* produced by Orca and initializes outer child if exists.
*
* ----------------------------------------------------------------
*/
PartitionSelectorState *
ExecInitPartitionSelector(PartitionSelector *node, EState *estate, int eflags)
{
/* check for unsupported flags */
Assert (!(eflags & (EXEC_FLAG_MARK | EXEC_FLAG_BACKWARD)));
PartitionSelectorState *psstate = initPartitionSelection(node, estate);
/* tuple table initialization */
ExecInitResultTupleSlot(estate, &psstate->ps);
ExecAssignResultTypeFromTL(&psstate->ps);
ExecAssignProjectionInfo(&psstate->ps, NULL);
/* initialize child nodes */
/* No inner plan for PartitionSelector */
Assert(NULL == innerPlan(node));
if (NULL != outerPlan(node))
{
outerPlanState(psstate) = ExecInitNode(outerPlan(node), estate, eflags);
}
/*
* Initialize projection, to produce a tuple that has the partitioning key
* columns at the same positions as in the partitioned table.
*/
if (node->partTabTargetlist)
{
List *exprStates;
exprStates = (List *) ExecInitExpr((Expr *) node->partTabTargetlist,
(PlanState *) psstate);
psstate->partTabDesc = ExecTypeFromTL(node->partTabTargetlist, false);
psstate->partTabSlot = MakeSingleTupleTableSlot(psstate->partTabDesc);
psstate->partTabProj = ExecBuildProjectionInfo(exprStates,
psstate->ps.ps_ExprContext,
psstate->partTabSlot,
ExecGetResultType(&psstate->ps));
}
initGpmonPktForPartitionSelector((Plan *)node, &psstate->ps.gpmon_pkt, estate);
return psstate;
}
/*
* CopyIntoStream
*
* COPY events to a stream from an input source
*/
void
CopyIntoStream(Relation rel, TupleDesc desc, HeapTuple *tuples, int ntuples)
{
bool snap = ActiveSnapshotSet();
ResultRelInfo rinfo;
StreamInsertState *sis;
MemSet(&rinfo, 0, sizeof(ResultRelInfo));
rinfo.ri_RangeTableIndex = 1; /* dummy */
rinfo.ri_TrigDesc = NULL;
rinfo.ri_RelationDesc = rel;
if (snap)
PopActiveSnapshot();
BeginStreamModify(NULL, &rinfo, list_make1(desc), 0, 0);
sis = (StreamInsertState *) rinfo.ri_FdwState;
Assert(sis);
if (sis->queries)
{
TupleTableSlot *slot = MakeSingleTupleTableSlot(RelationGetDescr(rel));
int i;
for (i = 0; i < ntuples; i++)
{
ExecStoreTuple(tuples[i], slot, InvalidBuffer, false);
ExecStreamInsert(NULL, &rinfo, slot, NULL);
ExecClearTuple(slot);
}
ExecDropSingleTupleTableSlot(slot);
Assert(sis->ntups == ntuples);
pgstat_increment_cq_write(ntuples, sis->nbytes);
}
EndStreamModify(NULL, &rinfo);
if (snap)
PushActiveSnapshot(GetTransactionSnapshot());
}
void
SpoolerOpen(Spooler *self,
Relation rel,
bool use_wal,
ON_DUPLICATE on_duplicate,
int64 max_dup_errors,
const char *dup_badfile)
{
memset(self, 0, sizeof(Spooler));
self->on_duplicate = on_duplicate;
self->use_wal = use_wal;
self->max_dup_errors = max_dup_errors;
self->dup_old = 0;
self->dup_new = 0;
self->dup_badfile = pstrdup(dup_badfile);
self->dup_fp = NULL;
self->relinfo = makeNode(ResultRelInfo);
self->relinfo->ri_RangeTableIndex = 1; /* dummy */
self->relinfo->ri_RelationDesc = rel;
self->relinfo->ri_TrigDesc = NULL; /* TRIGGER is not supported */
self->relinfo->ri_TrigInstrument = NULL;
ExecOpenIndices(self->relinfo);
self->estate = CreateExecutorState();
self->estate->es_num_result_relations = 1;
self->estate->es_result_relations = self->relinfo;
self->estate->es_result_relation_info = self->relinfo;
self->slot = MakeSingleTupleTableSlot(RelationGetDescr(rel));
self->spools = IndexSpoolBegin(self->relinfo,
max_dup_errors == 0);
}
/*
* CatalogIndexInsert - insert index entries for one catalog tuple
*
* This should be called for each inserted or updated catalog tuple.
*
* This is effectively a cut-down version of ExecInsertIndexTuples.
*/
void
CatalogIndexInsert(CatalogIndexState indstate, HeapTuple heapTuple)
{
int i;
int numIndexes;
RelationPtr relationDescs;
Relation heapRelation;
TupleTableSlot *slot;
IndexInfo **indexInfoArray;
Datum values[INDEX_MAX_KEYS];
bool isnull[INDEX_MAX_KEYS];
/* HOT update does not require index inserts */
if (HeapTupleIsHeapOnly(heapTuple))
return;
/*
* Get information from the state structure. Fall out if nothing to do.
*/
numIndexes = indstate->ri_NumIndices;
if (numIndexes == 0)
return;
relationDescs = indstate->ri_IndexRelationDescs;
indexInfoArray = indstate->ri_IndexRelationInfo;
heapRelation = indstate->ri_RelationDesc;
/* Need a slot to hold the tuple being examined */
slot = MakeSingleTupleTableSlot(RelationGetDescr(heapRelation));
ExecStoreTuple(heapTuple, slot, InvalidBuffer, false);
/*
* for each index, form and insert the index tuple
*/
for (i = 0; i < numIndexes; i++)
{
IndexInfo *indexInfo;
indexInfo = indexInfoArray[i];
/* If the index is marked as read-only, ignore it */
if (!indexInfo->ii_ReadyForInserts)
continue;
/*
* Expressional and partial indexes on system catalogs are not
* supported, nor exclusion constraints, nor deferred uniqueness
*/
Assert(indexInfo->ii_Expressions == NIL);
Assert(indexInfo->ii_Predicate == NIL);
Assert(indexInfo->ii_ExclusionOps == NULL);
Assert(relationDescs[i]->rd_index->indimmediate);
/*
* FormIndexDatum fills in its values and isnull parameters with the
* appropriate values for the column(s) of the index.
*/
FormIndexDatum(indexInfo,
slot,
NULL, /* no expression eval to do */
values,
isnull);
/*
* The index AM does the rest.
*/
index_insert(relationDescs[i], /* index relation */
values, /* array of index Datums */
isnull, /* is-null flags */
&(heapTuple->t_self), /* tid of heap tuple */
heapRelation,
relationDescs[i]->rd_index->indisunique ?
UNIQUE_CHECK_YES : UNIQUE_CHECK_NO);
}
ExecDropSingleTupleTableSlot(slot);
}
Datum partition_insert_trigger(PG_FUNCTION_ARGS)
{
TriggerData *trigdata = (TriggerData *) fcinfo->context;
char *date_time;
char child_table[sizeof(TABLE)+sizeof("2012_09_10")] = TABLE;
int partition_field;
Relation child_table_id;
Oid child_table_oid;
BulkInsertState bistate = GetBulkInsertState();
TupleTableSlot *slot;
EState *estate = CreateExecutorState();
ResultRelInfo *resultRelInfo = makeNode(ResultRelInfo);
List *recheckIndexes = NIL;
/* make sure it's called as a trigger at all */
if (!CALLED_AS_TRIGGER(fcinfo))
elog(ERROR, "partition_insert_trigger: not called by trigger manager");
/* Sanity checks */
if (!TRIGGER_FIRED_BY_INSERT(trigdata->tg_event) || !TRIGGER_FIRED_BEFORE(trigdata->tg_event))
elog(ERROR, "partition_insert_trigger: not called on insert before");
#ifdef DEBUG
elog(INFO, "Trigger Called for: %s", SPI_getrelname(trigdata->tg_relation));
#endif
// Get the field number for the partition
partition_field = SPI_fnumber(trigdata->tg_relation->rd_att, PARTITION_COLUMN);
// Get the value for the partition_field
date_time = SPI_getvalue(trigdata->tg_trigtuple, trigdata->tg_relation->rd_att, partition_field);
//make sure date was specified
if (!date_time)
elog(ERROR, "You cannot insert data without specifying a value for the column %s", PARTITION_COLUMN);
#ifdef DEBUG
elog(INFO, "Trying to insert date_time=%s", date_time);
#endif
//add date_time, child_table_2012_01_23
strncpy(child_table + sizeof(TABLE) -1 , date_time, 4); //2012
child_table[sizeof(TABLE) + 3] = SEPARATOR; //2012_
strncpy(child_table + sizeof(TABLE) + 4 , date_time + 5, 2); //2012_01
child_table[sizeof(TABLE) + 6] = SEPARATOR; //2012_01_
strncpy(child_table + sizeof(TABLE) + 7 , date_time + 8, 2); //2012_01_23
#ifdef DEBUG
elog(INFO, "New table will be %s", child_table);
#endif
pfree(date_time);
//if you care about triggers on the child tables, call ExecBRInsertTriggers
//don't care, continue
//get the OID of the table we are looking for to insert
child_table_oid = RelnameGetRelid(child_table); //Look for child child_table
if (child_table_oid == InvalidOid){
elog(INFO, "partition_insert_trigger: Invalid child table %s, inserting data to main table %s", child_table, TABLE);
return PointerGetDatum(trigdata->tg_trigtuple);
}
//get the descriptor of the table we are looking for
child_table_id = RelationIdGetRelation(child_table_oid); //Get the child relation descriptor
if (child_table_id == NULL){
elog(ERROR, "partition_insert_trigger: Failed to locate relation for child table %s, inserting data to main table %s", child_table, TABLE);
return PointerGetDatum(trigdata->tg_trigtuple);
}
//set the resultRelInfo
resultRelInfo->ri_RangeTableIndex = 1; /* dummy */
resultRelInfo->ri_RelationDesc = child_table_id;
//setup the estate, not sure why
estate->es_result_relations = resultRelInfo;
estate->es_num_result_relations = 1;
estate->es_result_relation_info = resultRelInfo;
/* Set up a tuple slot not sure why yet */
slot = MakeSingleTupleTableSlot(trigdata->tg_relation->rd_att);
ExecStoreTuple(trigdata->tg_trigtuple, slot, InvalidBuffer, false);
//heap_insert(child_table_id, trigdata->tg_trigtuple, GetCurrentCommandId(true), use_wal, bistate);
simple_heap_insert(child_table_id, trigdata->tg_trigtuple);
if (resultRelInfo->ri_NumIndices > 0)
recheckIndexes = ExecInsertIndexTuples(slot, &(trigdata->tg_trigtuple->t_self), estate);
// not sure if this would work CatalogUpdateIndexes(child_table_id, trigdata->tg_trigtuple)
//free the used memory
list_free(recheckIndexes);
ExecDropSingleTupleTableSlot(slot); //saw this somewhere :P
RelationClose(child_table_id); //Must be called to free the relation memory page
FreeBulkInsertState(bistate); // ? not sure if needed ?
//If return next line will add to the regular table
//return PointerGetDatum(trigdata->tg_trigtuple);
//Return Null data, still have to figure out to return a proper X rows affected
return PointerGetDatum(NULL);
}
TupleTableSlot* BuildModelInfoTupleTableSlot(ModelInfo *modelInfo){
TupleTableSlot *resultSlot;
HeapTuple tuple;
TupleDesc tdesc;
Datum *values;
bool *isNull;
Form_pg_attribute *attrs;
// open the relation pg_model to lend the TupleDescriptor from it
Relation rel = heap_open(ModelRelationId, 0);
// we have to copy the lent TupleDesc, otherwise the relation pg_model won't talk to us later
tdesc = CreateTupleDescCopy(rel->rd_att);
/*hsdesc = heap_beginscan(rel, SnapshotNow, 0, NULL);
tuple = heap_getnext(hsdesc, ForwardScanDirection);
resultSlot = MakeSingleTupleTableSlot(tdesc);*/
attrs = palloc(sizeof(FormData_pg_attribute)*7);
attrs[0] = tdesc->attrs[0]; //name
attrs[1] = tdesc->attrs[4]; //algorithm
attrs[2] = tdesc->attrs[2]; //time
attrs[3] = tdesc->attrs[3]; //measure
attrs[4] = tdesc->attrs[6]; //aggtype
attrs[5] = tdesc->attrs[5]; //granularity
attrs[6] = tdesc->attrs[9]; //timestamp
tdesc = CreateTupleDesc(7, false, attrs);
resultSlot = MakeSingleTupleTableSlot(tdesc);
values = (Datum *)palloc(sizeof(Datum)*7);
isNull = (bool *)palloc0(sizeof(bool)*7);
if(modelInfo->modelName){
values[0] = PointerGetDatum(modelInfo->modelName);
}else{
// if a model is not stored, it has no name
isNull[0] = true;
}
values[1] = PointerGetDatum(getModelTypeAsString((ModelType)(modelInfo->forecastMethod)));
isNull[1] = false;
values[2] = Int16GetDatum(modelInfo->time->resorigcol);
isNull[2] = false;
values[3] = Int16GetDatum(modelInfo->measure->resorigcol);
isNull[3] = false;
values[4] = ObjectIdGetDatum(modelInfo->aggType);
isNull[4] = false;
values[5] = Int16GetDatum(modelInfo->granularity);
isNull[5] = false;
values[6] = Int32GetDatum(modelInfo->timestamp);
isNull[6] = false;
tuple = heap_form_tuple(tdesc, values, isNull);
if(tuple){
ExecStoreTuple(tuple, resultSlot, InvalidBuffer, false);
}
// if(hsdesc){
// heap_endscan(hsdesc);
// }
heap_close(rel, 0);
return resultSlot;
}
/*
* ExecFilterRecommend
*
* This function just borrows a tuple descriptor from the RecView,
* but we create the data ourselves through various means.
*/
static TupleTableSlot*
ExecFilterRecommend(RecScanState *recnode,
ExecScanAccessMtd accessMtd,
ExecScanRecheckMtd recheckMtd)
{
ExprContext *econtext;
List *qual;
ProjectionInfo *projInfo;
ExprDoneCond isDone;
TupleTableSlot *resultSlot;
ScanState *node;
AttributeInfo *attributes;
node = recnode->subscan;
attributes = (AttributeInfo*) recnode->attributes;
/*
* Fetch data from node
*/
qual = node->ps.qual;
projInfo = node->ps.ps_ProjInfo;
econtext = node->ps.ps_ExprContext;
/*
* Check to see if we're still projecting out tuples from a previous scan
* tuple (because there is a function-returning-set in the projection
* expressions). If so, try to project another one.
*/
if (node->ps.ps_TupFromTlist)
{
Assert(projInfo); /* can't get here if not projecting */
resultSlot = ExecProject(projInfo, &isDone);
if (isDone == ExprMultipleResult)
return resultSlot;
/* Done with that source tuple... */
node->ps.ps_TupFromTlist = false;
}
/*
* Reset per-tuple memory context to free any expression evaluation
* storage allocated in the previous tuple cycle. Note this can't happen
* until we're done projecting out tuples from a scan tuple.
*/
ResetExprContext(econtext);
/*
* get a tuple from the access method. Loop until we obtain a tuple that
* passes the qualification.
*/
for (;;)
{
TupleTableSlot *slot;
int natts, i, userID, userindex, itemID, itemindex;
CHECK_FOR_INTERRUPTS();
slot = recnode->ss.ps.ps_ResultTupleSlot;
/* The first thing we need to do is initialize our recommender
* model and other things, if we haven't done so already. */
if (!recnode->initialized)
InitializeRecommender(recnode);
/*
* If we've exhausted our item list, then we're totally
* finished. We set a flag for this. It's possible that
* we'll be in the inner loop of a join, through poor
* planning, so we'll reset the appropriate data in case
* we have to do this again, though our JoinRecommend
* should assure this doesn't happen.
*/
if (recnode->finished) {
recnode->finished = false;
recnode->userNum = 0;
recnode->itemNum = 0;
return NULL;
}
/* We're only going to fetch one tuple and store its tuple
* descriptor. We can use this tuple descriptor to make as
* many new tuples as we want. */
if (recnode->base_slot == NULL) {
slot = ExecRecFetch(node, accessMtd, recheckMtd);
recnode->base_slot = CreateTupleDescCopy(slot->tts_tupleDescriptor);
}
/* Create a new slot to operate on. */
slot = MakeSingleTupleTableSlot(recnode->base_slot);
slot->tts_isempty = false;
/*
* place the current tuple into the expr context
*/
econtext->ecxt_scantuple = slot;
/* Mark all slots as usable. */
natts = slot->tts_tupleDescriptor->natts;
for (i = 0; i < natts; i++) {
/* Mark slot. */
slot->tts_values[i] = Int32GetDatum(0);
slot->tts_isnull[i] = false;
slot->tts_nvalid++;
}
/* While we're here, record what tuple attributes
* correspond to our key columns. This will save
* us unnecessary strcmp functions. */
if (recnode->useratt < 0) {
for (i = 0; i < natts; i++) {
char* col_name = slot->tts_tupleDescriptor->attrs[i]->attname.data;
//printf("%s\n",col_name);
if (strcmp(col_name,attributes->userkey) == 0)
recnode->useratt = i;
else if (strcmp(col_name,attributes->itemkey) == 0)
recnode->itematt = i;
else if (strcmp(col_name,attributes->eventval) == 0)
recnode->eventatt = i;
}
}
/*
* We now have a problem: we need to create prediction structures
* for a user before we do filtering, so that we can have a proper
* item list. But we also need to filter before creating those
* structures, so we don't end up taking forever with it. The
* solution is to filter twice.
*/
userID = -1; itemID = -1;
/* First, replace the user ID. */
userindex = recnode->userNum;
userID = recnode->userList[userindex];
/*
* We now have a blank tuple slot that we need to fill with data.
* We have a working user ID, but not a valid item list. We'd like to
* use the filter to determine if this is a good user, but we can't
* do that without an item, in many cases. The solution is to add in
* dummy items, then compare it against the filter. If a given user ID
* doesn't make it past the filter with any item ID, then that user is
* being filtered out, and we'll move on to the next.
*/
if (recnode->newUser) {
recnode->fullItemNum = 0;
itemindex = recnode->fullItemNum;
itemID = recnode->fullItemList[itemindex];
slot->tts_values[recnode->useratt] = Int32GetDatum(userID);
slot->tts_values[recnode->itematt] = Int32GetDatum(itemID);
slot->tts_values[recnode->eventatt] = Int32GetDatum(-1);
/* We have a preliminary slot - let's test it. */
while (qual && !ExecQual(qual, econtext, false)) {
/* We failed the test. Try the next item. */
recnode->fullItemNum++;
if (recnode->fullItemNum >= recnode->fullTotalItems) {
/* If we've reached the last item, move onto the next user.
* If we've reached the last user, we're done. */
InstrCountFiltered1(node, recnode->fullTotalItems);
recnode->userNum++;
recnode->newUser = true;
recnode->fullItemNum = 0;
if (recnode->userNum >= recnode->totalUsers) {
recnode->userNum = 0;
recnode->itemNum = 0;
return NULL;
}
userindex = recnode->userNum;
userID = recnode->userList[userindex];
}
itemindex = recnode->fullItemNum;
itemID = recnode->fullItemList[itemindex];
slot->tts_values[recnode->useratt] = Int32GetDatum(userID);
slot->tts_values[recnode->itematt] = Int32GetDatum(itemID);
}
/* If we get here, then we found a user who will be actually
* returned in the results. One quick reset here. */
recnode->fullItemNum = 0;
}
/* Mark the user ID and index. */
attributes->userID = userID;
recnode->userindex = userindex;
/* With the user ID determined, we need to investigate and see
* if this is a new user. If so, attempt to create prediction
* data structures, or report that this user is invalid. We have
* to do this here, so we can establish the item list. */
if (recnode->newUser) {
recnode->validUser = prepUserForRating(recnode,userID);
recnode->newUser = false;
}
/* Now replace the item ID, if the user is valid. Otherwise,
* leave the item ID as is, as it doesn't matter what it is. */
if (recnode->validUser)
itemID = recnode->itemList[recnode->itemNum];
while (recnode->fullItemList[recnode->fullItemNum] < itemID)
recnode->fullItemNum++;
itemindex = recnode->fullItemNum;
if (recnode->fullItemList[itemindex] > itemID)
elog(ERROR, "critical item mismatch in ExecRecommend");
/* Plug in the data, marking those columns full. We also need to
* mark the rating column with something temporary. */
slot->tts_values[recnode->useratt] = Int32GetDatum(userID);
slot->tts_values[recnode->itematt] = Int32GetDatum(itemID);
slot->tts_values[recnode->eventatt] = Int32GetDatum(-1);
/* It's possible our filter criteria involves the RecScore somehow.
* If that's the case, we need to calculate it before we do the
* qual filtering. Also, if we're doing a JoinRecommend, we should
* not calculate the RecScore in this node. In the current version
* of RecDB, an OP_NOFILTER shouldn't be allowed. */
if (attributes->opType == OP_NOFILTER)
applyRecScore(recnode, slot, itemID, itemindex);
/* Move onto the next item, for next time. If we're doing a RecJoin,
* though, we'll move onto the next user instead. */
recnode->itemNum++;
if (recnode->itemNum >= recnode->totalItems ||
attributes->opType == OP_JOIN ||
attributes->opType == OP_GENERATEJOIN) {
/* If we've reached the last item, move onto the next user.
* If we've reached the last user, we're done. */
recnode->userNum++;
recnode->newUser = true;
recnode->itemNum = 0;
recnode->fullItemNum = 0;
if (recnode->userNum >= recnode->totalUsers)
recnode->finished = true;
}
/*
* check that the current tuple satisfies the qual-clause
*
* check for non-nil qual here to avoid a function call to ExecQual()
* when the qual is nil ... saves only a few cycles, but they add up
* ...
*/
if (!qual || ExecQual(qual, econtext, false))
{
/*
* If this is an invalid user, then we'll skip this tuple,
* adding one to the filter count.
*/
if (!recnode->validUser) {
InstrCountFiltered1(node, 1);
ResetExprContext(econtext);
ExecDropSingleTupleTableSlot(slot);
continue;
}
/*
* Found a satisfactory scan tuple. This is usually when
* we will calculate and apply the RecScore.
*/
if (attributes->opType == OP_FILTER || attributes->opType == OP_GENERATE)
applyRecScore(recnode, slot, itemID, itemindex);
if (projInfo)
{
/*
* Form a projection tuple, store it in the result tuple slot
* and return it --- unless we find we can project no tuples
* from this scan tuple, in which case continue scan.
*/
resultSlot = ExecProject(projInfo, &isDone);
if (isDone != ExprEndResult)
{
node->ps.ps_TupFromTlist = (isDone == ExprMultipleResult);
return resultSlot;
}
}
else
{
/*
* Here, we aren't projecting, so just return scan tuple.
*/
return slot;
}
}
else
InstrCountFiltered1(node, 1);
/*
* Tuple fails qual, so free per-tuple memory and try again.
*/
ResetExprContext(econtext);
ExecDropSingleTupleTableSlot(slot);
}
}
Datum
pipeline_stream_insert(PG_FUNCTION_ARGS)
{
TriggerData *trigdata = (TriggerData *) fcinfo->context;
Trigger *trig = trigdata->tg_trigger;
HeapTuple tup;
List *fdw_private;
int i;
ResultRelInfo rinfo;
if (trig->tgnargs < 1)
elog(ERROR, "pipeline_stream_insert: must be provided a stream name");
/* make sure it's called as a trigger */
if (!CALLED_AS_TRIGGER(fcinfo))
ereport(ERROR,
(errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED),
errmsg("pipeline_stream_insert: must be called as trigger")));
/* and that it's called on update or insert */
if (!TRIGGER_FIRED_BY_UPDATE(trigdata->tg_event) && !TRIGGER_FIRED_BY_INSERT(trigdata->tg_event))
ereport(ERROR,
(errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED),
errmsg("pipeline_stream_insert: must be called on insert or update")));
/* and that it's called for each row */
if (!TRIGGER_FIRED_FOR_ROW(trigdata->tg_event))
ereport(ERROR,
(errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED),
errmsg("pipeline_stream_insert: must be called for each row")));
/* and that it's called after insert or update */
if (!TRIGGER_FIRED_AFTER(trigdata->tg_event))
ereport(ERROR,
(errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED),
errmsg("pipeline_stream_insert: must be called after insert or update")));
if (TRIGGER_FIRED_BY_UPDATE(trigdata->tg_event))
tup = trigdata->tg_newtuple;
else
tup = trigdata->tg_trigtuple;
fdw_private = list_make1(RelationGetDescr(trigdata->tg_relation));
MemSet(&rinfo, 0, sizeof(ResultRelInfo));
rinfo.ri_RangeTableIndex = 1; /* dummy */
rinfo.ri_TrigDesc = NULL;
for (i = 0; i < trig->tgnargs; i++)
{
RangeVar *stream;
Relation rel;
StreamInsertState *sis;
stream = makeRangeVarFromNameList(textToQualifiedNameList(cstring_to_text(trig->tgargs[i])));
rel = heap_openrv(stream, AccessShareLock);
rinfo.ri_RelationDesc = rel;
BeginStreamModify(NULL, &rinfo, fdw_private, 0, 0);
sis = (StreamInsertState *) rinfo.ri_FdwState;
Assert(sis);
if (sis->queries)
{
TupleTableSlot *slot = MakeSingleTupleTableSlot(RelationGetDescr(rel));
ExecStoreTuple(tup, slot, InvalidBuffer, false);
ExecStreamInsert(NULL, &rinfo, slot, NULL);
ExecClearTuple(slot);
ExecDropSingleTupleTableSlot(slot);
pgstat_report_streamstat(true);
}
EndStreamModify(NULL, &rinfo);
heap_close(rel, AccessShareLock);
}
return PointerGetDatum(tup);
}
/*
* Assumes that the segment file lock is already held.
* Assumes that the segment file should be compacted.
*/
static bool
AOCSSegmentFileFullCompaction(Relation aorel,
AOCSInsertDesc insertDesc,
AOCSFileSegInfo *fsinfo,
Snapshot snapshot)
{
const char *relname;
AppendOnlyVisimap visiMap;
AOCSScanDesc scanDesc;
TupleDesc tupDesc;
TupleTableSlot *slot;
int compact_segno;
int64 movedTupleCount = 0;
ResultRelInfo *resultRelInfo;
MemTupleBinding *mt_bind;
EState *estate;
bool *proj;
int i;
AOTupleId *aoTupleId;
int64 tupleCount = 0;
int64 tuplePerPage = INT_MAX;
Assert(Gp_role == GP_ROLE_EXECUTE || Gp_role == GP_ROLE_UTILITY);
Assert(RelationIsAoCols(aorel));
Assert(insertDesc);
compact_segno = fsinfo->segno;
if (fsinfo->varblockcount > 0)
{
tuplePerPage = fsinfo->total_tupcount / fsinfo->varblockcount;
}
relname = RelationGetRelationName(aorel);
AppendOnlyVisimap_Init(&visiMap,
aorel->rd_appendonly->visimaprelid,
aorel->rd_appendonly->visimapidxid,
ShareLock,
snapshot);
elogif(Debug_appendonly_print_compaction,
LOG, "Compact AO segfile %d, relation %sd",
compact_segno, relname);
proj = palloc0(sizeof(bool) * RelationGetNumberOfAttributes(aorel));
for (i = 0; i < RelationGetNumberOfAttributes(aorel); ++i)
{
proj[i] = true;
}
scanDesc = aocs_beginrangescan(aorel,
snapshot, snapshot,
&compact_segno, 1, NULL, proj);
tupDesc = RelationGetDescr(aorel);
slot = MakeSingleTupleTableSlot(tupDesc);
mt_bind = create_memtuple_binding(tupDesc);
/*
* We need a ResultRelInfo and an EState so we can use the regular
* executor's index-entry-making machinery.
*/
estate = CreateExecutorState();
resultRelInfo = makeNode(ResultRelInfo);
resultRelInfo->ri_RangeTableIndex = 1; /* dummy */
resultRelInfo->ri_RelationDesc = aorel;
resultRelInfo->ri_TrigDesc = NULL; /* we don't fire triggers */
ExecOpenIndices(resultRelInfo);
estate->es_result_relations = resultRelInfo;
estate->es_num_result_relations = 1;
estate->es_result_relation_info = resultRelInfo;
while (aocs_getnext(scanDesc, ForwardScanDirection, slot))
{
CHECK_FOR_INTERRUPTS();
aoTupleId = (AOTupleId *) slot_get_ctid(slot);
if (AppendOnlyVisimap_IsVisible(&scanDesc->visibilityMap, aoTupleId))
{
AOCSMoveTuple(slot,
insertDesc,
resultRelInfo,
estate);
movedTupleCount++;
}
else
{
/* Tuple is invisible and needs to be dropped */
AppendOnlyThrowAwayTuple(aorel,
slot,
mt_bind);
}
/*
* Check for vacuum delay point after approximatly a var block
*/
tupleCount++;
if (VacuumCostActive && tupleCount % tuplePerPage == 0)
{
vacuum_delay_point();
}
}
SetAOCSFileSegInfoState(aorel, compact_segno,
AOSEG_STATE_AWAITING_DROP);
AppendOnlyVisimap_DeleteSegmentFile(&visiMap,
compact_segno);
/* Delete all mini pages of the segment files if block directory exists */
if (OidIsValid(aorel->rd_appendonly->blkdirrelid))
{
AppendOnlyBlockDirectory_DeleteSegmentFile(aorel,
snapshot,
compact_segno,
0);
}
elogif(Debug_appendonly_print_compaction, LOG,
"Finished compaction: "
"AO segfile %d, relation %s, moved tuple count " INT64_FORMAT,
compact_segno, relname, movedTupleCount);
AppendOnlyVisimap_Finish(&visiMap, NoLock);
ExecCloseIndices(resultRelInfo);
FreeExecutorState(estate);
ExecDropSingleTupleTableSlot(slot);
destroy_memtuple_binding(mt_bind);
aocs_endscan(scanDesc);
pfree(proj);
return true;
}
/*
* Use the supplied ResultRelInfo to create an appropriately restructured
* version of the tuple in the supplied slot, if necessary.
*
* slot -- slot containing the input tuple
* resultRelInfo -- info pertaining to the target part of an insert
*
* If no restructuring is required, the result is the argument slot, else
* it is the slot from the argument result info updated to hold the
* restructured tuple.
*/
TupleTableSlot *
reconstructMatchingTupleSlot(TupleTableSlot *slot, ResultRelInfo *resultRelInfo)
{
int natts;
Datum *values;
bool *isnull;
AttrMap *map;
TupleTableSlot *partslot;
Datum *partvalues;
bool *partisnull;
map = resultRelInfo->ri_partInsertMap;
TupleDesc inputTupDesc = slot->tts_tupleDescriptor;
TupleDesc resultTupDesc = resultRelInfo->ri_RelationDesc->rd_att;
bool tupleDescMatch = (resultRelInfo->tupdesc_match == 1);
if (resultRelInfo->tupdesc_match == 0)
{
tupleDescMatch = equalTupleDescs(inputTupDesc, resultTupDesc, false);
if (tupleDescMatch)
{
resultRelInfo->tupdesc_match = 1;
}
else
{
resultRelInfo->tupdesc_match = -1;
}
}
/* No map and matching tuple descriptor means no restructuring needed. */
if (map == NULL && tupleDescMatch)
return slot;
/* Put the given tuple into attribute arrays. */
natts = slot->tts_tupleDescriptor->natts;
slot_getallattrs(slot);
values = slot_get_values(slot);
isnull = slot_get_isnull(slot);
/*
* Get the target slot ready. If this is a child partition table,
* set target slot to ri_partSlot. Otherwise, use ri_resultSlot.
*/
if (map != NULL)
{
Assert(resultRelInfo->ri_partSlot != NULL);
partslot = resultRelInfo->ri_partSlot;
}
else
{
if (resultRelInfo->ri_resultSlot == NULL)
{
resultRelInfo->ri_resultSlot = MakeSingleTupleTableSlot(resultTupDesc);
}
partslot = resultRelInfo->ri_resultSlot;
}
partslot = ExecStoreAllNullTuple(partslot);
partvalues = slot_get_values(partslot);
partisnull = slot_get_isnull(partslot);
/* Restructure the input tuple. Non-zero map entries are attribute
* numbers in the target tuple, however, not every attribute
* number of the input tuple need be present. In particular,
* attribute numbers corresponding to dropped attributes will be
* missing.
*/
reconstructTupleValues(map, values, isnull, natts,
partvalues, partisnull, partslot->tts_tupleDescriptor->natts);
partslot = ExecStoreVirtualTuple(partslot);
return partslot;
}
/*
* load_consumer_state
*
* Read consumer state from pipeline_kafka_consumers into the given struct
*/
static void
load_consumer_state(Oid worker_id, KafkaConsumer *consumer)
{
ScanKeyData skey[1];
HeapTuple tup = NULL;
HeapScanDesc scan;
Relation consumers = open_pipeline_kafka_consumers();
TupleTableSlot *slot = MakeSingleTupleTableSlot(RelationGetDescr(consumers));
Datum d;
bool isnull;
text *qualified;
MemoryContext old;
MemSet(consumer, 0, sizeof(KafkaConsumer));
ScanKeyInit(&skey[0], -2, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(worker_id));
scan = heap_beginscan(consumers, GetTransactionSnapshot(), 1, skey);
tup = heap_getnext(scan, ForwardScanDirection);
if (!HeapTupleIsValid(tup))
elog(ERROR, "kafka consumer %d not found", worker_id);
ExecStoreTuple(tup, slot, InvalidBuffer, false);
consumer->id = HeapTupleGetOid(tup);
d = slot_getattr(slot, CONSUMER_ATTR_RELATION, &isnull);
/* we don't want anything that's palloc'd to get freed when we commit */
old = MemoryContextSwitchTo(CacheMemoryContext);
/* target relation */
qualified = (text *) DatumGetPointer(d);
consumer->rel = makeRangeVarFromNameList(textToQualifiedNameList(qualified));
/* topic */
d = slot_getattr(slot, CONSUMER_ATTR_TOPIC, &isnull);
consumer->topic = TextDatumGetCString(d);
/* format */
d = slot_getattr(slot, CONSUMER_ATTR_FORMAT, &isnull);
consumer->format = TextDatumGetCString(d);
/* delimiter */
d = slot_getattr(slot, CONSUMER_ATTR_DELIMITER, &isnull);
if (!isnull)
consumer->delimiter = TextDatumGetCString(d);
else
consumer->delimiter = NULL;
/* quote character */
d = slot_getattr(slot, CONSUMER_ATTR_QUOTE, &isnull);
if (!isnull)
consumer->quote = TextDatumGetCString(d);
else
consumer->quote = NULL;
/* escape character */
d = slot_getattr(slot, CONSUMER_ATTR_ESCAPE, &isnull);
if (!isnull)
consumer->escape = TextDatumGetCString(d);
else
consumer->escape = NULL;
/* now load all brokers */
consumer->brokers = get_all_brokers();
MemoryContextSwitchTo(old);
d = slot_getattr(slot, CONSUMER_ATTR_PARALLELISM, &isnull);
consumer->parallelism = DatumGetInt32(d);
/* batch size */
d = slot_getattr(slot, CONSUMER_ATTR_BATCH_SIZE, &isnull);
consumer->batch_size = DatumGetInt32(d);
ExecDropSingleTupleTableSlot(slot);
heap_endscan(scan);
heap_close(consumers, NoLock);
}
/*
* Find or create a hashtable entry for the tuple group containing the
* given tuple. The tuple must be the same type as the hashtable entries.
*
* If isnew is NULL, we do not create new entries; we return NULL if no
* match is found.
*
* If isnew isn't NULL, then a new entry is created if no existing entry
* matches. On return, *isnew is true if the entry is newly created,
* false if it existed already. ->additional_data in the new entry has
* been zeroed.
*/
TupleHashEntry
LookupTupleHashEntry(TupleHashTable hashtable, TupleTableSlot *slot,
bool *isnew)
{
TupleHashEntryData *entry;
MemoryContext oldContext;
bool found;
MinimalTuple key;
/* If first time through, clone the input slot to make table slot */
if (hashtable->tableslot == NULL)
{
TupleDesc tupdesc;
oldContext = MemoryContextSwitchTo(hashtable->tablecxt);
/*
* We copy the input tuple descriptor just for safety --- we assume
* all input tuples will have equivalent descriptors.
*/
tupdesc = CreateTupleDescCopy(slot->tts_tupleDescriptor);
hashtable->tableslot = MakeSingleTupleTableSlot(tupdesc);
MemoryContextSwitchTo(oldContext);
}
/* Need to run the hash functions in short-lived context */
oldContext = MemoryContextSwitchTo(hashtable->tempcxt);
/* set up data needed by hash and match functions */
hashtable->inputslot = slot;
hashtable->in_hash_funcs = hashtable->tab_hash_funcs;
hashtable->cur_eq_funcs = hashtable->tab_eq_funcs;
key = NULL; /* flag to reference inputslot */
if (isnew)
{
entry = tuplehash_insert(hashtable->hashtab, key, &found);
if (found)
{
/* found pre-existing entry */
*isnew = false;
}
else
{
/* created new entry */
*isnew = true;
/* zero caller data */
entry->additional = NULL;
MemoryContextSwitchTo(hashtable->tablecxt);
/* Copy the first tuple into the table context */
entry->firstTuple = ExecCopySlotMinimalTuple(slot);
}
}
else
{
entry = tuplehash_lookup(hashtable->hashtab, key);
}
MemoryContextSwitchTo(oldContext);
return entry;
}
/*
* unique_key_recheck - trigger function to do a deferred uniqueness check.
*
* This now also does deferred exclusion-constraint checks, so the name is
* somewhat historical.
*
* This is invoked as an AFTER ROW trigger for both INSERT and UPDATE,
* for any rows recorded as potentially violating a deferrable unique
* or exclusion constraint.
*
* This may be an end-of-statement check, a commit-time check, or a
* check triggered by a SET CONSTRAINTS command.
*/
Datum
unique_key_recheck(PG_FUNCTION_ARGS)
{
TriggerData *trigdata = (TriggerData *) fcinfo->context;
const char *funcname = "unique_key_recheck";
HeapTuple new_row;
ItemPointerData tmptid;
Relation indexRel;
IndexInfo *indexInfo;
EState *estate;
ExprContext *econtext;
TupleTableSlot *slot;
Datum values[INDEX_MAX_KEYS];
bool isnull[INDEX_MAX_KEYS];
/*
* Make sure this is being called as an AFTER ROW trigger. Note:
* translatable error strings are shared with ri_triggers.c, so resist the
* temptation to fold the function name into them.
*/
if (!CALLED_AS_TRIGGER(fcinfo))
ereport(ERROR,
(errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED),
errmsg("function \"%s\" was not called by trigger manager",
funcname)));
if (!TRIGGER_FIRED_AFTER(trigdata->tg_event) ||
!TRIGGER_FIRED_FOR_ROW(trigdata->tg_event))
ereport(ERROR,
(errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED),
errmsg("function \"%s\" must be fired AFTER ROW",
funcname)));
/*
* Get the new data that was inserted/updated.
*/
if (TRIGGER_FIRED_BY_INSERT(trigdata->tg_event))
new_row = trigdata->tg_trigtuple;
else if (TRIGGER_FIRED_BY_UPDATE(trigdata->tg_event))
new_row = trigdata->tg_newtuple;
else
{
ereport(ERROR,
(errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED),
errmsg("function \"%s\" must be fired for INSERT or UPDATE",
funcname)));
new_row = NULL; /* keep compiler quiet */
}
/*
* If the new_row is now dead (ie, inserted and then deleted within our
* transaction), we can skip the check. However, we have to be careful,
* because this trigger gets queued only in response to index insertions;
* which means it does not get queued for HOT updates. The row we are
* called for might now be dead, but have a live HOT child, in which case
* we still need to make the check. Therefore we have to use
* heap_hot_search, not just HeapTupleSatisfiesVisibility as is done in
* the comparable test in RI_FKey_check.
*
* This might look like just an optimization, because the index AM will
* make this identical test before throwing an error. But it's actually
* needed for correctness, because the index AM will also throw an error
* if it doesn't find the index entry for the row. If the row's dead then
* it's possible the index entry has also been marked dead, and even
* removed.
*/
tmptid = new_row->t_self;
if (!heap_hot_search(&tmptid, trigdata->tg_relation, SnapshotSelf, NULL))
{
/*
* All rows in the HOT chain are dead, so skip the check.
*/
return PointerGetDatum(NULL);
}
/*
* Open the index, acquiring a RowExclusiveLock, just as if we were going
* to update it. (This protects against possible changes of the index
* schema, not against concurrent updates.)
*/
indexRel = index_open(trigdata->tg_trigger->tgconstrindid,
RowExclusiveLock);
indexInfo = BuildIndexInfo(indexRel);
/*
* The heap tuple must be put into a slot for FormIndexDatum.
*/
slot = MakeSingleTupleTableSlot(RelationGetDescr(trigdata->tg_relation));
ExecStoreTuple(new_row, slot, InvalidBuffer, false);
/*
* Typically the index won't have expressions, but if it does we need an
* EState to evaluate them. We need it for exclusion constraints too,
* even if they are just on simple columns.
*/
if (indexInfo->ii_Expressions != NIL ||
indexInfo->ii_ExclusionOps != NULL)
{
estate = CreateExecutorState();
econtext = GetPerTupleExprContext(estate);
econtext->ecxt_scantuple = slot;
}
else
estate = NULL;
/*
* Form the index values and isnull flags for the index entry that we need
* to check.
*
* Note: if the index uses functions that are not as immutable as they are
* supposed to be, this could produce an index tuple different from the
* original. The index AM can catch such errors by verifying that it
* finds a matching index entry with the tuple's TID. For exclusion
* constraints we check this in check_exclusion_constraint().
*/
FormIndexDatum(indexInfo, slot, estate, values, isnull);
/*
* Now do the appropriate check.
*/
if (indexInfo->ii_ExclusionOps == NULL)
{
/*
* Note: this is not a real insert; it is a check that the index entry
* that has already been inserted is unique.
*/
index_insert(indexRel, values, isnull, &(new_row->t_self),
trigdata->tg_relation, UNIQUE_CHECK_EXISTING);
}
else
{
/*
* For exclusion constraints we just do the normal check, but now it's
* okay to throw error.
*/
check_exclusion_constraint(trigdata->tg_relation, indexRel, indexInfo,
&(new_row->t_self), values, isnull,
estate, false, false);
}
/*
* If that worked, then this index entry is unique or non-excluded, and we
* are done.
*/
if (estate != NULL)
FreeExecutorState(estate);
ExecDropSingleTupleTableSlot(slot);
index_close(indexRel, RowExclusiveLock);
return PointerGetDatum(NULL);
}
/*
* Construct an empty TupleHashTable
*
* numCols, keyColIdx: identify the tuple fields to use as lookup key
* eqfunctions: equality comparison functions to use
* hashfunctions: datatype-specific hashing functions to use
* nbuckets: initial estimate of hashtable size
* additionalsize: size of data stored in ->additional
* tablecxt: memory context in which to store table and table entries
* tempcxt: short-lived context for evaluation hash and comparison functions
*
* The function arrays may be made with execTuplesHashPrepare(). Note they
* are not cross-type functions, but expect to see the table datatype(s)
* on both sides.
*
* Note that keyColIdx, eqfunctions, and hashfunctions must be allocated in
* storage that will live as long as the hashtable does.
*/
TupleHashTable
BuildTupleHashTable(PlanState *parent,
TupleDesc inputDesc,
int numCols, AttrNumber *keyColIdx,
Oid *eqfuncoids,
FmgrInfo *hashfunctions,
long nbuckets, Size additionalsize,
MemoryContext tablecxt, MemoryContext tempcxt,
bool use_variable_hash_iv)
{
TupleHashTable hashtable;
Size entrysize = sizeof(TupleHashEntryData) + additionalsize;
MemoryContext oldcontext;
Assert(nbuckets > 0);
/* Limit initial table size request to not more than work_mem */
nbuckets = Min(nbuckets, (long) ((work_mem * 1024L) / entrysize));
hashtable = (TupleHashTable)
MemoryContextAlloc(tablecxt, sizeof(TupleHashTableData));
hashtable->numCols = numCols;
hashtable->keyColIdx = keyColIdx;
hashtable->tab_hash_funcs = hashfunctions;
hashtable->tablecxt = tablecxt;
hashtable->tempcxt = tempcxt;
hashtable->entrysize = entrysize;
hashtable->tableslot = NULL; /* will be made on first lookup */
hashtable->inputslot = NULL;
hashtable->in_hash_funcs = NULL;
hashtable->cur_eq_func = NULL;
/*
* If parallelism is in use, even if the master backend is performing the
* scan itself, we don't want to create the hashtable exactly the same way
* in all workers. As hashtables are iterated over in keyspace-order,
* doing so in all processes in the same way is likely to lead to
* "unbalanced" hashtables when the table size initially is
* underestimated.
*/
if (use_variable_hash_iv)
hashtable->hash_iv = murmurhash32(ParallelWorkerNumber);
else
hashtable->hash_iv = 0;
hashtable->hashtab = tuplehash_create(tablecxt, nbuckets, hashtable);
oldcontext = MemoryContextSwitchTo(hashtable->tablecxt);
/*
* We copy the input tuple descriptor just for safety --- we assume all
* input tuples will have equivalent descriptors.
*/
hashtable->tableslot = MakeSingleTupleTableSlot(CreateTupleDescCopy(inputDesc),
&TTSOpsMinimalTuple);
/* build comparator for all columns */
/* XXX: should we support non-minimal tuples for the inputslot? */
hashtable->tab_eq_func = ExecBuildGroupingEqual(inputDesc, inputDesc,
&TTSOpsMinimalTuple, &TTSOpsMinimalTuple,
numCols,
keyColIdx, eqfuncoids,
parent);
MemoryContextSwitchTo(oldcontext);
hashtable->exprcontext = CreateExprContext(parent->state);
return hashtable;
}
/*
* Find or create a hashtable entry for the tuple group containing the
* given tuple.
*
* If isnew is NULL, we do not create new entries; we return NULL if no
* match is found.
*
* If isnew isn't NULL, then a new entry is created if no existing entry
* matches. On return, *isnew is true if the entry is newly created,
* false if it existed already. Any extra space in a new entry has been
* zeroed.
*/
TupleHashEntry
LookupTupleHashEntry(TupleHashTable hashtable, TupleTableSlot *slot,
bool *isnew)
{
TupleHashEntry entry;
MemoryContext oldContext;
TupleHashTable saveCurHT;
TupleHashEntryData dummy;
bool found;
/* If first time through, clone the input slot to make table slot */
if (hashtable->tableslot == NULL)
{
TupleDesc tupdesc;
oldContext = MemoryContextSwitchTo(hashtable->tablecxt);
/*
* We copy the input tuple descriptor just for safety --- we assume
* all input tuples will have equivalent descriptors.
*/
tupdesc = CreateTupleDescCopy(slot->tts_tupleDescriptor);
hashtable->tableslot = MakeSingleTupleTableSlot(tupdesc);
MemoryContextSwitchTo(oldContext);
}
/* Need to run the hash functions in short-lived context */
oldContext = MemoryContextSwitchTo(hashtable->tempcxt);
/*
* Set up data needed by hash and match functions
*
* We save and restore CurTupleHashTable just in case someone manages to
* invoke this code re-entrantly.
*/
hashtable->inputslot = slot;
saveCurHT = CurTupleHashTable;
CurTupleHashTable = hashtable;
/* Search the hash table */
dummy.firstTuple = NULL; /* flag to reference inputslot */
entry = (TupleHashEntry) hash_search(hashtable->hashtab,
&dummy,
isnew ? HASH_ENTER : HASH_FIND,
&found);
if (isnew)
{
if (found)
{
/* found pre-existing entry */
*isnew = false;
}
else
{
/*
* created new entry
*
* Zero any caller-requested space in the entry. (This zaps the
* "key data" dynahash.c copied into the new entry, but we don't
* care since we're about to overwrite it anyway.)
*/
MemSet(entry, 0, hashtable->entrysize);
/* Copy the first tuple into the table context */
MemoryContextSwitchTo(hashtable->tablecxt);
entry->firstTuple = ExecCopySlotMinimalTuple(slot);
*isnew = true;
}
}
CurTupleHashTable = saveCurHT;
MemoryContextSwitchTo(oldContext);
return entry;
}
/*
* save_consumer_state
*
* Saves the given consumer's state to pipeline_kafka_consumers
*/
static void
save_consumer_state(KafkaConsumer *consumer, int partition_group)
{
ScanKeyData skey[1];
HeapTuple tup = NULL;
HeapScanDesc scan;
Relation offsets = open_pipeline_kafka_offsets();
Datum values[OFFSETS_RELATION_NATTS];
bool nulls[OFFSETS_RELATION_NATTS];
bool replace[OFFSETS_RELATION_NATTS];
bool updated[consumer->num_partitions];
TupleTableSlot *slot = MakeSingleTupleTableSlot(RelationGetDescr(offsets));
int partition;
MemSet(updated, false, sizeof(updated));
ScanKeyInit(&skey[0], OFFSETS_ATTR_CONSUMER, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(consumer->id));
scan = heap_beginscan(offsets, GetTransactionSnapshot(), 1, skey);
/* update any existing offset rows */
while ((tup = heap_getnext(scan, ForwardScanDirection)) != NULL)
{
Datum d;
bool isnull;
int partition;
HeapTuple modified;
ExecStoreTuple(tup, slot, InvalidBuffer, false);
d = slot_getattr(slot, OFFSETS_ATTR_PARTITION, &isnull);
partition = DatumGetInt32(d);
/* we only want to update the offsets we're responsible for */
if (partition % consumer->parallelism != partition_group)
continue;
MemSet(nulls, false, sizeof(nulls));
MemSet(replace, false, sizeof(nulls));
values[OFFSETS_ATTR_OFFSET - 1] = Int64GetDatum(consumer->offsets[partition]);
replace[OFFSETS_ATTR_OFFSET - 1] = true;
updated[partition] = true;
modified = heap_modify_tuple(tup, RelationGetDescr(offsets), values, nulls, replace);
simple_heap_update(offsets, &modified->t_self, modified);
}
heap_endscan(scan);
/* now insert any offset rows that didn't already exist */
for (partition = 0; partition < consumer->num_partitions; partition++)
{
if (updated[partition])
continue;
if (partition % consumer->parallelism != partition_group)
continue;
values[OFFSETS_ATTR_CONSUMER - 1] = ObjectIdGetDatum(consumer->id);
values[OFFSETS_ATTR_PARTITION - 1] = Int32GetDatum(partition);
values[OFFSETS_ATTR_OFFSET - 1] = Int64GetDatum(consumer->offsets[partition]);
MemSet(nulls, false, sizeof(nulls));
tup = heap_form_tuple(RelationGetDescr(offsets), values, nulls);
simple_heap_insert(offsets, tup);
}
ExecDropSingleTupleTableSlot(slot);
heap_close(offsets, NoLock);
}
/*
* Assumes that the segment file lock is already held.
* Assumes that the segment file should be compacted.
*
*/
static void
AppendOnlySegmentFileFullCompaction(Relation aorel,
AppendOnlyEntry *aoEntry,
AppendOnlyInsertDesc insertDesc,
FileSegInfo* fsinfo)
{
const char* relname;
AppendOnlyVisimap visiMap;
AppendOnlyScanDesc scanDesc;
TupleDesc tupDesc;
MemTuple tuple;
TupleTableSlot *slot;
MemTupleBinding *mt_bind;
int compact_segno;
int64 movedTupleCount = 0;
ResultRelInfo *resultRelInfo;
EState *estate;
AOTupleId *aoTupleId;
int64 tupleCount = 0;
int64 tuplePerPage = INT_MAX;
Assert(Gp_role == GP_ROLE_EXECUTE || Gp_role == GP_ROLE_UTILITY);
Assert(RelationIsAoRows(aorel));
Assert(insertDesc);
compact_segno = fsinfo->segno;
if (fsinfo->varblockcount > 0)
{
tuplePerPage = fsinfo->total_tupcount / fsinfo->varblockcount;
}
relname = RelationGetRelationName(aorel);
AppendOnlyVisimap_Init(&visiMap,
aoEntry->visimaprelid,
aoEntry->visimapidxid,
ShareUpdateExclusiveLock,
SnapshotNow);
elogif(Debug_appendonly_print_compaction,
LOG, "Compact AO segno %d, relation %s, insert segno %d",
compact_segno, relname, insertDesc->storageWrite.segmentFileNum);
/*
* Todo: We need to limit the scan to one file and we need to avoid to
* lock the file again.
*
* We use SnapshotAny to get visible and invisible tuples.
*/
scanDesc = appendonly_beginrangescan(aorel,
SnapshotAny, SnapshotNow,
&compact_segno, 1, 0, NULL);
tupDesc = RelationGetDescr(aorel);
slot = MakeSingleTupleTableSlot(tupDesc);
mt_bind = create_memtuple_binding(tupDesc);
/*
* We need a ResultRelInfo and an EState so we can use the regular
* executor's index-entry-making machinery.
*/
estate = CreateExecutorState();
resultRelInfo = makeNode(ResultRelInfo);
resultRelInfo->ri_RangeTableIndex = 1; /* dummy */
resultRelInfo->ri_RelationDesc = aorel;
resultRelInfo->ri_TrigDesc = NULL; /* we don't fire triggers */
ExecOpenIndices(resultRelInfo);
estate->es_result_relations = resultRelInfo;
estate->es_num_result_relations = 1;
estate->es_result_relation_info = resultRelInfo;
/*
* Go through all visible tuples and move them to a new segfile.
*/
while ((tuple = appendonly_getnext(scanDesc, ForwardScanDirection, slot)) != NULL)
{
/* Check interrupts as this may take time. */
CHECK_FOR_INTERRUPTS();
aoTupleId = (AOTupleId*)slot_get_ctid(slot);
if (AppendOnlyVisimap_IsVisible(&scanDesc->visibilityMap, aoTupleId))
{
AppendOnlyMoveTuple(tuple,
slot,
mt_bind,
insertDesc,
resultRelInfo,
estate);
movedTupleCount++;
}
else
{
/* Tuple is invisible and needs to be dropped */
AppendOnlyThrowAwayTuple(aorel,
tuple,
slot,
mt_bind);
}
/*
* Check for vacuum delay point after approximatly a var block
*/
tupleCount++;
if (VacuumCostActive && tupleCount % tuplePerPage == 0)
{
vacuum_delay_point();
}
}
SetFileSegInfoState(aorel, aoEntry, compact_segno, AOSEG_STATE_AWAITING_DROP);
AppendOnlyVisimap_DeleteSegmentFile(&visiMap, compact_segno);
/* Delete all mini pages of the segment files if block directory exists */
if (OidIsValid(aoEntry->blkdirrelid))
{
AppendOnlyBlockDirectory_DeleteSegmentFile(
aoEntry,
SnapshotNow,
compact_segno,
0);
}
elogif(Debug_appendonly_print_compaction, LOG,
"Finished compaction: "
"AO segfile %d, relation %s, moved tuple count " INT64_FORMAT,
compact_segno, relname, movedTupleCount);
AppendOnlyVisimap_Finish(&visiMap, NoLock);
ExecCloseIndices(resultRelInfo);
FreeExecutorState(estate);
ExecDropSingleTupleTableSlot(slot);
destroy_memtuple_binding(mt_bind);
appendonly_endscan(scanDesc);
}
