Exemplo n.º 1
0
/*
 * get_all_brokers
 *
 * Return a list of all brokers in pipeline_kafka_brokers
 */
static List *
get_all_brokers(void)
{
	HeapTuple tup = NULL;
	HeapScanDesc scan;
	Relation brokers = open_pipeline_kafka_brokers();
	TupleTableSlot *slot = MakeSingleTupleTableSlot(RelationGetDescr(brokers));
	List *result = NIL;

	scan = heap_beginscan(brokers, GetTransactionSnapshot(), 0, NULL);
	while ((tup = heap_getnext(scan, ForwardScanDirection)) != NULL)
	{
		char *host;
		Datum d;
		bool isnull;

		ExecStoreTuple(tup, slot, InvalidBuffer, false);
		d = slot_getattr(slot, BROKER_ATTR_HOST, &isnull);
		host = TextDatumGetCString(d);

		result = lappend(result, host);
	}

	ExecDropSingleTupleTableSlot(slot);
	heap_endscan(scan);
	heap_close(brokers, NoLock);

	return result;
}
Exemplo n.º 2
0
void
CheckerTerm(Checker *checker)
{
	if (checker->slot)
		ExecDropSingleTupleTableSlot(checker->slot);

	if (checker->estate)
		FreeExecutorState(checker->estate);
}
Exemplo n.º 3
0
/*
 * load_consumer_offsets
 *
 * Load all offsets for all of this consumer's partitions
 */
static void
load_consumer_offsets(KafkaConsumer *consumer, struct rd_kafka_metadata_topic *meta, int64_t offset)
{
	MemoryContext old;
	ScanKeyData skey[1];
	HeapTuple tup = NULL;
	HeapScanDesc scan;
	Relation offsets = open_pipeline_kafka_offsets();
	TupleTableSlot *slot = MakeSingleTupleTableSlot(RelationGetDescr(offsets));
	int i;

	ScanKeyInit(&skey[0], OFFSETS_ATTR_CONSUMER, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(consumer->id));
	scan = heap_beginscan(offsets, GetTransactionSnapshot(), 1, skey);

	old = MemoryContextSwitchTo(CacheMemoryContext);
	consumer->offsets = palloc0(meta->partition_cnt * sizeof(int64_t));
	MemoryContextSwitchTo(old);

	/* by default, begin consuming from the end of a stream */
	for (i = 0; i < meta->partition_cnt; i++)
		consumer->offsets[i] = offset;

	consumer->num_partitions = meta->partition_cnt;

	while ((tup = heap_getnext(scan, ForwardScanDirection)) != NULL)
	{
		Datum d;
		bool isnull;
		int partition;
		ExecStoreTuple(tup, slot, InvalidBuffer, false);

		d = slot_getattr(slot, OFFSETS_ATTR_PARTITION, &isnull);
		partition = DatumGetInt32(d);

		if(partition > consumer->num_partitions)
			elog(ERROR, "invalid partition id: %d", partition);

		if (offset == RD_KAFKA_OFFSET_NULL)
		{
			d = slot_getattr(slot, OFFSETS_ATTR_OFFSET, &isnull);
			if (isnull)
				offset = RD_KAFKA_OFFSET_END;
			else
				offset = DatumGetInt64(d);
		}

		consumer->offsets[partition] = DatumGetInt64(offset);
	}

	ExecDropSingleTupleTableSlot(slot);
	heap_endscan(scan);
	heap_close(offsets, RowExclusiveLock);
}
Exemplo n.º 4
0
/*
 * CopyIntoStream
 *
 * COPY events to a stream from an input source
 */
void
CopyIntoStream(Relation rel, TupleDesc desc, HeapTuple *tuples, int ntuples)
{
	bool snap = ActiveSnapshotSet();
	ResultRelInfo rinfo;
	StreamInsertState *sis;

	MemSet(&rinfo, 0, sizeof(ResultRelInfo));
	rinfo.ri_RangeTableIndex = 1; /* dummy */
	rinfo.ri_TrigDesc = NULL;
	rinfo.ri_RelationDesc = rel;

	if (snap)
		PopActiveSnapshot();

	BeginStreamModify(NULL, &rinfo, list_make1(desc), 0, 0);
	sis = (StreamInsertState *) rinfo.ri_FdwState;
	Assert(sis);

	if (sis->queries)
	{
		TupleTableSlot *slot = MakeSingleTupleTableSlot(RelationGetDescr(rel));
		int i;

		for (i = 0; i < ntuples; i++)
		{
			ExecStoreTuple(tuples[i], slot, InvalidBuffer, false);
			ExecStreamInsert(NULL, &rinfo, slot, NULL);
			ExecClearTuple(slot);
		}

		ExecDropSingleTupleTableSlot(slot);

		Assert(sis->ntups == ntuples);
		pgstat_increment_cq_write(ntuples, sis->nbytes);
	}

	EndStreamModify(NULL, &rinfo);

	if (snap)
		PushActiveSnapshot(GetTransactionSnapshot());
}
Exemplo n.º 5
0
/*
 * To perform that check simply start an index scan, create the necessary
 * slot, do the heap lookup, and shut everything down again. This could be
 * optimized, but is unlikely to matter from a performance POV. If there
 * frequently are live index pointers also matching a unique index key, the
 * CPU overhead of this routine is unlikely to matter.
 */
bool
table_index_fetch_tuple_check(Relation rel,
							  ItemPointer tid,
							  Snapshot snapshot,
							  bool *all_dead)
{
	IndexFetchTableData *scan;
	TupleTableSlot *slot;
	bool		call_again = false;
	bool		found;

	slot = table_slot_create(rel, NULL);
	scan = table_index_fetch_begin(rel);
	found = table_index_fetch_tuple(scan, tid, snapshot, slot, &call_again,
									all_dead);
	table_index_fetch_end(scan);
	ExecDropSingleTupleTableSlot(slot);

	return found;
}
Exemplo n.º 6
0
void
SpoolerClose(Spooler *self)
{
	/* Merge indexes */
	if (self->spools != NULL)
		IndexSpoolEnd(self, true);

	/* Terminate spooler. */
	ExecDropSingleTupleTableSlot(self->slot);
	if (self->estate->es_result_relation_info)
		ExecCloseIndices(self->estate->es_result_relation_info);
	FreeExecutorState(self->estate);

	/* Close and release members. */
	if (self->dup_fp != NULL && FreeFile(self->dup_fp) < 0)
		ereport(WARNING,
				(errcode_for_file_access(),
				 errmsg("could not close duplicate bad file \"%s\": %m",
						self->dup_badfile)));
	if (self->dup_badfile != NULL)
		pfree(self->dup_badfile);
}
Exemplo n.º 7
0
/*
 * save_consumer_state
 *
 * Saves the given consumer's state to pipeline_kafka_consumers
 */
static void
save_consumer_state(KafkaConsumer *consumer, int partition_group)
{
	ScanKeyData skey[1];
	HeapTuple tup = NULL;
	HeapScanDesc scan;
	Relation offsets = open_pipeline_kafka_offsets();
	Datum values[OFFSETS_RELATION_NATTS];
	bool nulls[OFFSETS_RELATION_NATTS];
	bool replace[OFFSETS_RELATION_NATTS];
	bool updated[consumer->num_partitions];
	TupleTableSlot *slot = MakeSingleTupleTableSlot(RelationGetDescr(offsets));
	int partition;

	MemSet(updated, false, sizeof(updated));

	ScanKeyInit(&skey[0], OFFSETS_ATTR_CONSUMER, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(consumer->id));
	scan = heap_beginscan(offsets, GetTransactionSnapshot(), 1, skey);

	/* update any existing offset rows */
	while ((tup = heap_getnext(scan, ForwardScanDirection)) != NULL)
	{
		Datum d;
		bool isnull;
		int partition;
		HeapTuple modified;

		ExecStoreTuple(tup, slot, InvalidBuffer, false);
		d = slot_getattr(slot, OFFSETS_ATTR_PARTITION, &isnull);
		partition = DatumGetInt32(d);

		/* we only want to update the offsets we're responsible for */
		if (partition % consumer->parallelism != partition_group)
			continue;

		MemSet(nulls, false, sizeof(nulls));
		MemSet(replace, false, sizeof(nulls));

		values[OFFSETS_ATTR_OFFSET - 1] = Int64GetDatum(consumer->offsets[partition]);
		replace[OFFSETS_ATTR_OFFSET - 1] = true;
		updated[partition] = true;

		modified = heap_modify_tuple(tup, RelationGetDescr(offsets), values, nulls, replace);
		simple_heap_update(offsets, &modified->t_self, modified);
	}

	heap_endscan(scan);

	/* now insert any offset rows that didn't already exist */
	for (partition = 0; partition < consumer->num_partitions; partition++)
	{
		if (updated[partition])
			continue;

		if (partition % consumer->parallelism != partition_group)
			continue;

		values[OFFSETS_ATTR_CONSUMER - 1] = ObjectIdGetDatum(consumer->id);
		values[OFFSETS_ATTR_PARTITION - 1] = Int32GetDatum(partition);
		values[OFFSETS_ATTR_OFFSET - 1] = Int64GetDatum(consumer->offsets[partition]);

		MemSet(nulls, false, sizeof(nulls));

		tup = heap_form_tuple(RelationGetDescr(offsets), values, nulls);
		simple_heap_insert(offsets, tup);
	}

	ExecDropSingleTupleTableSlot(slot);
	heap_close(offsets, NoLock);
}
Exemplo n.º 8
0
/*
 * load_consumer_state
 *
 * Read consumer state from pipeline_kafka_consumers into the given struct
 */
static void
load_consumer_state(Oid worker_id, KafkaConsumer *consumer)
{
	ScanKeyData skey[1];
	HeapTuple tup = NULL;
	HeapScanDesc scan;
	Relation consumers = open_pipeline_kafka_consumers();
	TupleTableSlot *slot = MakeSingleTupleTableSlot(RelationGetDescr(consumers));
	Datum d;
	bool isnull;
	text *qualified;
	MemoryContext old;

	MemSet(consumer, 0, sizeof(KafkaConsumer));

	ScanKeyInit(&skey[0], -2, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(worker_id));
	scan = heap_beginscan(consumers, GetTransactionSnapshot(), 1, skey);
	tup = heap_getnext(scan, ForwardScanDirection);

	if (!HeapTupleIsValid(tup))
		elog(ERROR, "kafka consumer %d not found", worker_id);

	ExecStoreTuple(tup, slot, InvalidBuffer, false);
	consumer->id = HeapTupleGetOid(tup);

	d = slot_getattr(slot, CONSUMER_ATTR_RELATION, &isnull);

	/* we don't want anything that's palloc'd to get freed when we commit */
	old = MemoryContextSwitchTo(CacheMemoryContext);

	/* target relation */
	qualified = (text *) DatumGetPointer(d);
	consumer->rel = makeRangeVarFromNameList(textToQualifiedNameList(qualified));

	/* topic */
	d = slot_getattr(slot, CONSUMER_ATTR_TOPIC, &isnull);
	consumer->topic = TextDatumGetCString(d);

	/* format */
	d = slot_getattr(slot, CONSUMER_ATTR_FORMAT, &isnull);
	consumer->format = TextDatumGetCString(d);

	/* delimiter */
	d = slot_getattr(slot, CONSUMER_ATTR_DELIMITER, &isnull);
	if (!isnull)
		consumer->delimiter = TextDatumGetCString(d);
	else
		consumer->delimiter = NULL;

	/* quote character */
	d = slot_getattr(slot, CONSUMER_ATTR_QUOTE, &isnull);
	if (!isnull)
		consumer->quote = TextDatumGetCString(d);
	else
		consumer->quote = NULL;

	/* escape character */
	d = slot_getattr(slot, CONSUMER_ATTR_ESCAPE, &isnull);
	if (!isnull)
		consumer->escape = TextDatumGetCString(d);
	else
		consumer->escape = NULL;

	/* now load all brokers */
	consumer->brokers = get_all_brokers();
	MemoryContextSwitchTo(old);

	d = slot_getattr(slot, CONSUMER_ATTR_PARALLELISM, &isnull);
	consumer->parallelism = DatumGetInt32(d);

	/* batch size */
	d = slot_getattr(slot, CONSUMER_ATTR_BATCH_SIZE, &isnull);
	consumer->batch_size = DatumGetInt32(d);

	ExecDropSingleTupleTableSlot(slot);
	heap_endscan(scan);
	heap_close(consumers, NoLock);
}
Exemplo n.º 9
0
Datum
pipeline_stream_insert(PG_FUNCTION_ARGS)
{
	TriggerData *trigdata = (TriggerData *) fcinfo->context;
	Trigger *trig = trigdata->tg_trigger;
	HeapTuple tup;
	List *fdw_private;
	int i;
	ResultRelInfo rinfo;

	if (trig->tgnargs < 1)
		elog(ERROR, "pipeline_stream_insert: must be provided a stream name");

	/* make sure it's called as a trigger */
	if (!CALLED_AS_TRIGGER(fcinfo))
		ereport(ERROR,
				(errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED),
				 errmsg("pipeline_stream_insert: must be called as trigger")));

	/* and that it's called on update or insert */
	if (!TRIGGER_FIRED_BY_UPDATE(trigdata->tg_event) && !TRIGGER_FIRED_BY_INSERT(trigdata->tg_event))
		ereport(ERROR,
				(errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED),
				 errmsg("pipeline_stream_insert: must be called on insert or update")));

	/* and that it's called for each row */
	if (!TRIGGER_FIRED_FOR_ROW(trigdata->tg_event))
		ereport(ERROR,
				(errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED),
				 errmsg("pipeline_stream_insert: must be called for each row")));

	/* and that it's called after insert or update */
	if (!TRIGGER_FIRED_AFTER(trigdata->tg_event))
		ereport(ERROR,
				(errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED),
				 errmsg("pipeline_stream_insert: must be called after insert or update")));

	if (TRIGGER_FIRED_BY_UPDATE(trigdata->tg_event))
		tup = trigdata->tg_newtuple;
	else
		tup = trigdata->tg_trigtuple;

	fdw_private = list_make1(RelationGetDescr(trigdata->tg_relation));

	MemSet(&rinfo, 0, sizeof(ResultRelInfo));
	rinfo.ri_RangeTableIndex = 1; /* dummy */
	rinfo.ri_TrigDesc = NULL;

	for (i = 0; i < trig->tgnargs; i++)
	{
		RangeVar *stream;
		Relation rel;
		StreamInsertState *sis;

		stream = makeRangeVarFromNameList(textToQualifiedNameList(cstring_to_text(trig->tgargs[i])));
		rel = heap_openrv(stream, AccessShareLock);

		rinfo.ri_RelationDesc = rel;

		BeginStreamModify(NULL, &rinfo, fdw_private, 0, 0);
		sis = (StreamInsertState *) rinfo.ri_FdwState;
		Assert(sis);

		if (sis->queries)
		{
			TupleTableSlot *slot = MakeSingleTupleTableSlot(RelationGetDescr(rel));

			ExecStoreTuple(tup, slot, InvalidBuffer, false);
			ExecStreamInsert(NULL, &rinfo, slot, NULL);
			ExecClearTuple(slot);

			ExecDropSingleTupleTableSlot(slot);
			pgstat_report_streamstat(true);
		}

		EndStreamModify(NULL, &rinfo);
		heap_close(rel, AccessShareLock);
	}

	return PointerGetDatum(tup);
}
Exemplo n.º 10
0
/*
 * CatalogIndexInsert - insert index entries for one catalog tuple
 *
 * This should be called for each inserted or updated catalog tuple.
 *
 * This is effectively a cut-down version of ExecInsertIndexTuples.
 */
void
CatalogIndexInsert(CatalogIndexState indstate, HeapTuple heapTuple)
{
	int			i;
	int			numIndexes;
	RelationPtr relationDescs;
	Relation	heapRelation;
	TupleTableSlot *slot;
	IndexInfo **indexInfoArray;
	Datum		values[INDEX_MAX_KEYS];
	bool		isnull[INDEX_MAX_KEYS];

	/* HOT update does not require index inserts */
	if (HeapTupleIsHeapOnly(heapTuple))
		return;

	/*
	 * Get information from the state structure.  Fall out if nothing to do.
	 */
	numIndexes = indstate->ri_NumIndices;
	if (numIndexes == 0)
		return;
	relationDescs = indstate->ri_IndexRelationDescs;
	indexInfoArray = indstate->ri_IndexRelationInfo;
	heapRelation = indstate->ri_RelationDesc;

	/* Need a slot to hold the tuple being examined */
	slot = MakeSingleTupleTableSlot(RelationGetDescr(heapRelation));
	ExecStoreTuple(heapTuple, slot, InvalidBuffer, false);

	/*
	 * for each index, form and insert the index tuple
	 */
	for (i = 0; i < numIndexes; i++)
	{
		IndexInfo  *indexInfo;

		indexInfo = indexInfoArray[i];

		/* If the index is marked as read-only, ignore it */
		if (!indexInfo->ii_ReadyForInserts)
			continue;

		/*
		 * Expressional and partial indexes on system catalogs are not
		 * supported, nor exclusion constraints, nor deferred uniqueness
		 */
		Assert(indexInfo->ii_Expressions == NIL);
		Assert(indexInfo->ii_Predicate == NIL);
		Assert(indexInfo->ii_ExclusionOps == NULL);
		Assert(relationDescs[i]->rd_index->indimmediate);

		/*
		 * FormIndexDatum fills in its values and isnull parameters with the
		 * appropriate values for the column(s) of the index.
		 */
		FormIndexDatum(indexInfo,
					   slot,
					   NULL,	/* no expression eval to do */
					   values,
					   isnull);

		/*
		 * The index AM does the rest.
		 */
		index_insert(relationDescs[i],	/* index relation */
					 values,	/* array of index Datums */
					 isnull,	/* is-null flags */
					 &(heapTuple->t_self),		/* tid of heap tuple */
					 heapRelation,
					 relationDescs[i]->rd_index->indisunique ?
					 UNIQUE_CHECK_YES : UNIQUE_CHECK_NO);
	}

	ExecDropSingleTupleTableSlot(slot);
}
Exemplo n.º 11
0
/*
 * unique_key_recheck - trigger function to do a deferred uniqueness check.
 *
 * This now also does deferred exclusion-constraint checks, so the name is
 * somewhat historical.
 *
 * This is invoked as an AFTER ROW trigger for both INSERT and UPDATE,
 * for any rows recorded as potentially violating a deferrable unique
 * or exclusion constraint.
 *
 * This may be an end-of-statement check, a commit-time check, or a
 * check triggered by a SET CONSTRAINTS command.
 */
Datum
unique_key_recheck(PG_FUNCTION_ARGS)
{
	TriggerData *trigdata = (TriggerData *) fcinfo->context;
	const char *funcname = "unique_key_recheck";
	HeapTuple	new_row;
	ItemPointerData tmptid;
	Relation	indexRel;
	IndexInfo  *indexInfo;
	EState	   *estate;
	ExprContext *econtext;
	TupleTableSlot *slot;
	Datum		values[INDEX_MAX_KEYS];
	bool		isnull[INDEX_MAX_KEYS];

	/*
	 * Make sure this is being called as an AFTER ROW trigger.	Note:
	 * translatable error strings are shared with ri_triggers.c, so resist the
	 * temptation to fold the function name into them.
	 */
	if (!CALLED_AS_TRIGGER(fcinfo))
		ereport(ERROR,
				(errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED),
				 errmsg("function \"%s\" was not called by trigger manager",
						funcname)));

	if (!TRIGGER_FIRED_AFTER(trigdata->tg_event) ||
		!TRIGGER_FIRED_FOR_ROW(trigdata->tg_event))
		ereport(ERROR,
				(errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED),
				 errmsg("function \"%s\" must be fired AFTER ROW",
						funcname)));

	/*
	 * Get the new data that was inserted/updated.
	 */
	if (TRIGGER_FIRED_BY_INSERT(trigdata->tg_event))
		new_row = trigdata->tg_trigtuple;
	else if (TRIGGER_FIRED_BY_UPDATE(trigdata->tg_event))
		new_row = trigdata->tg_newtuple;
	else
	{
		ereport(ERROR,
				(errcode(ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED),
				 errmsg("function \"%s\" must be fired for INSERT or UPDATE",
						funcname)));
		new_row = NULL;			/* keep compiler quiet */
	}

	/*
	 * If the new_row is now dead (ie, inserted and then deleted within our
	 * transaction), we can skip the check.  However, we have to be careful,
	 * because this trigger gets queued only in response to index insertions;
	 * which means it does not get queued for HOT updates.	The row we are
	 * called for might now be dead, but have a live HOT child, in which case
	 * we still need to make the check.  Therefore we have to use
	 * heap_hot_search, not just HeapTupleSatisfiesVisibility as is done in
	 * the comparable test in RI_FKey_check.
	 *
	 * This might look like just an optimization, because the index AM will
	 * make this identical test before throwing an error.  But it's actually
	 * needed for correctness, because the index AM will also throw an error
	 * if it doesn't find the index entry for the row.  If the row's dead then
	 * it's possible the index entry has also been marked dead, and even
	 * removed.
	 */
	tmptid = new_row->t_self;
	if (!heap_hot_search(&tmptid, trigdata->tg_relation, SnapshotSelf, NULL))
	{
		/*
		 * All rows in the HOT chain are dead, so skip the check.
		 */
		return PointerGetDatum(NULL);
	}

	/*
	 * Open the index, acquiring a RowExclusiveLock, just as if we were going
	 * to update it.  (This protects against possible changes of the index
	 * schema, not against concurrent updates.)
	 */
	indexRel = index_open(trigdata->tg_trigger->tgconstrindid,
						  RowExclusiveLock);
	indexInfo = BuildIndexInfo(indexRel);

	/*
	 * The heap tuple must be put into a slot for FormIndexDatum.
	 */
	slot = MakeSingleTupleTableSlot(RelationGetDescr(trigdata->tg_relation));

	ExecStoreTuple(new_row, slot, InvalidBuffer, false);

	/*
	 * Typically the index won't have expressions, but if it does we need an
	 * EState to evaluate them.  We need it for exclusion constraints too,
	 * even if they are just on simple columns.
	 */
	if (indexInfo->ii_Expressions != NIL ||
		indexInfo->ii_ExclusionOps != NULL)
	{
		estate = CreateExecutorState();
		econtext = GetPerTupleExprContext(estate);
		econtext->ecxt_scantuple = slot;
	}
	else
		estate = NULL;

	/*
	 * Form the index values and isnull flags for the index entry that we need
	 * to check.
	 *
	 * Note: if the index uses functions that are not as immutable as they are
	 * supposed to be, this could produce an index tuple different from the
	 * original.  The index AM can catch such errors by verifying that it
	 * finds a matching index entry with the tuple's TID.  For exclusion
	 * constraints we check this in check_exclusion_constraint().
	 */
	FormIndexDatum(indexInfo, slot, estate, values, isnull);

	/*
	 * Now do the appropriate check.
	 */
	if (indexInfo->ii_ExclusionOps == NULL)
	{
		/*
		 * Note: this is not a real insert; it is a check that the index entry
		 * that has already been inserted is unique.
		 */
		index_insert(indexRel, values, isnull, &(new_row->t_self),
					 trigdata->tg_relation, UNIQUE_CHECK_EXISTING);
	}
	else
	{
		/*
		 * For exclusion constraints we just do the normal check, but now it's
		 * okay to throw error.
		 */
		check_exclusion_constraint(trigdata->tg_relation, indexRel, indexInfo,
								   &(new_row->t_self), values, isnull,
								   estate, false, false);
	}

	/*
	 * If that worked, then this index entry is unique or non-excluded, and we
	 * are done.
	 */
	if (estate != NULL)
		FreeExecutorState(estate);

	ExecDropSingleTupleTableSlot(slot);

	index_close(indexRel, RowExclusiveLock);

	return PointerGetDatum(NULL);
}
Exemplo n.º 12
0
/*
 * ExecFilterRecommend
 *
 * This function just borrows a tuple descriptor from the RecView,
 * but we create the data ourselves through various means.
 */
static TupleTableSlot*
ExecFilterRecommend(RecScanState *recnode,
					 ExecScanAccessMtd accessMtd,
					 ExecScanRecheckMtd recheckMtd)
{
	ExprContext *econtext;
	List	   *qual;
	ProjectionInfo *projInfo;
	ExprDoneCond isDone;
	TupleTableSlot *resultSlot;
	ScanState *node;
	AttributeInfo *attributes;

	node = recnode->subscan;
	attributes = (AttributeInfo*) recnode->attributes;

	/*
	 * Fetch data from node
	 */
	qual = node->ps.qual;
	projInfo = node->ps.ps_ProjInfo;
	econtext = node->ps.ps_ExprContext;

	/*
	 * Check to see if we're still projecting out tuples from a previous scan
	 * tuple (because there is a function-returning-set in the projection
	 * expressions).  If so, try to project another one.
	 */
	if (node->ps.ps_TupFromTlist)
	{
		Assert(projInfo);		/* can't get here if not projecting */
		resultSlot = ExecProject(projInfo, &isDone);
		if (isDone == ExprMultipleResult)
			return resultSlot;
		/* Done with that source tuple... */
		node->ps.ps_TupFromTlist = false;
	}

	/*
	 * Reset per-tuple memory context to free any expression evaluation
	 * storage allocated in the previous tuple cycle.  Note this can't happen
	 * until we're done projecting out tuples from a scan tuple.
	 */
	ResetExprContext(econtext);

	/*
	 * get a tuple from the access method.	Loop until we obtain a tuple that
	 * passes the qualification.
	 */
	for (;;)
	{
		TupleTableSlot *slot;
		int natts, i, userID, userindex, itemID, itemindex;

		CHECK_FOR_INTERRUPTS();

		slot = recnode->ss.ps.ps_ResultTupleSlot;

		/* The first thing we need to do is initialize our recommender
		 * model and other things, if we haven't done so already. */
		if (!recnode->initialized)
			InitializeRecommender(recnode);

		/*
		 * If we've exhausted our item list, then we're totally
		 * finished. We set a flag for this. It's possible that
		 * we'll be in the inner loop of a join, through poor
		 * planning, so we'll reset the appropriate data in case
		 * we have to do this again, though our JoinRecommend
		 * should assure this doesn't happen.
		 */
		if (recnode->finished) {
			recnode->finished = false;
			recnode->userNum = 0;
			recnode->itemNum = 0;
			return NULL;
		}

		/* We're only going to fetch one tuple and store its tuple
		 * descriptor. We can use this tuple descriptor to make as
		 * many new tuples as we want. */
		if (recnode->base_slot == NULL) {
			slot = ExecRecFetch(node, accessMtd, recheckMtd);
			recnode->base_slot = CreateTupleDescCopy(slot->tts_tupleDescriptor);
		}

		/* Create a new slot to operate on. */
		slot = MakeSingleTupleTableSlot(recnode->base_slot);
		slot->tts_isempty = false;

		/*
		 * place the current tuple into the expr context
		 */
		econtext->ecxt_scantuple = slot;

		/* Mark all slots as usable. */
		natts = slot->tts_tupleDescriptor->natts;
		for (i = 0; i < natts; i++) {
			/* Mark slot. */
			slot->tts_values[i] = Int32GetDatum(0);
			slot->tts_isnull[i] = false;
			slot->tts_nvalid++;
		}

		/* While we're here, record what tuple attributes
		 * correspond to our key columns. This will save
		 * us unnecessary strcmp functions. */
		if (recnode->useratt < 0) {
			for (i = 0; i < natts; i++) {
				char* col_name = slot->tts_tupleDescriptor->attrs[i]->attname.data;
//printf("%s\n",col_name);
				if (strcmp(col_name,attributes->userkey) == 0)
					recnode->useratt = i;
				else if (strcmp(col_name,attributes->itemkey) == 0)
					recnode->itematt = i;
				else if (strcmp(col_name,attributes->eventval) == 0)
					recnode->eventatt = i;
			}
		}

		/*
		 * We now have a problem: we need to create prediction structures
		 * for a user before we do filtering, so that we can have a proper
		 * item list. But we also need to filter before creating those
		 * structures, so we don't end up taking forever with it. The
		 * solution is to filter twice.
		 */
		userID = -1; itemID = -1;

		/* First, replace the user ID. */
		userindex = recnode->userNum;
		userID = recnode->userList[userindex];

		/*
		 * We now have a blank tuple slot that we need to fill with data.
		 * We have a working user ID, but not a valid item list. We'd like to
		 * use the filter to determine if this is a good user, but we can't
		 * do that without an item, in many cases. The solution is to add in
		 * dummy items, then compare it against the filter. If a given user ID
		 * doesn't make it past the filter with any item ID, then that user is
		 * being filtered out, and we'll move on to the next.
		 */
		if (recnode->newUser) {
			recnode->fullItemNum = 0;
			itemindex = recnode->fullItemNum;
			itemID = recnode->fullItemList[itemindex];

			slot->tts_values[recnode->useratt] = Int32GetDatum(userID);
			slot->tts_values[recnode->itematt] = Int32GetDatum(itemID);
			slot->tts_values[recnode->eventatt] = Int32GetDatum(-1);

			/* We have a preliminary slot - let's test it. */
			while (qual && !ExecQual(qual, econtext, false)) {
				/* We failed the test. Try the next item. */
				recnode->fullItemNum++;
				if (recnode->fullItemNum >= recnode->fullTotalItems) {
					/* If we've reached the last item, move onto the next user.
					 * If we've reached the last user, we're done. */
					InstrCountFiltered1(node, recnode->fullTotalItems);
					recnode->userNum++;
					recnode->newUser = true;
					recnode->fullItemNum = 0;
					if (recnode->userNum >= recnode->totalUsers) {
						recnode->userNum = 0;
						recnode->itemNum = 0;
						return NULL;
					}
					userindex = recnode->userNum;
					userID = recnode->userList[userindex];
				}

				itemindex = recnode->fullItemNum;
				itemID = recnode->fullItemList[itemindex];
				slot->tts_values[recnode->useratt] = Int32GetDatum(userID);
				slot->tts_values[recnode->itematt] = Int32GetDatum(itemID);
			}

			/* If we get here, then we found a user who will be actually
			 * returned in the results. One quick reset here. */
			recnode->fullItemNum = 0;
		}

		/* Mark the user ID and index. */
		attributes->userID = userID;
		recnode->userindex = userindex;

		/* With the user ID determined, we need to investigate and see
		 * if this is a new user. If so, attempt to create prediction
		 * data structures, or report that this user is invalid. We have
		 * to do this here, so we can establish the item list. */
		if (recnode->newUser) {
			recnode->validUser = prepUserForRating(recnode,userID);
			recnode->newUser = false;
		}

		/* Now replace the item ID, if the user is valid. Otherwise,
		 * leave the item ID as is, as it doesn't matter what it is. */
		if (recnode->validUser)
			itemID = recnode->itemList[recnode->itemNum];
		while (recnode->fullItemList[recnode->fullItemNum] < itemID)
			recnode->fullItemNum++;
		itemindex = recnode->fullItemNum;
		if (recnode->fullItemList[itemindex] > itemID)
			elog(ERROR, "critical item mismatch in ExecRecommend");

		/* Plug in the data, marking those columns full. We also need to
		 * mark the rating column with something temporary. */
		slot->tts_values[recnode->useratt] = Int32GetDatum(userID);
		slot->tts_values[recnode->itematt] = Int32GetDatum(itemID);
		slot->tts_values[recnode->eventatt] = Int32GetDatum(-1);

		/* It's possible our filter criteria involves the RecScore somehow.
		 * If that's the case, we need to calculate it before we do the
		 * qual filtering. Also, if we're doing a JoinRecommend, we should
		 * not calculate the RecScore in this node. In the current version
		 * of RecDB, an OP_NOFILTER shouldn't be allowed. */
		if (attributes->opType == OP_NOFILTER)
			applyRecScore(recnode, slot, itemID, itemindex);

		/* Move onto the next item, for next time. If we're doing a RecJoin,
		 * though, we'll move onto the next user instead. */
		recnode->itemNum++;
		if (recnode->itemNum >= recnode->totalItems ||
			attributes->opType == OP_JOIN ||
			attributes->opType == OP_GENERATEJOIN) {
			/* If we've reached the last item, move onto the next user.
			 * If we've reached the last user, we're done. */
			recnode->userNum++;
			recnode->newUser = true;
			recnode->itemNum = 0;
			recnode->fullItemNum = 0;
			if (recnode->userNum >= recnode->totalUsers)
				recnode->finished = true;
		}

		/*
		 * check that the current tuple satisfies the qual-clause
		 *
		 * check for non-nil qual here to avoid a function call to ExecQual()
		 * when the qual is nil ... saves only a few cycles, but they add up
		 * ...
		 */
		if (!qual || ExecQual(qual, econtext, false))
		{
			/*
			 * If this is an invalid user, then we'll skip this tuple,
			 * adding one to the filter count.
			 */
			if (!recnode->validUser) {
				InstrCountFiltered1(node, 1);
				ResetExprContext(econtext);
				ExecDropSingleTupleTableSlot(slot);
				continue;
			}

			/*
			 * Found a satisfactory scan tuple. This is usually when
			 * we will calculate and apply the RecScore.
			 */
			if (attributes->opType == OP_FILTER || attributes->opType == OP_GENERATE)
				applyRecScore(recnode, slot, itemID, itemindex);

			if (projInfo)
			{
				/*
				 * Form a projection tuple, store it in the result tuple slot
				 * and return it --- unless we find we can project no tuples
				 * from this scan tuple, in which case continue scan.
				 */
				resultSlot = ExecProject(projInfo, &isDone);
				if (isDone != ExprEndResult)
				{
					node->ps.ps_TupFromTlist = (isDone == ExprMultipleResult);

					return resultSlot;
				}
			}
			else
			{
				/*
				 * Here, we aren't projecting, so just return scan tuple.
				 */

				return slot;
			}
		}
		else
			InstrCountFiltered1(node, 1);

		/*
		 * Tuple fails qual, so free per-tuple memory and try again.
		 */
		ResetExprContext(econtext);
		ExecDropSingleTupleTableSlot(slot);
	}

}
Exemplo n.º 13
0
Datum partition_insert_trigger(PG_FUNCTION_ARGS)
{
    TriggerData *trigdata = (TriggerData *) fcinfo->context;
    char       *date_time;
    char        child_table[sizeof(TABLE)+sizeof("2012_09_10")] = TABLE;
    int         partition_field;
    Relation    child_table_id;
    Oid         child_table_oid;
    BulkInsertState bistate = GetBulkInsertState();
    TupleTableSlot  *slot;
    EState          *estate = CreateExecutorState();
    ResultRelInfo   *resultRelInfo = makeNode(ResultRelInfo);
    List       *recheckIndexes = NIL;
 
    /* make sure it's called as a trigger at all */
    if (!CALLED_AS_TRIGGER(fcinfo))
        elog(ERROR, "partition_insert_trigger: not called by trigger manager");
    /* Sanity checks */
    if (!TRIGGER_FIRED_BY_INSERT(trigdata->tg_event) || !TRIGGER_FIRED_BEFORE(trigdata->tg_event))
        elog(ERROR, "partition_insert_trigger: not called on insert before");
    #ifdef DEBUG
    elog(INFO, "Trigger Called for: %s", SPI_getrelname(trigdata->tg_relation));
    #endif
    // Get the field number for the partition
    partition_field = SPI_fnumber(trigdata->tg_relation->rd_att, PARTITION_COLUMN);
    // Get the value for the partition_field
    date_time = SPI_getvalue(trigdata->tg_trigtuple, trigdata->tg_relation->rd_att, partition_field);
    //make sure date was specified
    if (!date_time)
        elog(ERROR, "You cannot insert data without specifying a value for the column %s", PARTITION_COLUMN);
    #ifdef DEBUG
    elog(INFO, "Trying to insert date_time=%s", date_time);
    #endif
    //add date_time,  child_table_2012_01_23
    strncpy(child_table + sizeof(TABLE) -1  , date_time, 4);         //2012
    child_table[sizeof(TABLE) + 3] = SEPARATOR;                      //2012_
    strncpy(child_table + sizeof(TABLE) + 4  , date_time + 5, 2);    //2012_01
    child_table[sizeof(TABLE) + 6] =  SEPARATOR;                     //2012_01_
    strncpy(child_table + sizeof(TABLE) + 7  , date_time + 8, 2);    //2012_01_23
    #ifdef DEBUG
    elog(INFO, "New table will be %s", child_table);    
    #endif
    pfree(date_time);
 
    //if you care about triggers on the child tables, call ExecBRInsertTriggers
    //don't care, continue
     
    //get the OID of the table we are looking for to insert
    child_table_oid = RelnameGetRelid(child_table); //Look for child child_table
    if (child_table_oid == InvalidOid){
        elog(INFO, "partition_insert_trigger: Invalid child table %s, inserting data to main table %s", child_table, TABLE);
        return PointerGetDatum(trigdata->tg_trigtuple);
    }
    //get the descriptor of the table we are looking for
    child_table_id = RelationIdGetRelation(child_table_oid); //Get the child relation descriptor
    if (child_table_id == NULL){
        elog(ERROR, "partition_insert_trigger: Failed to locate relation for child table %s, inserting data to main table %s", child_table, TABLE);
        return PointerGetDatum(trigdata->tg_trigtuple);
    }
    //set the resultRelInfo
    resultRelInfo->ri_RangeTableIndex = 1;      /* dummy */
    resultRelInfo->ri_RelationDesc = child_table_id;
    //setup the estate, not sure why
    estate->es_result_relations = resultRelInfo;
    estate->es_num_result_relations = 1;
    estate->es_result_relation_info = resultRelInfo;
 
    /* Set up a tuple slot not sure why yet */
    slot = MakeSingleTupleTableSlot(trigdata->tg_relation->rd_att);
    ExecStoreTuple(trigdata->tg_trigtuple, slot, InvalidBuffer, false);
 
    //heap_insert(child_table_id, trigdata->tg_trigtuple, GetCurrentCommandId(true), use_wal, bistate); 
    simple_heap_insert(child_table_id, trigdata->tg_trigtuple); 
    if (resultRelInfo->ri_NumIndices > 0)
        recheckIndexes = ExecInsertIndexTuples(slot, &(trigdata->tg_trigtuple->t_self), estate);
    // not sure if this would work CatalogUpdateIndexes(child_table_id, trigdata->tg_trigtuple)
 
    //free the used memory
    list_free(recheckIndexes);
    ExecDropSingleTupleTableSlot(slot); //saw this somewhere :P
    RelationClose(child_table_id); //Must be called to free the relation memory page
    FreeBulkInsertState(bistate);  // ? not sure if needed ?
    //If return next line will add to the regular table
    //return PointerGetDatum(trigdata->tg_trigtuple); 
     
    //Return Null data, still have to figure out to return a proper X rows affected
    return PointerGetDatum(NULL);
}
Exemplo n.º 14
0
/*
 * Assumes that the segment file lock is already held.
 * Assumes that the segment file should be compacted.
 */
static bool
AOCSSegmentFileFullCompaction(Relation aorel,
							  AOCSInsertDesc insertDesc,
							  AOCSFileSegInfo *fsinfo,
							  Snapshot snapshot)
{
	const char *relname;
	AppendOnlyVisimap visiMap;
	AOCSScanDesc scanDesc;
	TupleDesc	tupDesc;
	TupleTableSlot *slot;
	int			compact_segno;
	int64		movedTupleCount = 0;
	ResultRelInfo *resultRelInfo;
	MemTupleBinding *mt_bind;
	EState	   *estate;
	bool	   *proj;
	int			i;
	AOTupleId  *aoTupleId;
	int64		tupleCount = 0;
	int64		tuplePerPage = INT_MAX;

	Assert(Gp_role == GP_ROLE_EXECUTE || Gp_role == GP_ROLE_UTILITY);
	Assert(RelationIsAoCols(aorel));
	Assert(insertDesc);

	compact_segno = fsinfo->segno;
	if (fsinfo->varblockcount > 0)
	{
		tuplePerPage = fsinfo->total_tupcount / fsinfo->varblockcount;
	}
	relname = RelationGetRelationName(aorel);

	AppendOnlyVisimap_Init(&visiMap,
						   aorel->rd_appendonly->visimaprelid,
						   aorel->rd_appendonly->visimapidxid,
						   ShareLock,
						   snapshot);

	elogif(Debug_appendonly_print_compaction,
		   LOG, "Compact AO segfile %d, relation %sd",
		   compact_segno, relname);

	proj = palloc0(sizeof(bool) * RelationGetNumberOfAttributes(aorel));
	for (i = 0; i < RelationGetNumberOfAttributes(aorel); ++i)
	{
		proj[i] = true;
	}
	scanDesc = aocs_beginrangescan(aorel,
								   snapshot, snapshot,
								   &compact_segno, 1, NULL, proj);

	tupDesc = RelationGetDescr(aorel);
	slot = MakeSingleTupleTableSlot(tupDesc);
	mt_bind = create_memtuple_binding(tupDesc);

	/*
	 * We need a ResultRelInfo and an EState so we can use the regular
	 * executor's index-entry-making machinery.
	 */
	estate = CreateExecutorState();
	resultRelInfo = makeNode(ResultRelInfo);
	resultRelInfo->ri_RangeTableIndex = 1;	/* dummy */
	resultRelInfo->ri_RelationDesc = aorel;
	resultRelInfo->ri_TrigDesc = NULL;	/* we don't fire triggers */
	ExecOpenIndices(resultRelInfo);
	estate->es_result_relations = resultRelInfo;
	estate->es_num_result_relations = 1;
	estate->es_result_relation_info = resultRelInfo;

	while (aocs_getnext(scanDesc, ForwardScanDirection, slot))
	{
		CHECK_FOR_INTERRUPTS();

		aoTupleId = (AOTupleId *) slot_get_ctid(slot);
		if (AppendOnlyVisimap_IsVisible(&scanDesc->visibilityMap, aoTupleId))
		{
			AOCSMoveTuple(slot,
						  insertDesc,
						  resultRelInfo,
						  estate);
			movedTupleCount++;
		}
		else
		{
			/* Tuple is invisible and needs to be dropped */
			AppendOnlyThrowAwayTuple(aorel,
									 slot,
									 mt_bind);
		}

		/*
		 * Check for vacuum delay point after approximatly a var block
		 */
		tupleCount++;
		if (VacuumCostActive && tupleCount % tuplePerPage == 0)
		{
			vacuum_delay_point();
		}
	}

	SetAOCSFileSegInfoState(aorel, compact_segno,
							AOSEG_STATE_AWAITING_DROP);

	AppendOnlyVisimap_DeleteSegmentFile(&visiMap,
										compact_segno);

	/* Delete all mini pages of the segment files if block directory exists */
	if (OidIsValid(aorel->rd_appendonly->blkdirrelid))
	{
		AppendOnlyBlockDirectory_DeleteSegmentFile(aorel,
												   snapshot,
												   compact_segno,
												   0);
	}

	elogif(Debug_appendonly_print_compaction, LOG,
		   "Finished compaction: "
		   "AO segfile %d, relation %s, moved tuple count " INT64_FORMAT,
		   compact_segno, relname, movedTupleCount);

	AppendOnlyVisimap_Finish(&visiMap, NoLock);

	ExecCloseIndices(resultRelInfo);
	FreeExecutorState(estate);

	ExecDropSingleTupleTableSlot(slot);
	destroy_memtuple_binding(mt_bind);

	aocs_endscan(scanDesc);
	pfree(proj);

	return true;
}
Exemplo n.º 15
0
/*
 * Assumes that the segment file lock is already held.
 * Assumes that the segment file should be compacted.
 *
 */
static void
AppendOnlySegmentFileFullCompaction(Relation aorel, 
		AppendOnlyEntry *aoEntry, 
		AppendOnlyInsertDesc insertDesc,
		FileSegInfo* fsinfo)
{
	const char* relname;
	AppendOnlyVisimap visiMap;
	AppendOnlyScanDesc scanDesc;
	TupleDesc tupDesc;
	MemTuple		tuple;
	TupleTableSlot	*slot;
	MemTupleBinding *mt_bind;
	int compact_segno;
	int64 movedTupleCount = 0;
	ResultRelInfo *resultRelInfo;
	EState *estate;
	AOTupleId *aoTupleId;
	int64 tupleCount = 0;
	int64 tuplePerPage = INT_MAX;

	Assert(Gp_role == GP_ROLE_EXECUTE || Gp_role == GP_ROLE_UTILITY);
	Assert(RelationIsAoRows(aorel));
	Assert(insertDesc);

	compact_segno = fsinfo->segno;
	if (fsinfo->varblockcount > 0)
	{
		tuplePerPage = fsinfo->total_tupcount / fsinfo->varblockcount;
	}
	relname = RelationGetRelationName(aorel);

	AppendOnlyVisimap_Init(&visiMap,
			aoEntry->visimaprelid,
			aoEntry->visimapidxid,
			ShareUpdateExclusiveLock,
			SnapshotNow);

	elogif(Debug_appendonly_print_compaction,
			LOG, "Compact AO segno %d, relation %s, insert segno %d", 
			compact_segno, relname, insertDesc->storageWrite.segmentFileNum);

	/*
	 * Todo: We need to limit the scan to one file and we need to avoid to
	 * lock the file again.
     *
     * We use SnapshotAny to get visible and invisible tuples.
	 */
	scanDesc = appendonly_beginrangescan(aorel,
			SnapshotAny, SnapshotNow,
			&compact_segno, 1, 0, NULL);

	tupDesc = RelationGetDescr(aorel);
	slot = MakeSingleTupleTableSlot(tupDesc);
	mt_bind = create_memtuple_binding(tupDesc);

	/*
	 * We need a ResultRelInfo and an EState so we can use the regular
	 * executor's index-entry-making machinery.
	 */
	estate = CreateExecutorState();
	resultRelInfo = makeNode(ResultRelInfo);
	resultRelInfo->ri_RangeTableIndex = 1;	/* dummy */
	resultRelInfo->ri_RelationDesc = aorel;
	resultRelInfo->ri_TrigDesc = NULL;		/* we don't fire triggers */
	ExecOpenIndices(resultRelInfo);
	estate->es_result_relations = resultRelInfo;
	estate->es_num_result_relations = 1;
	estate->es_result_relation_info = resultRelInfo;

	/*
	 * Go through all visible tuples and move them to a new segfile.
	 */
	while ((tuple = appendonly_getnext(scanDesc, ForwardScanDirection, slot)) != NULL)
	{
		/* Check interrupts as this may take time. */
		CHECK_FOR_INTERRUPTS();

		aoTupleId = (AOTupleId*)slot_get_ctid(slot);
		if (AppendOnlyVisimap_IsVisible(&scanDesc->visibilityMap, aoTupleId))
		{
			AppendOnlyMoveTuple(tuple,
							slot,
							mt_bind,
							insertDesc,
							resultRelInfo,
							estate);
			movedTupleCount++;
		}
		else
		{
			/* Tuple is invisible and needs to be dropped */
			AppendOnlyThrowAwayTuple(aorel, 
							tuple,
							slot,
							mt_bind);
		}

		/* 
		 * Check for vacuum delay point after approximatly a var block
		 */
		tupleCount++;
		if (VacuumCostActive && tupleCount % tuplePerPage == 0)
		{
			vacuum_delay_point();
		}
	}

	SetFileSegInfoState(aorel, aoEntry, compact_segno, AOSEG_STATE_AWAITING_DROP);

	AppendOnlyVisimap_DeleteSegmentFile(&visiMap, compact_segno);

	/* Delete all mini pages of the segment files if block directory exists */
	if (OidIsValid(aoEntry->blkdirrelid))
	{
		AppendOnlyBlockDirectory_DeleteSegmentFile(
			aoEntry,
			SnapshotNow,
			compact_segno,
			0);
	}

	elogif(Debug_appendonly_print_compaction, LOG,
		   "Finished compaction: "
		   "AO segfile %d, relation %s, moved tuple count " INT64_FORMAT,
		   compact_segno, relname, movedTupleCount);

	AppendOnlyVisimap_Finish(&visiMap, NoLock);

	ExecCloseIndices(resultRelInfo);
	FreeExecutorState(estate);

	ExecDropSingleTupleTableSlot(slot);
	destroy_memtuple_binding(mt_bind);

	appendonly_endscan(scanDesc);
}