/*
* GetNewRelFileNode
* Generate a new relfilenode number that is unique within the given
* tablespace.
*
* If the relfilenode will also be used as the relation's OID, pass the
* opened pg_class catalog, and this routine will guarantee that the result
* is also an unused OID within pg_class. If the result is to be used only
* as a relfilenode for an existing relation, pass NULL for pg_class.
*
* As with GetNewOid, there is some theoretical risk of a race condition,
* but it doesn't seem worth worrying about.
*
* Note: we don't support using this in bootstrap mode. All relations
* created by bootstrap have preassigned OIDs, so there's no need.
*/
Oid
GetNewRelFileNode(Oid reltablespace, Relation pg_class)
{
RelFileNode rnode;
char *rpath;
int fd;
bool collides;
/* This logic should match RelationInitPhysicalAddr */
rnode.spcNode = reltablespace ? reltablespace : MyDatabaseTableSpace;
rnode.dbNode = (rnode.spcNode == GLOBALTABLESPACE_OID) ? InvalidOid : MyDatabaseId;
do
{
CHECK_FOR_INTERRUPTS();
/* Generate the OID */
if (pg_class)
rnode.relNode = GetNewOid(pg_class);
else
rnode.relNode = GetNewObjectId();
/* Check for existing file of same name */
rpath = relpath(rnode, MAIN_FORKNUM);
fd = BasicOpenFile(rpath, O_RDONLY | PG_BINARY, 0);
if (fd >= 0)
{
/* definite collision */
close(fd);
collides = true;
}
else
{
/*
* Here we have a little bit of a dilemma: if errno is something
* other than ENOENT, should we declare a collision and loop? In
* particular one might think this advisable for, say, EPERM.
* However there really shouldn't be any unreadable files in a
* tablespace directory, and if the EPERM is actually complaining
* that we can't read the directory itself, we'd be in an infinite
* loop. In practice it seems best to go ahead regardless of the
* errno. If there is a colliding file we will get an smgr
* failure when we attempt to create the new relation file.
*/
collides = false;
}
pfree(rpath);
} while (collides);
return rnode.relNode;
}
/*
* GetNewRelFileNode
* Generate a new relfilenode number that is unique within the given
* tablespace.
*
* If the relfilenode will also be used as the relation's OID, pass the
* opened pg_class catalog, and this routine will guarantee that the result
* is also an unused OID within pg_class. If the result is to be used only
* as a relfilenode for an existing relation, pass NULL for pg_class.
*
* As with GetNewOid, there is some theoretical risk of a race condition,
* but it doesn't seem worth worrying about.
*
* Note: we don't support using this in bootstrap mode. All relations
* created by bootstrap have preassigned OIDs, so there's no need.
*/
Oid
GetNewRelFileNode(Oid reltablespace, Relation pg_class, char relpersistence)
{
RelFileNodeBackend rnode;
char *rpath;
int fd;
bool collides;
BackendId backend;
switch (relpersistence)
{
case RELPERSISTENCE_TEMP:
backend = MyBackendId;
break;
case RELPERSISTENCE_UNLOGGED:
case RELPERSISTENCE_PERMANENT:
backend = InvalidBackendId;
break;
default:
elog(ERROR, "invalid relpersistence: %c", relpersistence);
return InvalidOid; /* placate compiler */
}
/* This logic should match RelationInitPhysicalAddr */
rnode.node.spcNode = reltablespace ? reltablespace : MyDatabaseTableSpace;
rnode.node.dbNode = (rnode.node.spcNode == GLOBALTABLESPACE_OID) ? InvalidOid : MyDatabaseId;
/*
* The relpath will vary based on the backend ID, so we must initialize
* that properly here to make sure that any collisions based on filename
* are properly detected.
*/
rnode.backend = backend;
do
{
CHECK_FOR_INTERRUPTS();
/* Generate the OID */
if (pg_class)
rnode.node.relNode = GetNewOid(pg_class);
else
rnode.node.relNode = GetNewObjectId();
/* Check for existing file of same name */
rpath = relpath(rnode, MAIN_FORKNUM);
fd = BasicOpenFile(rpath, O_RDONLY | PG_BINARY, 0);
if (fd >= 0)
{
/* definite collision */
close(fd);
collides = true;
}
else
{
/*
* Here we have a little bit of a dilemma: if errno is something
* other than ENOENT, should we declare a collision and loop? In
* particular one might think this advisable for, say, EPERM.
* However there really shouldn't be any unreadable files in a
* tablespace directory, and if the EPERM is actually complaining
* that we can't read the directory itself, we'd be in an infinite
* loop. In practice it seems best to go ahead regardless of the
* errno. If there is a colliding file we will get an smgr
* failure when we attempt to create the new relation file.
*/
collides = false;
}
pfree(rpath);
} while (collides);
return rnode.node.relNode;
}
/*
* launch_consumer_group
*
* Launch a group of background worker process that will consume from the given topic
* into the given relation
*/
static bool
launch_consumer_group(Relation consumers, KafkaConsumer *consumer, int64 offset)
{
BackgroundWorker worker;
BackgroundWorkerHandle *handle;
KafkaConsumerGroup *group;
bool found;
int i;
group = (KafkaConsumerGroup *) hash_search(consumer_groups, &consumer->id, HASH_ENTER, &found);
if (found)
{
KafkaConsumerProc *proc;
HASH_SEQ_STATUS iter;
bool running = false;
hash_seq_init(&iter, consumer_procs);
while ((proc = (KafkaConsumerProc *) hash_seq_search(&iter)) != NULL)
{
if (proc->consumer_id == consumer->id)
{
running = true;
break;
}
}
hash_seq_term(&iter);
/* if there are already procs running, it's a noop */
if (running)
return true;
/* no procs actually running, so it's ok to launch new ones */
}
group->parallelism = consumer->parallelism;
for (i = 0; i < group->parallelism; i++)
{
/* we just need any unique OID here */
Oid id = GetNewOid(consumers);
KafkaConsumerProc *proc;
proc = (KafkaConsumerProc *) hash_search(consumer_procs, &id, HASH_ENTER, &found);
if (found)
continue;
worker.bgw_main_arg = DatumGetObjectId(id);
worker.bgw_flags = BGWORKER_BACKEND_DATABASE_CONNECTION | BGWORKER_SHMEM_ACCESS;
worker.bgw_start_time = BgWorkerStart_RecoveryFinished;
worker.bgw_restart_time = BGW_NEVER_RESTART;
worker.bgw_main = NULL;
worker.bgw_notify_pid = 0;
/* this module is loaded dynamically, so we can't use bgw_main */
sprintf(worker.bgw_library_name, PIPELINE_KAFKA_LIB);
sprintf(worker.bgw_function_name, KAFKA_CONSUME_MAIN);
snprintf(worker.bgw_name, BGW_MAXLEN, "[kafka consumer] %s <- %s", consumer->rel->relname, consumer->topic);
proc->consumer_id = consumer->id;
proc->partition_group = i;
proc->offset = offset;
namestrcpy(&proc->dbname, get_database_name(MyDatabaseId));
if (!RegisterDynamicBackgroundWorker(&worker, &handle))
return false;
proc->worker = *handle;
}
return true;
}
/*
* EnumValuesCreate
* Create an entry in pg_enum for each of the supplied enum values.
*
* vals is a list of Value strings.
*/
void
EnumValuesCreate(Oid enumTypeOid, List *vals)
{
Relation pg_enum;
NameData enumlabel;
Oid *oids;
int elemno,
num_elems;
Datum values[Natts_pg_enum];
bool nulls[Natts_pg_enum];
ListCell *lc;
HeapTuple tup;
num_elems = list_length(vals);
/*
* We do not bother to check the list of values for duplicates --- if you
* have any, you'll get a less-than-friendly unique-index violation. It is
* probably not worth trying harder.
*/
pg_enum = heap_open(EnumRelationId, RowExclusiveLock);
/*
* Allocate OIDs for the enum's members.
*
* While this method does not absolutely guarantee that we generate no
* duplicate OIDs (since we haven't entered each oid into the table before
* allocating the next), trouble could only occur if the OID counter wraps
* all the way around before we finish. Which seems unlikely.
*/
oids = (Oid *) palloc(num_elems * sizeof(Oid));
for (elemno = 0; elemno < num_elems; elemno++)
{
/*
* We assign even-numbered OIDs to all the new enum labels. This
* tells the comparison functions the OIDs are in the correct sort
* order and can be compared directly.
*/
Oid new_oid;
do
{
new_oid = GetNewOid(pg_enum);
} while (new_oid & 1);
oids[elemno] = new_oid;
}
/* sort them, just in case OID counter wrapped from high to low */
qsort(oids, num_elems, sizeof(Oid), oid_cmp);
/* and make the entries */
memset(nulls, false, sizeof(nulls));
elemno = 0;
foreach(lc, vals)
{
char *lab = strVal(lfirst(lc));
/*
* labels are stored in a name field, for easier syscache lookup, so
* check the length to make sure it's within range.
*/
if (strlen(lab) > (NAMEDATALEN - 1))
ereport(ERROR,
(errcode(ERRCODE_INVALID_NAME),
errmsg("invalid enum label \"%s\"", lab),
errdetail("Labels must be %d characters or less.",
NAMEDATALEN - 1)));
values[Anum_pg_enum_enumtypid - 1] = ObjectIdGetDatum(enumTypeOid);
values[Anum_pg_enum_enumsortorder - 1] = Float4GetDatum(elemno + 1);
namestrcpy(&enumlabel, lab);
values[Anum_pg_enum_enumlabel - 1] = NameGetDatum(&enumlabel);
tup = heap_form_tuple(RelationGetDescr(pg_enum), values, nulls);
HeapTupleSetOid(tup, oids[elemno]);
simple_heap_insert(pg_enum, tup);
CatalogUpdateIndexes(pg_enum, tup);
heap_freetuple(tup);
elemno++;
}
/*
* GetNewRelFileNode
* Generate a new relfilenode number that is unique within the given
* tablespace.
*
* If the relfilenode will also be used as the relation's OID, pass the
* opened pg_class catalog, and this routine will guarantee that the result
* is also an unused OID within pg_class. If the result is to be used only
* as a relfilenode for an existing relation, pass NULL for pg_class.
*
* As with GetNewOid, there is some theoretical risk of a race condition,
* but it doesn't seem worth worrying about.
*
* Note: we don't support using this in bootstrap mode. All relations
* created by bootstrap have preassigned OIDs, so there's no need.
*/
Oid
GetNewRelFileNode(Oid reltablespace, bool relisshared, Relation pg_class)
{
RelFileNode rnode;
char *rpath;
int fd;
bool collides = true;
/* This should match RelationInitPhysicalAddr */
rnode.spcNode = reltablespace ? reltablespace : MyDatabaseTableSpace;
rnode.dbNode = relisshared ? InvalidOid : MyDatabaseId;
do
{
CHECK_FOR_INTERRUPTS();
/* Generate the OID */
if (pg_class)
rnode.relNode = GetNewOid(pg_class);
else
rnode.relNode = GetNewObjectId();
if (!UseOidForRelFileNode(rnode.relNode))
continue;
/* Check for existing file of same name */
rpath = relpath(rnode);
fd = BasicOpenFile(rpath, O_RDONLY | PG_BINARY, 0);
if (fd >= 0)
{
/* definite collision */
gp_retry_close(fd);
collides = true;
}
else
{
/*
* Here we have a little bit of a dilemma: if errno is something
* other than ENOENT, should we declare a collision and loop? In
* particular one might think this advisable for, say, EPERM.
* However there really shouldn't be any unreadable files in a
* tablespace directory, and if the EPERM is actually complaining
* that we can't read the directory itself, we'd be in an infinite
* loop. In practice it seems best to go ahead regardless of the
* errno. If there is a colliding file we will get an smgr
* failure when we attempt to create the new relation file.
*/
collides = false;
}
pfree(rpath);
} while (collides);
if (Gp_role == GP_ROLE_EXECUTE)
Insist(!PointerIsValid(pg_class));
elog(DEBUG1, "Calling GetNewRelFileNode in %s mode %s pg_class. New relOid = %d",
(Gp_role == GP_ROLE_EXECUTE ? "execute" :
Gp_role == GP_ROLE_UTILITY ? "utility" :
"dispatch"), pg_class ? "with" : "without",
rnode.relNode);
return rnode.relNode;
}
/*
* EnumValuesCreate
* Create an entry in pg_enum for each of the supplied enum values.
*
* vals is a list of Value strings.
*/
void
EnumValuesCreate(Oid enumTypeOid, List *vals,
Oid binary_upgrade_next_pg_enum_oid)
{
Relation pg_enum;
TupleDesc tupDesc;
NameData enumlabel;
Oid *oids;
int elemno,
num_elems;
Datum values[Natts_pg_enum];
bool nulls[Natts_pg_enum];
ListCell *lc;
HeapTuple tup;
num_elems = list_length(vals);
/*
* XXX we do not bother to check the list of values for duplicates --- if
* you have any, you'll get a less-than-friendly unique-index violation.
* Is it worth trying harder?
*/
pg_enum = heap_open(EnumRelationId, RowExclusiveLock);
tupDesc = pg_enum->rd_att;
/*
* Allocate oids
*/
oids = (Oid *) palloc(num_elems * sizeof(Oid));
if (OidIsValid(binary_upgrade_next_pg_enum_oid))
{
if (num_elems != 1)
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("EnumValuesCreate() can only set a single OID")));
oids[0] = binary_upgrade_next_pg_enum_oid;
binary_upgrade_next_pg_enum_oid = InvalidOid;
}
else
{
/*
* While this method does not absolutely guarantee that we generate no
* duplicate oids (since we haven't entered each oid into the table
* before allocating the next), trouble could only occur if the oid
* counter wraps all the way around before we finish. Which seems
* unlikely.
*/
for (elemno = 0; elemno < num_elems; elemno++)
{
/*
* The pg_enum.oid is stored in user tables. This oid must be
* preserved by binary upgrades.
*/
oids[elemno] = GetNewOid(pg_enum);
}
/* sort them, just in case counter wrapped from high to low */
qsort(oids, num_elems, sizeof(Oid), oid_cmp);
}
/* and make the entries */
memset(nulls, false, sizeof(nulls));
elemno = 0;
foreach(lc, vals)
{
char *lab = strVal(lfirst(lc));
/*
* labels are stored in a name field, for easier syscache lookup, so
* check the length to make sure it's within range.
*/
if (strlen(lab) > (NAMEDATALEN - 1))
ereport(ERROR,
(errcode(ERRCODE_INVALID_NAME),
errmsg("invalid enum label \"%s\"", lab),
errdetail("Labels must be %d characters or less.",
NAMEDATALEN - 1)));
values[Anum_pg_enum_enumtypid - 1] = ObjectIdGetDatum(enumTypeOid);
namestrcpy(&enumlabel, lab);
values[Anum_pg_enum_enumlabel - 1] = NameGetDatum(&enumlabel);
tup = heap_form_tuple(tupDesc, values, nulls);
HeapTupleSetOid(tup, oids[elemno]);
simple_heap_insert(pg_enum, tup);
CatalogUpdateIndexes(pg_enum, tup);
heap_freetuple(tup);
elemno++;
}
/*
* EnumValuesCreate
* Create an entry in pg_enum for each of the supplied enum values.
*
* vals is a list of Value strings.
*/
void
EnumValuesCreate(Oid enumTypeOid, List *vals)
{
Relation pg_enum;
TupleDesc tupDesc;
NameData enumlabel;
Oid *oids;
int i,
n;
Datum values[Natts_pg_enum];
bool nulls[Natts_pg_enum];
ListCell *lc;
HeapTuple tup;
n = list_length(vals);
/*
* XXX we do not bother to check the list of values for duplicates --- if
* you have any, you'll get a less-than-friendly unique-index violation.
* Is it worth trying harder?
*/
pg_enum = heap_open(EnumRelationId, RowExclusiveLock);
tupDesc = pg_enum->rd_att;
/*
* Allocate oids. While this method does not absolutely guarantee that we
* generate no duplicate oids (since we haven't entered each oid into the
* table before allocating the next), trouble could only occur if the oid
* counter wraps all the way around before we finish. Which seems
* unlikely.
*/
oids = (Oid *) palloc(n * sizeof(Oid));
for (i = 0; i < n; i++)
{
/*
* In QE node, however, use the OIDs assigned by the master (they are delivered
* out-of-band, see oid_dispatch.c.
*/
if (Gp_role == GP_ROLE_EXECUTE)
oids[i] = InvalidOid;
else
oids[i] = GetNewOid(pg_enum);
}
/* sort them, just in case counter wrapped from high to low */
qsort(oids, n, sizeof(Oid), oid_cmp);
/* and make the entries */
memset(nulls, false, sizeof(nulls));
i = 0;
foreach(lc, vals)
{
char *lab = strVal(lfirst(lc));
/*
* labels are stored in a name field, for easier syscache lookup, so
* check the length to make sure it's within range.
*/
if (strlen(lab) > (NAMEDATALEN - 1))
ereport(ERROR,
(errcode(ERRCODE_INVALID_NAME),
errmsg("invalid enum label \"%s\"", lab),
errdetail("Labels must be %d characters or less.",
NAMEDATALEN - 1)));
values[Anum_pg_enum_enumtypid - 1] = ObjectIdGetDatum(enumTypeOid);
namestrcpy(&enumlabel, lab);
values[Anum_pg_enum_enumlabel - 1] = NameGetDatum(&enumlabel);
tup = heap_form_tuple(tupDesc, values, nulls);
HeapTupleSetOid(tup, oids[i]);
simple_heap_insert(pg_enum, tup);
CatalogUpdateIndexes(pg_enum, tup);
heap_freetuple(tup);
i++;
}
/**
* @brief Create LoadStatus file and load heap tuples directly.
* @return void
*/
static void
DirectWriterInsert(DirectWriter *self, HeapTuple tuple)
{
Page page;
OffsetNumber offnum;
ItemId itemId;
Item item;
LoadStatus *ls = &self->ls;
/* Compress the tuple data if needed. */
if (tuple->t_len > TOAST_TUPLE_THRESHOLD)
tuple = toast_insert_or_update(self->base.rel, tuple, NULL, 0);
BULKLOAD_PROFILE(&prof_writer_toast);
/* Assign oids if needed. */
if (self->base.rel->rd_rel->relhasoids)
{
Assert(!OidIsValid(HeapTupleGetOid(tuple)));
HeapTupleSetOid(tuple, GetNewOid(self->base.rel));
}
/* Assume the tuple has been toasted already. */
if (MAXALIGN(tuple->t_len) > MaxHeapTupleSize)
ereport(ERROR,
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
errmsg("row is too big: size %lu, maximum size %lu",
(unsigned long) tuple->t_len,
(unsigned long) MaxHeapTupleSize)));
/* Fill current page, or go to next page if the page is full. */
page = GetCurrentPage(self);
if (PageGetFreeSpace(page) < MAXALIGN(tuple->t_len) +
RelationGetTargetPageFreeSpace(self->base.rel, HEAP_DEFAULT_FILLFACTOR))
{
if (self->curblk < BLOCK_BUF_NUM - 1)
self->curblk++;
else
{
flush_pages(self);
self->curblk = 0; /* recycle from first block */
}
page = GetCurrentPage(self);
/* Initialize current block */
PageInit(page, BLCKSZ, 0);
PageSetTLI(page, ThisTimeLineID);
}
tuple->t_data->t_infomask &= ~(HEAP_XACT_MASK);
tuple->t_data->t_infomask2 &= ~(HEAP2_XACT_MASK);
tuple->t_data->t_infomask |= HEAP_XMAX_INVALID;
HeapTupleHeaderSetXmin(tuple->t_data, self->xid);
HeapTupleHeaderSetCmin(tuple->t_data, self->cid);
HeapTupleHeaderSetXmax(tuple->t_data, 0);
/* put the tuple on local page. */
offnum = PageAddItem(page, (Item) tuple->t_data,
tuple->t_len, InvalidOffsetNumber, false, true);
ItemPointerSet(&(tuple->t_self), LS_TOTAL_CNT(ls) + self->curblk, offnum);
itemId = PageGetItemId(page, offnum);
item = PageGetItem(page, itemId);
((HeapTupleHeader) item)->t_ctid = tuple->t_self;
BULKLOAD_PROFILE(&prof_writer_table);
SpoolerInsert(&self->spooler, tuple);
BULKLOAD_PROFILE(&prof_writer_index);
}