void
test_mem_index_read
(void)
{
redirect_stderr();
index_t * index = index_build("examples/repeats.fa", "test_base30");
test_assert_critical(index != NULL);
test_assert(index_ann_new(25, 1, 1, index) == 0);
// Set alloc failure rate to 0.1.
set_alloc_failure_rate_to(0.1);
for (int i = 0; i < 1000; i++) {
index_t * index_i = index_read("test_base30");
index_free(index_i);
}
reset_alloc();
// Set alloc countdown 0->10.
for (int i = 0; i <= 200; i++) {
set_alloc_failure_countdown_to(i);
index_t * index_i = index_read("test_base30");
index_free(index_i);
}
reset_alloc();
index_free(index);
unredirect_stderr();
}
void
test_mem_index_build
(void)
{
redirect_stderr();
// Set alloc failure rate to 0.1.
set_alloc_failure_rate_to(0.1);
for (int i = 0; i < 1000; i++) {
index_t * index = index_build("examples/repeats.fa", "test_base20");
index_free(index);
}
reset_alloc();
// Set alloc countdown 0->10.
for (int i = 0; i <= 200; i++) {
set_alloc_failure_countdown_to(i);
index_t * index = index_build("examples/repeats.fa", "test_base20");
index_free(index);
}
reset_alloc();
unredirect_stderr();
}
int main(int argc, char *argv[])
{
int re = 1;
if (argc < 2) return usage();
if (strcmp(argv[1], "index") == 0) re = index_build(argc - 1, argv + 1);
else if (strcmp(argv[1], "aln") == 0) re = load_input_map(argc - 1, argv + 1);
else if (strcmp(argv[1], "--help") == 0) return help_usage();
else {
fprintf(stderr, "wrong command: '%s'\n", argv[1]);
return 1;
}
if(re == 0)
{
fprintf(stderr, "Program finished\n");
}
}
void
test_mem_index_ann_new
(void)
{
redirect_stderr();
index_t * index = index_build("examples/repeats.fa", "test_base20");
test_assert_critical(index != NULL);
// Set alloc failure rate to 0.1.
set_alloc_failure_rate_to(0.1);
for (int i = 0; i < 1000; i++) {
if (index_ann_new(25, 1, 1, index) == 0) {
unlink("test_base20.ann.25.1");
ann_free(index->ann[0]);
free(index->ann);
index->ann = NULL;
index->ann_cnt = 0;
}
}
reset_alloc();
// Set alloc countdown 0->10.
for (int i = 0; i <= 1000; i++) {
set_alloc_failure_countdown_to(i);
if (index_ann_new(25, 1, 1, index) == 0) {
unlink("test_base20.ann.25.1");
ann_free(index->ann[0]);
free(index->ann);
index->ann = NULL;
index->ann_cnt = 0;
}
}
reset_alloc();
index_free(index);
unredirect_stderr();
}
static void PersistentBuild_PopulateGpRelationNode(
DatabaseInfo *info,
Oid defaultTablespace,
int64 *count)
{
Relation gp_relfile_node;
int r;
RelFileNode indexRelFileNode;
bool indexFound;
Relation gp_relfile_node_index;
struct IndexInfo *indexInfo;
if (Debug_persistent_print)
elog(Persistent_DebugPrintLevel(),
"PersistentBuild_PopulateGpRelationNode: Enter for dbOid %u",
info->database);
MemSet(&indexRelFileNode, 0, sizeof(RelFileNode));
indexFound = false;
gp_relfile_node =
DirectOpen_GpRelfileNodeOpen(
defaultTablespace,
info->database);
for (r = 0; r < info->dbInfoRelArrayCount; r++)
{
DbInfoRel *dbInfoRel = &info->dbInfoRelArray[r];
RelFileNode relFileNode;
PersistentFileSysRelStorageMgr relStorageMgr;
ItemPointerData persistentTid;
int64 persistentSerialNum;
if (dbInfoRel->reltablespace == GLOBALTABLESPACE_OID &&
info->database != TemplateDbOid)
continue;
relFileNode.spcNode = dbInfoRel->reltablespace;
relFileNode.dbNode =
(dbInfoRel->reltablespace == GLOBALTABLESPACE_OID ?
0 : info->database);
relFileNode.relNode = dbInfoRel->relfilenodeOid;
if (dbInfoRel->relationOid == GpRelfileNodeOidIndexId)
{
indexRelFileNode = relFileNode;
indexFound = true;
}
relStorageMgr = (
(dbInfoRel->relstorage == RELSTORAGE_AOROWS ||
dbInfoRel->relstorage == RELSTORAGE_PARQUET) ?
PersistentFileSysRelStorageMgr_AppendOnly :
PersistentFileSysRelStorageMgr_BufferPool);
/*
* The gp_relation_node mapping table is empty, so use the physical files as
* the guide.
*/
if (relStorageMgr == PersistentFileSysRelStorageMgr_BufferPool)
{
PersistentFileSysRelStorageMgr localRelStorageMgr;
PersistentFileSysRelBufpoolKind relBufpoolKind;
GpPersistentRelfileNode_GetRelfileInfo(
dbInfoRel->relkind,
dbInfoRel->relstorage,
dbInfoRel->relam,
&localRelStorageMgr,
&relBufpoolKind);
Assert(localRelStorageMgr == PersistentFileSysRelStorageMgr_BufferPool);
/*
* Heap tables only ever add a single segment_file_num=0 entry to
* gp_persistent_relation regardless of how many segment files there
* really are.
*/
PersistentRelfile_AddCreated(
&relFileNode,
/* segmentFileNum */ 0,
relStorageMgr,
relBufpoolKind,
dbInfoRel->relname,
&persistentTid,
&persistentSerialNum,
/* flushToXLog */ false);
InsertGpRelfileNodeTuple(
gp_relfile_node,
dbInfoRel->relationOid, // pg_class OID
dbInfoRel->relname,
relFileNode.relNode, // pg_class relfilenode
/* segmentFileNum */ 0,
/* updateIndex */ false,
&persistentTid,
persistentSerialNum);
}
else
{
int a;
int p;
/*
* Append-Only.
*/
/*if (dbInfoRel->physicalSegmentFilesCount == 0 ||
dbInfoRel->physicalSegmentFiles[0].segmentFileNum != 0)
{
elog(ERROR, "Physical segment file #0 missing for relation '%s'",
dbInfoRel->relname);
}*/
/*
* Merge physical file existence and ao[cs]seg catalog logical EOFs .
*/
a = 0;
for (p = 0; p < dbInfoRel->physicalSegmentFilesCount; p++)
{
int physicalSegmentFileNum = dbInfoRel->physicalSegmentFiles[p].segmentFileNum;
bool haveCatalogInfo;
int64 logicalEof;
/*
* There is mostly a 1:1 matching of physical files and logical
* files and we just have to match them up correctly. However
* there are several cases where this can diverge that we have
* to be able to handle.
*
* 1) Segment file 0 always exists as a physical file, but is
* only cataloged when it actually contains data - this only
* occurs for ao when data is inserted in utility mode.
*
* 2) Files created in aborted transactions where an initial
* frozen tuple never made it to disk may have a physical file
* with no logical file.
* XXX - These are leaked files that should probably be
* cleaned up at some point.
*
* 3) It is possible to have files that logically exist with a
* logical EOF of 0 but not exist in the filesystem.
* XXX - how does this happen, is it really safe?
*/
logicalEof = 0;
haveCatalogInfo = false;
/* If we exhaust the loop then we are in case 2 */
while (a < dbInfoRel->appendOnlyCatalogSegmentInfoCount)
{
DbInfoAppendOnlyCatalogSegmentInfo *logicalSegInfo = \
&dbInfoRel->appendOnlyCatalogSegmentInfo[a];
/* Normal Case: both exist */
if (logicalSegInfo->segmentFileNum == physicalSegmentFileNum)
{
logicalEof = logicalSegInfo->logicalEof;
haveCatalogInfo = true;
a++;
break; /* found */
}
/* case 0 or case 2 */
else if (logicalSegInfo->segmentFileNum > physicalSegmentFileNum)
{
logicalEof = 0;
haveCatalogInfo = false;
break; /* not found */
}
/* case 3 - skip over logical segments w/o physical files */
else if (logicalSegInfo->logicalEof == 0)
{
a++;
continue; /* keep looking */
}
/* otherwise it is an error */
else
{
elog(ERROR, "logical EOF greater than zero (" INT64_FORMAT ") for segment file #%d in relation '%s' but physical file is missing",
logicalSegInfo->logicalEof,
logicalSegInfo->segmentFileNum,
dbInfoRel->relname);
}
/* unreachable */
Assert(false);
}
/*
* case 2) Ignore segment file left over from pre-Release 4.0 aborted
* transaction whose initial frozen ao[cs]seg tuple never made it to
* disk. This will be a file that can result in an upgrade complaint...
*/
if (physicalSegmentFileNum > 0 && !haveCatalogInfo)
continue;
PersistentRelfile_AddCreated(
&relFileNode,
physicalSegmentFileNum,
relStorageMgr,
PersistentFileSysRelBufpoolKind_None,
dbInfoRel->relname,
&persistentTid,
&persistentSerialNum,
/* flushToXLog */ false);
InsertGpRelfileNodeTuple(
gp_relfile_node,
dbInfoRel->relationOid, // pg_class OID
dbInfoRel->relname,
relFileNode.relNode, // pg_class relfilenode
physicalSegmentFileNum,
/* updateIndex */ false,
&persistentTid,
persistentSerialNum);
}
}
(*count)++;
}
if (info->database != TemplateDbOid)
{
PersistentBuild_ScanGpPersistentRelationNodeForGlobal(
gp_relfile_node,
count);
}
/*
* Build the index for gp_relation_node.
*
* The problem is the session we are using is associated with one particular database
* of the cluster, but we need to iterate through all the databases. So, unfortunately,
* the solution has been to use the "Direct Open" stuff.
*
* We do this because MyDatabaseId, the default tablespace of the session should not be
* changed. The various caches and many other implicit things assume the object is for
* MyDatabaseId and the default tablespace. For example, we cannot use
* CatalogUpdateIndexes called in InsertGpRelationNodeTuple because it will not do
* the right thing.
*
* Also, if they re-indexed gp_relation_node, it will have a different relfilenode and so we
* must have found it (above) and open it with dynamically.
*/
Assert(indexFound);
PersistentBuild_NonTransactionTruncate(
&indexRelFileNode);
gp_relfile_node_index =
DirectOpen_GpRelfileNodeIndexOpenDynamic(
GpRelfileNodeOidIndexId,
indexRelFileNode.spcNode,
indexRelFileNode.dbNode,
indexRelFileNode.relNode);
indexInfo = makeNode(IndexInfo);
indexInfo->ii_NumIndexAttrs = Natts_gp_relfile_node_index;
indexInfo->ii_KeyAttrNumbers[0] = 1;
indexInfo->ii_KeyAttrNumbers[1] = 2;
indexInfo->ii_KeyAttrNumbers[2] = 6;
indexInfo->ii_Unique = true;
if (Debug_persistent_print)
elog(Persistent_DebugPrintLevel(),
"PersistentBuild_PopulateGpRelationNode: building gp_relfile_node_index %u/%u/%u for gp_relfile_node %u/%u/%u",
gp_relfile_node_index->rd_node.spcNode,
gp_relfile_node_index->rd_node.dbNode,
gp_relfile_node_index->rd_node.relNode,
gp_relfile_node->rd_node.spcNode,
gp_relfile_node->rd_node.dbNode,
gp_relfile_node->rd_node.relNode);
index_build(
gp_relfile_node,
gp_relfile_node_index,
indexInfo,
false);
DirectOpen_GpRelfileNodeIndexClose(gp_relfile_node_index);
DirectOpen_GpRelfileNodeClose(gp_relfile_node);
if (Debug_persistent_print)
elog(Persistent_DebugPrintLevel(),
"PersistentBuild_PopulateGpRelationNode: Exit for dbOid %u",
info->database);
}
/* ----------------------------------------------------------------
* index_create
*
* Returns OID of the created index.
* ----------------------------------------------------------------
*/
Oid
index_create(Oid heapRelationId,
const char *indexRelationName,
IndexInfo *indexInfo,
Oid accessMethodObjectId,
Oid *classObjectId,
bool primary,
bool isconstraint,
bool allow_system_table_mods)
{
Relation heapRelation;
Relation indexRelation;
TupleDesc indexTupDesc;
bool shared_relation;
Oid namespaceId;
Oid indexoid;
int i;
/*
* Only SELECT ... FOR UPDATE are allowed while doing this
*/
heapRelation = heap_open(heapRelationId, ShareLock);
/*
* The index will be in the same namespace as its parent table, and is
* shared across databases if and only if the parent is.
*/
namespaceId = RelationGetNamespace(heapRelation);
shared_relation = heapRelation->rd_rel->relisshared;
/*
* check parameters
*/
if (indexInfo->ii_NumIndexAttrs < 1)
elog(ERROR, "must index at least one column");
if (!allow_system_table_mods &&
IsSystemRelation(heapRelation) &&
IsNormalProcessingMode())
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("user-defined indexes on system catalog tables are not supported")));
/*
* We cannot allow indexing a shared relation after initdb (because
* there's no way to make the entry in other databases' pg_class).
* Unfortunately we can't distinguish initdb from a manually started
* standalone backend. However, we can at least prevent this mistake
* under normal multi-user operation.
*/
if (shared_relation && IsUnderPostmaster)
ereport(ERROR,
(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
errmsg("shared indexes cannot be created after initdb")));
if (get_relname_relid(indexRelationName, namespaceId))
ereport(ERROR,
(errcode(ERRCODE_DUPLICATE_TABLE),
errmsg("relation \"%s\" already exists",
indexRelationName)));
/*
* construct tuple descriptor for index tuples
*/
indexTupDesc = ConstructTupleDescriptor(heapRelation,
indexInfo,
classObjectId);
/*
* create the index relation's relcache entry and physical disk file.
* (If we fail further down, it's the smgr's responsibility to remove
* the disk file again.)
*/
indexRelation = heap_create(indexRelationName,
namespaceId,
indexTupDesc,
shared_relation,
true,
allow_system_table_mods);
/* Fetch the relation OID assigned by heap_create */
indexoid = RelationGetRelid(indexRelation);
/*
* Obtain exclusive lock on it. Although no other backends can see it
* until we commit, this prevents deadlock-risk complaints from lock
* manager in cases such as CLUSTER.
*/
LockRelation(indexRelation, AccessExclusiveLock);
/*
* Fill in fields of the index's pg_class entry that are not set
* correctly by heap_create.
*
* XXX should have a cleaner way to create cataloged indexes
*/
indexRelation->rd_rel->relowner = GetUserId();
indexRelation->rd_rel->relam = accessMethodObjectId;
indexRelation->rd_rel->relkind = RELKIND_INDEX;
indexRelation->rd_rel->relhasoids = false;
/*
* store index's pg_class entry
*/
UpdateRelationRelation(indexRelation);
/*
* now update the object id's of all the attribute tuple forms in the
* index relation's tuple descriptor
*/
InitializeAttributeOids(indexRelation,
indexInfo->ii_NumIndexAttrs,
indexoid);
/*
* append ATTRIBUTE tuples for the index
*/
AppendAttributeTuples(indexRelation, indexInfo->ii_NumIndexAttrs);
/* ----------------
* update pg_index
* (append INDEX tuple)
*
* Note that this stows away a representation of "predicate".
* (Or, could define a rule to maintain the predicate) --Nels, Feb '92
* ----------------
*/
UpdateIndexRelation(indexoid, heapRelationId, indexInfo,
classObjectId, primary);
/*
* Register constraint and dependencies for the index.
*
* If the index is from a CONSTRAINT clause, construct a pg_constraint
* entry. The index is then linked to the constraint, which in turn
* is linked to the table. If it's not a CONSTRAINT, make the
* dependency directly on the table.
*
* We don't need a dependency on the namespace, because there'll be an
* indirect dependency via our parent table.
*
* During bootstrap we can't register any dependencies, and we don't try
* to make a constraint either.
*/
if (!IsBootstrapProcessingMode())
{
ObjectAddress myself,
referenced;
myself.classId = RelOid_pg_class;
myself.objectId = indexoid;
myself.objectSubId = 0;
if (isconstraint)
{
char constraintType;
Oid conOid;
if (primary)
constraintType = CONSTRAINT_PRIMARY;
else if (indexInfo->ii_Unique)
constraintType = CONSTRAINT_UNIQUE;
else
{
elog(ERROR, "constraint must be PRIMARY or UNIQUE");
constraintType = 0; /* keep compiler quiet */
}
/* Shouldn't have any expressions */
if (indexInfo->ii_Expressions)
elog(ERROR, "constraints can't have index expressions");
conOid = CreateConstraintEntry(indexRelationName,
namespaceId,
constraintType,
false, /* isDeferrable */
false, /* isDeferred */
heapRelationId,
indexInfo->ii_KeyAttrNumbers,
indexInfo->ii_NumIndexAttrs,
InvalidOid, /* no domain */
InvalidOid, /* no foreign key */
NULL,
0,
' ',
' ',
' ',
InvalidOid, /* no associated index */
NULL, /* no check constraint */
NULL,
NULL);
referenced.classId = get_system_catalog_relid(ConstraintRelationName);
referenced.objectId = conOid;
referenced.objectSubId = 0;
recordDependencyOn(&myself, &referenced, DEPENDENCY_INTERNAL);
}
else
{
/* Create auto dependencies on simply-referenced columns */
for (i = 0; i < indexInfo->ii_NumIndexAttrs; i++)
{
if (indexInfo->ii_KeyAttrNumbers[i] != 0)
{
referenced.classId = RelOid_pg_class;
referenced.objectId = heapRelationId;
referenced.objectSubId = indexInfo->ii_KeyAttrNumbers[i];
recordDependencyOn(&myself, &referenced, DEPENDENCY_AUTO);
}
}
}
/* Store dependency on operator classes */
referenced.classId = get_system_catalog_relid(OperatorClassRelationName);
for (i = 0; i < indexInfo->ii_NumIndexAttrs; i++)
{
referenced.objectId = classObjectId[i];
referenced.objectSubId = 0;
recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL);
}
/* Store dependencies on anything mentioned in index expressions */
if (indexInfo->ii_Expressions)
{
recordDependencyOnSingleRelExpr(&myself,
(Node *) indexInfo->ii_Expressions,
heapRelationId,
DEPENDENCY_NORMAL,
DEPENDENCY_AUTO);
}
/* Store dependencies on anything mentioned in predicate */
if (indexInfo->ii_Predicate)
{
recordDependencyOnSingleRelExpr(&myself,
(Node *) indexInfo->ii_Predicate,
heapRelationId,
DEPENDENCY_NORMAL,
DEPENDENCY_AUTO);
}
}
/*
* Advance the command counter so that we can see the newly-entered
* catalog tuples for the index.
*/
CommandCounterIncrement();
/*
* In bootstrap mode, we have to fill in the index strategy structure
* with information from the catalogs. If we aren't bootstrapping,
* then the relcache entry has already been rebuilt thanks to sinval
* update during CommandCounterIncrement.
*/
if (IsBootstrapProcessingMode())
RelationInitIndexAccessInfo(indexRelation);
else
Assert(indexRelation->rd_indexcxt != NULL);
/*
* If this is bootstrap (initdb) time, then we don't actually fill in
* the index yet. We'll be creating more indexes and classes later,
* so we delay filling them in until just before we're done with
* bootstrapping. Otherwise, we call the routine that constructs the
* index.
*
* In normal processing mode, the heap and index relations are closed by
* index_build() --- but we continue to hold the ShareLock on the heap
* and the exclusive lock on the index that we acquired above, until
* end of transaction.
*/
if (IsBootstrapProcessingMode())
{
index_register(heapRelationId, indexoid, indexInfo);
/* XXX shouldn't we close the heap and index rels here? */
}
else
index_build(heapRelation, indexRelation, indexInfo);
return indexoid;
}
/*
* reindex_index - This routine is used to recreate a single index
*/
void
reindex_index(Oid indexId)
{
Relation iRel,
heapRelation;
IndexInfo *indexInfo;
Oid heapId;
bool inplace;
/*
* Open our index relation and get an exclusive lock on it.
*
* Note: for REINDEX INDEX, doing this before opening the parent heap
* relation means there's a possibility for deadlock failure against
* another xact that is doing normal accesses to the heap and index.
* However, it's not real clear why you'd be wanting to do REINDEX INDEX
* on a table that's in active use, so I'd rather have the protection of
* making sure the index is locked down. In the REINDEX TABLE and
* REINDEX DATABASE cases, there is no problem because caller already
* holds exclusive lock on the parent table.
*/
iRel = index_open(indexId);
LockRelation(iRel, AccessExclusiveLock);
/* Get OID of index's parent table */
heapId = iRel->rd_index->indrelid;
/* Open and lock the parent heap relation */
heapRelation = heap_open(heapId, AccessExclusiveLock);
SetReindexProcessing(heapId, indexId);
/*
* If it's a shared index, we must do inplace processing (because we
* have no way to update relfilenode in other databases). Otherwise
* we can do it the normal transaction-safe way.
*
* Since inplace processing isn't crash-safe, we only allow it in a
* standalone backend. (In the REINDEX TABLE and REINDEX DATABASE cases,
* the caller should have detected this.)
*/
inplace = iRel->rd_rel->relisshared;
if (inplace && IsUnderPostmaster)
ereport(ERROR,
(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
errmsg("shared index \"%s\" can only be reindexed in stand-alone mode",
RelationGetRelationName(iRel))));
/* Fetch info needed for index_build */
indexInfo = BuildIndexInfo(iRel);
if (inplace)
{
/*
* Release any buffers associated with this index. If they're
* dirty, they're just dropped without bothering to flush to disk.
*/
DropRelationBuffers(iRel);
/* Now truncate the actual data and set blocks to zero */
smgrtruncate(DEFAULT_SMGR, iRel, 0);
iRel->rd_nblocks = 0;
iRel->rd_targblock = InvalidBlockNumber;
}
else
{
/*
* We'll build a new physical relation for the index.
*/
setNewRelfilenode(iRel);
}
/* Initialize the index and rebuild */
index_build(heapRelation, iRel, indexInfo);
/*
* index_build will close both the heap and index relations (but not
* give up the locks we hold on them). So we're done.
*/
SetReindexProcessing(InvalidOid, InvalidOid);
}
