/*
* Performs database specific PersistentTables-Catalog verification
* Return true if all the verifications pass else returns false
*/
bool Persistent_DBSpecificPTCatVerification (void)
{
int querynum = 0;
bool testSucceeded = true;
/*DataState is needed because some cross cons. queries should be run only in InSync mode*/
getFileRepRoleAndState(&fileRepRole, &segmentState, &dataState, NULL, NULL);
bool isDataInSync = (dataState == DataStateInSync);
Persistent_Pre_ExecuteQuery();
PG_TRY();
{
for (querynum = 0; querynum < DB_SPECIFIC_PTCAT_VERIFICATION_NUM_QUERIES; querynum++)
{
if (!isDataInSync && DB_PTCat_Veritifcation_queries[querynum].executeWhenInSyncOnly)
continue;
elog(LOG,"%s",DB_PTCat_Veritifcation_queries[querynum].quertTitle);
if (Persistent_ExecuteQuery(DB_PTCat_Veritifcation_queries[querynum].queryStr, true) > 0)
testSucceeded = false;
}
}
PG_CATCH();
{
Persistent_ExecuteQuery_Cleanup();
elog(FATAL, "PersistentTable-Cat DBSpecific Verification: Failure");
}
PG_END_TRY();
Persistent_Post_ExecuteQuery();
return testSucceeded;
}
static int
FaultInjector_NewHashEntry(
FaultInjectorEntry_s *entry)
{
FaultInjectorEntry_s *entryLocal=NULL;
bool exists;
int status = STATUS_OK;
FiLockAcquire();
if ((faultInjectorShmem->faultInjectorSlots + 1) >= FAULTINJECTOR_MAX_SLOTS) {
FiLockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection, no slots available"
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, max slots:'%d' reached",
FAULTINJECTOR_MAX_SLOTS);
goto exit;
}
if (entry->faultInjectorType == FaultInjectorTypeSkip)
{
switch (entry->faultInjectorIdentifier)
{
case Checkpoint:
case ChangeTrackingDisable:
case FileRepVerification:
case FinishPreparedTransactionCommitPass1FromCreatePendingToCreated:
case FinishPreparedTransactionCommitPass2FromCreatePendingToCreated:
case FinishPreparedTransactionCommitPass1FromDropInMemoryToDropPending:
case FinishPreparedTransactionCommitPass2FromDropInMemoryToDropPending:
case FinishPreparedTransactionCommitPass1AbortingCreateNeeded:
case FinishPreparedTransactionCommitPass2AbortingCreateNeeded:
case FinishPreparedTransactionAbortPass1FromCreatePendingToAbortingCreate:
case FinishPreparedTransactionAbortPass2FromCreatePendingToAbortingCreate:
case FinishPreparedTransactionAbortPass1AbortingCreateNeeded:
case FinishPreparedTransactionAbortPass2AbortingCreateNeeded:
break;
default:
FiLockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection, fault type not supported"
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, fault type not supported");
goto exit;
}
}
/* check role */
getFileRepRoleAndState(&fileRepRole, &segmentState, &dataState, NULL, NULL);
switch (entry->faultInjectorIdentifier)
{
case ChangeTrackingDisable:
case FileRepConsumerVerification:
case StartPrepareTx:
case FileRepResync:
case FileRepResyncInProgress:
case FileRepResyncWorker:
case FileRepResyncWorkerRead:
case FileRepTransitionToInResyncMirrorReCreate:
case FileRepTransitionToInResyncMarkReCreated:
case FileRepTransitionToInResyncMarkCompleted:
case FileRepTransitionToInSyncBegin:
case FileRepTransitionToInSync:
case FileRepTransitionToInSyncMarkCompleted:
case FileRepTransitionToInSyncBeforeCheckpoint:
case FileRepTransitionToChangeTracking:
case FileRepIsOperationCompleted:
case FileRepImmediateShutdownRequested:
case TransactionCommitPass1FromCreatePendingToCreated:
case TransactionCommitPass1FromDropInMemoryToDropPending:
case TransactionCommitPass1FromAbortingCreateNeededToAbortingCreate:
case TransactionAbortPass1FromCreatePendingToAbortingCreate:
case TransactionAbortPass1FromAbortingCreateNeededToAbortingCreate:
case TransactionCommitPass2FromDropInMemoryToDropPending:
case TransactionCommitPass2FromAbortingCreateNeededToAbortingCreate:
case TransactionAbortPass2FromCreatePendingToAbortingCreate:
case TransactionAbortPass2FromAbortingCreateNeededToAbortingCreate:
case FinishPreparedTransactionCommitPass1FromCreatePendingToCreated:
case FinishPreparedTransactionCommitPass2FromCreatePendingToCreated:
case FinishPreparedTransactionCommitPass1FromDropInMemoryToDropPending:
case FinishPreparedTransactionCommitPass2FromDropInMemoryToDropPending:
case FinishPreparedTransactionCommitPass1AbortingCreateNeeded:
case FinishPreparedTransactionCommitPass2AbortingCreateNeeded:
case FinishPreparedTransactionAbortPass1FromCreatePendingToAbortingCreate:
// case FinishPreparedTransactionAbortPass2FromCreatePendingToAbortingCreate:
case FinishPreparedTransactionAbortPass1AbortingCreateNeeded:
case FinishPreparedTransactionAbortPass2AbortingCreateNeeded:
case TwoPhaseTransactionCommitPrepared:
case TwoPhaseTransactionAbortPrepared:
case ExecSortMKSortMergeRuns:
// case SubtransactionFlushToFile:
// case SubtransactionReadFromFile:
// case SubtransactionRelease:
// case SubtransactionRollback:
/* Ashwin */
case FileRepChangeTrackingCompacting:
/* We do not use vmem on master. Therefore, we only attempt large palloc on segments. */
case MultiExecHashLargeVmem:
case FaultInBackgroundWriterMain:
if (fileRepRole != FileRepPrimaryRole)
{
FiLockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection entry into table, segment not in primary role"
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, segment not in primary role");
goto exit;
}
break;
case FaultBeforePendingDeleteRelationEntry:
case FaultBeforePendingDeleteDatabaseEntry:
case FaultBeforePendingDeleteTablespaceEntry:
case FaultBeforePendingDeleteFilespaceEntry:
case FileRepConsumer:
case FileRepSender:
case FileRepReceiver:
case FileRepFlush:
case Postmaster:
case PgControl:
case PgXlog:
case SegmentTransitionRequest:
case SegmentProbeResponse:
if (fileRepRole != FileRepPrimaryRole && fileRepRole != FileRepMirrorRole)
{
FiLockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection entry into table, "
"segment not in primary or mirror role, "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, segment not in primary or mirror role");
goto exit;
}
break;
case TransactionAbortAfterDistributedPrepared:
case DtmBroadcastPrepare:
case DtmBroadcastCommitPrepared:
case DtmBroadcastAbortPrepared:
case DtmXLogDistributedCommit:
case OptTaskAllocateStringBuffer:
case OptRelcacheTranslatorCatalogAccess:
if (fileRepRole != FileRepNoRoleConfigured)
{
FiLockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection entry into table, "
"segment not in master role, "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, segment not in master role");
goto exit;
}
break;
case LocalTmRecordTransactionCommit:
case Checkpoint:
case AbortTransactionFail:
case UpdateCommittedEofInPersistentTable:
case ExecSortBeforeSorting:
case FaultDuringExecDynamicTableScan:
case FaultExecHashJoinNewBatch:
case RunawayCleanup:
if (fileRepRole != FileRepNoRoleConfigured && fileRepRole != FileRepPrimaryRole)
{
FiLockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection entry into table, "
"segment not in primary or master role, "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, segment not in primary or master role");
goto exit;
}
break;
default:
break;
}
entryLocal = FaultInjector_InsertHashEntry(entry->faultInjectorIdentifier, &exists);
if (entryLocal == NULL) {
FiLockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection entry into table, no memory, "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, no memory");
goto exit;
}
if (exists) {
FiLockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection entry into table, "
"entry already exists, "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, entry already exists");
goto exit;
}
entryLocal->faultInjectorType = entry->faultInjectorType;
entryLocal->sleepTime = entry->sleepTime;
entryLocal->ddlStatement = entry->ddlStatement;
if (entry->occurrence != 0)
{
entryLocal->occurrence = entry->occurrence;
}
else
{
entryLocal->occurrence = FILEREP_UNDEFINED;
}
strcpy(entryLocal->databaseName, entry->databaseName);
strcpy(entryLocal->tableName, entry->tableName);
entryLocal->faultInjectorState = FaultInjectorStateWaiting;
faultInjectorShmem->faultInjectorSlots++;
FiLockRelease();
elog(DEBUG1, "FaultInjector_NewHashEntry() identifier:'%s'",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier]);
exit:
return status;
}
FaultInjectorType_e
FaultInjector_InjectFaultIfSet(
FaultInjectorIdentifier_e identifier,
DDLStatement_e ddlStatement,
char* databaseName,
char* tableName)
{
FaultInjectorEntry_s *entryShared, localEntry,
*entryLocal = &localEntry;
char databaseNameLocal[NAMEDATALEN];
char tableNameLocal[NAMEDATALEN];
int ii = 0;
int cnt = 3600;
/*
* Return immediately if no fault has been injected ever. It is
* important to not touch the spinlock, especially if this is the
* postmaster process. If one of the backend processes dies while
* holding the spin lock, and postmaster comes here before resetting
* the shared memory, it waits without holder process and eventually
* goes into PANIC. Also this saves a few cycles to acquire the spin
* lock and look into the shared hash table.
*
* Although this is a race condition without lock, a false negative is
* ok given this framework is purely for dev/testing.
*/
if (faultInjectorShmem->faultInjectorSlots == 0)
return FALSE;
getFileRepRoleAndState(&fileRepRole, &segmentState, &dataState, NULL, NULL);
FiLockAcquire();
entryShared = FaultInjector_LookupHashEntry(identifier);
if (entryShared != NULL)
memcpy(entryLocal, entryShared, sizeof(FaultInjectorEntry_s));
FiLockRelease();
/* Verify if fault injection is set */
if (entryShared == NULL)
/* fault injection is not set */
return FALSE;
if (entryLocal->ddlStatement != ddlStatement)
/* fault injection is not set for the specified DDL */
return FALSE;
snprintf(databaseNameLocal, sizeof(databaseNameLocal), "%s", databaseName);
if (strcmp(entryLocal->databaseName, databaseNameLocal) != 0)
/* fault injection is not set for the specified database name */
return FALSE;
snprintf(tableNameLocal, sizeof(tableNameLocal), "%s", tableName);
if (strcmp(entryLocal->tableName, tableNameLocal) != 0)
/* fault injection is not set for the specified table name */
return FALSE;
if (entryLocal->faultInjectorState == FaultInjectorStateTriggered ||
entryLocal->faultInjectorState == FaultInjectorStateCompleted ||
entryLocal->faultInjectorState == FaultInjectorStateFailed) {
/* fault injection was already executed */
return FALSE;
}
/* Update the injection fault entry in hash table */
if (entryLocal->occurrence != FILEREP_UNDEFINED)
{
if (entryLocal->occurrence > 1)
{
entryLocal->occurrence--;
FaultInjector_UpdateHashEntry(entryLocal);
return FALSE;
}
entryLocal->faultInjectorState = FaultInjectorStateTriggered;
FaultInjector_UpdateHashEntry(entryLocal);
}
/* Inject fault */
switch (entryLocal->faultInjectorType) {
case FaultInjectorTypeNotSpecified:
break;
case FaultInjectorTypeSleep:
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
pg_usleep(entryLocal->sleepTime * 1000000L);
break;
case FaultInjectorTypeFault:
switch (entryLocal->faultInjectorIdentifier)
{
case FileRepConsumer:
case FileRepConsumerVerification:
case FileRepSender:
case FileRepReceiver:
case FileRepResync:
case FileRepResyncInProgress:
case FileRepResyncWorker:
case FileRepResyncWorkerRead:
case FileRepTransitionToInResyncMirrorReCreate:
case FileRepTransitionToInResyncMarkReCreated:
case FileRepTransitionToInResyncMarkCompleted:
case FileRepTransitionToInSyncBegin:
case FileRepTransitionToInSync:
case FileRepTransitionToInSyncMarkCompleted:
case FileRepTransitionToInSyncBeforeCheckpoint:
case FileRepIsOperationCompleted:
FileRep_SetSegmentState(SegmentStateFault, FaultTypeMirror);
break;
case FileRepTransitionToChangeTracking:
FileRep_SetPostmasterReset();
break;
default:
FileRep_SetSegmentState(SegmentStateFault, FaultTypeIO);
break;
}
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypeFatal:
/*
* If it's one time occurrence then disable the fault before it's
* actually triggered because this fault errors out the transaction
* and hence we wont get a chance to disable it or put it in completed
* state.
*/
if (entryLocal->occurrence != FILEREP_UNDEFINED)
{
entryLocal->faultInjectorState = FaultInjectorStateCompleted;
FaultInjector_UpdateHashEntry(entryLocal);
}
ereport(FATAL,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypePanic:
/*
* If it's one time occurrence then disable the fault before it's
* actually triggered because this fault errors out the transaction
* and hence we wont get a chance to disable it or put it in completed
* state. For PANIC it may be unnecessary though.
*/
if (entryLocal->occurrence != FILEREP_UNDEFINED)
{
entryLocal->faultInjectorState = FaultInjectorStateCompleted;
FaultInjector_UpdateHashEntry(entryLocal);
}
ereport(PANIC,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypeError:
/*
* If it's one time occurrence then disable the fault before it's
* actually triggered because this fault errors out the transaction
* and hence we wont get a chance to disable it or put it in completed
* state.
*/
if (entryLocal->occurrence != FILEREP_UNDEFINED)
{
entryLocal->faultInjectorState = FaultInjectorStateCompleted;
FaultInjector_UpdateHashEntry(entryLocal);
}
ereport(ERROR,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypeInfiniteLoop:
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
if (entryLocal->faultInjectorIdentifier == FileRepImmediateShutdownRequested)
cnt = entryLocal->sleepTime;
for (ii=0; ii < cnt; ii++)
{
pg_usleep(1000000L); // sleep for 1 sec (1 sec * 3600 = 1 hour)
getFileRepRoleAndState(NULL, &segmentState, NULL, NULL, NULL);
if ((entryLocal->faultInjectorIdentifier != FileRepImmediateShutdownRequested) &&
(segmentState == SegmentStateShutdownFilerepBackends ||
segmentState == SegmentStateImmediateShutdown ||
segmentState == SegmentStateShutdown ||
IsFtsShudownRequested()))
{
break;
}
}
break;
case FaultInjectorTypeDataCorruption:
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypeSuspend:
{
FaultInjectorEntry_s *entry;
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
while ((entry = FaultInjector_LookupHashEntry(entryLocal->faultInjectorIdentifier)) != NULL &&
entry->faultInjectorType != FaultInjectorTypeResume)
{
pg_usleep(1000000L); // 1 sec
}
if (entry != NULL)
{
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
}
else
{
ereport(LOG,
(errmsg("fault 'NULL', fault name:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier])));
/*
* Since the entry is gone already, we should NOT update
* the entry below. (There could be other places in this
* function that are under the same situation, but I'm too
* tired to look for them...)
*/
return entryLocal->faultInjectorType;
}
break;
}
case FaultInjectorTypeSkip:
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypeMemoryFull:
{
char *buffer = NULL;
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
buffer = (char*) palloc(BLCKSZ);
while (buffer != NULL)
{
buffer = (char*) palloc(BLCKSZ);
}
break;
}
case FaultInjectorTypeReset:
case FaultInjectorTypeStatus:
ereport(LOG,
(errmsg("unexpected error, fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
Assert(0);
break;
case FaultInjectorTypeResume:
break;
case FaultInjectorTypeSegv:
{
*(int *) 0 = 1234;
break;
}
case FaultInjectorTypeInterrupt:
{
/*
* The place where this type of fault is injected must have
* has HOLD_INTERRUPTS() .. RESUME_INTERRUPTS() around it, otherwise
* the interrupt could be handled inside the fault injector itself
*/
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
InterruptPending = true;
QueryCancelPending = true;
break;
}
case FaultInjectorTypeCheckpointAndPanic:
{
if (entryLocal->occurrence != FILEREP_UNDEFINED)
{
entryLocal->faultInjectorState = FaultInjectorStateCompleted;
FaultInjector_UpdateHashEntry(entryLocal);
}
RequestCheckpoint(true, false);
ereport(PANIC,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
}
default:
ereport(LOG,
(errmsg("unexpected error, fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
Assert(0);
break;
}
if (entryLocal->occurrence != FILEREP_UNDEFINED)
{
entryLocal->faultInjectorState = FaultInjectorStateCompleted;
}
FaultInjector_UpdateHashEntry(entryLocal);
return (entryLocal->faultInjectorType);
}
int
FaultInjector_SetFaultInjection(
FaultInjectorEntry_s *entry)
{
int status = STATUS_OK;
bool isRemoved = FALSE;
getFileRepRoleAndState(&fileRepRole, &segmentState, &dataState, NULL, NULL);
switch (entry->faultInjectorType) {
case FaultInjectorTypeReset:
{
HASH_SEQ_STATUS hash_status;
FaultInjectorEntry_s *entryLocal;
if (entry->faultInjectorIdentifier == FaultInjectorIdAll)
{
hash_seq_init(&hash_status, faultInjectorShmem->hash);
FiLockAcquire();
while ((entryLocal = (FaultInjectorEntry_s *) hash_seq_search(&hash_status)) != NULL) {
isRemoved = FaultInjector_RemoveHashEntry(entryLocal->faultInjectorIdentifier);
if (isRemoved == TRUE) {
faultInjectorShmem->faultInjectorSlots--;
}
}
FiLockRelease();
Assert(faultInjectorShmem->faultInjectorSlots == 0);
}
else
{
FiLockAcquire();
isRemoved = FaultInjector_RemoveHashEntry(entry->faultInjectorIdentifier);
if (isRemoved == TRUE) {
faultInjectorShmem->faultInjectorSlots--;
}
FiLockRelease();
}
if (isRemoved == FALSE) {
ereport(DEBUG1,
(errmsg("LOG(fault injector): could not remove fault injection from hash"
"identifier:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier])));
}
break;
}
case FaultInjectorTypeStatus:
{
HASH_SEQ_STATUS hash_status;
FaultInjectorEntry_s *entryLocal;
bool found = FALSE;
if (faultInjectorShmem->hash == NULL) {
status = STATUS_ERROR;
break;
}
snprintf(entry->bufOutput, sizeof(entry->bufOutput), "Success: ");
if (entry->faultInjectorIdentifier == ChangeTrackingCompactingReport)
{
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"Success: compacting in progress %s",
ChangeTrackingIsCompactingInProgress() ? "true" : "false");
break;
}
hash_seq_init(&hash_status, faultInjectorShmem->hash);
while ((entryLocal = (FaultInjectorEntry_s *) hash_seq_search(&hash_status)) != NULL) {
ereport(LOG,
(errmsg("fault injector status: "
"fault name:'%s' "
"fault type:'%s' "
"ddl statement:'%s' "
"database name:'%s' "
"table name:'%s' "
"occurrence:'%d' "
"sleep time:'%d' "
"fault injection state:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType],
FaultInjectorDDLEnumToString[entryLocal->ddlStatement],
entryLocal->databaseName,
entryLocal->tableName,
entryLocal->occurrence,
entryLocal->sleepTime,
FaultInjectorStateEnumToString[entryLocal->faultInjectorState])));
if (entry->faultInjectorIdentifier == entryLocal->faultInjectorIdentifier ||
entry->faultInjectorIdentifier == FaultInjectorIdAll) {
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"%s \n"
"fault name:'%s' "
"fault type:'%s' "
"ddl statement:'%s' "
"database name:'%s' "
"table name:'%s' "
"occurrence:'%d' "
"sleep time:'%d' "
"fault injection state:'%s' ",
entry->bufOutput,
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType],
FaultInjectorDDLEnumToString[entryLocal->ddlStatement],
entryLocal->databaseName,
entryLocal->tableName,
entryLocal->occurrence,
entryLocal->sleepTime,
FaultInjectorStateEnumToString[entryLocal->faultInjectorState]);
found = TRUE;
}
}
if (found == FALSE) {
snprintf(entry->bufOutput, sizeof(entry->bufOutput), "Failure: "
"fault name:'%s' not set",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier]);
}
break;
}
case FaultInjectorTypeResume:
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
FaultInjector_UpdateHashEntry(entry);
break;
default:
status = FaultInjector_NewHashEntry(entry);
break;
}
return status;
}
bool
FileRepSubProcess_IsStateTransitionRequested(void)
{
bool isStateTransitionRequested = FALSE;
getFileRepRoleAndState(&fileRepRole, &segmentState, &dataState, NULL, NULL);
switch (fileRepProcessType)
{
case FileRepProcessTypeMain:
/* Handle Shutdown request */
if (segmentState == SegmentStateImmediateShutdown)
{
isStateTransitionRequested = TRUE;
}
break;
case FileRepProcessTypeNotInitialized:
if (segmentState == SegmentStateShutdownFilerepBackends &&
fileRepShmemArray[0]->state == FileRepStateFault)
{
FileRep_InsertConfigLogEntry("failure is detected in segment mirroring during backend shutdown, abort requested");
}
/* no break */
default:
if (fileRepProcessType != FileRepProcessTypeNotInitialized)
{
FileRepSubProcess_ProcessSignals();
}
if (dataState == DataStateInChangeTracking)
{
isStateTransitionRequested = TRUE;
}
switch (segmentState)
{
case SegmentStateFault:
case SegmentStateImmediateShutdown:
case SegmentStateShutdown:
isStateTransitionRequested = TRUE;
break;
default:
break;
}
break;
}
if (isStateTransitionRequested)
{
FileRep_InsertConfigLogEntry("state transition requested ");
}
return isStateTransitionRequested;
}
/*
* FileRepSubProcess_ProcessSignals()
*
*/
bool
FileRepSubProcess_ProcessSignals()
{
bool processExit = false;
if (reloadConfigFile)
{
reloadConfigFile = false;
ProcessConfigFile(PGC_SIGHUP);
FileRep_SetFileRepRetry();
}
if (shutdownRequested)
{
SegmentState_e segmentState;
getPrimaryMirrorStatusCodes(NULL, &segmentState, NULL, NULL);
shutdownRequested = false;
if (segmentState == SegmentStateShutdownFilerepBackends)
{
processExit = FileRepIsBackendSubProcess(fileRepProcessType);
FileRepSubProcess_SetState(FileRepStateShutdownBackends);
}
else
{
processExit = true;
FileRepSubProcess_SetState(FileRepStateShutdown);
}
}
/*
* Immediate shutdown if postmaster or main filerep process (parent) is
* not alive to avoid manual cleanup.
*/
if (!PostmasterIsAlive(false /* amDirectChild */ ) || !ParentProcIsAlive())
{
quickdie_impl();
}
for (;;)
{
/* check to see if change required */
sig_atomic_t curStateChangeRequestCounter = stateChangeRequestCounter;
if (curStateChangeRequestCounter == lastChangeRequestProcessCounterValue)
break;
lastChangeRequestProcessCounterValue = curStateChangeRequestCounter;
/* do the change in local memory */
getFileRepRoleAndState(&fileRepRole, &segmentState, &dataState, NULL, NULL);
switch (segmentState)
{
case SegmentStateNotInitialized:
FileRepSubProcess_SetState(FileRepStateNotInitialized);
break;
case SegmentStateInitialization:
FileRepSubProcess_SetState(FileRepStateInitialization);
break;
case SegmentStateInResyncTransition:
FileRepSubProcess_SetState(FileRepStateInitialization);
break;
case SegmentStateInChangeTrackingTransition:
case SegmentStateInSyncTransition:
/* fileRepState remains Ready */
break;
case SegmentStateChangeTrackingDisabled:
case SegmentStateReady:
FileRepSubProcess_SetState(FileRepStateReady);
break;
case SegmentStateFault:
FileRepSubProcess_SetState(FileRepStateFault);
break;
case SegmentStateShutdownFilerepBackends:
if (fileRepRole == FileRepPrimaryRole)
{
FileRepSubProcess_SetState(FileRepStateShutdownBackends);
}
else
{
processExit = true;
FileRepSubProcess_SetState(FileRepStateShutdown);
}
break;
case SegmentStateImmediateShutdown:
case SegmentStateShutdown:
processExit = true;
FileRepSubProcess_SetState(FileRepStateShutdown);
break;
default:
Assert(0);
break;
} //switch ()
if (processExit == true)
{
FileRep_IpcSignalAll();
}
}
return (processExit);
}
/*
* SIGUSR2 signal from main file rep process
*/
static void
FileRepSubProcess_ShutdownHandler(SIGNAL_ARGS)
{
bool isInTransition = FALSE;
DataState_e dataStateTransition;
shutdownRequested = true;
/*
* Exit the process if recv() call is hanging or compacting is running.
* Compacting can take many minutes.
*/
if (fileRepProcessType == FileRepProcessTypePrimaryReceiverAck ||
fileRepProcessType == FileRepProcessTypeMirrorReceiver ||
fileRepProcessType == FileRepProcessTypePrimaryRecovery)
{
/* workaround for gcov testing */
if (Debug_filerep_gcov)
{
getFileRepRoleAndState(&fileRepRole, &segmentState, &dataState, &isInTransition, &dataStateTransition);
if (isInTransition == TRUE &&
dataStateTransition == DataStateInChangeTracking)
{
proc_exit(0);
return;
}
}
die(PASS_SIGNAL_ARGS);
}
if (FileRepIsBackendSubProcess(fileRepProcessType))
{
if (FileRepPrimary_IsResyncManagerOrWorker())
{
getFileRepRoleAndState(&fileRepRole, &segmentState, &dataState, &isInTransition, &dataStateTransition);
if (isInTransition == TRUE &&
dataStateTransition == DataStateInChangeTracking)
{
/*
* Resync workers and manager may be waiting on lock that is
* acquired by backend process that is suspended during
* transition to Change Tracking and so FileRep backend
* shutdown may never be completed.
*/
if (fileRepProcessType == FileRepProcessTypeResyncManager)
{
FileRepResync_Cleanup();
}
else
{
LockReleaseAll(DEFAULT_LOCKMETHOD, false);
}
/*
* We remove ourself from LW waiter list (if applicable).
*
* If the current backend is waiting on a LWLock and exits w/o
* any cleanup (remove from waiters list) it can cause a
* breakage in the LWlock's waiters linked list after it dies.
* This can lead to unpleasant issues causing starvation for
* subsequent waiters because the current backend is already
* dead without assigning the LWLock to the next waiter.
*
* XXX Side note - Although implemented here, avoid exiting
* inside an signal handler.
*/
LWLockWaitCancel();
LWLockReleaseAll();
proc_exit(0);
return;
}
}
/*
* call the normal postgres die so that it requests query
* cancel/procdie
*/
die(PASS_SIGNAL_ARGS);
}
}
static int
FaultInjector_NewHashEntry(
FaultInjectorEntry_s *entry)
{
FaultInjectorEntry_s *entryLocal=NULL;
bool exists;
int status = STATUS_OK;
LockAcquire();
if ((faultInjectorShmem->faultInjectorSlots + 1) >= FAULTINJECTOR_MAX_SLOTS) {
LockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection, no slots available"
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, max slots:'%d' reached",
FAULTINJECTOR_MAX_SLOTS);
goto exit;
}
if (entry->faultInjectorType == FaultInjectorTypeSkip)
{
switch (entry->faultInjectorIdentifier)
{
case Checkpoint:
case ChangeTrackingDisable:
case FileRepVerification:
case FinishPreparedTransactionCommitPass1FromCreatePendingToCreated:
case FinishPreparedTransactionCommitPass2FromCreatePendingToCreated:
case FinishPreparedTransactionCommitPass1FromDropInMemoryToDropPending:
case FinishPreparedTransactionCommitPass2FromDropInMemoryToDropPending:
case FinishPreparedTransactionCommitPass1AbortingCreateNeeded:
case FinishPreparedTransactionCommitPass2AbortingCreateNeeded:
case FinishPreparedTransactionAbortPass1FromCreatePendingToAbortingCreate:
case FinishPreparedTransactionAbortPass2FromCreatePendingToAbortingCreate:
case FinishPreparedTransactionAbortPass1AbortingCreateNeeded:
case FinishPreparedTransactionAbortPass2AbortingCreateNeeded:
case SyncPersistentTable:
case XLOGInsert:
break;
default:
LockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection, fault type not supported"
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, fault type not supported");
goto exit;
}
}
/* check role */
getFileRepRoleAndState(&fileRepRole, &segmentState, &dataState, NULL, NULL);
switch (entry->faultInjectorIdentifier)
{
case ChangeTrackingDisable:
case FileRepConsumerVerification:
case FileRepResync:
case FileRepResyncInProgress:
case FileRepResyncWorker:
case FileRepResyncWorkerRead:
case FileRepTransitionToInResyncMirrorReCreate:
case FileRepTransitionToInResyncMarkReCreated:
case FileRepTransitionToInResyncMarkCompleted:
case FileRepTransitionToInSyncBegin:
case FileRepTransitionToInSync:
case FileRepTransitionToInSyncMarkCompleted:
case FileRepTransitionToInSyncBeforeCheckpoint:
case FileRepTransitionToChangeTracking:
case FileRepConsumer:
case FileRepSender:
case FileRepReceiver:
case FileRepFlush:
/* Ashwin */
case FileRepChangeTrackingCompacting:
case FinishPreparedTransactionCommitPass1FromCreatePendingToCreated:
case FinishPreparedTransactionCommitPass2FromCreatePendingToCreated:
case FinishPreparedTransactionCommitPass1FromDropInMemoryToDropPending:
case FinishPreparedTransactionCommitPass2FromDropInMemoryToDropPending:
case FinishPreparedTransactionCommitPass1AbortingCreateNeeded:
case FinishPreparedTransactionCommitPass2AbortingCreateNeeded:
case FinishPreparedTransactionAbortPass1FromCreatePendingToAbortingCreate:
// case FinishPreparedTransactionAbortPass2FromCreatePendingToAbortingCreate:
case FinishPreparedTransactionAbortPass1AbortingCreateNeeded:
case FinishPreparedTransactionAbortPass2AbortingCreateNeeded:
case TwoPhaseTransactionCommitPrepared:
case TwoPhaseTransactionAbortPrepared:
/* This kind of fault injection has not been supported yet. */
LockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("This kind of fault injection has not been supported yet. "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"This kind of fault injection has not been supported yet. "
"Please check faultname");
goto exit;
// case SubtransactionFlushToFile:
// case SubtransactionReadFromFile:
// case SubtransactionRelease:
// case SubtransactionRollback:
case StartPrepareTx:
case TransactionCommitPass1FromCreatePendingToCreated:
case TransactionCommitPass1FromDropInMemoryToDropPending:
case TransactionCommitPass1FromAbortingCreateNeededToAbortingCreate:
case TransactionAbortPass1FromCreatePendingToAbortingCreate:
case TransactionAbortPass1FromAbortingCreateNeededToAbortingCreate:
case TransactionCommitPass2FromDropInMemoryToDropPending:
case TransactionCommitPass2FromAbortingCreateNeededToAbortingCreate:
case TransactionAbortPass2FromCreatePendingToAbortingCreate:
case TransactionAbortPass2FromAbortingCreateNeededToAbortingCreate:
case FaultBeforePendingDeleteRelationEntry:
case FaultBeforePendingDeleteDatabaseEntry:
case FaultBeforePendingDeleteTablespaceEntry:
case FaultBeforePendingDeleteFilespaceEntry:
case TransactionAbortAfterDistributedPrepared:
case DtmBroadcastPrepare:
case DtmBroadcastCommitPrepared:
case DtmBroadcastAbortPrepared:
case DtmXLogDistributedCommit:
case AnalyzeSubxactError:
case OptTaskAllocateStringBuffer:
case ConnectionFailAfterGangCreation:
case CreateCdbDispathResultObject:
case WorkerManagerSubmitJob:
case FaillQeAfterConnection:
/* These faults are designed for master. */
if(!AmIMaster())
{
LockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection entry into table, "
"This kind of fault injection should be sent to master. "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection. "
"This kind of fault injection should be sent to master. "
"Please use \"-r master\"");
goto exit;
}
break;
case SegmentTransitionRequest:
case SegmentProbeResponse:
/* We do not use vmem on master. Therefore, we only attempt large palloc on segments. */
case MultiExecHashLargeVmem:
case FailQeWhenDoQuery:
case FailQeWhenBeginParquetScan:
/* SEGMENT */
if(!AmISegment())
{
LockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection entry into table, "
"This kind of fault injection should be sent to segment. "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection. "
"This kind of fault injection should be sent to segment. "
"Please use \"-r primary\"");
goto exit;
}
break;
case LocalTmRecordTransactionCommit:
case Checkpoint:
case AbortTransactionFail:
case UpdateCommittedEofInPersistentTable:
case FaultDuringExecDynamicTableScan:
case ExecSortBeforeSorting:
case FaultExecHashJoinNewBatch:
case WorkfileCleanupSet:
case RunawayCleanup:
/* MASTER OR SEGMENT */
if(AmIStandby())
{
LockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection entry into table, "
"segment not in primary or master role, "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, segment not in master or segment role");
goto exit;
}
break;
default:
break;
}
entryLocal = FaultInjector_InsertHashEntry(entry->faultInjectorIdentifier, &exists);
if (entryLocal == NULL) {
LockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection entry into table, no memory, "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, no memory");
goto exit;
}
if (exists) {
LockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection entry into table, "
"entry already exists, "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, entry already exists");
goto exit;
}
entryLocal->faultInjectorType = entry->faultInjectorType;
entryLocal->sleepTime = entry->sleepTime;
entryLocal->ddlStatement = entry->ddlStatement;
if (entry->occurrence != 0)
{
entryLocal->occurrence = entry->occurrence;
}
else
{
entryLocal->occurrence = FILEREP_UNDEFINED;
}
strcpy(entryLocal->databaseName, entry->databaseName);
strcpy(entryLocal->tableName, entry->tableName);
entryLocal->faultInjectorState = FaultInjectorStateWaiting;
faultInjectorShmem->faultInjectorSlots++;
LockRelease();
elog(DEBUG1, "FaultInjector_NewHashEntry() identifier:'%s'",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier]);
exit:
return status;
}
FaultInjectorType_e
FaultInjector_InjectFaultIfSet(
FaultInjectorIdentifier_e identifier,
DDLStatement_e ddlStatement,
char* databaseName,
char* tableName)
{
FaultInjectorEntry_s *entryLocal;
char databaseNameLocal[NAMEDATALEN];
char tableNameLocal[NAMEDATALEN];
int ii = 0;
/*
* Return immediately if no fault has been injected ever. It is
* important to not touch the spinlock, especially if this is the
* postmaster process. If one of the backend processes dies while
* holding the spin lock, and postmaster comes here before resetting
* the shared memory, it waits without holder process and eventually
* goes into PANIC. Also this saves a few cycles to acquire the spin
* lock and look into the shared hash table.
*
* Although this is a race condition without lock, a false negative is
* ok given this framework is purely for dev/testing.
*/
if (faultInjectorShmem->faultInjectorSlots == 0)
return FALSE;
getFileRepRoleAndState(&fileRepRole, &segmentState, &dataState, NULL, NULL);
LockAcquire();
entryLocal = FaultInjector_LookupHashEntry(identifier);
LockRelease();
/* Verify if fault injection is set */
if (entryLocal == NULL)
/* fault injection is not set */
return FALSE;
if (entryLocal->ddlStatement != ddlStatement)
/* fault injection is not set for the specified DDL */
return FALSE;
snprintf(databaseNameLocal, sizeof(databaseNameLocal), "%s", databaseName);
if (strcmp(entryLocal->databaseName, databaseNameLocal) != 0)
/* fault injection is not set for the specified database name */
return FALSE;
snprintf(tableNameLocal, sizeof(tableNameLocal), "%s", tableName);
if (strcmp(entryLocal->tableName, tableNameLocal) != 0)
/* fault injection is not set for the specified table name */
return FALSE;
if (entryLocal->faultInjectorState == FaultInjectorStateTriggered ||
entryLocal->faultInjectorState == FaultInjectorStateCompleted ||
entryLocal->faultInjectorState == FaultInjectorStateFailed) {
/* fault injection was already executed */
return FALSE;
}
/* Update the injection fault entry in hash table */
if (entryLocal->occurrence != FILEREP_UNDEFINED)
{
if (entryLocal->occurrence > 1)
{
entryLocal->occurrence--;
return FALSE;
}
else
entryLocal->faultInjectorState = FaultInjectorStateTriggered;
}
FaultInjector_UpdateHashEntry(entryLocal);
/* Inject fault */
switch (entryLocal->faultInjectorType) {
case FaultInjectorTypeNotSpecified:
break;
case FaultInjectorTypeSleep:
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
pg_usleep(entryLocal->sleepTime * 1000000L);
break;
case FaultInjectorTypeFault:
switch (entryLocal->faultInjectorIdentifier)
{
case FileRepConsumer:
case FileRepConsumerVerification:
case FileRepSender:
case FileRepReceiver:
case FileRepResync:
case FileRepResyncInProgress:
case FileRepResyncWorker:
case FileRepResyncWorkerRead:
case FileRepTransitionToInResyncMirrorReCreate:
case FileRepTransitionToInResyncMarkReCreated:
case FileRepTransitionToInResyncMarkCompleted:
case FileRepTransitionToInSyncBegin:
case FileRepTransitionToInSync:
case FileRepTransitionToInSyncMarkCompleted:
case FileRepTransitionToInSyncBeforeCheckpoint:
/*
* Since we have removed all the file replication related
* functions, so the following cases should be avoided.
*/
/* FileRep_SetSegmentState(SegmentStateFault, FaultTypeMirror); */
break;
case FileRepTransitionToChangeTracking:
/* FileRep_SetPostmasterReset(); */
break;
default:
/* FileRep_SetSegmentState(SegmentStateFault, FaultTypeIO); */
break;
}
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypeFatal:
if (entryLocal->occurrence != FILEREP_UNDEFINED)
{
entryLocal->faultInjectorState = FaultInjectorStateCompleted;
}
FaultInjector_UpdateHashEntry(entryLocal);
ereport(FATAL,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypePanic:
if (entryLocal->occurrence != FILEREP_UNDEFINED)
{
entryLocal->faultInjectorState = FaultInjectorStateCompleted;
}
FaultInjector_UpdateHashEntry(entryLocal);
ereport(PANIC,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypeError:
if (entryLocal->occurrence != FILEREP_UNDEFINED)
{
entryLocal->faultInjectorState = FaultInjectorStateCompleted;
}
FaultInjector_UpdateHashEntry(entryLocal);
ereport(ERROR,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypeInfiniteLoop:
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
for (ii=0; ii < 3600; ii++)
{
pg_usleep(1000000L); // sleep for 1 sec (1 sec * 3600 = 1 hour)
getFileRepRoleAndState(NULL, &segmentState, NULL, NULL, NULL);
if (segmentState == SegmentStateShutdownFilerepBackends ||
segmentState == SegmentStateImmediateShutdown ||
segmentState == SegmentStateShutdown)
{
break;
}
}
break;
case FaultInjectorTypeDataCorruption:
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypeSuspend:
{
FaultInjectorEntry_s *entry;
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
while ((entry = FaultInjector_LookupHashEntry(entryLocal->faultInjectorIdentifier)) != NULL &&
entry->faultInjectorType != FaultInjectorTypeResume)
{
pg_usleep(1000000L); // 1 sec
}
if (entry != NULL)
{
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
}
else
{
ereport(LOG,
(errmsg("fault 'NULL', fault name:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier])));
}
break;
}
case FaultInjectorTypeSkip:
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypeMemoryFull:
{
char *buffer = NULL;
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
buffer = (char*) palloc(BLCKSZ);
while (buffer != NULL)
{
buffer = (char*) palloc(BLCKSZ);
}
break;
}
case FaultInjectorTypeReset:
case FaultInjectorTypeStatus:
ereport(LOG,
(errmsg("unexpected error, fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
Assert(0);
break;
case FaultInjectorTypeResume:
break;
case FaultInjectorTypePanicSuppress:
{
DECLARE_SAVE_SUPPRESS_PANIC();
entryLocal->faultInjectorState = FaultInjectorStateCompleted;
FaultInjector_UpdateHashEntry(entryLocal);
SUPPRESS_PANIC();
ereport(FATAL,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
}
case FaultInjectorTypeSegv:
{
*(int *) 0 = 1234;
break;
}
case FaultInjectorTypeCreateThreadFail:
case FaultInjectorTypeConnectionNull:
case FaultInjectorTypeConnectionNullInRestoreMode:
case FaultInjectorTypeUserCancel:
case FaultInjectorTypeProcDie:
case FaultInjectorTypeInterrupt:
{
/*
* The place where this type of fault is injected must have
* has HOLD_INTERRUPTS() .. RESUME_INTERRUPTS() around it, otherwise
* the interrupt could be handled inside the fault injector itself
*/
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
InterruptPending = true;
QueryCancelPending = true;
break;
}
case FaultInjectorTypeTimeOut:
case FaultInjectorTypeDispatchError:
{
break;
}
default:
ereport(LOG,
(errmsg("unexpected error, fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
Assert(0);
break;
}
if (entryLocal->occurrence != FILEREP_UNDEFINED)
{
entryLocal->faultInjectorState = FaultInjectorStateCompleted;
}
FaultInjector_UpdateHashEntry(entryLocal);
return (entryLocal->faultInjectorType);
}
/*
*
* FileRepPrimary_StartRecoveryInChangeTracking()
*
*/
static void
FileRepPrimary_StartRecoveryInChangeTracking(void)
{
FileRep_InsertConfigLogEntry("run recovery");
while (1) {
while (FileRepSubProcess_GetState() == FileRepStateFault) {
FileRepSubProcess_ProcessSignals();
pg_usleep(50000L); /* 50 ms */
}
if (FileRepSubProcess_GetState() == FileRepStateShutdown ||
FileRepSubProcess_GetState() == FileRepStateShutdownBackends) {
break;
}
Insist(fileRepRole == FileRepPrimaryRole);
Insist(dataState == DataStateInChangeTracking);
Insist(FileRepSubProcess_GetState() != FileRepStateReady);
if (ChangeTracking_RetrieveIsTransitionToInsync())
{
ChangeTracking_DropAll();
}
else
{
if (ChangeTracking_RetrieveIsTransitionToResync() == FALSE &&
isFullResync())
{
ChangeTracking_MarkFullResync();
/* segmentState == SegmentStateChangeTrackingDisabled */
getFileRepRoleAndState(&fileRepRole, &segmentState, &dataState, NULL, NULL);
Assert(segmentState == SegmentStateChangeTrackingDisabled);
/* database is resumed */
primaryMirrorSetIOSuspended(FALSE);
FileRep_InsertConfigLogEntry("change tracking recovery completed");
break;
}
else
{
ChangeTracking_MarkIncrResync();
}
}
XLogInChangeTrackingTransition();
/* NOTE: Any error during change tracking will result in disabling Change Tracking */
FileRepSubProcess_SetState(FileRepStateReady);
/* database is resumed */
primaryMirrorSetIOSuspended(FALSE);
FileRep_InsertConfigLogEntry("change tracking recovery completed");
break;
} // while(1)
}