/*
* update hash entry with state
*/
static int
FaultInjector_UpdateHashEntry(
FaultInjectorEntry_s *entry)
{
FaultInjectorEntry_s *entryLocal;
bool exists;
int status = STATUS_OK;
FiLockAcquire();
entryLocal = FaultInjector_InsertHashEntry(entry->faultInjectorIdentifier, &exists);
/* entry should be found since fault has not been injected yet */
Assert(entryLocal != NULL);
if (!exists) {
FiLockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not update fault injection hash entry with fault injection status, "
"no entry found, "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
goto exit;
}
if (entry->faultInjectorType == FaultInjectorTypeResume)
{
entryLocal->faultInjectorType = FaultInjectorTypeResume;
}
else
{
entryLocal->faultInjectorState = entry->faultInjectorState;
entryLocal->occurrence = entry->occurrence;
}
FiLockRelease();
ereport(DEBUG1,
(errmsg("LOG(fault injector): update fault injection hash entry "
"identifier:'%s' state:'%s' occurrence:'%d' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorStateEnumToString[entryLocal->faultInjectorState],
entry->occurrence)));
exit:
return status;
}
/*
* update hash entry with state
*/
static int
FaultInjector_UpdateHashEntry(
FaultInjectorEntry_s *entry)
{
FaultInjectorEntry_s *entryLocal;
int status = STATUS_OK;
FiLockAcquire();
entryLocal = FaultInjector_LookupHashEntry(entry->faultName);
if (entryLocal == NULL)
{
FiLockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not update fault injection hash entry with fault injection status, "
"no entry found, "
"fault name:'%s' fault type:'%s' ",
entry->faultName,
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
goto exit;
}
if (entry->faultInjectorType == FaultInjectorTypeResume)
{
entryLocal->faultInjectorType = FaultInjectorTypeResume;
}
else
{
entryLocal->faultInjectorState = entry->faultInjectorState;
entryLocal->occurrence = entry->occurrence;
}
FiLockRelease();
ereport(DEBUG1,
(errmsg("LOG(fault injector): update fault injection hash entry "
"identifier:'%s' state:'%s' occurrence:'%d' ",
entry->faultName,
FaultInjectorStateEnumToString[entryLocal->faultInjectorState],
entry->occurrence)));
exit:
return status;
}
bool
FaultInjector_IsFaultInjected(
FaultInjectorIdentifier_e identifier)
{
FaultInjectorEntry_s *entry = NULL;
bool isCompleted = FALSE;
bool retval = FALSE;
bool isRemoved;
FiLockAcquire();
entry = FaultInjector_LookupHashEntry(identifier);
if (entry == NULL) {
retval = TRUE;
isCompleted = TRUE;
goto exit;
}
switch (entry->faultInjectorState) {
case FaultInjectorStateWaiting:
/* No operation */
break;
case FaultInjectorStateTriggered:
/* No operation */
break;
case FaultInjectorStateCompleted:
retval = TRUE;
/* NO break */
case FaultInjectorStateFailed:
isCompleted = TRUE;
isRemoved = FaultInjector_RemoveHashEntry(identifier);
if (isRemoved == FALSE) {
ereport(DEBUG1,
(errmsg("LOG(fault injector): could not remove fault injection from hash"
"identifier:'%s' ",
FaultInjectorIdentifierEnumToString[identifier])));
} else {
faultInjectorShmem->faultInjectorSlots--;
}
break;
default:
Assert(0);
}
exit:
FiLockRelease();
if ((isCompleted == TRUE) && (retval == FALSE)) {
ereport(WARNING,
(errmsg("could not complete fault injection, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[identifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
}
return isCompleted;
}
static int
FaultInjector_NewHashEntry(
FaultInjectorEntry_s *entry)
{
FaultInjectorEntry_s *entryLocal=NULL;
bool exists;
int status = STATUS_OK;
FiLockAcquire();
if ((faultInjectorShmem->faultInjectorSlots + 1) >= FAULTINJECTOR_MAX_SLOTS) {
FiLockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection, no slots available"
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, max slots:'%d' reached",
FAULTINJECTOR_MAX_SLOTS);
goto exit;
}
if (entry->faultInjectorType == FaultInjectorTypeSkip)
{
switch (entry->faultInjectorIdentifier)
{
case Checkpoint:
case ChangeTrackingDisable:
case FileRepVerification:
case FinishPreparedTransactionCommitPass1FromCreatePendingToCreated:
case FinishPreparedTransactionCommitPass2FromCreatePendingToCreated:
case FinishPreparedTransactionCommitPass1FromDropInMemoryToDropPending:
case FinishPreparedTransactionCommitPass2FromDropInMemoryToDropPending:
case FinishPreparedTransactionCommitPass1AbortingCreateNeeded:
case FinishPreparedTransactionCommitPass2AbortingCreateNeeded:
case FinishPreparedTransactionAbortPass1FromCreatePendingToAbortingCreate:
case FinishPreparedTransactionAbortPass2FromCreatePendingToAbortingCreate:
case FinishPreparedTransactionAbortPass1AbortingCreateNeeded:
case FinishPreparedTransactionAbortPass2AbortingCreateNeeded:
break;
default:
FiLockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection, fault type not supported"
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, fault type not supported");
goto exit;
}
}
/* check role */
getFileRepRoleAndState(&fileRepRole, &segmentState, &dataState, NULL, NULL);
switch (entry->faultInjectorIdentifier)
{
case ChangeTrackingDisable:
case FileRepConsumerVerification:
case StartPrepareTx:
case FileRepResync:
case FileRepResyncInProgress:
case FileRepResyncWorker:
case FileRepResyncWorkerRead:
case FileRepTransitionToInResyncMirrorReCreate:
case FileRepTransitionToInResyncMarkReCreated:
case FileRepTransitionToInResyncMarkCompleted:
case FileRepTransitionToInSyncBegin:
case FileRepTransitionToInSync:
case FileRepTransitionToInSyncMarkCompleted:
case FileRepTransitionToInSyncBeforeCheckpoint:
case FileRepTransitionToChangeTracking:
case FileRepIsOperationCompleted:
case FileRepImmediateShutdownRequested:
case TransactionCommitPass1FromCreatePendingToCreated:
case TransactionCommitPass1FromDropInMemoryToDropPending:
case TransactionCommitPass1FromAbortingCreateNeededToAbortingCreate:
case TransactionAbortPass1FromCreatePendingToAbortingCreate:
case TransactionAbortPass1FromAbortingCreateNeededToAbortingCreate:
case TransactionCommitPass2FromDropInMemoryToDropPending:
case TransactionCommitPass2FromAbortingCreateNeededToAbortingCreate:
case TransactionAbortPass2FromCreatePendingToAbortingCreate:
case TransactionAbortPass2FromAbortingCreateNeededToAbortingCreate:
case FinishPreparedTransactionCommitPass1FromCreatePendingToCreated:
case FinishPreparedTransactionCommitPass2FromCreatePendingToCreated:
case FinishPreparedTransactionCommitPass1FromDropInMemoryToDropPending:
case FinishPreparedTransactionCommitPass2FromDropInMemoryToDropPending:
case FinishPreparedTransactionCommitPass1AbortingCreateNeeded:
case FinishPreparedTransactionCommitPass2AbortingCreateNeeded:
case FinishPreparedTransactionAbortPass1FromCreatePendingToAbortingCreate:
// case FinishPreparedTransactionAbortPass2FromCreatePendingToAbortingCreate:
case FinishPreparedTransactionAbortPass1AbortingCreateNeeded:
case FinishPreparedTransactionAbortPass2AbortingCreateNeeded:
case TwoPhaseTransactionCommitPrepared:
case TwoPhaseTransactionAbortPrepared:
case ExecSortMKSortMergeRuns:
// case SubtransactionFlushToFile:
// case SubtransactionReadFromFile:
// case SubtransactionRelease:
// case SubtransactionRollback:
/* Ashwin */
case FileRepChangeTrackingCompacting:
/* We do not use vmem on master. Therefore, we only attempt large palloc on segments. */
case MultiExecHashLargeVmem:
case FaultInBackgroundWriterMain:
if (fileRepRole != FileRepPrimaryRole)
{
FiLockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection entry into table, segment not in primary role"
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, segment not in primary role");
goto exit;
}
break;
case FaultBeforePendingDeleteRelationEntry:
case FaultBeforePendingDeleteDatabaseEntry:
case FaultBeforePendingDeleteTablespaceEntry:
case FaultBeforePendingDeleteFilespaceEntry:
case FileRepConsumer:
case FileRepSender:
case FileRepReceiver:
case FileRepFlush:
case Postmaster:
case PgControl:
case PgXlog:
case SegmentTransitionRequest:
case SegmentProbeResponse:
if (fileRepRole != FileRepPrimaryRole && fileRepRole != FileRepMirrorRole)
{
FiLockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection entry into table, "
"segment not in primary or mirror role, "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, segment not in primary or mirror role");
goto exit;
}
break;
case TransactionAbortAfterDistributedPrepared:
case DtmBroadcastPrepare:
case DtmBroadcastCommitPrepared:
case DtmBroadcastAbortPrepared:
case DtmXLogDistributedCommit:
case OptTaskAllocateStringBuffer:
case OptRelcacheTranslatorCatalogAccess:
if (fileRepRole != FileRepNoRoleConfigured)
{
FiLockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection entry into table, "
"segment not in master role, "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, segment not in master role");
goto exit;
}
break;
case LocalTmRecordTransactionCommit:
case Checkpoint:
case AbortTransactionFail:
case UpdateCommittedEofInPersistentTable:
case ExecSortBeforeSorting:
case FaultDuringExecDynamicTableScan:
case FaultExecHashJoinNewBatch:
case RunawayCleanup:
if (fileRepRole != FileRepNoRoleConfigured && fileRepRole != FileRepPrimaryRole)
{
FiLockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection entry into table, "
"segment not in primary or master role, "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, segment not in primary or master role");
goto exit;
}
break;
default:
break;
}
entryLocal = FaultInjector_InsertHashEntry(entry->faultInjectorIdentifier, &exists);
if (entryLocal == NULL) {
FiLockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection entry into table, no memory, "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, no memory");
goto exit;
}
if (exists) {
FiLockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection entry into table, "
"entry already exists, "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, entry already exists");
goto exit;
}
entryLocal->faultInjectorType = entry->faultInjectorType;
entryLocal->sleepTime = entry->sleepTime;
entryLocal->ddlStatement = entry->ddlStatement;
if (entry->occurrence != 0)
{
entryLocal->occurrence = entry->occurrence;
}
else
{
entryLocal->occurrence = FILEREP_UNDEFINED;
}
strcpy(entryLocal->databaseName, entry->databaseName);
strcpy(entryLocal->tableName, entry->tableName);
entryLocal->faultInjectorState = FaultInjectorStateWaiting;
faultInjectorShmem->faultInjectorSlots++;
FiLockRelease();
elog(DEBUG1, "FaultInjector_NewHashEntry() identifier:'%s'",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier]);
exit:
return status;
}
FaultInjectorType_e
FaultInjector_InjectFaultIfSet(
FaultInjectorIdentifier_e identifier,
DDLStatement_e ddlStatement,
char* databaseName,
char* tableName)
{
FaultInjectorEntry_s *entryShared, localEntry,
*entryLocal = &localEntry;
char databaseNameLocal[NAMEDATALEN];
char tableNameLocal[NAMEDATALEN];
int ii = 0;
int cnt = 3600;
/*
* Return immediately if no fault has been injected ever. It is
* important to not touch the spinlock, especially if this is the
* postmaster process. If one of the backend processes dies while
* holding the spin lock, and postmaster comes here before resetting
* the shared memory, it waits without holder process and eventually
* goes into PANIC. Also this saves a few cycles to acquire the spin
* lock and look into the shared hash table.
*
* Although this is a race condition without lock, a false negative is
* ok given this framework is purely for dev/testing.
*/
if (faultInjectorShmem->faultInjectorSlots == 0)
return FALSE;
getFileRepRoleAndState(&fileRepRole, &segmentState, &dataState, NULL, NULL);
FiLockAcquire();
entryShared = FaultInjector_LookupHashEntry(identifier);
if (entryShared != NULL)
memcpy(entryLocal, entryShared, sizeof(FaultInjectorEntry_s));
FiLockRelease();
/* Verify if fault injection is set */
if (entryShared == NULL)
/* fault injection is not set */
return FALSE;
if (entryLocal->ddlStatement != ddlStatement)
/* fault injection is not set for the specified DDL */
return FALSE;
snprintf(databaseNameLocal, sizeof(databaseNameLocal), "%s", databaseName);
if (strcmp(entryLocal->databaseName, databaseNameLocal) != 0)
/* fault injection is not set for the specified database name */
return FALSE;
snprintf(tableNameLocal, sizeof(tableNameLocal), "%s", tableName);
if (strcmp(entryLocal->tableName, tableNameLocal) != 0)
/* fault injection is not set for the specified table name */
return FALSE;
if (entryLocal->faultInjectorState == FaultInjectorStateTriggered ||
entryLocal->faultInjectorState == FaultInjectorStateCompleted ||
entryLocal->faultInjectorState == FaultInjectorStateFailed) {
/* fault injection was already executed */
return FALSE;
}
/* Update the injection fault entry in hash table */
if (entryLocal->occurrence != FILEREP_UNDEFINED)
{
if (entryLocal->occurrence > 1)
{
entryLocal->occurrence--;
FaultInjector_UpdateHashEntry(entryLocal);
return FALSE;
}
entryLocal->faultInjectorState = FaultInjectorStateTriggered;
FaultInjector_UpdateHashEntry(entryLocal);
}
/* Inject fault */
switch (entryLocal->faultInjectorType) {
case FaultInjectorTypeNotSpecified:
break;
case FaultInjectorTypeSleep:
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
pg_usleep(entryLocal->sleepTime * 1000000L);
break;
case FaultInjectorTypeFault:
switch (entryLocal->faultInjectorIdentifier)
{
case FileRepConsumer:
case FileRepConsumerVerification:
case FileRepSender:
case FileRepReceiver:
case FileRepResync:
case FileRepResyncInProgress:
case FileRepResyncWorker:
case FileRepResyncWorkerRead:
case FileRepTransitionToInResyncMirrorReCreate:
case FileRepTransitionToInResyncMarkReCreated:
case FileRepTransitionToInResyncMarkCompleted:
case FileRepTransitionToInSyncBegin:
case FileRepTransitionToInSync:
case FileRepTransitionToInSyncMarkCompleted:
case FileRepTransitionToInSyncBeforeCheckpoint:
case FileRepIsOperationCompleted:
FileRep_SetSegmentState(SegmentStateFault, FaultTypeMirror);
break;
case FileRepTransitionToChangeTracking:
FileRep_SetPostmasterReset();
break;
default:
FileRep_SetSegmentState(SegmentStateFault, FaultTypeIO);
break;
}
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypeFatal:
/*
* If it's one time occurrence then disable the fault before it's
* actually triggered because this fault errors out the transaction
* and hence we wont get a chance to disable it or put it in completed
* state.
*/
if (entryLocal->occurrence != FILEREP_UNDEFINED)
{
entryLocal->faultInjectorState = FaultInjectorStateCompleted;
FaultInjector_UpdateHashEntry(entryLocal);
}
ereport(FATAL,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypePanic:
/*
* If it's one time occurrence then disable the fault before it's
* actually triggered because this fault errors out the transaction
* and hence we wont get a chance to disable it or put it in completed
* state. For PANIC it may be unnecessary though.
*/
if (entryLocal->occurrence != FILEREP_UNDEFINED)
{
entryLocal->faultInjectorState = FaultInjectorStateCompleted;
FaultInjector_UpdateHashEntry(entryLocal);
}
ereport(PANIC,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypeError:
/*
* If it's one time occurrence then disable the fault before it's
* actually triggered because this fault errors out the transaction
* and hence we wont get a chance to disable it or put it in completed
* state.
*/
if (entryLocal->occurrence != FILEREP_UNDEFINED)
{
entryLocal->faultInjectorState = FaultInjectorStateCompleted;
FaultInjector_UpdateHashEntry(entryLocal);
}
ereport(ERROR,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypeInfiniteLoop:
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
if (entryLocal->faultInjectorIdentifier == FileRepImmediateShutdownRequested)
cnt = entryLocal->sleepTime;
for (ii=0; ii < cnt; ii++)
{
pg_usleep(1000000L); // sleep for 1 sec (1 sec * 3600 = 1 hour)
getFileRepRoleAndState(NULL, &segmentState, NULL, NULL, NULL);
if ((entryLocal->faultInjectorIdentifier != FileRepImmediateShutdownRequested) &&
(segmentState == SegmentStateShutdownFilerepBackends ||
segmentState == SegmentStateImmediateShutdown ||
segmentState == SegmentStateShutdown ||
IsFtsShudownRequested()))
{
break;
}
}
break;
case FaultInjectorTypeDataCorruption:
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypeSuspend:
{
FaultInjectorEntry_s *entry;
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
while ((entry = FaultInjector_LookupHashEntry(entryLocal->faultInjectorIdentifier)) != NULL &&
entry->faultInjectorType != FaultInjectorTypeResume)
{
pg_usleep(1000000L); // 1 sec
}
if (entry != NULL)
{
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
}
else
{
ereport(LOG,
(errmsg("fault 'NULL', fault name:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier])));
/*
* Since the entry is gone already, we should NOT update
* the entry below. (There could be other places in this
* function that are under the same situation, but I'm too
* tired to look for them...)
*/
return entryLocal->faultInjectorType;
}
break;
}
case FaultInjectorTypeSkip:
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypeMemoryFull:
{
char *buffer = NULL;
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
buffer = (char*) palloc(BLCKSZ);
while (buffer != NULL)
{
buffer = (char*) palloc(BLCKSZ);
}
break;
}
case FaultInjectorTypeReset:
case FaultInjectorTypeStatus:
ereport(LOG,
(errmsg("unexpected error, fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
Assert(0);
break;
case FaultInjectorTypeResume:
break;
case FaultInjectorTypeSegv:
{
*(int *) 0 = 1234;
break;
}
case FaultInjectorTypeInterrupt:
{
/*
* The place where this type of fault is injected must have
* has HOLD_INTERRUPTS() .. RESUME_INTERRUPTS() around it, otherwise
* the interrupt could be handled inside the fault injector itself
*/
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
InterruptPending = true;
QueryCancelPending = true;
break;
}
case FaultInjectorTypeCheckpointAndPanic:
{
if (entryLocal->occurrence != FILEREP_UNDEFINED)
{
entryLocal->faultInjectorState = FaultInjectorStateCompleted;
FaultInjector_UpdateHashEntry(entryLocal);
}
RequestCheckpoint(true, false);
ereport(PANIC,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
}
default:
ereport(LOG,
(errmsg("unexpected error, fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
Assert(0);
break;
}
if (entryLocal->occurrence != FILEREP_UNDEFINED)
{
entryLocal->faultInjectorState = FaultInjectorStateCompleted;
}
FaultInjector_UpdateHashEntry(entryLocal);
return (entryLocal->faultInjectorType);
}
int
FaultInjector_SetFaultInjection(
FaultInjectorEntry_s *entry)
{
int status = STATUS_OK;
bool isRemoved = FALSE;
getFileRepRoleAndState(&fileRepRole, &segmentState, &dataState, NULL, NULL);
switch (entry->faultInjectorType) {
case FaultInjectorTypeReset:
{
HASH_SEQ_STATUS hash_status;
FaultInjectorEntry_s *entryLocal;
if (entry->faultInjectorIdentifier == FaultInjectorIdAll)
{
hash_seq_init(&hash_status, faultInjectorShmem->hash);
FiLockAcquire();
while ((entryLocal = (FaultInjectorEntry_s *) hash_seq_search(&hash_status)) != NULL) {
isRemoved = FaultInjector_RemoveHashEntry(entryLocal->faultInjectorIdentifier);
if (isRemoved == TRUE) {
faultInjectorShmem->faultInjectorSlots--;
}
}
FiLockRelease();
Assert(faultInjectorShmem->faultInjectorSlots == 0);
}
else
{
FiLockAcquire();
isRemoved = FaultInjector_RemoveHashEntry(entry->faultInjectorIdentifier);
if (isRemoved == TRUE) {
faultInjectorShmem->faultInjectorSlots--;
}
FiLockRelease();
}
if (isRemoved == FALSE) {
ereport(DEBUG1,
(errmsg("LOG(fault injector): could not remove fault injection from hash"
"identifier:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier])));
}
break;
}
case FaultInjectorTypeStatus:
{
HASH_SEQ_STATUS hash_status;
FaultInjectorEntry_s *entryLocal;
bool found = FALSE;
if (faultInjectorShmem->hash == NULL) {
status = STATUS_ERROR;
break;
}
snprintf(entry->bufOutput, sizeof(entry->bufOutput), "Success: ");
if (entry->faultInjectorIdentifier == ChangeTrackingCompactingReport)
{
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"Success: compacting in progress %s",
ChangeTrackingIsCompactingInProgress() ? "true" : "false");
break;
}
hash_seq_init(&hash_status, faultInjectorShmem->hash);
while ((entryLocal = (FaultInjectorEntry_s *) hash_seq_search(&hash_status)) != NULL) {
ereport(LOG,
(errmsg("fault injector status: "
"fault name:'%s' "
"fault type:'%s' "
"ddl statement:'%s' "
"database name:'%s' "
"table name:'%s' "
"occurrence:'%d' "
"sleep time:'%d' "
"fault injection state:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType],
FaultInjectorDDLEnumToString[entryLocal->ddlStatement],
entryLocal->databaseName,
entryLocal->tableName,
entryLocal->occurrence,
entryLocal->sleepTime,
FaultInjectorStateEnumToString[entryLocal->faultInjectorState])));
if (entry->faultInjectorIdentifier == entryLocal->faultInjectorIdentifier ||
entry->faultInjectorIdentifier == FaultInjectorIdAll) {
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"%s \n"
"fault name:'%s' "
"fault type:'%s' "
"ddl statement:'%s' "
"database name:'%s' "
"table name:'%s' "
"occurrence:'%d' "
"sleep time:'%d' "
"fault injection state:'%s' ",
entry->bufOutput,
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType],
FaultInjectorDDLEnumToString[entryLocal->ddlStatement],
entryLocal->databaseName,
entryLocal->tableName,
entryLocal->occurrence,
entryLocal->sleepTime,
FaultInjectorStateEnumToString[entryLocal->faultInjectorState]);
found = TRUE;
}
}
if (found == FALSE) {
snprintf(entry->bufOutput, sizeof(entry->bufOutput), "Failure: "
"fault name:'%s' not set",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier]);
}
break;
}
case FaultInjectorTypeResume:
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
FaultInjector_UpdateHashEntry(entry);
break;
default:
status = FaultInjector_NewHashEntry(entry);
break;
}
return status;
}
