bool
FaultInjector_IsFaultInjected(
FaultInjectorIdentifier_e identifier)
{
FaultInjectorEntry_s *entry = NULL;
bool isCompleted = FALSE;
bool retval = FALSE;
bool isRemoved;
FiLockAcquire();
entry = FaultInjector_LookupHashEntry(identifier);
if (entry == NULL) {
retval = TRUE;
isCompleted = TRUE;
goto exit;
}
switch (entry->faultInjectorState) {
case FaultInjectorStateWaiting:
/* No operation */
break;
case FaultInjectorStateTriggered:
/* No operation */
break;
case FaultInjectorStateCompleted:
retval = TRUE;
/* NO break */
case FaultInjectorStateFailed:
isCompleted = TRUE;
isRemoved = FaultInjector_RemoveHashEntry(identifier);
if (isRemoved == FALSE) {
ereport(DEBUG1,
(errmsg("LOG(fault injector): could not remove fault injection from hash"
"identifier:'%s' ",
FaultInjectorIdentifierEnumToString[identifier])));
} else {
faultInjectorShmem->faultInjectorSlots--;
}
break;
default:
Assert(0);
}
exit:
FiLockRelease();
if ((isCompleted == TRUE) && (retval == FALSE)) {
ereport(WARNING,
(errmsg("could not complete fault injection, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[identifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
}
return isCompleted;
}
/*
* update hash entry with state
*/
static int
FaultInjector_UpdateHashEntry(
FaultInjectorEntry_s *entry)
{
FaultInjectorEntry_s *entryLocal;
int status = STATUS_OK;
FiLockAcquire();
entryLocal = FaultInjector_LookupHashEntry(entry->faultName);
if (entryLocal == NULL)
{
FiLockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not update fault injection hash entry with fault injection status, "
"no entry found, "
"fault name:'%s' fault type:'%s' ",
entry->faultName,
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
goto exit;
}
if (entry->faultInjectorType == FaultInjectorTypeResume)
{
entryLocal->faultInjectorType = FaultInjectorTypeResume;
}
else
{
entryLocal->faultInjectorState = entry->faultInjectorState;
entryLocal->occurrence = entry->occurrence;
}
FiLockRelease();
ereport(DEBUG1,
(errmsg("LOG(fault injector): update fault injection hash entry "
"identifier:'%s' state:'%s' occurrence:'%d' ",
entry->faultName,
FaultInjectorStateEnumToString[entryLocal->faultInjectorState],
entry->occurrence)));
exit:
return status;
}
FaultInjectorType_e
FaultInjector_InjectFaultIfSet(
FaultInjectorIdentifier_e identifier,
DDLStatement_e ddlStatement,
char* databaseName,
char* tableName)
{
FaultInjectorEntry_s *entryShared, localEntry,
*entryLocal = &localEntry;
char databaseNameLocal[NAMEDATALEN];
char tableNameLocal[NAMEDATALEN];
int ii = 0;
int cnt = 3600;
/*
* Return immediately if no fault has been injected ever. It is
* important to not touch the spinlock, especially if this is the
* postmaster process. If one of the backend processes dies while
* holding the spin lock, and postmaster comes here before resetting
* the shared memory, it waits without holder process and eventually
* goes into PANIC. Also this saves a few cycles to acquire the spin
* lock and look into the shared hash table.
*
* Although this is a race condition without lock, a false negative is
* ok given this framework is purely for dev/testing.
*/
if (faultInjectorShmem->faultInjectorSlots == 0)
return FALSE;
getFileRepRoleAndState(&fileRepRole, &segmentState, &dataState, NULL, NULL);
FiLockAcquire();
entryShared = FaultInjector_LookupHashEntry(identifier);
if (entryShared != NULL)
memcpy(entryLocal, entryShared, sizeof(FaultInjectorEntry_s));
FiLockRelease();
/* Verify if fault injection is set */
if (entryShared == NULL)
/* fault injection is not set */
return FALSE;
if (entryLocal->ddlStatement != ddlStatement)
/* fault injection is not set for the specified DDL */
return FALSE;
snprintf(databaseNameLocal, sizeof(databaseNameLocal), "%s", databaseName);
if (strcmp(entryLocal->databaseName, databaseNameLocal) != 0)
/* fault injection is not set for the specified database name */
return FALSE;
snprintf(tableNameLocal, sizeof(tableNameLocal), "%s", tableName);
if (strcmp(entryLocal->tableName, tableNameLocal) != 0)
/* fault injection is not set for the specified table name */
return FALSE;
if (entryLocal->faultInjectorState == FaultInjectorStateTriggered ||
entryLocal->faultInjectorState == FaultInjectorStateCompleted ||
entryLocal->faultInjectorState == FaultInjectorStateFailed) {
/* fault injection was already executed */
return FALSE;
}
/* Update the injection fault entry in hash table */
if (entryLocal->occurrence != FILEREP_UNDEFINED)
{
if (entryLocal->occurrence > 1)
{
entryLocal->occurrence--;
FaultInjector_UpdateHashEntry(entryLocal);
return FALSE;
}
entryLocal->faultInjectorState = FaultInjectorStateTriggered;
FaultInjector_UpdateHashEntry(entryLocal);
}
/* Inject fault */
switch (entryLocal->faultInjectorType) {
case FaultInjectorTypeNotSpecified:
break;
case FaultInjectorTypeSleep:
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
pg_usleep(entryLocal->sleepTime * 1000000L);
break;
case FaultInjectorTypeFault:
switch (entryLocal->faultInjectorIdentifier)
{
case FileRepConsumer:
case FileRepConsumerVerification:
case FileRepSender:
case FileRepReceiver:
case FileRepResync:
case FileRepResyncInProgress:
case FileRepResyncWorker:
case FileRepResyncWorkerRead:
case FileRepTransitionToInResyncMirrorReCreate:
case FileRepTransitionToInResyncMarkReCreated:
case FileRepTransitionToInResyncMarkCompleted:
case FileRepTransitionToInSyncBegin:
case FileRepTransitionToInSync:
case FileRepTransitionToInSyncMarkCompleted:
case FileRepTransitionToInSyncBeforeCheckpoint:
case FileRepIsOperationCompleted:
FileRep_SetSegmentState(SegmentStateFault, FaultTypeMirror);
break;
case FileRepTransitionToChangeTracking:
FileRep_SetPostmasterReset();
break;
default:
FileRep_SetSegmentState(SegmentStateFault, FaultTypeIO);
break;
}
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypeFatal:
/*
* If it's one time occurrence then disable the fault before it's
* actually triggered because this fault errors out the transaction
* and hence we wont get a chance to disable it or put it in completed
* state.
*/
if (entryLocal->occurrence != FILEREP_UNDEFINED)
{
entryLocal->faultInjectorState = FaultInjectorStateCompleted;
FaultInjector_UpdateHashEntry(entryLocal);
}
ereport(FATAL,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypePanic:
/*
* If it's one time occurrence then disable the fault before it's
* actually triggered because this fault errors out the transaction
* and hence we wont get a chance to disable it or put it in completed
* state. For PANIC it may be unnecessary though.
*/
if (entryLocal->occurrence != FILEREP_UNDEFINED)
{
entryLocal->faultInjectorState = FaultInjectorStateCompleted;
FaultInjector_UpdateHashEntry(entryLocal);
}
ereport(PANIC,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypeError:
/*
* If it's one time occurrence then disable the fault before it's
* actually triggered because this fault errors out the transaction
* and hence we wont get a chance to disable it or put it in completed
* state.
*/
if (entryLocal->occurrence != FILEREP_UNDEFINED)
{
entryLocal->faultInjectorState = FaultInjectorStateCompleted;
FaultInjector_UpdateHashEntry(entryLocal);
}
ereport(ERROR,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypeInfiniteLoop:
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
if (entryLocal->faultInjectorIdentifier == FileRepImmediateShutdownRequested)
cnt = entryLocal->sleepTime;
for (ii=0; ii < cnt; ii++)
{
pg_usleep(1000000L); // sleep for 1 sec (1 sec * 3600 = 1 hour)
getFileRepRoleAndState(NULL, &segmentState, NULL, NULL, NULL);
if ((entryLocal->faultInjectorIdentifier != FileRepImmediateShutdownRequested) &&
(segmentState == SegmentStateShutdownFilerepBackends ||
segmentState == SegmentStateImmediateShutdown ||
segmentState == SegmentStateShutdown ||
IsFtsShudownRequested()))
{
break;
}
}
break;
case FaultInjectorTypeDataCorruption:
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypeSuspend:
{
FaultInjectorEntry_s *entry;
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
while ((entry = FaultInjector_LookupHashEntry(entryLocal->faultInjectorIdentifier)) != NULL &&
entry->faultInjectorType != FaultInjectorTypeResume)
{
pg_usleep(1000000L); // 1 sec
}
if (entry != NULL)
{
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
}
else
{
ereport(LOG,
(errmsg("fault 'NULL', fault name:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier])));
/*
* Since the entry is gone already, we should NOT update
* the entry below. (There could be other places in this
* function that are under the same situation, but I'm too
* tired to look for them...)
*/
return entryLocal->faultInjectorType;
}
break;
}
case FaultInjectorTypeSkip:
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypeMemoryFull:
{
char *buffer = NULL;
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
buffer = (char*) palloc(BLCKSZ);
while (buffer != NULL)
{
buffer = (char*) palloc(BLCKSZ);
}
break;
}
case FaultInjectorTypeReset:
case FaultInjectorTypeStatus:
ereport(LOG,
(errmsg("unexpected error, fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
Assert(0);
break;
case FaultInjectorTypeResume:
break;
case FaultInjectorTypeSegv:
{
*(int *) 0 = 1234;
break;
}
case FaultInjectorTypeInterrupt:
{
/*
* The place where this type of fault is injected must have
* has HOLD_INTERRUPTS() .. RESUME_INTERRUPTS() around it, otherwise
* the interrupt could be handled inside the fault injector itself
*/
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
InterruptPending = true;
QueryCancelPending = true;
break;
}
case FaultInjectorTypeCheckpointAndPanic:
{
if (entryLocal->occurrence != FILEREP_UNDEFINED)
{
entryLocal->faultInjectorState = FaultInjectorStateCompleted;
FaultInjector_UpdateHashEntry(entryLocal);
}
RequestCheckpoint(true, false);
ereport(PANIC,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
}
default:
ereport(LOG,
(errmsg("unexpected error, fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
Assert(0);
break;
}
if (entryLocal->occurrence != FILEREP_UNDEFINED)
{
entryLocal->faultInjectorState = FaultInjectorStateCompleted;
}
FaultInjector_UpdateHashEntry(entryLocal);
return (entryLocal->faultInjectorType);
}
FaultInjectorType_e
FaultInjector_InjectFaultIfSet(
FaultInjectorIdentifier_e identifier,
DDLStatement_e ddlStatement,
char* databaseName,
char* tableName)
{
FaultInjectorEntry_s *entryLocal;
char databaseNameLocal[NAMEDATALEN];
char tableNameLocal[NAMEDATALEN];
int ii = 0;
/*
* Return immediately if no fault has been injected ever. It is
* important to not touch the spinlock, especially if this is the
* postmaster process. If one of the backend processes dies while
* holding the spin lock, and postmaster comes here before resetting
* the shared memory, it waits without holder process and eventually
* goes into PANIC. Also this saves a few cycles to acquire the spin
* lock and look into the shared hash table.
*
* Although this is a race condition without lock, a false negative is
* ok given this framework is purely for dev/testing.
*/
if (faultInjectorShmem->faultInjectorSlots == 0)
return FALSE;
getFileRepRoleAndState(&fileRepRole, &segmentState, &dataState, NULL, NULL);
LockAcquire();
entryLocal = FaultInjector_LookupHashEntry(identifier);
LockRelease();
/* Verify if fault injection is set */
if (entryLocal == NULL)
/* fault injection is not set */
return FALSE;
if (entryLocal->ddlStatement != ddlStatement)
/* fault injection is not set for the specified DDL */
return FALSE;
snprintf(databaseNameLocal, sizeof(databaseNameLocal), "%s", databaseName);
if (strcmp(entryLocal->databaseName, databaseNameLocal) != 0)
/* fault injection is not set for the specified database name */
return FALSE;
snprintf(tableNameLocal, sizeof(tableNameLocal), "%s", tableName);
if (strcmp(entryLocal->tableName, tableNameLocal) != 0)
/* fault injection is not set for the specified table name */
return FALSE;
if (entryLocal->faultInjectorState == FaultInjectorStateTriggered ||
entryLocal->faultInjectorState == FaultInjectorStateCompleted ||
entryLocal->faultInjectorState == FaultInjectorStateFailed) {
/* fault injection was already executed */
return FALSE;
}
/* Update the injection fault entry in hash table */
if (entryLocal->occurrence != FILEREP_UNDEFINED)
{
if (entryLocal->occurrence > 1)
{
entryLocal->occurrence--;
return FALSE;
}
else
entryLocal->faultInjectorState = FaultInjectorStateTriggered;
}
FaultInjector_UpdateHashEntry(entryLocal);
/* Inject fault */
switch (entryLocal->faultInjectorType) {
case FaultInjectorTypeNotSpecified:
break;
case FaultInjectorTypeSleep:
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
pg_usleep(entryLocal->sleepTime * 1000000L);
break;
case FaultInjectorTypeFault:
switch (entryLocal->faultInjectorIdentifier)
{
case FileRepConsumer:
case FileRepConsumerVerification:
case FileRepSender:
case FileRepReceiver:
case FileRepResync:
case FileRepResyncInProgress:
case FileRepResyncWorker:
case FileRepResyncWorkerRead:
case FileRepTransitionToInResyncMirrorReCreate:
case FileRepTransitionToInResyncMarkReCreated:
case FileRepTransitionToInResyncMarkCompleted:
case FileRepTransitionToInSyncBegin:
case FileRepTransitionToInSync:
case FileRepTransitionToInSyncMarkCompleted:
case FileRepTransitionToInSyncBeforeCheckpoint:
/*
* Since we have removed all the file replication related
* functions, so the following cases should be avoided.
*/
/* FileRep_SetSegmentState(SegmentStateFault, FaultTypeMirror); */
break;
case FileRepTransitionToChangeTracking:
/* FileRep_SetPostmasterReset(); */
break;
default:
/* FileRep_SetSegmentState(SegmentStateFault, FaultTypeIO); */
break;
}
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypeFatal:
if (entryLocal->occurrence != FILEREP_UNDEFINED)
{
entryLocal->faultInjectorState = FaultInjectorStateCompleted;
}
FaultInjector_UpdateHashEntry(entryLocal);
ereport(FATAL,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypePanic:
if (entryLocal->occurrence != FILEREP_UNDEFINED)
{
entryLocal->faultInjectorState = FaultInjectorStateCompleted;
}
FaultInjector_UpdateHashEntry(entryLocal);
ereport(PANIC,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypeError:
if (entryLocal->occurrence != FILEREP_UNDEFINED)
{
entryLocal->faultInjectorState = FaultInjectorStateCompleted;
}
FaultInjector_UpdateHashEntry(entryLocal);
ereport(ERROR,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypeInfiniteLoop:
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
for (ii=0; ii < 3600; ii++)
{
pg_usleep(1000000L); // sleep for 1 sec (1 sec * 3600 = 1 hour)
getFileRepRoleAndState(NULL, &segmentState, NULL, NULL, NULL);
if (segmentState == SegmentStateShutdownFilerepBackends ||
segmentState == SegmentStateImmediateShutdown ||
segmentState == SegmentStateShutdown)
{
break;
}
}
break;
case FaultInjectorTypeDataCorruption:
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypeSuspend:
{
FaultInjectorEntry_s *entry;
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
while ((entry = FaultInjector_LookupHashEntry(entryLocal->faultInjectorIdentifier)) != NULL &&
entry->faultInjectorType != FaultInjectorTypeResume)
{
pg_usleep(1000000L); // 1 sec
}
if (entry != NULL)
{
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
}
else
{
ereport(LOG,
(errmsg("fault 'NULL', fault name:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier])));
}
break;
}
case FaultInjectorTypeSkip:
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
case FaultInjectorTypeMemoryFull:
{
char *buffer = NULL;
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
buffer = (char*) palloc(BLCKSZ);
while (buffer != NULL)
{
buffer = (char*) palloc(BLCKSZ);
}
break;
}
case FaultInjectorTypeReset:
case FaultInjectorTypeStatus:
ereport(LOG,
(errmsg("unexpected error, fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
Assert(0);
break;
case FaultInjectorTypeResume:
break;
case FaultInjectorTypePanicSuppress:
{
DECLARE_SAVE_SUPPRESS_PANIC();
entryLocal->faultInjectorState = FaultInjectorStateCompleted;
FaultInjector_UpdateHashEntry(entryLocal);
SUPPRESS_PANIC();
ereport(FATAL,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
break;
}
case FaultInjectorTypeSegv:
{
*(int *) 0 = 1234;
break;
}
case FaultInjectorTypeCreateThreadFail:
case FaultInjectorTypeConnectionNull:
case FaultInjectorTypeConnectionNullInRestoreMode:
case FaultInjectorTypeUserCancel:
case FaultInjectorTypeProcDie:
case FaultInjectorTypeInterrupt:
{
/*
* The place where this type of fault is injected must have
* has HOLD_INTERRUPTS() .. RESUME_INTERRUPTS() around it, otherwise
* the interrupt could be handled inside the fault injector itself
*/
ereport(LOG,
(errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
InterruptPending = true;
QueryCancelPending = true;
break;
}
case FaultInjectorTypeTimeOut:
case FaultInjectorTypeDispatchError:
{
break;
}
default:
ereport(LOG,
(errmsg("unexpected error, fault triggered, fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType])));
Assert(0);
break;
}
if (entryLocal->occurrence != FILEREP_UNDEFINED)
{
entryLocal->faultInjectorState = FaultInjectorStateCompleted;
}
FaultInjector_UpdateHashEntry(entryLocal);
return (entryLocal->faultInjectorType);
}
int
FaultInjector_SetFaultInjection(
FaultInjectorEntry_s *entry)
{
int status = STATUS_OK;
bool isRemoved = FALSE;
getFileRepRoleAndState(&fileRepRole, &segmentState, &dataState, NULL, NULL);
switch (entry->faultInjectorType) {
case FaultInjectorTypeReset:
{
HASH_SEQ_STATUS hash_status;
FaultInjectorEntry_s *entryLocal;
if (entry->faultInjectorIdentifier == FaultInjectorIdAll)
{
hash_seq_init(&hash_status, faultInjectorShmem->hash);
FiLockAcquire();
while ((entryLocal = (FaultInjectorEntry_s *) hash_seq_search(&hash_status)) != NULL) {
isRemoved = FaultInjector_RemoveHashEntry(entryLocal->faultName);
if (isRemoved == TRUE) {
faultInjectorShmem->faultInjectorSlots--;
}
}
FiLockRelease();
Assert(faultInjectorShmem->faultInjectorSlots == 0);
}
else
{
FiLockAcquire();
isRemoved = FaultInjector_RemoveHashEntry(entry->faultName);
if (isRemoved == TRUE) {
faultInjectorShmem->faultInjectorSlots--;
}
FiLockRelease();
}
if (isRemoved == FALSE) {
ereport(DEBUG1,
(errmsg("LOG(fault injector): could not remove fault injection from hash"
"identifier:'%s' ",
entry->faultName)));
}
break;
}
case FaultInjectorTypeWaitUntilTriggered:
{
FaultInjectorEntry_s *entryLocal;
while ((entryLocal = FaultInjector_LookupHashEntry(entry->faultName)) != NULL &&
entry->occurrence > entryLocal->numTimesTriggered)
{
pg_usleep(1000000L); // 1 sec
}
if (entryLocal != NULL)
{
ereport(LOG,
(errcode(ERRCODE_FAULT_INJECT),
errmsg("fault triggered %d times, fault name:'%s' fault type:'%s' ",
entry->occurrence,
entryLocal->faultName,
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
status = STATUS_OK;
}
else
{
ereport(LOG,
(errcode(ERRCODE_FAULT_INJECT),
errmsg("fault 'NULL', fault name:'%s' ",
entryLocal->faultName)));
status = STATUS_ERROR;
}
break;
}
case FaultInjectorTypeStatus:
{
HASH_SEQ_STATUS hash_status;
FaultInjectorEntry_s *entryLocal;
bool found = FALSE;
int length;
if (faultInjectorShmem->hash == NULL) {
status = STATUS_ERROR;
break;
}
length = snprintf(entry->bufOutput, sizeof(entry->bufOutput), "Success: ");
if (entry->faultInjectorIdentifier == ChangeTrackingCompactingReport)
{
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"Success: compacting in progress %s",
ChangeTrackingIsCompactingInProgress() ? "true" : "false");
break;
}
hash_seq_init(&hash_status, faultInjectorShmem->hash);
while ((entryLocal = (FaultInjectorEntry_s *) hash_seq_search(&hash_status)) != NULL) {
ereport(LOG,
(errmsg("fault injector status: "
"fault name:'%s' "
"fault type:'%s' "
"ddl statement:'%s' "
"database name:'%s' "
"table name:'%s' "
"occurrence:'%d' "
"sleep time:'%d' "
"fault injection state:'%s' "
"num times hit:'%d' ",
entryLocal->faultName,
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType],
FaultInjectorDDLEnumToString[entryLocal->ddlStatement],
entryLocal->databaseName,
entryLocal->tableName,
entryLocal->occurrence,
entryLocal->sleepTime,
FaultInjectorStateEnumToString[entryLocal->faultInjectorState],
entryLocal->numTimesTriggered)));
if (entry->faultInjectorIdentifier == entryLocal->faultInjectorIdentifier ||
entry->faultInjectorIdentifier == FaultInjectorIdAll)
{
length = snprintf((entry->bufOutput + length), sizeof(entry->bufOutput) - length,
"fault name:'%s' "
"fault type:'%s' "
"ddl statement:'%s' "
"database name:'%s' "
"table name:'%s' "
"occurrence:'%d' "
"sleep time:'%d' "
"fault injection state:'%s' "
"num times hit:'%d' \n",
entryLocal->faultName,
FaultInjectorTypeEnumToString[entryLocal->faultInjectorType],
FaultInjectorDDLEnumToString[entryLocal->ddlStatement],
entryLocal->databaseName,
entryLocal->tableName,
entryLocal->occurrence,
entryLocal->sleepTime,
FaultInjectorStateEnumToString[entryLocal->faultInjectorState],
entryLocal->numTimesTriggered);
found = TRUE;
}
}
if (found == FALSE) {
snprintf(entry->bufOutput, sizeof(entry->bufOutput), "Failure: "
"fault name:'%s' not set",
entry->faultName);
}
break;
}
case FaultInjectorTypeResume:
ereport(LOG,
(errcode(ERRCODE_FAULT_INJECT),
errmsg("fault triggered, fault name:'%s' fault type:'%s' ",
entry->faultName,
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
FaultInjector_UpdateHashEntry(entry);
break;
default:
status = FaultInjector_NewHashEntry(entry);
break;
}
return status;
}
