/*
* update hash entry with state
*/
static int
FaultInjector_UpdateHashEntry(
FaultInjectorEntry_s *entry)
{
FaultInjectorEntry_s *entryLocal;
bool exists;
int status = STATUS_OK;
FiLockAcquire();
entryLocal = FaultInjector_InsertHashEntry(entry->faultInjectorIdentifier, &exists);
/* entry should be found since fault has not been injected yet */
Assert(entryLocal != NULL);
if (!exists) {
FiLockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not update fault injection hash entry with fault injection status, "
"no entry found, "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
goto exit;
}
if (entry->faultInjectorType == FaultInjectorTypeResume)
{
entryLocal->faultInjectorType = FaultInjectorTypeResume;
}
else
{
entryLocal->faultInjectorState = entry->faultInjectorState;
entryLocal->occurrence = entry->occurrence;
}
FiLockRelease();
ereport(DEBUG1,
(errmsg("LOG(fault injector): update fault injection hash entry "
"identifier:'%s' state:'%s' occurrence:'%d' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorStateEnumToString[entryLocal->faultInjectorState],
entry->occurrence)));
exit:
return status;
}
static int
FaultInjector_NewHashEntry(
FaultInjectorEntry_s *entry)
{
FaultInjectorEntry_s *entryLocal=NULL;
bool exists;
int status = STATUS_OK;
FiLockAcquire();
if ((faultInjectorShmem->faultInjectorSlots + 1) >= FAULTINJECTOR_MAX_SLOTS) {
FiLockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection, no slots available"
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, max slots:'%d' reached",
FAULTINJECTOR_MAX_SLOTS);
goto exit;
}
if (entry->faultInjectorType == FaultInjectorTypeSkip)
{
switch (entry->faultInjectorIdentifier)
{
case Checkpoint:
case ChangeTrackingDisable:
case FileRepVerification:
case FinishPreparedTransactionCommitPass1FromCreatePendingToCreated:
case FinishPreparedTransactionCommitPass2FromCreatePendingToCreated:
case FinishPreparedTransactionCommitPass1FromDropInMemoryToDropPending:
case FinishPreparedTransactionCommitPass2FromDropInMemoryToDropPending:
case FinishPreparedTransactionCommitPass1AbortingCreateNeeded:
case FinishPreparedTransactionCommitPass2AbortingCreateNeeded:
case FinishPreparedTransactionAbortPass1FromCreatePendingToAbortingCreate:
case FinishPreparedTransactionAbortPass2FromCreatePendingToAbortingCreate:
case FinishPreparedTransactionAbortPass1AbortingCreateNeeded:
case FinishPreparedTransactionAbortPass2AbortingCreateNeeded:
break;
default:
FiLockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection, fault type not supported"
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, fault type not supported");
goto exit;
}
}
/* check role */
getFileRepRoleAndState(&fileRepRole, &segmentState, &dataState, NULL, NULL);
switch (entry->faultInjectorIdentifier)
{
case ChangeTrackingDisable:
case FileRepConsumerVerification:
case StartPrepareTx:
case FileRepResync:
case FileRepResyncInProgress:
case FileRepResyncWorker:
case FileRepResyncWorkerRead:
case FileRepTransitionToInResyncMirrorReCreate:
case FileRepTransitionToInResyncMarkReCreated:
case FileRepTransitionToInResyncMarkCompleted:
case FileRepTransitionToInSyncBegin:
case FileRepTransitionToInSync:
case FileRepTransitionToInSyncMarkCompleted:
case FileRepTransitionToInSyncBeforeCheckpoint:
case FileRepTransitionToChangeTracking:
case FileRepIsOperationCompleted:
case FileRepImmediateShutdownRequested:
case TransactionCommitPass1FromCreatePendingToCreated:
case TransactionCommitPass1FromDropInMemoryToDropPending:
case TransactionCommitPass1FromAbortingCreateNeededToAbortingCreate:
case TransactionAbortPass1FromCreatePendingToAbortingCreate:
case TransactionAbortPass1FromAbortingCreateNeededToAbortingCreate:
case TransactionCommitPass2FromDropInMemoryToDropPending:
case TransactionCommitPass2FromAbortingCreateNeededToAbortingCreate:
case TransactionAbortPass2FromCreatePendingToAbortingCreate:
case TransactionAbortPass2FromAbortingCreateNeededToAbortingCreate:
case FinishPreparedTransactionCommitPass1FromCreatePendingToCreated:
case FinishPreparedTransactionCommitPass2FromCreatePendingToCreated:
case FinishPreparedTransactionCommitPass1FromDropInMemoryToDropPending:
case FinishPreparedTransactionCommitPass2FromDropInMemoryToDropPending:
case FinishPreparedTransactionCommitPass1AbortingCreateNeeded:
case FinishPreparedTransactionCommitPass2AbortingCreateNeeded:
case FinishPreparedTransactionAbortPass1FromCreatePendingToAbortingCreate:
// case FinishPreparedTransactionAbortPass2FromCreatePendingToAbortingCreate:
case FinishPreparedTransactionAbortPass1AbortingCreateNeeded:
case FinishPreparedTransactionAbortPass2AbortingCreateNeeded:
case TwoPhaseTransactionCommitPrepared:
case TwoPhaseTransactionAbortPrepared:
case ExecSortMKSortMergeRuns:
// case SubtransactionFlushToFile:
// case SubtransactionReadFromFile:
// case SubtransactionRelease:
// case SubtransactionRollback:
/* Ashwin */
case FileRepChangeTrackingCompacting:
/* We do not use vmem on master. Therefore, we only attempt large palloc on segments. */
case MultiExecHashLargeVmem:
case FaultInBackgroundWriterMain:
if (fileRepRole != FileRepPrimaryRole)
{
FiLockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection entry into table, segment not in primary role"
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, segment not in primary role");
goto exit;
}
break;
case FaultBeforePendingDeleteRelationEntry:
case FaultBeforePendingDeleteDatabaseEntry:
case FaultBeforePendingDeleteTablespaceEntry:
case FaultBeforePendingDeleteFilespaceEntry:
case FileRepConsumer:
case FileRepSender:
case FileRepReceiver:
case FileRepFlush:
case Postmaster:
case PgControl:
case PgXlog:
case SegmentTransitionRequest:
case SegmentProbeResponse:
if (fileRepRole != FileRepPrimaryRole && fileRepRole != FileRepMirrorRole)
{
FiLockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection entry into table, "
"segment not in primary or mirror role, "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, segment not in primary or mirror role");
goto exit;
}
break;
case TransactionAbortAfterDistributedPrepared:
case DtmBroadcastPrepare:
case DtmBroadcastCommitPrepared:
case DtmBroadcastAbortPrepared:
case DtmXLogDistributedCommit:
case OptTaskAllocateStringBuffer:
case OptRelcacheTranslatorCatalogAccess:
if (fileRepRole != FileRepNoRoleConfigured)
{
FiLockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection entry into table, "
"segment not in master role, "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, segment not in master role");
goto exit;
}
break;
case LocalTmRecordTransactionCommit:
case Checkpoint:
case AbortTransactionFail:
case UpdateCommittedEofInPersistentTable:
case ExecSortBeforeSorting:
case FaultDuringExecDynamicTableScan:
case FaultExecHashJoinNewBatch:
case RunawayCleanup:
if (fileRepRole != FileRepNoRoleConfigured && fileRepRole != FileRepPrimaryRole)
{
FiLockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection entry into table, "
"segment not in primary or master role, "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, segment not in primary or master role");
goto exit;
}
break;
default:
break;
}
entryLocal = FaultInjector_InsertHashEntry(entry->faultInjectorIdentifier, &exists);
if (entryLocal == NULL) {
FiLockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection entry into table, no memory, "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, no memory");
goto exit;
}
if (exists) {
FiLockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection entry into table, "
"entry already exists, "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, entry already exists");
goto exit;
}
entryLocal->faultInjectorType = entry->faultInjectorType;
entryLocal->sleepTime = entry->sleepTime;
entryLocal->ddlStatement = entry->ddlStatement;
if (entry->occurrence != 0)
{
entryLocal->occurrence = entry->occurrence;
}
else
{
entryLocal->occurrence = FILEREP_UNDEFINED;
}
strcpy(entryLocal->databaseName, entry->databaseName);
strcpy(entryLocal->tableName, entry->tableName);
entryLocal->faultInjectorState = FaultInjectorStateWaiting;
faultInjectorShmem->faultInjectorSlots++;
FiLockRelease();
elog(DEBUG1, "FaultInjector_NewHashEntry() identifier:'%s'",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier]);
exit:
return status;
}
static int
FaultInjector_NewHashEntry(
FaultInjectorEntry_s *entry)
{
FaultInjectorEntry_s *entryLocal=NULL;
bool exists;
int status = STATUS_OK;
LockAcquire();
if ((faultInjectorShmem->faultInjectorSlots + 1) >= FAULTINJECTOR_MAX_SLOTS) {
LockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection, no slots available"
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, max slots:'%d' reached",
FAULTINJECTOR_MAX_SLOTS);
goto exit;
}
if (entry->faultInjectorType == FaultInjectorTypeSkip)
{
switch (entry->faultInjectorIdentifier)
{
case Checkpoint:
case ChangeTrackingDisable:
case FileRepVerification:
case FinishPreparedTransactionCommitPass1FromCreatePendingToCreated:
case FinishPreparedTransactionCommitPass2FromCreatePendingToCreated:
case FinishPreparedTransactionCommitPass1FromDropInMemoryToDropPending:
case FinishPreparedTransactionCommitPass2FromDropInMemoryToDropPending:
case FinishPreparedTransactionCommitPass1AbortingCreateNeeded:
case FinishPreparedTransactionCommitPass2AbortingCreateNeeded:
case FinishPreparedTransactionAbortPass1FromCreatePendingToAbortingCreate:
case FinishPreparedTransactionAbortPass2FromCreatePendingToAbortingCreate:
case FinishPreparedTransactionAbortPass1AbortingCreateNeeded:
case FinishPreparedTransactionAbortPass2AbortingCreateNeeded:
case SyncPersistentTable:
case XLOGInsert:
break;
default:
LockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection, fault type not supported"
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, fault type not supported");
goto exit;
}
}
/* check role */
getFileRepRoleAndState(&fileRepRole, &segmentState, &dataState, NULL, NULL);
switch (entry->faultInjectorIdentifier)
{
case ChangeTrackingDisable:
case FileRepConsumerVerification:
case FileRepResync:
case FileRepResyncInProgress:
case FileRepResyncWorker:
case FileRepResyncWorkerRead:
case FileRepTransitionToInResyncMirrorReCreate:
case FileRepTransitionToInResyncMarkReCreated:
case FileRepTransitionToInResyncMarkCompleted:
case FileRepTransitionToInSyncBegin:
case FileRepTransitionToInSync:
case FileRepTransitionToInSyncMarkCompleted:
case FileRepTransitionToInSyncBeforeCheckpoint:
case FileRepTransitionToChangeTracking:
case FileRepConsumer:
case FileRepSender:
case FileRepReceiver:
case FileRepFlush:
/* Ashwin */
case FileRepChangeTrackingCompacting:
case FinishPreparedTransactionCommitPass1FromCreatePendingToCreated:
case FinishPreparedTransactionCommitPass2FromCreatePendingToCreated:
case FinishPreparedTransactionCommitPass1FromDropInMemoryToDropPending:
case FinishPreparedTransactionCommitPass2FromDropInMemoryToDropPending:
case FinishPreparedTransactionCommitPass1AbortingCreateNeeded:
case FinishPreparedTransactionCommitPass2AbortingCreateNeeded:
case FinishPreparedTransactionAbortPass1FromCreatePendingToAbortingCreate:
// case FinishPreparedTransactionAbortPass2FromCreatePendingToAbortingCreate:
case FinishPreparedTransactionAbortPass1AbortingCreateNeeded:
case FinishPreparedTransactionAbortPass2AbortingCreateNeeded:
case TwoPhaseTransactionCommitPrepared:
case TwoPhaseTransactionAbortPrepared:
/* This kind of fault injection has not been supported yet. */
LockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("This kind of fault injection has not been supported yet. "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"This kind of fault injection has not been supported yet. "
"Please check faultname");
goto exit;
// case SubtransactionFlushToFile:
// case SubtransactionReadFromFile:
// case SubtransactionRelease:
// case SubtransactionRollback:
case StartPrepareTx:
case TransactionCommitPass1FromCreatePendingToCreated:
case TransactionCommitPass1FromDropInMemoryToDropPending:
case TransactionCommitPass1FromAbortingCreateNeededToAbortingCreate:
case TransactionAbortPass1FromCreatePendingToAbortingCreate:
case TransactionAbortPass1FromAbortingCreateNeededToAbortingCreate:
case TransactionCommitPass2FromDropInMemoryToDropPending:
case TransactionCommitPass2FromAbortingCreateNeededToAbortingCreate:
case TransactionAbortPass2FromCreatePendingToAbortingCreate:
case TransactionAbortPass2FromAbortingCreateNeededToAbortingCreate:
case FaultBeforePendingDeleteRelationEntry:
case FaultBeforePendingDeleteDatabaseEntry:
case FaultBeforePendingDeleteTablespaceEntry:
case FaultBeforePendingDeleteFilespaceEntry:
case TransactionAbortAfterDistributedPrepared:
case DtmBroadcastPrepare:
case DtmBroadcastCommitPrepared:
case DtmBroadcastAbortPrepared:
case DtmXLogDistributedCommit:
case AnalyzeSubxactError:
case OptTaskAllocateStringBuffer:
case ConnectionFailAfterGangCreation:
case CreateCdbDispathResultObject:
case WorkerManagerSubmitJob:
case FaillQeAfterConnection:
/* These faults are designed for master. */
if(!AmIMaster())
{
LockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection entry into table, "
"This kind of fault injection should be sent to master. "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection. "
"This kind of fault injection should be sent to master. "
"Please use \"-r master\"");
goto exit;
}
break;
case SegmentTransitionRequest:
case SegmentProbeResponse:
/* We do not use vmem on master. Therefore, we only attempt large palloc on segments. */
case MultiExecHashLargeVmem:
case FailQeWhenDoQuery:
case FailQeWhenBeginParquetScan:
/* SEGMENT */
if(!AmISegment())
{
LockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection entry into table, "
"This kind of fault injection should be sent to segment. "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection. "
"This kind of fault injection should be sent to segment. "
"Please use \"-r primary\"");
goto exit;
}
break;
case LocalTmRecordTransactionCommit:
case Checkpoint:
case AbortTransactionFail:
case UpdateCommittedEofInPersistentTable:
case FaultDuringExecDynamicTableScan:
case ExecSortBeforeSorting:
case FaultExecHashJoinNewBatch:
case WorkfileCleanupSet:
case RunawayCleanup:
/* MASTER OR SEGMENT */
if(AmIStandby())
{
LockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection entry into table, "
"segment not in primary or master role, "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, segment not in master or segment role");
goto exit;
}
break;
default:
break;
}
entryLocal = FaultInjector_InsertHashEntry(entry->faultInjectorIdentifier, &exists);
if (entryLocal == NULL) {
LockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection entry into table, no memory, "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, no memory");
goto exit;
}
if (exists) {
LockRelease();
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("could not insert fault injection entry into table, "
"entry already exists, "
"fault name:'%s' fault type:'%s' ",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier],
FaultInjectorTypeEnumToString[entry->faultInjectorType])));
snprintf(entry->bufOutput, sizeof(entry->bufOutput),
"could not insert fault injection, entry already exists");
goto exit;
}
entryLocal->faultInjectorType = entry->faultInjectorType;
entryLocal->sleepTime = entry->sleepTime;
entryLocal->ddlStatement = entry->ddlStatement;
if (entry->occurrence != 0)
{
entryLocal->occurrence = entry->occurrence;
}
else
{
entryLocal->occurrence = FILEREP_UNDEFINED;
}
strcpy(entryLocal->databaseName, entry->databaseName);
strcpy(entryLocal->tableName, entry->tableName);
entryLocal->faultInjectorState = FaultInjectorStateWaiting;
faultInjectorShmem->faultInjectorSlots++;
LockRelease();
elog(DEBUG1, "FaultInjector_NewHashEntry() identifier:'%s'",
FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier]);
exit:
return status;
}
