/* * update hash entry with state */ static int FaultInjector_UpdateHashEntry( FaultInjectorEntry_s *entry) { FaultInjectorEntry_s *entryLocal; bool exists; int status = STATUS_OK; FiLockAcquire(); entryLocal = FaultInjector_InsertHashEntry(entry->faultInjectorIdentifier, &exists); /* entry should be found since fault has not been injected yet */ Assert(entryLocal != NULL); if (!exists) { FiLockRelease(); status = STATUS_ERROR; ereport(WARNING, (errmsg("could not update fault injection hash entry with fault injection status, " "no entry found, " "fault name:'%s' fault type:'%s' ", FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier], FaultInjectorTypeEnumToString[entry->faultInjectorType]))); goto exit; } if (entry->faultInjectorType == FaultInjectorTypeResume) { entryLocal->faultInjectorType = FaultInjectorTypeResume; } else { entryLocal->faultInjectorState = entry->faultInjectorState; entryLocal->occurrence = entry->occurrence; } FiLockRelease(); ereport(DEBUG1, (errmsg("LOG(fault injector): update fault injection hash entry " "identifier:'%s' state:'%s' occurrence:'%d' ", FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier], FaultInjectorStateEnumToString[entryLocal->faultInjectorState], entry->occurrence))); exit: return status; }
/* * update hash entry with state */ static int FaultInjector_UpdateHashEntry( FaultInjectorEntry_s *entry) { FaultInjectorEntry_s *entryLocal; int status = STATUS_OK; FiLockAcquire(); entryLocal = FaultInjector_LookupHashEntry(entry->faultName); if (entryLocal == NULL) { FiLockRelease(); status = STATUS_ERROR; ereport(WARNING, (errmsg("could not update fault injection hash entry with fault injection status, " "no entry found, " "fault name:'%s' fault type:'%s' ", entry->faultName, FaultInjectorTypeEnumToString[entry->faultInjectorType]))); goto exit; } if (entry->faultInjectorType == FaultInjectorTypeResume) { entryLocal->faultInjectorType = FaultInjectorTypeResume; } else { entryLocal->faultInjectorState = entry->faultInjectorState; entryLocal->occurrence = entry->occurrence; } FiLockRelease(); ereport(DEBUG1, (errmsg("LOG(fault injector): update fault injection hash entry " "identifier:'%s' state:'%s' occurrence:'%d' ", entry->faultName, FaultInjectorStateEnumToString[entryLocal->faultInjectorState], entry->occurrence))); exit: return status; }
bool FaultInjector_IsFaultInjected( FaultInjectorIdentifier_e identifier) { FaultInjectorEntry_s *entry = NULL; bool isCompleted = FALSE; bool retval = FALSE; bool isRemoved; FiLockAcquire(); entry = FaultInjector_LookupHashEntry(identifier); if (entry == NULL) { retval = TRUE; isCompleted = TRUE; goto exit; } switch (entry->faultInjectorState) { case FaultInjectorStateWaiting: /* No operation */ break; case FaultInjectorStateTriggered: /* No operation */ break; case FaultInjectorStateCompleted: retval = TRUE; /* NO break */ case FaultInjectorStateFailed: isCompleted = TRUE; isRemoved = FaultInjector_RemoveHashEntry(identifier); if (isRemoved == FALSE) { ereport(DEBUG1, (errmsg("LOG(fault injector): could not remove fault injection from hash" "identifier:'%s' ", FaultInjectorIdentifierEnumToString[identifier]))); } else { faultInjectorShmem->faultInjectorSlots--; } break; default: Assert(0); } exit: FiLockRelease(); if ((isCompleted == TRUE) && (retval == FALSE)) { ereport(WARNING, (errmsg("could not complete fault injection, fault name:'%s' fault type:'%s' ", FaultInjectorIdentifierEnumToString[identifier], FaultInjectorTypeEnumToString[entry->faultInjectorType]))); } return isCompleted; }
static int FaultInjector_NewHashEntry( FaultInjectorEntry_s *entry) { FaultInjectorEntry_s *entryLocal=NULL; bool exists; int status = STATUS_OK; FiLockAcquire(); if ((faultInjectorShmem->faultInjectorSlots + 1) >= FAULTINJECTOR_MAX_SLOTS) { FiLockRelease(); status = STATUS_ERROR; ereport(WARNING, (errmsg("could not insert fault injection, no slots available" "fault name:'%s' fault type:'%s' ", FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier], FaultInjectorTypeEnumToString[entry->faultInjectorType]))); snprintf(entry->bufOutput, sizeof(entry->bufOutput), "could not insert fault injection, max slots:'%d' reached", FAULTINJECTOR_MAX_SLOTS); goto exit; } if (entry->faultInjectorType == FaultInjectorTypeSkip) { switch (entry->faultInjectorIdentifier) { case Checkpoint: case ChangeTrackingDisable: case FileRepVerification: case FinishPreparedTransactionCommitPass1FromCreatePendingToCreated: case FinishPreparedTransactionCommitPass2FromCreatePendingToCreated: case FinishPreparedTransactionCommitPass1FromDropInMemoryToDropPending: case FinishPreparedTransactionCommitPass2FromDropInMemoryToDropPending: case FinishPreparedTransactionCommitPass1AbortingCreateNeeded: case FinishPreparedTransactionCommitPass2AbortingCreateNeeded: case FinishPreparedTransactionAbortPass1FromCreatePendingToAbortingCreate: case FinishPreparedTransactionAbortPass2FromCreatePendingToAbortingCreate: case FinishPreparedTransactionAbortPass1AbortingCreateNeeded: case FinishPreparedTransactionAbortPass2AbortingCreateNeeded: break; default: FiLockRelease(); status = STATUS_ERROR; ereport(WARNING, (errmsg("could not insert fault injection, fault type not supported" "fault name:'%s' fault type:'%s' ", FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier], FaultInjectorTypeEnumToString[entry->faultInjectorType]))); snprintf(entry->bufOutput, sizeof(entry->bufOutput), "could not insert fault injection, fault type not supported"); goto exit; } } /* check role */ getFileRepRoleAndState(&fileRepRole, &segmentState, &dataState, NULL, NULL); switch (entry->faultInjectorIdentifier) { case ChangeTrackingDisable: case FileRepConsumerVerification: case StartPrepareTx: case FileRepResync: case FileRepResyncInProgress: case FileRepResyncWorker: case FileRepResyncWorkerRead: case FileRepTransitionToInResyncMirrorReCreate: case FileRepTransitionToInResyncMarkReCreated: case FileRepTransitionToInResyncMarkCompleted: case FileRepTransitionToInSyncBegin: case FileRepTransitionToInSync: case FileRepTransitionToInSyncMarkCompleted: case FileRepTransitionToInSyncBeforeCheckpoint: case FileRepTransitionToChangeTracking: case FileRepIsOperationCompleted: case FileRepImmediateShutdownRequested: case TransactionCommitPass1FromCreatePendingToCreated: case TransactionCommitPass1FromDropInMemoryToDropPending: case TransactionCommitPass1FromAbortingCreateNeededToAbortingCreate: case TransactionAbortPass1FromCreatePendingToAbortingCreate: case TransactionAbortPass1FromAbortingCreateNeededToAbortingCreate: case TransactionCommitPass2FromDropInMemoryToDropPending: case TransactionCommitPass2FromAbortingCreateNeededToAbortingCreate: case TransactionAbortPass2FromCreatePendingToAbortingCreate: case TransactionAbortPass2FromAbortingCreateNeededToAbortingCreate: case FinishPreparedTransactionCommitPass1FromCreatePendingToCreated: case FinishPreparedTransactionCommitPass2FromCreatePendingToCreated: case FinishPreparedTransactionCommitPass1FromDropInMemoryToDropPending: case FinishPreparedTransactionCommitPass2FromDropInMemoryToDropPending: case FinishPreparedTransactionCommitPass1AbortingCreateNeeded: case FinishPreparedTransactionCommitPass2AbortingCreateNeeded: case FinishPreparedTransactionAbortPass1FromCreatePendingToAbortingCreate: // case FinishPreparedTransactionAbortPass2FromCreatePendingToAbortingCreate: case FinishPreparedTransactionAbortPass1AbortingCreateNeeded: case FinishPreparedTransactionAbortPass2AbortingCreateNeeded: case TwoPhaseTransactionCommitPrepared: case TwoPhaseTransactionAbortPrepared: case ExecSortMKSortMergeRuns: // case SubtransactionFlushToFile: // case SubtransactionReadFromFile: // case SubtransactionRelease: // case SubtransactionRollback: /* Ashwin */ case FileRepChangeTrackingCompacting: /* We do not use vmem on master. Therefore, we only attempt large palloc on segments. */ case MultiExecHashLargeVmem: case FaultInBackgroundWriterMain: if (fileRepRole != FileRepPrimaryRole) { FiLockRelease(); status = STATUS_ERROR; ereport(WARNING, (errmsg("could not insert fault injection entry into table, segment not in primary role" "fault name:'%s' fault type:'%s' ", FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier], FaultInjectorTypeEnumToString[entry->faultInjectorType]))); snprintf(entry->bufOutput, sizeof(entry->bufOutput), "could not insert fault injection, segment not in primary role"); goto exit; } break; case FaultBeforePendingDeleteRelationEntry: case FaultBeforePendingDeleteDatabaseEntry: case FaultBeforePendingDeleteTablespaceEntry: case FaultBeforePendingDeleteFilespaceEntry: case FileRepConsumer: case FileRepSender: case FileRepReceiver: case FileRepFlush: case Postmaster: case PgControl: case PgXlog: case SegmentTransitionRequest: case SegmentProbeResponse: if (fileRepRole != FileRepPrimaryRole && fileRepRole != FileRepMirrorRole) { FiLockRelease(); status = STATUS_ERROR; ereport(WARNING, (errmsg("could not insert fault injection entry into table, " "segment not in primary or mirror role, " "fault name:'%s' fault type:'%s' ", FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier], FaultInjectorTypeEnumToString[entry->faultInjectorType]))); snprintf(entry->bufOutput, sizeof(entry->bufOutput), "could not insert fault injection, segment not in primary or mirror role"); goto exit; } break; case TransactionAbortAfterDistributedPrepared: case DtmBroadcastPrepare: case DtmBroadcastCommitPrepared: case DtmBroadcastAbortPrepared: case DtmXLogDistributedCommit: case OptTaskAllocateStringBuffer: case OptRelcacheTranslatorCatalogAccess: if (fileRepRole != FileRepNoRoleConfigured) { FiLockRelease(); status = STATUS_ERROR; ereport(WARNING, (errmsg("could not insert fault injection entry into table, " "segment not in master role, " "fault name:'%s' fault type:'%s' ", FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier], FaultInjectorTypeEnumToString[entry->faultInjectorType]))); snprintf(entry->bufOutput, sizeof(entry->bufOutput), "could not insert fault injection, segment not in master role"); goto exit; } break; case LocalTmRecordTransactionCommit: case Checkpoint: case AbortTransactionFail: case UpdateCommittedEofInPersistentTable: case ExecSortBeforeSorting: case FaultDuringExecDynamicTableScan: case FaultExecHashJoinNewBatch: case RunawayCleanup: if (fileRepRole != FileRepNoRoleConfigured && fileRepRole != FileRepPrimaryRole) { FiLockRelease(); status = STATUS_ERROR; ereport(WARNING, (errmsg("could not insert fault injection entry into table, " "segment not in primary or master role, " "fault name:'%s' fault type:'%s' ", FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier], FaultInjectorTypeEnumToString[entry->faultInjectorType]))); snprintf(entry->bufOutput, sizeof(entry->bufOutput), "could not insert fault injection, segment not in primary or master role"); goto exit; } break; default: break; } entryLocal = FaultInjector_InsertHashEntry(entry->faultInjectorIdentifier, &exists); if (entryLocal == NULL) { FiLockRelease(); status = STATUS_ERROR; ereport(WARNING, (errmsg("could not insert fault injection entry into table, no memory, " "fault name:'%s' fault type:'%s' ", FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier], FaultInjectorTypeEnumToString[entry->faultInjectorType]))); snprintf(entry->bufOutput, sizeof(entry->bufOutput), "could not insert fault injection, no memory"); goto exit; } if (exists) { FiLockRelease(); status = STATUS_ERROR; ereport(WARNING, (errmsg("could not insert fault injection entry into table, " "entry already exists, " "fault name:'%s' fault type:'%s' ", FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier], FaultInjectorTypeEnumToString[entry->faultInjectorType]))); snprintf(entry->bufOutput, sizeof(entry->bufOutput), "could not insert fault injection, entry already exists"); goto exit; } entryLocal->faultInjectorType = entry->faultInjectorType; entryLocal->sleepTime = entry->sleepTime; entryLocal->ddlStatement = entry->ddlStatement; if (entry->occurrence != 0) { entryLocal->occurrence = entry->occurrence; } else { entryLocal->occurrence = FILEREP_UNDEFINED; } strcpy(entryLocal->databaseName, entry->databaseName); strcpy(entryLocal->tableName, entry->tableName); entryLocal->faultInjectorState = FaultInjectorStateWaiting; faultInjectorShmem->faultInjectorSlots++; FiLockRelease(); elog(DEBUG1, "FaultInjector_NewHashEntry() identifier:'%s'", FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier]); exit: return status; }
FaultInjectorType_e FaultInjector_InjectFaultIfSet( FaultInjectorIdentifier_e identifier, DDLStatement_e ddlStatement, char* databaseName, char* tableName) { FaultInjectorEntry_s *entryShared, localEntry, *entryLocal = &localEntry; char databaseNameLocal[NAMEDATALEN]; char tableNameLocal[NAMEDATALEN]; int ii = 0; int cnt = 3600; /* * Return immediately if no fault has been injected ever. It is * important to not touch the spinlock, especially if this is the * postmaster process. If one of the backend processes dies while * holding the spin lock, and postmaster comes here before resetting * the shared memory, it waits without holder process and eventually * goes into PANIC. Also this saves a few cycles to acquire the spin * lock and look into the shared hash table. * * Although this is a race condition without lock, a false negative is * ok given this framework is purely for dev/testing. */ if (faultInjectorShmem->faultInjectorSlots == 0) return FALSE; getFileRepRoleAndState(&fileRepRole, &segmentState, &dataState, NULL, NULL); FiLockAcquire(); entryShared = FaultInjector_LookupHashEntry(identifier); if (entryShared != NULL) memcpy(entryLocal, entryShared, sizeof(FaultInjectorEntry_s)); FiLockRelease(); /* Verify if fault injection is set */ if (entryShared == NULL) /* fault injection is not set */ return FALSE; if (entryLocal->ddlStatement != ddlStatement) /* fault injection is not set for the specified DDL */ return FALSE; snprintf(databaseNameLocal, sizeof(databaseNameLocal), "%s", databaseName); if (strcmp(entryLocal->databaseName, databaseNameLocal) != 0) /* fault injection is not set for the specified database name */ return FALSE; snprintf(tableNameLocal, sizeof(tableNameLocal), "%s", tableName); if (strcmp(entryLocal->tableName, tableNameLocal) != 0) /* fault injection is not set for the specified table name */ return FALSE; if (entryLocal->faultInjectorState == FaultInjectorStateTriggered || entryLocal->faultInjectorState == FaultInjectorStateCompleted || entryLocal->faultInjectorState == FaultInjectorStateFailed) { /* fault injection was already executed */ return FALSE; } /* Update the injection fault entry in hash table */ if (entryLocal->occurrence != FILEREP_UNDEFINED) { if (entryLocal->occurrence > 1) { entryLocal->occurrence--; FaultInjector_UpdateHashEntry(entryLocal); return FALSE; } entryLocal->faultInjectorState = FaultInjectorStateTriggered; FaultInjector_UpdateHashEntry(entryLocal); } /* Inject fault */ switch (entryLocal->faultInjectorType) { case FaultInjectorTypeNotSpecified: break; case FaultInjectorTypeSleep: ereport(LOG, (errmsg("fault triggered, fault name:'%s' fault type:'%s' ", FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier], FaultInjectorTypeEnumToString[entryLocal->faultInjectorType]))); pg_usleep(entryLocal->sleepTime * 1000000L); break; case FaultInjectorTypeFault: switch (entryLocal->faultInjectorIdentifier) { case FileRepConsumer: case FileRepConsumerVerification: case FileRepSender: case FileRepReceiver: case FileRepResync: case FileRepResyncInProgress: case FileRepResyncWorker: case FileRepResyncWorkerRead: case FileRepTransitionToInResyncMirrorReCreate: case FileRepTransitionToInResyncMarkReCreated: case FileRepTransitionToInResyncMarkCompleted: case FileRepTransitionToInSyncBegin: case FileRepTransitionToInSync: case FileRepTransitionToInSyncMarkCompleted: case FileRepTransitionToInSyncBeforeCheckpoint: case FileRepIsOperationCompleted: FileRep_SetSegmentState(SegmentStateFault, FaultTypeMirror); break; case FileRepTransitionToChangeTracking: FileRep_SetPostmasterReset(); break; default: FileRep_SetSegmentState(SegmentStateFault, FaultTypeIO); break; } ereport(LOG, (errmsg("fault triggered, fault name:'%s' fault type:'%s' ", FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier], FaultInjectorTypeEnumToString[entryLocal->faultInjectorType]))); break; case FaultInjectorTypeFatal: /* * If it's one time occurrence then disable the fault before it's * actually triggered because this fault errors out the transaction * and hence we wont get a chance to disable it or put it in completed * state. */ if (entryLocal->occurrence != FILEREP_UNDEFINED) { entryLocal->faultInjectorState = FaultInjectorStateCompleted; FaultInjector_UpdateHashEntry(entryLocal); } ereport(FATAL, (errmsg("fault triggered, fault name:'%s' fault type:'%s' ", FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier], FaultInjectorTypeEnumToString[entryLocal->faultInjectorType]))); break; case FaultInjectorTypePanic: /* * If it's one time occurrence then disable the fault before it's * actually triggered because this fault errors out the transaction * and hence we wont get a chance to disable it or put it in completed * state. For PANIC it may be unnecessary though. */ if (entryLocal->occurrence != FILEREP_UNDEFINED) { entryLocal->faultInjectorState = FaultInjectorStateCompleted; FaultInjector_UpdateHashEntry(entryLocal); } ereport(PANIC, (errmsg("fault triggered, fault name:'%s' fault type:'%s' ", FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier], FaultInjectorTypeEnumToString[entryLocal->faultInjectorType]))); break; case FaultInjectorTypeError: /* * If it's one time occurrence then disable the fault before it's * actually triggered because this fault errors out the transaction * and hence we wont get a chance to disable it or put it in completed * state. */ if (entryLocal->occurrence != FILEREP_UNDEFINED) { entryLocal->faultInjectorState = FaultInjectorStateCompleted; FaultInjector_UpdateHashEntry(entryLocal); } ereport(ERROR, (errmsg("fault triggered, fault name:'%s' fault type:'%s' ", FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier], FaultInjectorTypeEnumToString[entryLocal->faultInjectorType]))); break; case FaultInjectorTypeInfiniteLoop: ereport(LOG, (errmsg("fault triggered, fault name:'%s' fault type:'%s' ", FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier], FaultInjectorTypeEnumToString[entryLocal->faultInjectorType]))); if (entryLocal->faultInjectorIdentifier == FileRepImmediateShutdownRequested) cnt = entryLocal->sleepTime; for (ii=0; ii < cnt; ii++) { pg_usleep(1000000L); // sleep for 1 sec (1 sec * 3600 = 1 hour) getFileRepRoleAndState(NULL, &segmentState, NULL, NULL, NULL); if ((entryLocal->faultInjectorIdentifier != FileRepImmediateShutdownRequested) && (segmentState == SegmentStateShutdownFilerepBackends || segmentState == SegmentStateImmediateShutdown || segmentState == SegmentStateShutdown || IsFtsShudownRequested())) { break; } } break; case FaultInjectorTypeDataCorruption: ereport(LOG, (errmsg("fault triggered, fault name:'%s' fault type:'%s' ", FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier], FaultInjectorTypeEnumToString[entryLocal->faultInjectorType]))); break; case FaultInjectorTypeSuspend: { FaultInjectorEntry_s *entry; ereport(LOG, (errmsg("fault triggered, fault name:'%s' fault type:'%s' ", FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier], FaultInjectorTypeEnumToString[entryLocal->faultInjectorType]))); while ((entry = FaultInjector_LookupHashEntry(entryLocal->faultInjectorIdentifier)) != NULL && entry->faultInjectorType != FaultInjectorTypeResume) { pg_usleep(1000000L); // 1 sec } if (entry != NULL) { ereport(LOG, (errmsg("fault triggered, fault name:'%s' fault type:'%s' ", FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier], FaultInjectorTypeEnumToString[entry->faultInjectorType]))); } else { ereport(LOG, (errmsg("fault 'NULL', fault name:'%s' ", FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier]))); /* * Since the entry is gone already, we should NOT update * the entry below. (There could be other places in this * function that are under the same situation, but I'm too * tired to look for them...) */ return entryLocal->faultInjectorType; } break; } case FaultInjectorTypeSkip: ereport(LOG, (errmsg("fault triggered, fault name:'%s' fault type:'%s' ", FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier], FaultInjectorTypeEnumToString[entryLocal->faultInjectorType]))); break; case FaultInjectorTypeMemoryFull: { char *buffer = NULL; ereport(LOG, (errmsg("fault triggered, fault name:'%s' fault type:'%s' ", FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier], FaultInjectorTypeEnumToString[entryLocal->faultInjectorType]))); buffer = (char*) palloc(BLCKSZ); while (buffer != NULL) { buffer = (char*) palloc(BLCKSZ); } break; } case FaultInjectorTypeReset: case FaultInjectorTypeStatus: ereport(LOG, (errmsg("unexpected error, fault triggered, fault name:'%s' fault type:'%s' ", FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier], FaultInjectorTypeEnumToString[entryLocal->faultInjectorType]))); Assert(0); break; case FaultInjectorTypeResume: break; case FaultInjectorTypeSegv: { *(int *) 0 = 1234; break; } case FaultInjectorTypeInterrupt: { /* * The place where this type of fault is injected must have * has HOLD_INTERRUPTS() .. RESUME_INTERRUPTS() around it, otherwise * the interrupt could be handled inside the fault injector itself */ ereport(LOG, (errmsg("fault triggered, fault name:'%s' fault type:'%s' ", FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier], FaultInjectorTypeEnumToString[entryLocal->faultInjectorType]))); InterruptPending = true; QueryCancelPending = true; break; } case FaultInjectorTypeCheckpointAndPanic: { if (entryLocal->occurrence != FILEREP_UNDEFINED) { entryLocal->faultInjectorState = FaultInjectorStateCompleted; FaultInjector_UpdateHashEntry(entryLocal); } RequestCheckpoint(true, false); ereport(PANIC, (errmsg("fault triggered, fault name:'%s' fault type:'%s' ", FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier], FaultInjectorTypeEnumToString[entryLocal->faultInjectorType]))); break; } default: ereport(LOG, (errmsg("unexpected error, fault triggered, fault name:'%s' fault type:'%s' ", FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier], FaultInjectorTypeEnumToString[entryLocal->faultInjectorType]))); Assert(0); break; } if (entryLocal->occurrence != FILEREP_UNDEFINED) { entryLocal->faultInjectorState = FaultInjectorStateCompleted; } FaultInjector_UpdateHashEntry(entryLocal); return (entryLocal->faultInjectorType); }
int FaultInjector_SetFaultInjection( FaultInjectorEntry_s *entry) { int status = STATUS_OK; bool isRemoved = FALSE; getFileRepRoleAndState(&fileRepRole, &segmentState, &dataState, NULL, NULL); switch (entry->faultInjectorType) { case FaultInjectorTypeReset: { HASH_SEQ_STATUS hash_status; FaultInjectorEntry_s *entryLocal; if (entry->faultInjectorIdentifier == FaultInjectorIdAll) { hash_seq_init(&hash_status, faultInjectorShmem->hash); FiLockAcquire(); while ((entryLocal = (FaultInjectorEntry_s *) hash_seq_search(&hash_status)) != NULL) { isRemoved = FaultInjector_RemoveHashEntry(entryLocal->faultInjectorIdentifier); if (isRemoved == TRUE) { faultInjectorShmem->faultInjectorSlots--; } } FiLockRelease(); Assert(faultInjectorShmem->faultInjectorSlots == 0); } else { FiLockAcquire(); isRemoved = FaultInjector_RemoveHashEntry(entry->faultInjectorIdentifier); if (isRemoved == TRUE) { faultInjectorShmem->faultInjectorSlots--; } FiLockRelease(); } if (isRemoved == FALSE) { ereport(DEBUG1, (errmsg("LOG(fault injector): could not remove fault injection from hash" "identifier:'%s' ", FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier]))); } break; } case FaultInjectorTypeStatus: { HASH_SEQ_STATUS hash_status; FaultInjectorEntry_s *entryLocal; bool found = FALSE; if (faultInjectorShmem->hash == NULL) { status = STATUS_ERROR; break; } snprintf(entry->bufOutput, sizeof(entry->bufOutput), "Success: "); if (entry->faultInjectorIdentifier == ChangeTrackingCompactingReport) { snprintf(entry->bufOutput, sizeof(entry->bufOutput), "Success: compacting in progress %s", ChangeTrackingIsCompactingInProgress() ? "true" : "false"); break; } hash_seq_init(&hash_status, faultInjectorShmem->hash); while ((entryLocal = (FaultInjectorEntry_s *) hash_seq_search(&hash_status)) != NULL) { ereport(LOG, (errmsg("fault injector status: " "fault name:'%s' " "fault type:'%s' " "ddl statement:'%s' " "database name:'%s' " "table name:'%s' " "occurrence:'%d' " "sleep time:'%d' " "fault injection state:'%s' ", FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier], FaultInjectorTypeEnumToString[entryLocal->faultInjectorType], FaultInjectorDDLEnumToString[entryLocal->ddlStatement], entryLocal->databaseName, entryLocal->tableName, entryLocal->occurrence, entryLocal->sleepTime, FaultInjectorStateEnumToString[entryLocal->faultInjectorState]))); if (entry->faultInjectorIdentifier == entryLocal->faultInjectorIdentifier || entry->faultInjectorIdentifier == FaultInjectorIdAll) { snprintf(entry->bufOutput, sizeof(entry->bufOutput), "%s \n" "fault name:'%s' " "fault type:'%s' " "ddl statement:'%s' " "database name:'%s' " "table name:'%s' " "occurrence:'%d' " "sleep time:'%d' " "fault injection state:'%s' ", entry->bufOutput, FaultInjectorIdentifierEnumToString[entryLocal->faultInjectorIdentifier], FaultInjectorTypeEnumToString[entryLocal->faultInjectorType], FaultInjectorDDLEnumToString[entryLocal->ddlStatement], entryLocal->databaseName, entryLocal->tableName, entryLocal->occurrence, entryLocal->sleepTime, FaultInjectorStateEnumToString[entryLocal->faultInjectorState]); found = TRUE; } } if (found == FALSE) { snprintf(entry->bufOutput, sizeof(entry->bufOutput), "Failure: " "fault name:'%s' not set", FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier]); } break; } case FaultInjectorTypeResume: ereport(LOG, (errmsg("fault triggered, fault name:'%s' fault type:'%s' ", FaultInjectorIdentifierEnumToString[entry->faultInjectorIdentifier], FaultInjectorTypeEnumToString[entry->faultInjectorType]))); FaultInjector_UpdateHashEntry(entry); break; default: status = FaultInjector_NewHashEntry(entry); break; } return status; }