/*
* FileRepPrimary_StartResyncWorker()
*/
void
FileRepPrimary_StartResyncWorker(void)
{
int status = STATUS_OK;
FileRep_InsertConfigLogEntry("start resync worker");
Insist(fileRepRole == FileRepPrimaryRole);
while (1) {
if (status != STATUS_OK)
{
FileRep_SetSegmentState(SegmentStateFault, FaultTypeMirror);
FileRepSubProcess_SetState(FileRepStateFault);
}
/*
* We are waiting for following conditions to move forward:
*
* Database is running
* And
* if dataState is InResync, we wait for FileRepSubProcess to Ready state
* else don't wait
*/
while (!isDatabaseRunning() ||
!(dataState == DataStateInResync ? FileRepSubProcess_GetState() == FileRepStateReady : true))
{
FileRepSubProcess_ProcessSignals();
if (FileRepSubProcess_GetState() == FileRepStateShutdown ||
FileRepSubProcess_GetState() == FileRepStateShutdownBackends)
{
break;
}
pg_usleep(50000L); /* 50 ms */
}
if (FileRepSubProcess_GetState() == FileRepStateShutdown ||
FileRepSubProcess_GetState() == FileRepStateShutdownBackends) {
break;
}
FileRepSubProcess_InitHeapAccess();
status = FileRepPrimary_RunResyncWorker();
if (status != STATUS_OK) {
continue;
}
break;
} // while(1)
}
/*
*
* FileRepAckPrimary_StartConsumer
*/
void
FileRepAckPrimary_StartConsumer(void)
{
int status = STATUS_OK;
FileRep_InsertConfigLogEntry("run consumer");
while (1) {
if (status != STATUS_OK)
{
FileRep_SetSegmentState(SegmentStateFault, FaultTypeMirror);
FileRepSubProcess_SetState(FileRepStateFault);
}
while (FileRepSubProcess_GetState() == FileRepStateFault ||
(fileRepShmemArray[0]->state == FileRepStateNotInitialized &&
FileRepSubProcess_GetState() != FileRepStateShutdown)) {
FileRepSubProcess_ProcessSignals();
pg_usleep(50000L); /* 50 ms */
}
if (FileRepSubProcess_GetState() == FileRepStateShutdown) {
break;
}
status = FileRepAckPrimary_RunConsumer();
} // while(1)
if (FileRepSubProcess_GetState() == FileRepStateShutdown) {
/* perform graceful shutdown */
}
LWLockAcquire(FileRepAckHashShmemLock, LW_EXCLUSIVE);
FileRep_IpcSignal(fileRepIpcArray[fileRepAckHashShmem->ipcArrayIndex]->semP,
&fileRepIpcArray[fileRepAckHashShmem->ipcArrayIndex]->refCountSemP);
LWLockRelease(FileRepAckHashShmemLock);
/* NOTE free memory (if any) */
return;
}
/*
*
* FileRepPrimary_StartRecoveryInSync()
*
*
*/
static void
FileRepPrimary_StartRecoveryInSync(void)
{
int status = STATUS_OK;
FileRep_InsertConfigLogEntry("run recovery");
while (1) {
if (status != STATUS_OK)
{
FileRep_SetSegmentState(SegmentStateFault, FaultTypeMirror);
FileRepSubProcess_SetState(FileRepStateFault);
}
while (FileRepSubProcess_GetState() == FileRepStateFault ||
(fileRepShmemArray[0]->state == FileRepStateNotInitialized &&
FileRepSubProcess_GetState() != FileRepStateShutdownBackends &&
FileRepSubProcess_GetState() != FileRepStateShutdown)) {
FileRepSubProcess_ProcessSignals();
pg_usleep(50000L); /* 50 ms */
}
if (FileRepSubProcess_GetState() == FileRepStateShutdown ||
FileRepSubProcess_GetState() == FileRepStateShutdownBackends) {
break;
}
if (FileRepSubProcess_GetState() == FileRepStateReady) {
break;
}
Insist(fileRepRole == FileRepPrimaryRole);
Insist(dataState == DataStateInSync);
status = FileRepPrimary_RunRecoveryInSync();
} // while(1)
}
/*
* FileRepPrimary_StartResyncWorker()
*/
void
FileRepPrimary_StartResyncWorker(void)
{
int status = STATUS_OK;
FileRep_InsertConfigLogEntry("start resync worker");
Insist(fileRepRole == FileRepPrimaryRole);
while (1) {
if (status != STATUS_OK)
{
FileRep_SetSegmentState(SegmentStateFault, FaultTypeMirror);
FileRepSubProcess_SetState(FileRepStateFault);
}
while (FileRepSubProcess_GetState() != FileRepStateShutdown &&
FileRepSubProcess_GetState() != FileRepStateShutdownBackends &&
! (FileRepSubProcess_GetState() == FileRepStateReady &&
dataState == DataStateInResync)) {
FileRepSubProcess_ProcessSignals();
pg_usleep(50000L); /* 50 ms */
}
if (FileRepSubProcess_GetState() == FileRepStateShutdown ||
FileRepSubProcess_GetState() == FileRepStateShutdownBackends) {
break;
}
status = FileRepPrimary_RunResyncWorker();
if (status != STATUS_OK) {
continue;
}
break;
} // while(1)
}
/*
* FileRepSubProcess_ProcessSignals()
*
*/
bool
FileRepSubProcess_ProcessSignals()
{
bool processExit = false;
if (reloadConfigFile)
{
reloadConfigFile = false;
ProcessConfigFile(PGC_SIGHUP);
FileRep_SetFileRepRetry();
}
if (shutdownRequested)
{
SegmentState_e segmentState;
getPrimaryMirrorStatusCodes(NULL, &segmentState, NULL, NULL);
shutdownRequested = false;
if (segmentState == SegmentStateShutdownFilerepBackends)
{
processExit = FileRepIsBackendSubProcess(fileRepProcessType);
FileRepSubProcess_SetState(FileRepStateShutdownBackends);
}
else
{
processExit = true;
FileRepSubProcess_SetState(FileRepStateShutdown);
}
}
/*
* Immediate shutdown if postmaster or main filerep process (parent) is
* not alive to avoid manual cleanup.
*/
if (!PostmasterIsAlive(false /* amDirectChild */ ) || !ParentProcIsAlive())
{
quickdie_impl();
}
for (;;)
{
/* check to see if change required */
sig_atomic_t curStateChangeRequestCounter = stateChangeRequestCounter;
if (curStateChangeRequestCounter == lastChangeRequestProcessCounterValue)
break;
lastChangeRequestProcessCounterValue = curStateChangeRequestCounter;
/* do the change in local memory */
getFileRepRoleAndState(&fileRepRole, &segmentState, &dataState, NULL, NULL);
switch (segmentState)
{
case SegmentStateNotInitialized:
FileRepSubProcess_SetState(FileRepStateNotInitialized);
break;
case SegmentStateInitialization:
FileRepSubProcess_SetState(FileRepStateInitialization);
break;
case SegmentStateInResyncTransition:
FileRepSubProcess_SetState(FileRepStateInitialization);
break;
case SegmentStateInChangeTrackingTransition:
case SegmentStateInSyncTransition:
/* fileRepState remains Ready */
break;
case SegmentStateChangeTrackingDisabled:
case SegmentStateReady:
FileRepSubProcess_SetState(FileRepStateReady);
break;
case SegmentStateFault:
FileRepSubProcess_SetState(FileRepStateFault);
break;
case SegmentStateShutdownFilerepBackends:
if (fileRepRole == FileRepPrimaryRole)
{
FileRepSubProcess_SetState(FileRepStateShutdownBackends);
}
else
{
processExit = true;
FileRepSubProcess_SetState(FileRepStateShutdown);
}
break;
case SegmentStateImmediateShutdown:
case SegmentStateShutdown:
processExit = true;
FileRepSubProcess_SetState(FileRepStateShutdown);
break;
default:
Assert(0);
break;
} //switch ()
if (processExit == true)
{
FileRep_IpcSignalAll();
}
}
return (processExit);
}
/*
*
* FileRepPrimary_StartRecoveryInChangeTracking()
*
*/
static void
FileRepPrimary_StartRecoveryInChangeTracking(void)
{
FileRep_InsertConfigLogEntry("run recovery");
while (1) {
while (FileRepSubProcess_GetState() == FileRepStateFault) {
FileRepSubProcess_ProcessSignals();
pg_usleep(50000L); /* 50 ms */
}
if (FileRepSubProcess_GetState() == FileRepStateShutdown ||
FileRepSubProcess_GetState() == FileRepStateShutdownBackends) {
break;
}
Insist(fileRepRole == FileRepPrimaryRole);
Insist(dataState == DataStateInChangeTracking);
Insist(FileRepSubProcess_GetState() != FileRepStateReady);
if (ChangeTracking_RetrieveIsTransitionToInsync())
{
ChangeTracking_DropAll();
}
else
{
if (ChangeTracking_RetrieveIsTransitionToResync() == FALSE &&
isFullResync())
{
ChangeTracking_MarkFullResync();
/* segmentState == SegmentStateChangeTrackingDisabled */
getFileRepRoleAndState(&fileRepRole, &segmentState, &dataState, NULL, NULL);
Assert(segmentState == SegmentStateChangeTrackingDisabled);
/* database is resumed */
primaryMirrorSetIOSuspended(FALSE);
FileRep_InsertConfigLogEntry("change tracking recovery completed");
break;
}
else
{
ChangeTracking_MarkIncrResync();
}
}
XLogInChangeTrackingTransition();
/* NOTE: Any error during change tracking will result in disabling Change Tracking */
FileRepSubProcess_SetState(FileRepStateReady);
/* database is resumed */
primaryMirrorSetIOSuspended(FALSE);
FileRep_InsertConfigLogEntry("change tracking recovery completed");
break;
} // while(1)
}
/*
*
* FileRepPrimary_RunRecoveryInSync()
*
* 1) Recover Flat Files
* a) pg_control file
* b) pg_database file
* c) pg_auth file
* d) pg_twophase directory
* e) Slru directories
* *) pg_clog
* *) pg_multixact
* *) pg_distributedlog
* *) pg_distributedxidmap
* *) pg_subtrans
*
* 2) Reconcile xlog EOF
*
*/
static int
FileRepPrimary_RunRecoveryInSync(void)
{
int status = STATUS_OK;
FileRep_InsertConfigLogEntry("run recovery of flat files");
while (1) {
status = XLogRecoverMirrorControlFile();
if (status != STATUS_OK) {
break;
}
FileRepSubProcess_ProcessSignals();
if (FileRepSubProcess_GetState() != FileRepStateInitialization) {
break;
}
status = XLogReconcileEofPrimary();
if (status != STATUS_OK) {
break;
}
FileRepSubProcess_ProcessSignals();
if (FileRepSubProcess_GetState() != FileRepStateInitialization) {
break;
}
MirroredFlatFile_DropTemporaryFiles();
FileRepSubProcess_ProcessSignals();
if (FileRepSubProcess_GetState() != FileRepStateInitialization) {
break;
}
MirroredFlatFile_MirrorDropTemporaryFiles();
FileRepSubProcess_ProcessSignals();
if (FileRepSubProcess_GetState() != FileRepStateInitialization) {
break;
}
status = FlatFilesRecoverMirror();
if (status != STATUS_OK) {
break;
}
FileRepSubProcess_ProcessSignals();
if (FileRepSubProcess_GetState() != FileRepStateInitialization) {
break;
}
status = TwoPhaseRecoverMirror();
if (status != STATUS_OK) {
break;
}
FileRepSubProcess_ProcessSignals();
if (FileRepSubProcess_GetState() != FileRepStateInitialization) {
break;
}
status = SlruRecoverMirror();
if (status != STATUS_OK) {
break;
}
FileRepSubProcess_ProcessSignals();
if (FileRepSubProcess_GetState() != FileRepStateInitialization) {
break;
}
FileRepSubProcess_SetState(FileRepStateReady);
break;
}
return status;
}
/*
*
* FileRepPrimary_StartSender
*/
void
FileRepAckMirror_StartSender(void)
{
int status = STATUS_OK;
int retry = 0;
struct timeval currentTime;
pg_time_t beginTime = 0;
pg_time_t endTime = 0;
FileRep_InsertConfigLogEntry("start sender ack");
while (1) {
if (status != STATUS_OK) {
FileRep_SetSegmentState(SegmentStateFault, FaultTypeMirror);
FileRepSubProcess_SetState(FileRepStateFault);
}
while (FileRepSubProcess_GetState() == FileRepStateInitialization ||
FileRepSubProcess_GetState() == FileRepStateFault ||
(fileRepShmemArray[0]->state == FileRepStateNotInitialized &&
FileRepSubProcess_GetState() != FileRepStateShutdown )) {
FileRepSubProcess_ProcessSignals();
pg_usleep(50000L); /* 50 ms */
}
if (FileRepSubProcess_GetState() == FileRepStateShutdown) {
break;
}
{
char tmpBuf[FILEREP_MAX_LOG_DESCRIPTION_LEN];
snprintf(tmpBuf, sizeof(tmpBuf), "primary address(port) '%s(%d)' mirror address(port) '%s(%d)' ",
fileRepPrimaryHostAddress,
fileRepPrimaryPort,
fileRepMirrorHostAddress,
fileRepMirrorPort);
FileRep_InsertConfigLogEntry(tmpBuf);
}
Insist(fileRepRole == FileRepMirrorRole);
status = FileRepConnClient_EstablishConnection(
fileRepPrimaryHostAddress,
fileRepPrimaryPort,
FALSE /* reportError */);
if (status != STATUS_OK)
{
gettimeofday(¤tTime, NULL);
beginTime = (pg_time_t) currentTime.tv_sec;
}
while (status != STATUS_OK &&
FileRep_IsRetry(retry) &&
(endTime - beginTime) < gp_segment_connect_timeout)
{
FileRep_Sleep10ms(retry);
FileRep_IncrementRetry(retry);
gettimeofday(¤tTime, NULL);
endTime = (pg_time_t) currentTime.tv_sec;
status = FileRepConnClient_EstablishConnection(
fileRepPrimaryHostAddress,
fileRepPrimaryPort,
(retry == file_rep_retry && file_rep_retry != 0) ||
((endTime - beginTime) > gp_segment_connect_timeout) ? TRUE : FALSE);
if (FileRepSubProcess_IsStateTransitionRequested())
{
break;
}
}
if (status != STATUS_OK) {
continue;
}
FileRep_SetFileRepRetry();
status = FileRepAckMirror_RunSender();
} // while(1)
FileRepConnClient_CloseConnection();
return;
}
/*
*
* FileRepAckPrimary_StartReceiver
*/
void
FileRepAckPrimary_StartReceiver(void)
{
int status = STATUS_OK;
struct timeval currentTime;
pg_time_t beginTime = 0;
pg_time_t endTime = 0;
int retval = 0;
FileRep_InsertConfigLogEntry("start receiver ack");
{
char tmpBuf[FILEREP_MAX_LOG_DESCRIPTION_LEN];
snprintf(tmpBuf, sizeof(tmpBuf), "primary address(port) '%s(%d)' mirror address(port) '%s(%d)' ",
fileRepPrimaryHostAddress,
fileRepPrimaryPort,
fileRepMirrorHostAddress,
fileRepMirrorPort);
FileRep_InsertConfigLogEntry(tmpBuf);
}
FileRepAckPrimary_ShmemReInit();
Insist(fileRepRole == FileRepPrimaryRole);
if (filerep_inject_listener_fault)
{
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("mirror failure, "
"injected fault by guc filerep_inject_listener_fault, "
"failover requested"),
FileRep_errcontext()));
FileRep_SetSegmentState(SegmentStateFault, FaultTypeMirror);
FileRepSubProcess_SetState(FileRepStateFault);
FileRepSubProcess_ProcessSignals();
return;
}
status = FileRepConnServer_StartListener(
fileRepPrimaryHostAddress,
fileRepPrimaryPort);
gettimeofday(¤tTime, NULL);
beginTime = (pg_time_t) currentTime.tv_sec;
while (1) {
if (status != STATUS_OK)
{
FileRep_SetSegmentState(SegmentStateFault, FaultTypeMirror);
FileRepSubProcess_SetState(FileRepStateFault);
}
while (FileRepSubProcess_GetState() == FileRepStateFault) {
FileRepSubProcess_ProcessSignals();
pg_usleep(50000L); /* 50 ms */
}
if (FileRepSubProcess_GetState() == FileRepStateShutdown) {
break;
}
PG_SETMASK(&BlockSig);
retval = FileRepConnServer_Select();
PG_SETMASK(&UnBlockSig);
gettimeofday(¤tTime, NULL);
endTime = (pg_time_t) currentTime.tv_sec;
if ((endTime - beginTime) > gp_segment_connect_timeout)
{
ereport(WARNING,
(errmsg("mirror failure, "
"no connection was established from client from mirror, "
"primary address(port) '%s(%d)' mirror address(port) '%s(%d)' timeout reached '%d' "
"failover requested",
fileRepPrimaryHostAddress,
fileRepPrimaryPort,
fileRepMirrorHostAddress,
fileRepMirrorPort,
gp_segment_connect_timeout),
errSendAlert(true),
FileRep_errcontext()));
status = STATUS_ERROR;
continue;
}
/*
* check and process any signals received
* The routine returns TRUE if the received signal requests
* process shutdown.
*/
if (FileRepSubProcess_ProcessSignals()) {
continue;
}
if (retval < 0) {
status = STATUS_ERROR;
continue;
}
if (retval == 0) {
continue;
}
Assert(retval > 0);
status = FileRepConnServer_CreateConnection();
if (status != STATUS_OK) {
continue;
}
status = FileRepConnServer_ReceiveStartupPacket();
if (status != STATUS_OK) {
continue;
}
fileRepShmemArray[0]->state = FileRepStateInitialization;
status = FileRepAckPrimary_RunReceiver();
} // while(1)
FileRepConnServer_CloseConnection();
return;
}
