/*
* SenderLoop
*
*/
static int
FileRepAckMirror_RunSender(void)
{
FileRepShmemMessageDescr_s *fileRepShmemMessageDescr=NULL;
char *fileRepMessage;
int status = STATUS_OK;
bool movePositionConsume = FALSE;
FileRepConsumerProcIndex_e messageType;
FileRepMessageHeader_s *fileRepMessageHeader;
FileRepShmem_s *fileRepAckShmem = NULL;
FileRep_InsertConfigLogEntry("run sender ack");
fileRepAckShmem = fileRepAckShmemArray[FILEREP_OUTGOING_MESSAGE_QUEUE];
while (1) {
LWLockAcquire(FileRepAckShmemLock, LW_EXCLUSIVE);
if (movePositionConsume) {
fileRepAckShmem->positionConsume =
fileRepAckShmem->positionConsume +
fileRepShmemMessageDescr->messageLength +
sizeof(FileRepShmemMessageDescr_s);
if (fileRepAckShmem->positionConsume == fileRepAckShmem->positionWraparound &&
fileRepAckShmem->positionInsert != fileRepAckShmem->positionWraparound) {
fileRepAckShmem->positionConsume = fileRepAckShmem->positionBegin;
fileRepAckShmem->positionWraparound = fileRepAckShmem->positionEnd;
}
FileRep_IpcSignal(fileRepIpcArray[fileRepAckShmem->ipcArrayIndex]->semP,
&fileRepIpcArray[fileRepAckShmem->ipcArrayIndex]->refCountSemP);
}
fileRepShmemMessageDescr =
(FileRepShmemMessageDescr_s*) fileRepAckShmem->positionConsume;
while ((fileRepAckShmem->positionConsume == fileRepAckShmem->positionInsert) ||
((fileRepAckShmem->positionConsume != fileRepAckShmem->positionInsert) &&
(fileRepShmemMessageDescr->messageState != FileRepShmemMessageStateReady))) {
fileRepIpcArray[fileRepAckShmem->ipcArrayIndex]->refCountSemC++;
LWLockRelease(FileRepAckShmemLock);
FileRepSubProcess_ProcessSignals();
if (FileRepSubProcess_GetState() != FileRepStateReady) {
LWLockAcquire(FileRepAckShmemLock, LW_EXCLUSIVE);
break;
}
FileRep_IpcWait(fileRepIpcArray[fileRepAckShmem->ipcArrayIndex]->semC, &fileRepIpcArray[fileRepAckShmem->ipcArrayIndex]->refCountSemC, FileRepAckShmemLock);
LWLockAcquire(FileRepAckShmemLock, LW_EXCLUSIVE);
if (fileRepAckShmem->positionConsume == fileRepAckShmem->positionWraparound &&
fileRepAckShmem->positionInsert != fileRepAckShmem->positionWraparound) {
fileRepAckShmem->positionConsume = fileRepAckShmem->positionBegin;
fileRepAckShmem->positionWraparound = fileRepAckShmem->positionEnd;
}
/* Re-assign to find if messageState is changed */
fileRepShmemMessageDescr =
(FileRepShmemMessageDescr_s*) fileRepAckShmem->positionConsume;
} // while internal
fileRepAckShmem->consumeCount++;
LWLockRelease(FileRepAckShmemLock);
FileRepSubProcess_ProcessSignals();
if (FileRepSubProcess_GetState() != FileRepStateReady) {
break;
}
FileRep_InsertLogEntry(
"M_RunSenderAck",
FileRep_GetFlatFileIdentifier("", ""),
FileRepRelationTypeNotSpecified,
FileRepOperationNotSpecified,
FILEREP_UNDEFINED,
FILEREP_UNDEFINED,
FileRepAckStateNotInitialized,
FILEREP_UNDEFINED,
FILEREP_UNDEFINED);
#ifdef FAULT_INJECTOR
FaultInjector_InjectFaultIfSet(
FileRepSender,
DDLNotSpecified,
"", //databaseName
""); // tableName
#endif
fileRepMessage = (char*) (fileRepAckShmem->positionConsume +
sizeof(FileRepShmemMessageDescr_s));
fileRepMessageHeader = (FileRepMessageHeader_s*) (fileRepAckShmem->positionConsume +
sizeof(FileRepShmemMessageDescr_s));
messageType = FileRepMessageTypeXLog;
if (! FileRepConnClient_SendMessage(
messageType,
fileRepShmemMessageDescr->messageSync,
fileRepMessage,
fileRepShmemMessageDescr->messageLength))
{
ereport(WARNING,
(errcode_for_socket_access(),
errmsg("mirror failure, "
"could not sent ack message to primary : %m, "
"failover requested"),
errhint("run gprecoverseg to re-establish mirror connectivity"),
FileRep_errdetail_ShmemAck(),
FileRep_errcontext()));
status = STATUS_ERROR;
break;
}
movePositionConsume = TRUE;
} // while(1)
FileRepConnClient_CloseConnection();
return status;
}
/*
* Backend checks if acknowledgement that its operation is completed
* is received from mirror.
* If acknowledgement is received (state == FileRepAckStateCompleted) then
* a) entry is removed from hash
* b) TRUE is returned
*/
bool
FileRepAckPrimary_IsOperationCompleted(
FileRepIdentifier_u fileRepIdentifier,
FileRepRelationType_e fileRepRelationType)
{
FileRepAckHashEntry_s *entry = NULL;
bool isCompleted = FALSE;
bool isRemoved;
FileName fileName = NULL;
int retry = 0;
bool retval = FALSE;
bool wait = FALSE;
fileName = FileRep_GetFileName(fileRepIdentifier, fileRepRelationType);
while ((isCompleted == FALSE) && FileRep_IsRetry(retry))
{
LWLockAcquire(FileRepAckHashShmemLock, LW_EXCLUSIVE);
entry = FileRepAckPrimary_LookupHashEntry(fileName);
if (entry == NULL)
{
LWLockRelease(FileRepAckHashShmemLock);
break;
}
if (! FileRep_IsIpcSleep(entry->fileRepOperation))
{
if (wait == TRUE)
{
wait = FALSE;
}
}
switch (entry->fileRepAckState)
{
case FileRepAckStateWaiting:
/* No Operation */
break;
case FileRepAckStateCompleted:
retval = TRUE;
xLogEof = entry->xLogEof;
mirrorStatus = entry->mirrorStatus;
/* no BREAK */
case FileRepAckStateMirrorInFault:
isCompleted = TRUE;
isRemoved = FileRepAckPrimary_RemoveHashEntry(fileName);
Assert(isRemoved == TRUE);
break;
default:
break;
}
if (isCompleted == false)
{
if (! FileRep_IsIpcSleep(entry->fileRepOperation))
{
fileRepIpcArray[fileRepAckHashShmem->ipcArrayIndex]->refCountSemP++;
wait = TRUE;
}
}
LWLockRelease(FileRepAckHashShmemLock);
if (isCompleted == false)
{
if (FileRepSubProcess_IsStateTransitionRequested())
{
break;
}
if (FileRep_IsIpcSleep(entry->fileRepOperation))
{
FileRep_Sleep1ms(retry);
if (retry == (3 * file_rep_retry / 4))
ereport(WARNING,
(errmsg("threshold '75' percent of 'gp_segment_connect_timeout=%d' is reached, "
"mirror may not be able to keep up with primary, "
"primary may transition to change tracking",
gp_segment_connect_timeout),
errhint("increase guc 'gp_segment_connect_timeout' by 'gpconfig' and 'gpstop -u' "),
errSendAlert(true)));
FileRep_IncrementRetry(retry);
}
else
{
FileRep_IpcWait(fileRepIpcArray[fileRepAckHashShmem->ipcArrayIndex]->semP, &fileRepIpcArray[fileRepAckHashShmem->ipcArrayIndex]->refCountSemP, FileRepAckHashShmemLock);
}
/*
* if the message was from the main filerep process then it is a
* graceful shutdown message to the mirror. We don't want to stall
* shutdown if the mirror is unavailable so we wait a smaller amount
* of time
*/
if ( entry->fileRepOperation == FileRepOperationShutdown &&
retry == 50)
{
FileRepAckPrimary_RemoveHashEntry(fileName);
break;
}
}
}
if (retval == FALSE)
{
mirrorStatus = FileRepStatusMirrorLossOccurred;
if (! primaryMirrorIsIOSuspended())
{
ereport(WARNING,
(errmsg("mirror failure, "
"could not complete mirrored request identifier '%s' ack state '%s', "
"failover requested",
(fileName == NULL) ? "<null>" : fileName,
(entry == NULL) ? "<entry not found>" : FileRepAckStateToString[entry->fileRepAckState]),
errhint("run gprecoverseg to re-establish mirror connectivity"),
FileRep_errdetail_ShmemAck(),
FileRep_errcontext()));
}
}
if (fileName)
{
pfree(fileName);
fileName = NULL;
}
return retval;
}
/*
* FileRepAckPrimary_RunConsumer()
*/
static int
FileRepAckPrimary_RunConsumer(void)
{
FileRepShmemMessageDescr_s *fileRepShmemMessageDescr = NULL;
FileRepMessageHeader_s *fileRepMessageHeader = NULL;
pg_crc32 *fileRepMessageHeaderCrc;
pg_crc32 messageHeaderCrcLocal = 0;
int status = STATUS_OK;
bool movePositionConsume = FALSE;
FileRepShmem_s *fileRepAckShmem = NULL;
FileRep_InsertConfigLogEntry("run consumer");
fileRepAckShmem = fileRepAckShmemArray[FILEREP_ACKSHMEM_MESSAGE_SLOT_PRIMARY_ACK];
while (1) {
LWLockAcquire(FileRepAckShmemLock, LW_EXCLUSIVE);
if (movePositionConsume) {
fileRepAckShmem->positionConsume =
fileRepAckShmem->positionConsume +
fileRepShmemMessageDescr->messageLength +
sizeof(FileRepShmemMessageDescr_s);
if (fileRepAckShmem->positionConsume == fileRepAckShmem->positionWraparound &&
fileRepAckShmem->positionInsert != fileRepAckShmem->positionWraparound) {
fileRepAckShmem->positionConsume = fileRepAckShmem->positionBegin;
fileRepAckShmem->positionWraparound = fileRepAckShmem->positionEnd;
}
FileRep_IpcSignal(fileRepIpcArray[fileRepAckShmem->ipcArrayIndex]->semP,
&fileRepIpcArray[fileRepAckShmem->ipcArrayIndex]->refCountSemP);
}
fileRepShmemMessageDescr =
(FileRepShmemMessageDescr_s*) fileRepAckShmem->positionConsume;
while ((fileRepAckShmem->positionConsume == fileRepAckShmem->positionInsert) ||
((fileRepAckShmem->positionConsume != fileRepAckShmem->positionInsert) &&
(fileRepShmemMessageDescr->messageState != FileRepShmemMessageStateReady))) {
fileRepIpcArray[fileRepAckShmem->ipcArrayIndex]->refCountSemC++;
LWLockRelease(FileRepAckShmemLock);
FileRepSubProcess_ProcessSignals();
if (FileRepSubProcess_GetState() != FileRepStateReady &&
FileRepSubProcess_GetState() != FileRepStateInitialization) {
LWLockAcquire(FileRepAckShmemLock, LW_EXCLUSIVE);
break;
}
FileRep_IpcWait(fileRepIpcArray[fileRepAckShmem->ipcArrayIndex]->semC, &fileRepIpcArray[fileRepAckShmem->ipcArrayIndex]->refCountSemC, FileRepAckShmemLock);
LWLockAcquire(FileRepAckShmemLock, LW_EXCLUSIVE);
if (fileRepAckShmem->positionConsume == fileRepAckShmem->positionWraparound &&
fileRepAckShmem->positionInsert != fileRepAckShmem->positionWraparound) {
fileRepAckShmem->positionConsume = fileRepAckShmem->positionBegin;
fileRepAckShmem->positionWraparound = fileRepAckShmem->positionEnd;
}
/* Re-assign to find if messageState is changed */
fileRepShmemMessageDescr =
(FileRepShmemMessageDescr_s*) fileRepAckShmem->positionConsume;
} // internal while
fileRepAckShmem->consumeCount++;
LWLockRelease(FileRepAckShmemLock);
FileRepSubProcess_ProcessSignals();
if (FileRepSubProcess_GetState() != FileRepStateReady &&
FileRepSubProcess_GetState() != FileRepStateInitialization) {
break;
}
SIMPLE_FAULT_INJECTOR(FileRepConsumer);
/* Calculate and compare FileRepMessageHeader_s Crc */
fileRepMessageHeader = (FileRepMessageHeader_s*) (fileRepAckShmem->positionConsume +
sizeof(FileRepShmemMessageDescr_s));
FileRep_CalculateCrc((char *) fileRepMessageHeader,
sizeof(FileRepMessageHeader_s),
&messageHeaderCrcLocal);
fileRepMessageHeaderCrc =
(pg_crc32 *) (fileRepAckShmem->positionConsume +
sizeof(FileRepMessageHeader_s) +
sizeof(FileRepShmemMessageDescr_s));
if (*fileRepMessageHeaderCrc != messageHeaderCrcLocal)
{
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("mirror failure, "
"could not match ack message header checksum between primary '%u' and mirror '%u', "
"failover requested",
*fileRepMessageHeaderCrc,
messageHeaderCrcLocal),
errhint("run gprecoverseg to re-establish mirror connectivity"),
FileRep_errdetail(fileRepMessageHeader->fileRepIdentifier,
fileRepMessageHeader->fileRepRelationType,
fileRepMessageHeader->fileRepOperation,
fileRepMessageHeader->messageCount),
FileRep_errdetail_ShmemAck(),
FileRep_errcontext()));
break;
}
/* Write operation is never acknowledged.
* That means message should never have body.
* CRC of body should be always 0.
*/
Assert(fileRepMessageHeader->fileRepOperation != FileRepOperationWrite);
Assert(fileRepMessageHeader->fileRepMessageBodyCrc == 0);
switch (fileRepMessageHeader->fileRepOperation)
{
case FileRepOperationReconcileXLogEof:
xLogEof = fileRepMessageHeader->fileRepOperationDescription.reconcile.xLogEof;
if (Debug_filerep_print)
ereport(LOG,
(errmsg("ack reconcile xlogid '%d' xrecoff '%d' ",
xLogEof.xlogid,
xLogEof.xrecoff)));
break;
case FileRepOperationValidation:
mirrorStatus = fileRepMessageHeader->fileRepOperationDescription.validation.mirrorStatus;
if (Debug_filerep_print)
ereport(LOG,
(errmsg("ack validation status '%s' ",
FileRepStatusToString[mirrorStatus])));
break;
case FileRepOperationCreate:
mirrorStatus = fileRepMessageHeader->fileRepOperationDescription.create.mirrorStatus;
if (Debug_filerep_print)
ereport(LOG,
(errmsg("ack create status '%s' ",
FileRepStatusToString[mirrorStatus])));
break;
case FileRepOperationStartSlruChecksum:
mirrorStatus =
fileRepMessageHeader->fileRepOperationDescription.startChecksum.mirrorStatus;
if (Debug_filerep_print)
{
ereport(LOG,
(errmsg("ack start SLRU checksum: status = '%s', directory = '%s' ",
FileRepStatusToString[mirrorStatus],
fileRepMessageHeader->fileRepIdentifier.fileRepFlatFileIdentifier.directorySimpleName)));
}
break;
case FileRepOperationVerifySlruDirectoryChecksum:
mirrorStatus =
fileRepMessageHeader->fileRepOperationDescription.verifyDirectoryChecksum.mirrorStatus;
if (Debug_filerep_print)
{
ereport(LOG,
(errmsg("ack verify SLRU directory checksum: status = '%s', directory = '%s' ",
FileRepStatusToString[mirrorStatus],
fileRepMessageHeader->fileRepIdentifier.fileRepFlatFileIdentifier.directorySimpleName)));
}
break;
default:
break;
}
if (fileRepMessageHeader->fileRepAckState != FileRepAckStateCompleted) {
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("mirror failure, "
"could not complete operation on mirror ack state '%s', "
"failover requested",
FileRepAckStateToString[fileRepMessageHeader->fileRepAckState]),
errhint("run gprecoverseg to re-establish mirror connectivity"),
errSendAlert(true),
FileRep_errdetail(fileRepMessageHeader->fileRepIdentifier,
fileRepMessageHeader->fileRepRelationType,
fileRepMessageHeader->fileRepOperation,
fileRepMessageHeader->messageCount),
FileRep_errdetail_Shmem(),
FileRep_errdetail_ShmemAck(),
FileRep_errcontext()));
/*
* FAULT has to be set before entry is updated in ack hash table
* in order to suspend backend process.
*/
FileRep_SetSegmentState(SegmentStateFault, FaultTypeMirror);
FileRepSubProcess_ProcessSignals();
}
if (FileRepAckPrimary_UpdateHashEntry(
fileRepMessageHeader->fileRepIdentifier,
fileRepMessageHeader->fileRepRelationType,
fileRepMessageHeader->fileRepAckState) != STATUS_OK) {
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("mirror failure, "
"could not update ack state '%s' in ack hash table, "
"failover requested",
FileRepAckStateToString[fileRepMessageHeader->fileRepAckState]),
errhint("run gprecoverseg to re-establish mirror connectivity"),
errSendAlert(true),
FileRep_errdetail(fileRepMessageHeader->fileRepIdentifier,
fileRepMessageHeader->fileRepRelationType,
fileRepMessageHeader->fileRepOperation,
fileRepMessageHeader->messageCount),
FileRep_errdetail_Shmem(),
FileRep_errdetail_ShmemAck(),
FileRep_errcontext()));
}
FileRep_InsertLogEntry(
"P_RunConsumer",
fileRepMessageHeader->fileRepIdentifier,
fileRepMessageHeader->fileRepRelationType,
fileRepMessageHeader->fileRepOperation,
messageHeaderCrcLocal,
fileRepMessageHeader->fileRepMessageBodyCrc,
fileRepMessageHeader->fileRepAckState,
FILEREP_UNDEFINED,
fileRepMessageHeader->messageCount);
if (status != STATUS_OK) {
break;
}
movePositionConsume = TRUE;
} // while(1)
return status;
}
/*
* It is called by backend process to insert new ack entry into hash table.
*/
int
FileRepAckPrimary_NewHashEntry(
FileRepIdentifier_u fileRepIdentifier,
FileRepOperation_e fileRepOperation,
FileRepRelationType_e fileRepRelationType)
{
FileRepAckHashEntry_s *entry=NULL;
bool exists = FALSE;
FileName fileName = NULL;
int status = STATUS_OK;
int retry = 0;
bool wait = FALSE;
fileName = FileRep_GetFileName(fileRepIdentifier, fileRepRelationType);
while (FileRep_IsRetry(retry)) {
LWLockAcquire(FileRepAckHashShmemLock, LW_EXCLUSIVE);
if (! FileRep_IsIpcSleep(fileRepOperation))
{
if (wait == TRUE)
{
wait = FALSE;
}
}
entry = FileRepAckPrimary_InsertHashEntry(fileName, &exists);
if (entry == NULL)
{
LWLockRelease(FileRepAckHashShmemLock);
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("mirror failure, "
"could not insert ack entry into ack table, no memory "
"failover requested"),
errhint("run gprecoverseg to re-establish mirror connectivity"),
FileRep_errdetail(fileRepIdentifier,
fileRepRelationType,
fileRepOperation,
FILEREP_UNDEFINED),
FileRep_errcontext()));
goto exit;
}
if (exists)
{
if (! FileRep_IsIpcSleep(fileRepOperation))
{
fileRepIpcArray[fileRepAckHashShmem->ipcArrayIndex]->refCountSemC++;
wait = TRUE;
}
LWLockRelease(FileRepAckHashShmemLock);
if (FileRepSubProcess_IsStateTransitionRequested())
{
status = STATUS_ERROR;
break;
}
if (FileRep_IsIpcSleep(fileRepOperation))
{
FileRep_Sleep1ms(retry);
FileRep_IncrementRetry(retry);
}
else
{
FileRep_IpcWait(fileRepIpcArray[fileRepAckHashShmem->ipcArrayIndex]->semC, &fileRepIpcArray[fileRepAckHashShmem->ipcArrayIndex]->refCountSemC, FileRepAckHashShmemLock);
}
continue;
}
entry->fileRepOperation = fileRepOperation;
entry->fileRepRelationType = fileRepRelationType;
entry->fileRepAckState = FileRepAckStateWaiting;
entry->xLogEof = xLogEof;
entry->mirrorStatus = FileRepStatusSuccess;
LWLockRelease(FileRepAckHashShmemLock);
break;
}
if (exists) {
status = STATUS_ERROR;
ereport(WARNING,
(errmsg("mirror failure, "
"could not insert ack entry into ack table, entry exists "
"failover requested"),
errhint("run gprecoverseg to re-establish mirror connectivity"),
FileRep_errdetail(fileRepIdentifier,
fileRepRelationType,
fileRepOperation,
FILEREP_UNDEFINED),
FileRep_errcontext()));
}
exit:
if (fileName)
{
pfree(fileName);
fileName = NULL;
}
return status;
}
