/* * SenderLoop * */ static int FileRepAckMirror_RunSender(void) { FileRepShmemMessageDescr_s *fileRepShmemMessageDescr=NULL; char *fileRepMessage; int status = STATUS_OK; bool movePositionConsume = FALSE; FileRepConsumerProcIndex_e messageType; FileRepMessageHeader_s *fileRepMessageHeader; FileRepShmem_s *fileRepAckShmem = NULL; FileRep_InsertConfigLogEntry("run sender ack"); fileRepAckShmem = fileRepAckShmemArray[FILEREP_OUTGOING_MESSAGE_QUEUE]; while (1) { LWLockAcquire(FileRepAckShmemLock, LW_EXCLUSIVE); if (movePositionConsume) { fileRepAckShmem->positionConsume = fileRepAckShmem->positionConsume + fileRepShmemMessageDescr->messageLength + sizeof(FileRepShmemMessageDescr_s); if (fileRepAckShmem->positionConsume == fileRepAckShmem->positionWraparound && fileRepAckShmem->positionInsert != fileRepAckShmem->positionWraparound) { fileRepAckShmem->positionConsume = fileRepAckShmem->positionBegin; fileRepAckShmem->positionWraparound = fileRepAckShmem->positionEnd; } FileRep_IpcSignal(fileRepIpcArray[fileRepAckShmem->ipcArrayIndex]->semP, &fileRepIpcArray[fileRepAckShmem->ipcArrayIndex]->refCountSemP); } fileRepShmemMessageDescr = (FileRepShmemMessageDescr_s*) fileRepAckShmem->positionConsume; while ((fileRepAckShmem->positionConsume == fileRepAckShmem->positionInsert) || ((fileRepAckShmem->positionConsume != fileRepAckShmem->positionInsert) && (fileRepShmemMessageDescr->messageState != FileRepShmemMessageStateReady))) { fileRepIpcArray[fileRepAckShmem->ipcArrayIndex]->refCountSemC++; LWLockRelease(FileRepAckShmemLock); FileRepSubProcess_ProcessSignals(); if (FileRepSubProcess_GetState() != FileRepStateReady) { LWLockAcquire(FileRepAckShmemLock, LW_EXCLUSIVE); break; } FileRep_IpcWait(fileRepIpcArray[fileRepAckShmem->ipcArrayIndex]->semC, &fileRepIpcArray[fileRepAckShmem->ipcArrayIndex]->refCountSemC, FileRepAckShmemLock); LWLockAcquire(FileRepAckShmemLock, LW_EXCLUSIVE); if (fileRepAckShmem->positionConsume == fileRepAckShmem->positionWraparound && fileRepAckShmem->positionInsert != fileRepAckShmem->positionWraparound) { fileRepAckShmem->positionConsume = fileRepAckShmem->positionBegin; fileRepAckShmem->positionWraparound = fileRepAckShmem->positionEnd; } /* Re-assign to find if messageState is changed */ fileRepShmemMessageDescr = (FileRepShmemMessageDescr_s*) fileRepAckShmem->positionConsume; } // while internal fileRepAckShmem->consumeCount++; LWLockRelease(FileRepAckShmemLock); FileRepSubProcess_ProcessSignals(); if (FileRepSubProcess_GetState() != FileRepStateReady) { break; } FileRep_InsertLogEntry( "M_RunSenderAck", FileRep_GetFlatFileIdentifier("", ""), FileRepRelationTypeNotSpecified, FileRepOperationNotSpecified, FILEREP_UNDEFINED, FILEREP_UNDEFINED, FileRepAckStateNotInitialized, FILEREP_UNDEFINED, FILEREP_UNDEFINED); #ifdef FAULT_INJECTOR FaultInjector_InjectFaultIfSet( FileRepSender, DDLNotSpecified, "", //databaseName ""); // tableName #endif fileRepMessage = (char*) (fileRepAckShmem->positionConsume + sizeof(FileRepShmemMessageDescr_s)); fileRepMessageHeader = (FileRepMessageHeader_s*) (fileRepAckShmem->positionConsume + sizeof(FileRepShmemMessageDescr_s)); messageType = FileRepMessageTypeXLog; if (! FileRepConnClient_SendMessage( messageType, fileRepShmemMessageDescr->messageSync, fileRepMessage, fileRepShmemMessageDescr->messageLength)) { ereport(WARNING, (errcode_for_socket_access(), errmsg("mirror failure, " "could not sent ack message to primary : %m, " "failover requested"), errhint("run gprecoverseg to re-establish mirror connectivity"), FileRep_errdetail_ShmemAck(), FileRep_errcontext())); status = STATUS_ERROR; break; } movePositionConsume = TRUE; } // while(1) FileRepConnClient_CloseConnection(); return status; }
/* * Backend checks if acknowledgement that its operation is completed * is received from mirror. * If acknowledgement is received (state == FileRepAckStateCompleted) then * a) entry is removed from hash * b) TRUE is returned */ bool FileRepAckPrimary_IsOperationCompleted( FileRepIdentifier_u fileRepIdentifier, FileRepRelationType_e fileRepRelationType) { FileRepAckHashEntry_s *entry = NULL; bool isCompleted = FALSE; bool isRemoved; FileName fileName = NULL; int retry = 0; bool retval = FALSE; bool wait = FALSE; fileName = FileRep_GetFileName(fileRepIdentifier, fileRepRelationType); while ((isCompleted == FALSE) && FileRep_IsRetry(retry)) { LWLockAcquire(FileRepAckHashShmemLock, LW_EXCLUSIVE); entry = FileRepAckPrimary_LookupHashEntry(fileName); if (entry == NULL) { LWLockRelease(FileRepAckHashShmemLock); break; } if (! FileRep_IsIpcSleep(entry->fileRepOperation)) { if (wait == TRUE) { wait = FALSE; } } switch (entry->fileRepAckState) { case FileRepAckStateWaiting: /* No Operation */ break; case FileRepAckStateCompleted: retval = TRUE; xLogEof = entry->xLogEof; mirrorStatus = entry->mirrorStatus; /* no BREAK */ case FileRepAckStateMirrorInFault: isCompleted = TRUE; isRemoved = FileRepAckPrimary_RemoveHashEntry(fileName); Assert(isRemoved == TRUE); break; default: break; } if (isCompleted == false) { if (! FileRep_IsIpcSleep(entry->fileRepOperation)) { fileRepIpcArray[fileRepAckHashShmem->ipcArrayIndex]->refCountSemP++; wait = TRUE; } } LWLockRelease(FileRepAckHashShmemLock); if (isCompleted == false) { if (FileRepSubProcess_IsStateTransitionRequested()) { break; } if (FileRep_IsIpcSleep(entry->fileRepOperation)) { FileRep_Sleep1ms(retry); if (retry == (3 * file_rep_retry / 4)) ereport(WARNING, (errmsg("threshold '75' percent of 'gp_segment_connect_timeout=%d' is reached, " "mirror may not be able to keep up with primary, " "primary may transition to change tracking", gp_segment_connect_timeout), errhint("increase guc 'gp_segment_connect_timeout' by 'gpconfig' and 'gpstop -u' "), errSendAlert(true))); FileRep_IncrementRetry(retry); } else { FileRep_IpcWait(fileRepIpcArray[fileRepAckHashShmem->ipcArrayIndex]->semP, &fileRepIpcArray[fileRepAckHashShmem->ipcArrayIndex]->refCountSemP, FileRepAckHashShmemLock); } /* * if the message was from the main filerep process then it is a * graceful shutdown message to the mirror. We don't want to stall * shutdown if the mirror is unavailable so we wait a smaller amount * of time */ if ( entry->fileRepOperation == FileRepOperationShutdown && retry == 50) { FileRepAckPrimary_RemoveHashEntry(fileName); break; } } } if (retval == FALSE) { mirrorStatus = FileRepStatusMirrorLossOccurred; if (! primaryMirrorIsIOSuspended()) { ereport(WARNING, (errmsg("mirror failure, " "could not complete mirrored request identifier '%s' ack state '%s', " "failover requested", (fileName == NULL) ? "<null>" : fileName, (entry == NULL) ? "<entry not found>" : FileRepAckStateToString[entry->fileRepAckState]), errhint("run gprecoverseg to re-establish mirror connectivity"), FileRep_errdetail_ShmemAck(), FileRep_errcontext())); } } if (fileName) { pfree(fileName); fileName = NULL; } return retval; }
/* * FileRepAckPrimary_RunConsumer() */ static int FileRepAckPrimary_RunConsumer(void) { FileRepShmemMessageDescr_s *fileRepShmemMessageDescr = NULL; FileRepMessageHeader_s *fileRepMessageHeader = NULL; pg_crc32 *fileRepMessageHeaderCrc; pg_crc32 messageHeaderCrcLocal = 0; int status = STATUS_OK; bool movePositionConsume = FALSE; FileRepShmem_s *fileRepAckShmem = NULL; FileRep_InsertConfigLogEntry("run consumer"); fileRepAckShmem = fileRepAckShmemArray[FILEREP_ACKSHMEM_MESSAGE_SLOT_PRIMARY_ACK]; while (1) { LWLockAcquire(FileRepAckShmemLock, LW_EXCLUSIVE); if (movePositionConsume) { fileRepAckShmem->positionConsume = fileRepAckShmem->positionConsume + fileRepShmemMessageDescr->messageLength + sizeof(FileRepShmemMessageDescr_s); if (fileRepAckShmem->positionConsume == fileRepAckShmem->positionWraparound && fileRepAckShmem->positionInsert != fileRepAckShmem->positionWraparound) { fileRepAckShmem->positionConsume = fileRepAckShmem->positionBegin; fileRepAckShmem->positionWraparound = fileRepAckShmem->positionEnd; } FileRep_IpcSignal(fileRepIpcArray[fileRepAckShmem->ipcArrayIndex]->semP, &fileRepIpcArray[fileRepAckShmem->ipcArrayIndex]->refCountSemP); } fileRepShmemMessageDescr = (FileRepShmemMessageDescr_s*) fileRepAckShmem->positionConsume; while ((fileRepAckShmem->positionConsume == fileRepAckShmem->positionInsert) || ((fileRepAckShmem->positionConsume != fileRepAckShmem->positionInsert) && (fileRepShmemMessageDescr->messageState != FileRepShmemMessageStateReady))) { fileRepIpcArray[fileRepAckShmem->ipcArrayIndex]->refCountSemC++; LWLockRelease(FileRepAckShmemLock); FileRepSubProcess_ProcessSignals(); if (FileRepSubProcess_GetState() != FileRepStateReady && FileRepSubProcess_GetState() != FileRepStateInitialization) { LWLockAcquire(FileRepAckShmemLock, LW_EXCLUSIVE); break; } FileRep_IpcWait(fileRepIpcArray[fileRepAckShmem->ipcArrayIndex]->semC, &fileRepIpcArray[fileRepAckShmem->ipcArrayIndex]->refCountSemC, FileRepAckShmemLock); LWLockAcquire(FileRepAckShmemLock, LW_EXCLUSIVE); if (fileRepAckShmem->positionConsume == fileRepAckShmem->positionWraparound && fileRepAckShmem->positionInsert != fileRepAckShmem->positionWraparound) { fileRepAckShmem->positionConsume = fileRepAckShmem->positionBegin; fileRepAckShmem->positionWraparound = fileRepAckShmem->positionEnd; } /* Re-assign to find if messageState is changed */ fileRepShmemMessageDescr = (FileRepShmemMessageDescr_s*) fileRepAckShmem->positionConsume; } // internal while fileRepAckShmem->consumeCount++; LWLockRelease(FileRepAckShmemLock); FileRepSubProcess_ProcessSignals(); if (FileRepSubProcess_GetState() != FileRepStateReady && FileRepSubProcess_GetState() != FileRepStateInitialization) { break; } SIMPLE_FAULT_INJECTOR(FileRepConsumer); /* Calculate and compare FileRepMessageHeader_s Crc */ fileRepMessageHeader = (FileRepMessageHeader_s*) (fileRepAckShmem->positionConsume + sizeof(FileRepShmemMessageDescr_s)); FileRep_CalculateCrc((char *) fileRepMessageHeader, sizeof(FileRepMessageHeader_s), &messageHeaderCrcLocal); fileRepMessageHeaderCrc = (pg_crc32 *) (fileRepAckShmem->positionConsume + sizeof(FileRepMessageHeader_s) + sizeof(FileRepShmemMessageDescr_s)); if (*fileRepMessageHeaderCrc != messageHeaderCrcLocal) { status = STATUS_ERROR; ereport(WARNING, (errmsg("mirror failure, " "could not match ack message header checksum between primary '%u' and mirror '%u', " "failover requested", *fileRepMessageHeaderCrc, messageHeaderCrcLocal), errhint("run gprecoverseg to re-establish mirror connectivity"), FileRep_errdetail(fileRepMessageHeader->fileRepIdentifier, fileRepMessageHeader->fileRepRelationType, fileRepMessageHeader->fileRepOperation, fileRepMessageHeader->messageCount), FileRep_errdetail_ShmemAck(), FileRep_errcontext())); break; } /* Write operation is never acknowledged. * That means message should never have body. * CRC of body should be always 0. */ Assert(fileRepMessageHeader->fileRepOperation != FileRepOperationWrite); Assert(fileRepMessageHeader->fileRepMessageBodyCrc == 0); switch (fileRepMessageHeader->fileRepOperation) { case FileRepOperationReconcileXLogEof: xLogEof = fileRepMessageHeader->fileRepOperationDescription.reconcile.xLogEof; if (Debug_filerep_print) ereport(LOG, (errmsg("ack reconcile xlogid '%d' xrecoff '%d' ", xLogEof.xlogid, xLogEof.xrecoff))); break; case FileRepOperationValidation: mirrorStatus = fileRepMessageHeader->fileRepOperationDescription.validation.mirrorStatus; if (Debug_filerep_print) ereport(LOG, (errmsg("ack validation status '%s' ", FileRepStatusToString[mirrorStatus]))); break; case FileRepOperationCreate: mirrorStatus = fileRepMessageHeader->fileRepOperationDescription.create.mirrorStatus; if (Debug_filerep_print) ereport(LOG, (errmsg("ack create status '%s' ", FileRepStatusToString[mirrorStatus]))); break; case FileRepOperationStartSlruChecksum: mirrorStatus = fileRepMessageHeader->fileRepOperationDescription.startChecksum.mirrorStatus; if (Debug_filerep_print) { ereport(LOG, (errmsg("ack start SLRU checksum: status = '%s', directory = '%s' ", FileRepStatusToString[mirrorStatus], fileRepMessageHeader->fileRepIdentifier.fileRepFlatFileIdentifier.directorySimpleName))); } break; case FileRepOperationVerifySlruDirectoryChecksum: mirrorStatus = fileRepMessageHeader->fileRepOperationDescription.verifyDirectoryChecksum.mirrorStatus; if (Debug_filerep_print) { ereport(LOG, (errmsg("ack verify SLRU directory checksum: status = '%s', directory = '%s' ", FileRepStatusToString[mirrorStatus], fileRepMessageHeader->fileRepIdentifier.fileRepFlatFileIdentifier.directorySimpleName))); } break; default: break; } if (fileRepMessageHeader->fileRepAckState != FileRepAckStateCompleted) { status = STATUS_ERROR; ereport(WARNING, (errmsg("mirror failure, " "could not complete operation on mirror ack state '%s', " "failover requested", FileRepAckStateToString[fileRepMessageHeader->fileRepAckState]), errhint("run gprecoverseg to re-establish mirror connectivity"), errSendAlert(true), FileRep_errdetail(fileRepMessageHeader->fileRepIdentifier, fileRepMessageHeader->fileRepRelationType, fileRepMessageHeader->fileRepOperation, fileRepMessageHeader->messageCount), FileRep_errdetail_Shmem(), FileRep_errdetail_ShmemAck(), FileRep_errcontext())); /* * FAULT has to be set before entry is updated in ack hash table * in order to suspend backend process. */ FileRep_SetSegmentState(SegmentStateFault, FaultTypeMirror); FileRepSubProcess_ProcessSignals(); } if (FileRepAckPrimary_UpdateHashEntry( fileRepMessageHeader->fileRepIdentifier, fileRepMessageHeader->fileRepRelationType, fileRepMessageHeader->fileRepAckState) != STATUS_OK) { status = STATUS_ERROR; ereport(WARNING, (errmsg("mirror failure, " "could not update ack state '%s' in ack hash table, " "failover requested", FileRepAckStateToString[fileRepMessageHeader->fileRepAckState]), errhint("run gprecoverseg to re-establish mirror connectivity"), errSendAlert(true), FileRep_errdetail(fileRepMessageHeader->fileRepIdentifier, fileRepMessageHeader->fileRepRelationType, fileRepMessageHeader->fileRepOperation, fileRepMessageHeader->messageCount), FileRep_errdetail_Shmem(), FileRep_errdetail_ShmemAck(), FileRep_errcontext())); } FileRep_InsertLogEntry( "P_RunConsumer", fileRepMessageHeader->fileRepIdentifier, fileRepMessageHeader->fileRepRelationType, fileRepMessageHeader->fileRepOperation, messageHeaderCrcLocal, fileRepMessageHeader->fileRepMessageBodyCrc, fileRepMessageHeader->fileRepAckState, FILEREP_UNDEFINED, fileRepMessageHeader->messageCount); if (status != STATUS_OK) { break; } movePositionConsume = TRUE; } // while(1) return status; }
/* * It is called by backend process to insert new ack entry into hash table. */ int FileRepAckPrimary_NewHashEntry( FileRepIdentifier_u fileRepIdentifier, FileRepOperation_e fileRepOperation, FileRepRelationType_e fileRepRelationType) { FileRepAckHashEntry_s *entry=NULL; bool exists = FALSE; FileName fileName = NULL; int status = STATUS_OK; int retry = 0; bool wait = FALSE; fileName = FileRep_GetFileName(fileRepIdentifier, fileRepRelationType); while (FileRep_IsRetry(retry)) { LWLockAcquire(FileRepAckHashShmemLock, LW_EXCLUSIVE); if (! FileRep_IsIpcSleep(fileRepOperation)) { if (wait == TRUE) { wait = FALSE; } } entry = FileRepAckPrimary_InsertHashEntry(fileName, &exists); if (entry == NULL) { LWLockRelease(FileRepAckHashShmemLock); status = STATUS_ERROR; ereport(WARNING, (errmsg("mirror failure, " "could not insert ack entry into ack table, no memory " "failover requested"), errhint("run gprecoverseg to re-establish mirror connectivity"), FileRep_errdetail(fileRepIdentifier, fileRepRelationType, fileRepOperation, FILEREP_UNDEFINED), FileRep_errcontext())); goto exit; } if (exists) { if (! FileRep_IsIpcSleep(fileRepOperation)) { fileRepIpcArray[fileRepAckHashShmem->ipcArrayIndex]->refCountSemC++; wait = TRUE; } LWLockRelease(FileRepAckHashShmemLock); if (FileRepSubProcess_IsStateTransitionRequested()) { status = STATUS_ERROR; break; } if (FileRep_IsIpcSleep(fileRepOperation)) { FileRep_Sleep1ms(retry); FileRep_IncrementRetry(retry); } else { FileRep_IpcWait(fileRepIpcArray[fileRepAckHashShmem->ipcArrayIndex]->semC, &fileRepIpcArray[fileRepAckHashShmem->ipcArrayIndex]->refCountSemC, FileRepAckHashShmemLock); } continue; } entry->fileRepOperation = fileRepOperation; entry->fileRepRelationType = fileRepRelationType; entry->fileRepAckState = FileRepAckStateWaiting; entry->xLogEof = xLogEof; entry->mirrorStatus = FileRepStatusSuccess; LWLockRelease(FileRepAckHashShmemLock); break; } if (exists) { status = STATUS_ERROR; ereport(WARNING, (errmsg("mirror failure, " "could not insert ack entry into ack table, entry exists " "failover requested"), errhint("run gprecoverseg to re-establish mirror connectivity"), FileRep_errdetail(fileRepIdentifier, fileRepRelationType, fileRepOperation, FILEREP_UNDEFINED), FileRep_errcontext())); } exit: if (fileName) { pfree(fileName); fileName = NULL; } return status; }