EFIAPI
AcquireSpinLock (
IN OUT SPIN_LOCK *SpinLock
)
{
UINT64 Tick;
UINT64 Start, End;
UINT64 Timeout;
Tick = 0;
Start = 0;
End = 0;
if (PcdGet32 (PcdSpinLockTimeout) > 0) {
Tick = GetPerformanceCounter ();
Timeout = DivU64x32 (
MultU64x32 (
GetPerformanceCounterProperties (&Start, &End),
PcdGet32 (PcdSpinLockTimeout)
),
1000000
);
if (Start < End) {
Tick += Timeout;
} else {
Tick -= Timeout;
}
}
while (!AcquireSpinLockOrFail (SpinLock)) {
CpuPause ();
ASSERT ((Start < End) ^ (Tick <= GetPerformanceCounter ()));
}
return SpinLock;
}
/**
Acquire a spin lock when Multi-processor supported.
It will block in the function if cannot get the access control.
If Multi-processor is not supported, return directly.
@param[in, out] MpSpinLock A pointer to the spin lock.
**/
VOID
AcquireMpSpinLock (
IN OUT SPIN_LOCK *MpSpinLock
)
{
if (!MultiProcessorDebugSupport()) {
return;
}
while (TRUE) {
if (AcquireSpinLockOrFail (MpSpinLock)) {
break;
}
CpuPause ();
continue;
}
}
EFIAPI
AcquireSpinLock (
IN OUT SPIN_LOCK *SpinLock
)
{
UINT64 Current;
UINT64 Previous;
UINT64 Total;
UINT64 Start;
UINT64 End;
UINT64 Timeout;
INT64 Cycle;
INT64 Delta;
if (PcdGet32 (PcdSpinLockTimeout) > 0) {
//
// Get the current timer value
//
Current = GetPerformanceCounter();
//
// Initialize local variables
//
Start = 0;
End = 0;
Total = 0;
//
// Retrieve the performance counter properties and compute the number of performance
// counter ticks required to reach the timeout
//
Timeout = DivU64x32 (
MultU64x32 (
GetPerformanceCounterProperties (&Start, &End),
PcdGet32 (PcdSpinLockTimeout)
),
1000000
);
Cycle = End - Start;
if (Cycle < 0) {
Cycle = -Cycle;
}
Cycle++;
while (!AcquireSpinLockOrFail (SpinLock)) {
CpuPause ();
Previous = Current;
Current = GetPerformanceCounter();
Delta = (INT64) (Current - Previous);
if (Start > End) {
Delta = -Delta;
}
if (Delta < 0) {
Delta += Cycle;
}
Total += Delta;
ASSERT (Total < Timeout);
}
} else {
while (!AcquireSpinLockOrFail (SpinLock)) {
CpuPause ();
}
}
return SpinLock;
}
/**
Internal worker function for common exception handler.
@param ExceptionType Exception type.
@param SystemContext Pointer to EFI_SYSTEM_CONTEXT.
@param ExceptionHandlerData Pointer to exception handler data.
**/
VOID
CommonExceptionHandlerWorker (
IN EFI_EXCEPTION_TYPE ExceptionType,
IN EFI_SYSTEM_CONTEXT SystemContext,
IN EXCEPTION_HANDLER_DATA *ExceptionHandlerData
)
{
EXCEPTION_HANDLER_CONTEXT *ExceptionHandlerContext;
RESERVED_VECTORS_DATA *ReservedVectors;
EFI_CPU_INTERRUPT_HANDLER *ExternalInterruptHandler;
ExceptionHandlerContext = (EXCEPTION_HANDLER_CONTEXT *) (UINTN) (SystemContext.SystemContextIa32);
ReservedVectors = ExceptionHandlerData->ReservedVectors;
ExternalInterruptHandler = ExceptionHandlerData->ExternalInterruptHandler;
switch (ReservedVectors[ExceptionType].Attribute) {
case EFI_VECTOR_HANDOFF_HOOK_BEFORE:
//
// Need to jmp to old IDT handler after this exception handler
//
ExceptionHandlerContext->ExceptionDataFlag = (mErrorCodeFlag & (1 << ExceptionType)) ? TRUE : FALSE;
ExceptionHandlerContext->OldIdtHandler = ReservedVectors[ExceptionType].ExceptonHandler;
break;
case EFI_VECTOR_HANDOFF_HOOK_AFTER:
while (TRUE) {
//
// If if anyone has gotten SPIN_LOCK for owner running hook after
//
if (AcquireSpinLockOrFail (&ReservedVectors[ExceptionType].SpinLock)) {
//
// Need to execute old IDT handler before running this exception handler
//
ReservedVectors[ExceptionType].ApicId = GetApicId ();
ArchSaveExceptionContext (ExceptionType, SystemContext, ExceptionHandlerData);
ExceptionHandlerContext->ExceptionDataFlag = (mErrorCodeFlag & (1 << ExceptionType)) ? TRUE : FALSE;
ExceptionHandlerContext->OldIdtHandler = ReservedVectors[ExceptionType].ExceptonHandler;
return;
}
//
// If failed to acquire SPIN_LOCK, check if it was locked by processor itself
//
if (ReservedVectors[ExceptionType].ApicId == GetApicId ()) {
//
// Old IDT handler has been executed, then restore CPU exception content to
// run new exception handler.
//
ArchRestoreExceptionContext (ExceptionType, SystemContext, ExceptionHandlerData);
//
// Rlease spin lock for ApicId
//
ReleaseSpinLock (&ReservedVectors[ExceptionType].SpinLock);
break;
}
CpuPause ();
}
break;
case 0xffffffff:
break;
default:
//
// It should never reach here
//
CpuDeadLoop ();
break;
}
if (ExternalInterruptHandler != NULL &&
ExternalInterruptHandler[ExceptionType] != NULL) {
(ExternalInterruptHandler[ExceptionType]) (ExceptionType, SystemContext);
} else if (ExceptionType < CPU_EXCEPTION_NUM) {
//
// Get Spinlock to display CPU information
//
while (!AcquireSpinLockOrFail (&ExceptionHandlerData->DisplayMessageSpinLock)) {
CpuPause ();
}
//
// Display ExceptionType, CPU information and Image information
//
DumpCpuContent (ExceptionType, SystemContext);
//
// Release Spinlock of output message
//
ReleaseSpinLock (&ExceptionHandlerData->DisplayMessageSpinLock);
//
// Enter a dead loop if needn't to execute old IDT handler further
//
if (ReservedVectors[ExceptionType].Attribute != EFI_VECTOR_HANDOFF_HOOK_BEFORE) {
CpuDeadLoop ();
}
}
}
/**
SMI handler for AP.
@param CpuIndex AP processor Index.
@param ValidSmi Indicates that current SMI is a valid SMI or not.
@param SyncMode SMM MP sync mode.
**/
VOID
APHandler (
IN UINTN CpuIndex,
IN BOOLEAN ValidSmi,
IN SMM_CPU_SYNC_MODE SyncMode
)
{
UINT64 Timer;
UINTN BspIndex;
MTRR_SETTINGS Mtrrs;
//
// Timeout BSP
//
for (Timer = StartSyncTimer ();
!IsSyncTimerTimeout (Timer) &&
!mSmmMpSyncData->InsideSmm;
) {
CpuPause ();
}
if (!mSmmMpSyncData->InsideSmm) {
//
// BSP timeout in the first round
//
if (mSmmMpSyncData->BspIndex != -1) {
//
// BSP Index is known
//
BspIndex = mSmmMpSyncData->BspIndex;
ASSERT (CpuIndex != BspIndex);
//
// Send SMI IPI to bring BSP in
//
SendSmiIpi ((UINT32)gSmmCpuPrivate->ProcessorInfo[BspIndex].ProcessorId);
//
// Now clock BSP for the 2nd time
//
for (Timer = StartSyncTimer ();
!IsSyncTimerTimeout (Timer) &&
!mSmmMpSyncData->InsideSmm;
) {
CpuPause ();
}
if (!mSmmMpSyncData->InsideSmm) {
//
// Give up since BSP is unable to enter SMM
// and signal the completion of this AP
WaitForSemaphore (&mSmmMpSyncData->Counter);
return;
}
} else {
//
// Don't know BSP index. Give up without sending IPI to BSP.
//
WaitForSemaphore (&mSmmMpSyncData->Counter);
return;
}
}
//
// BSP is available
//
BspIndex = mSmmMpSyncData->BspIndex;
ASSERT (CpuIndex != BspIndex);
//
// Mark this processor's presence
//
mSmmMpSyncData->CpuData[CpuIndex].Present = TRUE;
if (SyncMode == SmmCpuSyncModeTradition || SmmCpuFeaturesNeedConfigureMtrrs()) {
//
// Notify BSP of arrival at this point
//
ReleaseSemaphore (&mSmmMpSyncData->CpuData[BspIndex].Run);
}
if (SmmCpuFeaturesNeedConfigureMtrrs()) {
//
// Wait for the signal from BSP to backup MTRRs
//
WaitForSemaphore (&mSmmMpSyncData->CpuData[CpuIndex].Run);
//
// Backup OS MTRRs
//
MtrrGetAllMtrrs(&Mtrrs);
//
// Signal BSP the completion of this AP
//
ReleaseSemaphore (&mSmmMpSyncData->CpuData[BspIndex].Run);
//
// Wait for BSP's signal to program MTRRs
//
WaitForSemaphore (&mSmmMpSyncData->CpuData[CpuIndex].Run);
//
// Replace OS MTRRs with SMI MTRRs
//
ReplaceOSMtrrs (CpuIndex);
//
// Signal BSP the completion of this AP
//
ReleaseSemaphore (&mSmmMpSyncData->CpuData[BspIndex].Run);
}
while (TRUE) {
//
// Wait for something to happen
//
WaitForSemaphore (&mSmmMpSyncData->CpuData[CpuIndex].Run);
//
// Check if BSP wants to exit SMM
//
if (!mSmmMpSyncData->InsideSmm) {
break;
}
//
// BUSY should be acquired by SmmStartupThisAp()
//
ASSERT (
!AcquireSpinLockOrFail (&mSmmMpSyncData->CpuData[CpuIndex].Busy)
);
//
// Invoke the scheduled procedure
//
(*mSmmMpSyncData->CpuData[CpuIndex].Procedure) (
(VOID*)mSmmMpSyncData->CpuData[CpuIndex].Parameter
);
//
// Release BUSY
//
ReleaseSpinLock (&mSmmMpSyncData->CpuData[CpuIndex].Busy);
}
if (SmmCpuFeaturesNeedConfigureMtrrs()) {
//
// Notify BSP the readiness of this AP to program MTRRs
//
ReleaseSemaphore (&mSmmMpSyncData->CpuData[BspIndex].Run);
//
// Wait for the signal from BSP to program MTRRs
//
WaitForSemaphore (&mSmmMpSyncData->CpuData[CpuIndex].Run);
//
// Restore OS MTRRs
//
SmmCpuFeaturesReenableSmrr ();
MtrrSetAllMtrrs(&Mtrrs);
}
//
// Notify BSP the readiness of this AP to Reset states/semaphore for this processor
//
ReleaseSemaphore (&mSmmMpSyncData->CpuData[BspIndex].Run);
//
// Wait for the signal from BSP to Reset states/semaphore for this processor
//
WaitForSemaphore (&mSmmMpSyncData->CpuData[CpuIndex].Run);
//
// Reset states/semaphore for this processor
//
mSmmMpSyncData->CpuData[CpuIndex].Present = FALSE;
//
// Notify BSP the readiness of this AP to exit SMM
//
ReleaseSemaphore (&mSmmMpSyncData->CpuData[BspIndex].Run);
}
/**
SMI handler for BSP.
@param CpuIndex BSP processor Index
@param SyncMode SMM MP sync mode
**/
VOID
BSPHandler (
IN UINTN CpuIndex,
IN SMM_CPU_SYNC_MODE SyncMode
)
{
UINTN Index;
MTRR_SETTINGS Mtrrs;
UINTN ApCount;
BOOLEAN ClearTopLevelSmiResult;
UINTN PresentCount;
ASSERT (CpuIndex == mSmmMpSyncData->BspIndex);
ApCount = 0;
//
// Flag BSP's presence
//
mSmmMpSyncData->InsideSmm = TRUE;
//
// Initialize Debug Agent to start source level debug in BSP handler
//
InitializeDebugAgent (DEBUG_AGENT_INIT_ENTER_SMI, NULL, NULL);
//
// Mark this processor's presence
//
mSmmMpSyncData->CpuData[CpuIndex].Present = TRUE;
//
// Clear platform top level SMI status bit before calling SMI handlers. If
// we cleared it after SMI handlers are run, we would miss the SMI that
// occurs after SMI handlers are done and before SMI status bit is cleared.
//
ClearTopLevelSmiResult = ClearTopLevelSmiStatus();
ASSERT (ClearTopLevelSmiResult == TRUE);
//
// Set running processor index
//
gSmmCpuPrivate->SmmCoreEntryContext.CurrentlyExecutingCpu = CpuIndex;
//
// If Traditional Sync Mode or need to configure MTRRs: gather all available APs.
//
if (SyncMode == SmmCpuSyncModeTradition || SmmCpuFeaturesNeedConfigureMtrrs()) {
//
// Wait for APs to arrive
//
SmmWaitForApArrival();
//
// Lock the counter down and retrieve the number of APs
//
mSmmMpSyncData->AllCpusInSync = TRUE;
ApCount = LockdownSemaphore (&mSmmMpSyncData->Counter) - 1;
//
// Wait for all APs to get ready for programming MTRRs
//
WaitForAllAPs (ApCount);
if (SmmCpuFeaturesNeedConfigureMtrrs()) {
//
// Signal all APs it's time for backup MTRRs
//
ReleaseAllAPs ();
//
// WaitForSemaphore() may wait for ever if an AP happens to enter SMM at
// exactly this point. Please make sure PcdCpuSmmMaxSyncLoops has been set
// to a large enough value to avoid this situation.
// Note: For HT capable CPUs, threads within a core share the same set of MTRRs.
// We do the backup first and then set MTRR to avoid race condition for threads
// in the same core.
//
MtrrGetAllMtrrs(&Mtrrs);
//
// Wait for all APs to complete their MTRR saving
//
WaitForAllAPs (ApCount);
//
// Let all processors program SMM MTRRs together
//
ReleaseAllAPs ();
//
// WaitForSemaphore() may wait for ever if an AP happens to enter SMM at
// exactly this point. Please make sure PcdCpuSmmMaxSyncLoops has been set
// to a large enough value to avoid this situation.
//
ReplaceOSMtrrs (CpuIndex);
//
// Wait for all APs to complete their MTRR programming
//
WaitForAllAPs (ApCount);
}
}
//
// The BUSY lock is initialized to Acquired state
//
AcquireSpinLockOrFail (&mSmmMpSyncData->CpuData[CpuIndex].Busy);
//
// Restore SMM Configuration in S3 boot path.
//
if (mRestoreSmmConfigurationInS3) {
//
// Configure SMM Code Access Check feature if available.
//
ConfigSmmCodeAccessCheck ();
mRestoreSmmConfigurationInS3 = FALSE;
}
//
// Invoke SMM Foundation EntryPoint with the processor information context.
//
gSmmCpuPrivate->SmmCoreEntry (&gSmmCpuPrivate->SmmCoreEntryContext);
//
// Make sure all APs have completed their pending none-block tasks
//
for (Index = mMaxNumberOfCpus; Index-- > 0;) {
if (Index != CpuIndex && mSmmMpSyncData->CpuData[Index].Present) {
AcquireSpinLock (&mSmmMpSyncData->CpuData[Index].Busy);
ReleaseSpinLock (&mSmmMpSyncData->CpuData[Index].Busy);;
}
}
//
// Perform the remaining tasks
//
PerformRemainingTasks ();
//
// If Relaxed-AP Sync Mode: gather all available APs after BSP SMM handlers are done, and
// make those APs to exit SMI synchronously. APs which arrive later will be excluded and
// will run through freely.
//
if (SyncMode != SmmCpuSyncModeTradition && !SmmCpuFeaturesNeedConfigureMtrrs()) {
//
// Lock the counter down and retrieve the number of APs
//
mSmmMpSyncData->AllCpusInSync = TRUE;
ApCount = LockdownSemaphore (&mSmmMpSyncData->Counter) - 1;
//
// Make sure all APs have their Present flag set
//
while (TRUE) {
PresentCount = 0;
for (Index = mMaxNumberOfCpus; Index-- > 0;) {
if (mSmmMpSyncData->CpuData[Index].Present) {
PresentCount ++;
}
}
if (PresentCount > ApCount) {
break;
}
}
}
//
// Notify all APs to exit
//
mSmmMpSyncData->InsideSmm = FALSE;
ReleaseAllAPs ();
//
// Wait for all APs to complete their pending tasks
//
WaitForAllAPs (ApCount);
if (SmmCpuFeaturesNeedConfigureMtrrs()) {
//
// Signal APs to restore MTRRs
//
ReleaseAllAPs ();
//
// Restore OS MTRRs
//
SmmCpuFeaturesReenableSmrr ();
MtrrSetAllMtrrs(&Mtrrs);
//
// Wait for all APs to complete MTRR programming
//
WaitForAllAPs (ApCount);
}
//
// Stop source level debug in BSP handler, the code below will not be
// debugged.
//
InitializeDebugAgent (DEBUG_AGENT_INIT_EXIT_SMI, NULL, NULL);
//
// Signal APs to Reset states/semaphore for this processor
//
ReleaseAllAPs ();
//
// Perform pending operations for hot-plug
//
SmmCpuUpdate ();
//
// Clear the Present flag of BSP
//
mSmmMpSyncData->CpuData[CpuIndex].Present = FALSE;
//
// Gather APs to exit SMM synchronously. Note the Present flag is cleared by now but
// WaitForAllAps does not depend on the Present flag.
//
WaitForAllAPs (ApCount);
//
// Reset BspIndex to -1, meaning BSP has not been elected.
//
if (FeaturePcdGet (PcdCpuSmmEnableBspElection)) {
mSmmMpSyncData->BspIndex = (UINT32)-1;
}
//
// Allow APs to check in from this point on
//
mSmmMpSyncData->Counter = 0;
mSmmMpSyncData->AllCpusInSync = FALSE;
}
