void KiPageFaultHandler(int trap, ppc_trap_frame_t *frame)
{
NTSTATUS Status = STATUS_SUCCESS;
KPROCESSOR_MODE Mode;
EXCEPTION_RECORD Er;
KTRAP_FRAME Tf;
BOOLEAN AccessFault = !!(frame->dsisr & (1<<28));
vaddr_t VirtualAddr;
PVOID TrapInfo = NULL;
/* get the faulting address */
if (trap == 4) /* Instruction miss */
VirtualAddr = frame->srr0;
else /* Data miss */
VirtualAddr = frame->dar;
/* MSR_PR */
Mode = frame->srr1 & 0x4000 ? UserMode : KernelMode;
DPRINT("Page Fault at %08x\n", frame->srr0);
/* handle the fault */
if (AccessFault)
{
Status = MmAccessFault(Mode, (PVOID)VirtualAddr, FALSE, TrapInfo);
}
else
{
Status = MmNotPresentFault(Mode, VirtualAddr, FALSE);
}
if (NT_SUCCESS(Status))
{
MmuCallbackRet();
}
if (KeGetCurrentThread()->ApcState.UserApcPending)
{
KIRQL oldIrql;
KeRaiseIrql(APC_LEVEL, &oldIrql);
KiDeliverApc(UserMode, NULL, NULL);
KeLowerIrql(oldIrql);
}
MmpPpcTrapFrameToTrapFrame(frame, &Tf);
Er.ExceptionCode = STATUS_ACCESS_VIOLATION;
Er.ExceptionFlags = 0;
Er.ExceptionRecord = NULL;
Er.ExceptionAddress = (PVOID)frame->srr0;
Er.NumberParameters = 2;
Er.ExceptionInformation[0] = AccessFault;
Er.ExceptionInformation[1] = VirtualAddr;
/* FIXME: Which exceptions are noncontinuable? */
Er.ExceptionFlags = 0;
KiDispatchException(&Er, 0, &Tf, Mode, TRUE);
MmuCallbackRet();
}
/*++
* @name KiCheckForKernelApcDelivery
* @implemented NT 5.2
*
* The KiCheckForKernelApcDelivery routine is called whenever APCs have
* just been re-enabled in Kernel Mode, such as after leaving a Critical or
* Guarded Region. It delivers APCs if the environment is right.
*
* @param None.
*
* @return None.
*
* @remarks This routine allows KeLeave/EnterCritical/GuardedRegion to be used
* as macros from inside WIN32K or other Drivers, which will then only
* have to do an Import API call in the case where APCs are enabled again.
*
*--*/
VOID
NTAPI
KiCheckForKernelApcDelivery(VOID)
{
KIRQL OldIrql;
/* We should only deliver at passive */
if (KeGetCurrentIrql() == PASSIVE_LEVEL)
{
/* Raise to APC and Deliver APCs, then lower back to Passive */
KeRaiseIrql(APC_LEVEL, &OldIrql);
KiDeliverApc(KernelMode, 0, 0);
KeLowerIrql(PASSIVE_LEVEL);
}
else
{
/*
* If we're not at passive level it means someone raised IRQL
* to APC level before the critical or guarded section was entered
* (e.g) by a fast mutex). This implies that the APCs shouldn't
* be delivered now, but after the IRQL is lowered to passive
* level again.
*/
KeGetCurrentThread()->ApcState.KernelApcPending = TRUE;
HalRequestSoftwareInterrupt(APC_LEVEL);
}
}
VOID
HalpLowerIrql(KIRQL NewIrql)
{
if (NewIrql >= PROFILE_LEVEL)
{
KeGetPcr()->Irql = NewIrql;
return;
}
HalpExecuteIrqs(NewIrql);
if (NewIrql >= DISPATCH_LEVEL)
{
KeGetPcr()->Irql = NewIrql;
return;
}
KeGetPcr()->Irql = DISPATCH_LEVEL;
if (((PKIPCR)KeGetPcr())->HalReserved[HAL_DPC_REQUEST])
{
((PKIPCR)KeGetPcr())->HalReserved[HAL_DPC_REQUEST] = FALSE;
KiDispatchInterrupt();
}
KeGetPcr()->Irql = APC_LEVEL;
if (NewIrql == APC_LEVEL)
{
return;
}
if (KeGetCurrentThread() != NULL &&
KeGetCurrentThread()->ApcState.KernelApcPending)
{
KiDeliverApc(KernelMode, NULL, NULL);
}
KeGetPcr()->Irql = PASSIVE_LEVEL;
}
VOID
DECLSPEC_NORETURN
FASTCALL
HalpApcInterruptHandler(IN PKTRAP_FRAME TrapFrame)
{
KPROCESSOR_MODE ProcessorMode;
KIRQL OldIrql;
ASSERT(ApicGetProcessorIrql() == APC_LEVEL);
/* Enter trap */
KiEnterInterruptTrap(TrapFrame);
#ifdef APIC_LAZY_IRQL
if (!HalBeginSystemInterrupt(APC_LEVEL, APC_VECTOR, &OldIrql))
{
/* "Spurious" interrupt, exit the interrupt */
KiEoiHelper(TrapFrame);
}
#else
/* Save the old IRQL */
OldIrql = ApicGetCurrentIrql();
ASSERT(OldIrql < APC_LEVEL);
#endif
/* Raise to APC_LEVEL */
ApicRaiseIrql(APC_LEVEL);
/* End the interrupt */
ApicSendEOI();
/* Kernel or user APC? */
if (KiUserTrap(TrapFrame)) ProcessorMode = UserMode;
else if (TrapFrame->EFlags & EFLAGS_V86_MASK) ProcessorMode = UserMode;
else ProcessorMode = KernelMode;
/* Enable interrupts and call the kernel's APC interrupt handler */
_enable();
KiDeliverApc(ProcessorMode, NULL, TrapFrame);
/* Disable interrupts */
_disable();
/* Restore the old IRQL */
ApicLowerIrql(OldIrql);
/* Exit the interrupt */
KiEoiHelper(TrapFrame);
}
VOID
HalpLowerIrql(KIRQL NewIrql, BOOLEAN FromHalEndSystemInterrupt)
{
ULONG Flags;
UCHAR DpcRequested;
if (NewIrql >= DISPATCH_LEVEL)
{
KeSetCurrentIrql (NewIrql);
APICWrite(APIC_TPR, IRQL2TPR (NewIrql) & APIC_TPR_PRI);
return;
}
Flags = __readeflags();
if (KeGetCurrentIrql() > APC_LEVEL)
{
KeSetCurrentIrql (DISPATCH_LEVEL);
APICWrite(APIC_TPR, IRQL2TPR (DISPATCH_LEVEL) & APIC_TPR_PRI);
DpcRequested = __readfsbyte(FIELD_OFFSET(KIPCR, HalReserved[HAL_DPC_REQUEST]));
if (FromHalEndSystemInterrupt || DpcRequested)
{
__writefsbyte(FIELD_OFFSET(KIPCR, HalReserved[HAL_DPC_REQUEST]), 0);
_enable();
KiDispatchInterrupt();
if (!(Flags & EFLAGS_INTERRUPT_MASK))
{
_disable();
}
}
KeSetCurrentIrql (APC_LEVEL);
}
if (NewIrql == APC_LEVEL)
{
return;
}
if (KeGetCurrentThread () != NULL &&
KeGetCurrentThread ()->ApcState.KernelApcPending)
{
_enable();
KiDeliverApc(KernelMode, NULL, NULL);
if (!(Flags & EFLAGS_INTERRUPT_MASK))
{
_disable();
}
}
KeSetCurrentIrql (PASSIVE_LEVEL);
}
NTSTATUS
KiContinueEx (
IN PCONTEXT ContextRecord,
IN BOOLEAN TestAlert,
IN PKEXCEPTION_FRAME ExceptionFrame,
IN PKTRAP_FRAME TrapFrame
)
/*++
Routine Description:
This function is called to copy the specified context frame to the
specified exception and trap frames for the continue system service.
N.B. If the previous mode is user mode, alerts are being tested, a user
APC is available to execute, and the specified context record was
previously placed on the stack by the user APC initialization routine,
then bypass context record copy to the kernel frames and back again.
Arguments:
ContextRecord - Supplies a pointer to a context record.
TestAlert - Supplies a boolean value that determines whether a test alert
is to be performed.
ExceptionFrame - Supplies a pointer to an exception frame.
TrapFrame - Supplies a pointer to a trap frame.
Return Value:
STATUS_ACCESS_VIOLATION is returned if the context record is not readable
from user mode.
STATUS_DATATYPE_MISALIGNMENT is returned if the context record is not
properly aligned.
STATUS_SUCCESS + 1 is returned if the context frame is copied successfully
to the specified exception and trap frames.
STATUS_SUCCESS is returned if the context record copy was bypassed and
another user APC is ready to be delivered.
--*/
{
KIRQL OldIrql;
NTSTATUS Status;
PKTHREAD Thread;
ULONG64 UserStack;
//
// Synchronize with other context operations.
//
// If the current IRQL is less than APC_LEVEL, then raise IRQL to APC level.
//
OldIrql = KeGetCurrentIrql();
if (OldIrql < APC_LEVEL) {
KfRaiseIrql(APC_LEVEL);
}
//
// If the previous mode was not kernel mode, then attempt to bypass
// the copy of the context record. Otherwise, copy context to kernel
// frames directly.
//
Status = STATUS_SUCCESS + 1;
Thread = KeGetCurrentThread();
if (Thread->PreviousMode != KernelMode) {
try {
if (TestAlert != FALSE) {
ProbeForWriteSmallStructure(ContextRecord,
sizeof(CONTEXT),
CONTEXT_ALIGN);
KeTestAlertThread(UserMode);
UserStack = (ContextRecord->Rsp - sizeof(MACHINE_FRAME)) & ~STACK_ROUND;
if (((UserStack - sizeof(CONTEXT)) == (ULONG64)ContextRecord) &&
(Thread->ApcState.UserApcPending != FALSE)) {
//
// Save the context record and exception frame addresses
// in the trap frame and deliver the user APC bypassing
// the context copy.
//
TrapFrame->ContextRecord = (ULONG64)ContextRecord;
TrapFrame->ExceptionFrame = (ULONG64)ExceptionFrame;
KiDeliverApc(UserMode, NULL, TrapFrame);
Status = STATUS_SUCCESS;
leave;
}
} else {
ProbeForReadSmallStructure(ContextRecord,
sizeof(CONTEXT),
CONTEXT_ALIGN);
}
KiContinuePreviousModeUser(ContextRecord,
ExceptionFrame,
TrapFrame);
} except(EXCEPTION_EXECUTE_HANDLER) {
Status = GetExceptionCode();
}
} else {
LONG_PTR
FASTCALL
KiSwapThread(IN PKTHREAD CurrentThread,
IN PKPRCB Prcb)
{
BOOLEAN ApcState = FALSE;
KIRQL WaitIrql;
LONG_PTR WaitStatus;
PKTHREAD NextThread;
ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL);
/* Acquire the PRCB lock */
KiAcquirePrcbLock(Prcb);
/* Get the next thread */
NextThread = Prcb->NextThread;
if (NextThread)
{
/* Already got a thread, set it up */
Prcb->NextThread = NULL;
Prcb->CurrentThread = NextThread;
NextThread->State = Running;
}
else
{
/* Try to find a ready thread */
NextThread = KiSelectReadyThread(0, Prcb);
if (NextThread)
{
/* Switch to it */
Prcb->CurrentThread = NextThread;
NextThread->State = Running;
}
else
{
/* Set the idle summary */
InterlockedOrSetMember(&KiIdleSummary, Prcb->SetMember);
/* Schedule the idle thread */
NextThread = Prcb->IdleThread;
Prcb->CurrentThread = NextThread;
NextThread->State = Running;
}
}
/* Sanity check and release the PRCB */
ASSERT(CurrentThread != Prcb->IdleThread);
KiReleasePrcbLock(Prcb);
/* Save the wait IRQL */
WaitIrql = CurrentThread->WaitIrql;
/* Swap contexts */
ApcState = KiSwapContext(WaitIrql, CurrentThread);
/* Get the wait status */
WaitStatus = CurrentThread->WaitStatus;
/* Check if we need to deliver APCs */
if (ApcState)
{
/* Lower to APC_LEVEL */
KeLowerIrql(APC_LEVEL);
/* Deliver APCs */
KiDeliverApc(KernelMode, NULL, NULL);
ASSERT(WaitIrql == 0);
}
/* Lower IRQL back to what it was and return the wait status */
KeLowerIrql(WaitIrql);
return WaitStatus;
}
