/*
* @implemented
*/
BOOLEAN
NTAPI
KeSynchronizeExecution(IN OUT PKINTERRUPT Interrupt,
IN PKSYNCHRONIZE_ROUTINE SynchronizeRoutine,
IN PVOID SynchronizeContext OPTIONAL)
{
BOOLEAN Success;
KIRQL OldIrql;
/* Raise IRQL */
OldIrql = KfRaiseIrql(Interrupt->SynchronizeIrql);
/* Acquire interrupt spinlock */
KeAcquireSpinLockAtDpcLevel(Interrupt->ActualLock);
/* Call the routine */
Success = SynchronizeRoutine(SynchronizeContext);
/* Release lock */
KeReleaseSpinLockFromDpcLevel(Interrupt->ActualLock);
/* Lower IRQL */
KfLowerIrql(OldIrql);
/* Return status */
return Success;
}
VOID
NTAPI
INIT_FUNCTION
KiSystemStartupBootStack(VOID)
{
PKTHREAD Thread;
/* Initialize the kernel for the current CPU */
KiInitializeKernel(&KiInitialProcess.Pcb,
(PKTHREAD)KeLoaderBlock->Thread,
(PVOID)(KeLoaderBlock->KernelStack & ~3),
(PKPRCB)__readfsdword(KPCR_PRCB),
KeNumberProcessors - 1,
KeLoaderBlock);
/* Set the priority of this thread to 0 */
Thread = KeGetCurrentThread();
Thread->Priority = 0;
/* Force interrupts enabled and lower IRQL back to DISPATCH_LEVEL */
_enable();
KfLowerIrql(DISPATCH_LEVEL);
/* Set the right wait IRQL */
Thread->WaitIrql = DISPATCH_LEVEL;
/* Jump into the idle loop */
KiIdleLoop();
}
/*
* @implemented
*/
VOID
NTAPI
KeLowerIrql(KIRQL NewIrql)
{
/* Call the fastcall function */
KfLowerIrql(NewIrql);
}
/*
* @implemented
*/
VOID
FASTCALL
KeReleaseInStackQueuedSpinLock(IN PKLOCK_QUEUE_HANDLE LockHandle)
{
/* Simply lower IRQL back */
KfLowerIrql(LockHandle->OldIrql);
}
/*
* @implemented
*/
VOID
FASTCALL
KeReleaseQueuedSpinLock(IN KSPIN_LOCK_QUEUE_NUMBER LockNumber,
IN KIRQL OldIrql)
{
/* Simply lower IRQL back */
KfLowerIrql(OldIrql);
}
VOID
HalpPCIReleaseType2Lock (
PKSPIN_LOCK SpinLock,
KIRQL Irql
)
{
if (!HalpDoingCrashDump) {
KiReleaseSpinLock (SpinLock);
KfLowerIrql (Irql);
}
}
// ******************************************************************
// * 0x00A3 - KiUnlockDispatcherDatabase()
// ******************************************************************
XBSYSAPI EXPORTNUM(163) xboxkrnl::VOID FASTCALL xboxkrnl::KiUnlockDispatcherDatabase
(
IN KIRQL OldIrql
)
{
LOG_FUNC_ONE_ARG_TYPE(KIRQL_TYPE, OldIrql);
if (!(KeGetCurrentPrcb()->DpcRoutineActive)) // Avoid KeIsExecutingDpc(), as that logs
HalRequestSoftwareInterrupt(DISPATCH_LEVEL);
LOG_INCOMPLETE(); // TODO : Thread-switch?
KfLowerIrql(OldIrql);
}
DECLSPEC_NORETURN
VOID
FASTCALL
KiTrap0DHandler(IN PKTRAP_FRAME TrapFrame)
{
ULONG i, j, Iopl;
BOOLEAN Privileged = FALSE;
PUCHAR Instructions;
UCHAR Instruction = 0;
KIRQL OldIrql;
/* Check for V86 GPF */
if (__builtin_expect(KiV86Trap(TrapFrame), 1))
{
/* Enter V86 trap */
KiEnterV86Trap(TrapFrame);
/* Must be a VDM process */
if (__builtin_expect(!PsGetCurrentProcess()->VdmObjects, 0))
{
/* Enable interrupts */
_enable();
/* Setup illegal instruction fault */
KiDispatchException0Args(STATUS_ILLEGAL_INSTRUCTION,
TrapFrame->Eip,
TrapFrame);
}
/* Go to APC level */
OldIrql = KfRaiseIrql(APC_LEVEL);
_enable();
/* Handle the V86 opcode */
if (__builtin_expect(Ki386HandleOpcodeV86(TrapFrame) == 0xFF, 0))
{
/* Should only happen in VDM mode */
UNIMPLEMENTED;
while (TRUE);
}
/* Bring IRQL back */
KfLowerIrql(OldIrql);
_disable();
/* Do a quick V86 exit if possible */
KiExitV86Trap(TrapFrame);
}
/* Save trap frame */
KiEnterTrap(TrapFrame);
/* Check for user-mode GPF */
if (KiUserTrap(TrapFrame))
{
/* Should not be VDM */
ASSERT(KiVdmTrap(TrapFrame) == FALSE);
/* Enable interrupts and check error code */
_enable();
if (!TrapFrame->ErrCode)
{
/* FIXME: Use SEH */
Instructions = (PUCHAR)TrapFrame->Eip;
/* Scan next 15 bytes */
for (i = 0; i < 15; i++)
{
/* Skip prefix instructions */
for (j = 0; j < sizeof(KiTrapPrefixTable); j++)
{
/* Is this a prefix instruction? */
if (Instructions[i] == KiTrapPrefixTable[j])
{
/* Stop looking */
break;
}
}
/* Is this NOT any prefix instruction? */
if (j == sizeof(KiTrapPrefixTable))
{
/* We can go ahead and handle the fault now */
Instruction = Instructions[i];
break;
}
}
/* If all we found was prefixes, then this instruction is too long */
if (i == 15)
{
/* Setup illegal instruction fault */
KiDispatchException0Args(STATUS_ILLEGAL_INSTRUCTION,
TrapFrame->Eip,
TrapFrame);
}
/* Check for privileged instructions */
if (Instruction == 0xF4) // HLT
{
/* HLT is privileged */
Privileged = TRUE;
}
else if (Instruction == 0x0F)
{
/* Test if it's any of the privileged two-byte opcodes */
if (((Instructions[i + 1] == 0x00) && // LLDT or LTR
(((Instructions[i + 2] & 0x38) == 0x10) || // LLDT
(Instructions[i + 2] == 0x18))) || // LTR
((Instructions[i + 1] == 0x01) && // LGDT or LIDT or LMSW
(((Instructions[i + 2] & 0x38) == 0x10) || // LGDT
(Instructions[i + 2] == 0x18) || // LIDT
(Instructions[i + 2] == 0x30))) || // LMSW
(Instructions[i + 1] == 0x08) || // INVD
(Instructions[i + 1] == 0x09) || // WBINVD
(Instructions[i + 1] == 0x35) || // SYSEXIT
(Instructions[i + 1] == 0x26) || // MOV DR, XXX
(Instructions[i + 1] == 0x06) || // CLTS
(Instructions[i + 1] == 0x20) || // MOV CR, XXX
(Instructions[i + 1] == 0x24) || // MOV YYY, DR
(Instructions[i + 1] == 0x30) || // WRMSR
(Instructions[i + 1] == 0x33)) // RDPMC
// INVLPG, INVLPGA, SYSRET
{
/* These are all privileged */
Privileged = TRUE;
}
}
else
{
/* Get the IOPL and compare with the RPL mask */
Iopl = (TrapFrame->EFlags & EFLAGS_IOPL) >> 12;
if ((TrapFrame->SegCs & RPL_MASK) > Iopl)
{
/* I/O privilege error -- check for known instructions */
if ((Instruction == 0xFA) || (Instruction == 0xFB)) // CLI or STI
{
/* These are privileged */
Privileged = TRUE;
}
else
{
/* Last hope: an IN/OUT instruction */
for (j = 0; j < sizeof(KiTrapIoTable); j++)
{
/* Is this an I/O instruction? */
if (Instruction == KiTrapIoTable[j])
{
/* Then it's privileged */
Privileged = TRUE;
break;
}
}
}
}
}
/* So now... was the instruction privileged or not? */
if (Privileged)
{
/* Whew! We have a privileged instruction, so dispatch the fault */
KiDispatchException0Args(STATUS_PRIVILEGED_INSTRUCTION,
TrapFrame->Eip,
TrapFrame);
}
}
/* If we got here, send an access violation */
KiDispatchException2Args(STATUS_ACCESS_VIOLATION,
TrapFrame->Eip,
0,
0xFFFFFFFF,
TrapFrame);
}
DECLSPEC_NORETURN
VOID
FASTCALL
KiTrap06Handler(IN PKTRAP_FRAME TrapFrame)
{
PUCHAR Instruction;
ULONG i;
KIRQL OldIrql;
/* Check for V86 GPF */
if (__builtin_expect(KiV86Trap(TrapFrame), 1))
{
/* Enter V86 trap */
KiEnterV86Trap(TrapFrame);
/* Must be a VDM process */
if (__builtin_expect(!PsGetCurrentProcess()->VdmObjects, 0))
{
/* Enable interrupts */
_enable();
/* Setup illegal instruction fault */
KiDispatchException0Args(STATUS_ILLEGAL_INSTRUCTION,
TrapFrame->Eip,
TrapFrame);
}
/* Go to APC level */
OldIrql = KfRaiseIrql(APC_LEVEL);
_enable();
/* Check for BOP */
if (!VdmDispatchBop(TrapFrame))
{
/* Should only happen in VDM mode */
UNIMPLEMENTED;
while (TRUE);
}
/* Bring IRQL back */
KfLowerIrql(OldIrql);
_disable();
/* Do a quick V86 exit if possible */
KiExitV86Trap(TrapFrame);
}
/* Save trap frame */
KiEnterTrap(TrapFrame);
/* Enable interrupts */
Instruction = (PUCHAR)TrapFrame->Eip;
_enable();
/* Check for user trap */
if (KiUserTrap(TrapFrame))
{
/* FIXME: Use SEH */
/* Scan next 4 opcodes */
for (i = 0; i < 4; i++)
{
/* Check for LOCK instruction */
if (Instruction[i] == 0xF0)
{
/* Send invalid lock sequence exception */
KiDispatchException0Args(STATUS_INVALID_LOCK_SEQUENCE,
TrapFrame->Eip,
TrapFrame);
}
}
/* FIXME: SEH ends here */
}
/* Kernel-mode or user-mode fault (but not LOCK) */
KiDispatchException0Args(STATUS_ILLEGAL_INSTRUCTION,
TrapFrame->Eip,
TrapFrame);
}
VOID
FASTCALL
KiTimedChainedDispatch2ndLvl(
PKINTERRUPT Interrupt
)
/*++
Routine Description:
This function performs the same function as KiChainedDispatch2ndLvl
except that it is written in C instead of assembly code and includes
code for timing ISRs.
I'd be interested in seeing some benchmarks to show if the assembly
code is actually faster. The Acquire/Release spinlock could be
inlined fairly easily.
Arguments:
Interrupt - Supplies a pointer to a control object of type interrupt.
Return Value:
None.
--*/
{
BOOLEAN Handled = FALSE;
PVOID ListEnd = &Interrupt->InterruptListEntry.Flink;
//
//BEGINTIMING
PKPRCB Prcb = KeGetCurrentPrcb();
ULONGLONG StartTimeHigher;
ULONGLONG StartTime;
ULONGLONG TimeHigher;
ULONGLONG ElapsedTime;
//BEGINTIMINGend
//
// For each interrupt on this chain.
//
do {
//
// If the current IRQL (IRQL raised to by nature of taking this
// interrupt) is not equal to the Synchronization IRQL required
// for this interrupt, raise to the appropriate level.
//
if (Interrupt->Irql != Interrupt->SynchronizeIrql) {
KfRaiseIrql(Interrupt->SynchronizeIrql);
}
//BEGINTIMING
StartTimeHigher = Prcb->IsrTime;
StartTime = RDTSC();
//BEGINTIMINGend
//
// Acquire the interrupt lock.
//
KiAcquireSpinLock(Interrupt->ActualLock);
//
// Call the Interrupt Service Routine.
//
Handled |= Interrupt->ServiceRoutine(Interrupt,
Interrupt->ServiceContext);
//
// Release the interrupt lock.
//
KiReleaseSpinLock(Interrupt->ActualLock);
//ENDTIMING
//
// ElapsedTime is time since we started looking at this element
// on the chain. (ie the current interrupt object).
//
ElapsedTime = RDTSC() - StartTime;
//
// TimeHigher is the amount Prcb->IsrTime has increased since we
// begin servicing this interrupt object, ie the amount of time
// spent in higher level ISRs.
//
TimeHigher = Prcb->IsrTime - StartTimeHigher;
//
// Adjust ElapsedTime to time spent on this interrupt object, excluding
// higher level ISRs.
//
ElapsedTime -= TimeHigher;
if (ElapsedTime > KiIsrTscLimit) {
//
// If there is a debugger attached, breakin. Otherwise do nothing.
// N.B. bugchecking is another possibility.
//
if (KdDebuggerEnabled) {
DbgPrint("KE; ISR time limit exceeded (intobj %p)\n",
Interrupt);
DbgBreakPoint();
}
}
//
// Update time spent processing interrupts. This doesn't need
// to be atomic as it doesn't matter if it's a little bit lossy.
// (Though a simple atomic add would do, it's per processor and
// at IRQL > DISPATCH_LEVEL so it doesn't need to be locked).
//
Prcb->IsrTime += ElapsedTime;
//ENDTIMINGend
//
// If IRQL was raised, lower to the previous level.
//
if (Interrupt->Irql != Interrupt->SynchronizeIrql) {
KfLowerIrql(Interrupt->Irql);
}
if ((Handled != FALSE) &&
(Interrupt->Mode == LevelSensitive)) {
//
// The interrupt has been handled.
//
return;
}
//
// If this is the last entry on the chain, get out, otherwise
// advance to the next entry.
//
if (Interrupt->InterruptListEntry.Flink == ListEnd) {
ASSERT(Interrupt->Mode != LevelSensitive);
//
// We should only get to the end of the list if
// (a) interrupts are on this chain are level sensitive and
// no ISR handled the request. This is a system fatal
// condition, or,
// (b) the chain has edge triggered interrupts in which case
// we must run the chain repeatedly until no ISR services
// the request.
//
// Question: Do we actually have chained edge triggered
// interrupts anymore?
//
if (Handled == FALSE) {
break;
}
}
Interrupt = CONTAINING_RECORD(Interrupt->InterruptListEntry.Flink,
KINTERRUPT,
InterruptListEntry);
} while (TRUE);
}
VOID
FASTCALL
KiChainedDispatch(IN PKTRAP_FRAME TrapFrame,
IN PKINTERRUPT Interrupt)
{
KIRQL OldIrql, OldInterruptIrql = 0;
BOOLEAN Handled;
PLIST_ENTRY NextEntry, ListHead;
/* Increase interrupt count */
KeGetCurrentPrcb()->InterruptCount++;
/* Begin the interrupt, making sure it's not spurious */
if (HalBeginSystemInterrupt(Interrupt->Irql,
Interrupt->Vector,
&OldIrql))
{
/* Get list pointers */
ListHead = &Interrupt->InterruptListEntry;
NextEntry = ListHead; /* The head is an entry! */
while (TRUE)
{
/* Check if this interrupt's IRQL is higher than the current one */
if (Interrupt->SynchronizeIrql > Interrupt->Irql)
{
/* Raise to higher IRQL */
OldInterruptIrql = KfRaiseIrql(Interrupt->SynchronizeIrql);
}
/* Acquire interrupt lock */
KxAcquireSpinLock(Interrupt->ActualLock);
/* Call the ISR */
Handled = Interrupt->ServiceRoutine(Interrupt,
Interrupt->ServiceContext);
/* Release interrupt lock */
KxReleaseSpinLock(Interrupt->ActualLock);
/* Check if this interrupt's IRQL is higher than the current one */
if (Interrupt->SynchronizeIrql > Interrupt->Irql)
{
/* Lower the IRQL back */
ASSERT(OldInterruptIrql == Interrupt->Irql);
KfLowerIrql(OldInterruptIrql);
}
/* Check if the interrupt got handled and it's level */
if ((Handled) && (Interrupt->Mode == LevelSensitive)) break;
/* What's next? */
NextEntry = NextEntry->Flink;
/* Is this the last one? */
if (NextEntry == ListHead)
{
/* Level should not have gotten here */
if (Interrupt->Mode == LevelSensitive) break;
/* As for edge, we can only exit once nobody can handle the interrupt */
if (!Handled) break;
}
/* Get the interrupt object for the next pass */
Interrupt = CONTAINING_RECORD(NextEntry, KINTERRUPT, InterruptListEntry);
}
/* Now call the epilogue code */
KiExitInterrupt(TrapFrame, OldIrql, FALSE);
}
else
{
/* Now call the epilogue code */
KiExitInterrupt(TrapFrame, OldIrql, TRUE);
}
}
