/* virtual */ bool IOWorkLoop::runEventSources()
{
bool res = false;
bool traceWL = (gIOKitTrace & kIOTraceWorkLoops) ? true : false;
bool traceES = (gIOKitTrace & kIOTraceEventSources) ? true : false;
closeGate();
if (ISSETP(&fFlags, kLoopTerminate))
goto abort;
if (traceWL)
IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this);
bool more;
do {
CLRP(&fFlags, kLoopRestart);
more = false;
IOInterruptState is = IOSimpleLockLockDisableInterrupt(workToDoLock);
workToDo = false;
IOSimpleLockUnlockEnableInterrupt(workToDoLock, is);
/* NOTE: only loop over event sources in eventChain. Bypass "passive" event sources for performance */
for (IOEventSource *evnt = eventChain; evnt; evnt = evnt->getNext()) {
if (traceES)
IOTimeStampStartConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt);
more |= evnt->checkForWork();
if (traceES)
IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_CLIENT), (uintptr_t) this, (uintptr_t) evnt);
if (ISSETP(&fFlags, kLoopTerminate))
goto abort;
else if (fFlags & kLoopRestart) {
more = true;
break;
}
}
} while (more);
res = true;
if (traceWL)
IOTimeStampEndConstant(IODBG_WORKLOOP(IOWL_WORK), (uintptr_t) this);
abort:
openGate();
return res;
}
void IOInterruptController::timeStampInterruptHandlerInternal(bool isStart, IOInterruptVectorNumber vectorNumber, IOInterruptVector *vector)
{
uint64_t providerID = 0;
vm_offset_t unslidHandler = 0;
vm_offset_t unslidTarget = 0;
IOService * provider = getProvider();
if (provider) {
providerID = provider->getRegistryEntryID();
}
if (vector) {
unslidHandler = VM_KERNEL_UNSLIDE((vm_offset_t)vector->handler);
unslidTarget = VM_KERNEL_UNSLIDE_OR_PERM((vm_offset_t)vector->target);
}
if (isStart) {
IOTimeStampStartConstant(IODBG_INTC(IOINTC_HANDLER), (uintptr_t)vectorNumber, (uintptr_t)unslidHandler,
(uintptr_t)unslidTarget, (uintptr_t)providerID);
} else {
IOTimeStampEndConstant(IODBG_INTC(IOINTC_HANDLER), (uintptr_t)vectorNumber, (uintptr_t)unslidHandler,
(uintptr_t)unslidTarget, (uintptr_t)providerID);
}
}
void IOFilterInterruptEventSource::signalInterrupt()
{
bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false;
IOStatisticsInterrupt();
producerCount++;
if (trace)
IOTimeStampStartConstant(IODBG_INTES(IOINTES_SEMA), (uintptr_t) this, (uintptr_t) owner);
signalWorkAvailable();
if (trace)
IOTimeStampEndConstant(IODBG_INTES(IOINTES_SEMA), (uintptr_t) this, (uintptr_t) owner);
}
IOReturn IOSharedInterruptController::handleInterrupt(void * /*refCon*/,
IOService * nub,
int /*source*/)
{
IOInterruptVectorNumber vectorNumber;
IOInterruptVector *vector;
for (vectorNumber = 0; vectorNumber < numVectors; vectorNumber++) {
vector = &vectors[vectorNumber];
vector->interruptActive = 1;
if (!vector->interruptDisabledSoft) {
// Call the handler if it exists.
if (vector->interruptRegistered) {
bool trace = (gIOKitTrace & kIOTraceInterrupts) ? true : false;
if (trace)
IOTimeStampStartConstant(IODBG_INTC(IOINTC_HANDLER),
(uintptr_t) vectorNumber, (uintptr_t) vector->handler, (uintptr_t)vector->target);
// Call handler.
vector->handler(vector->target, vector->refCon, vector->nub, vector->source);
if (trace)
IOTimeStampEndConstant(IODBG_INTC(IOINTC_HANDLER),
(uintptr_t) vectorNumber, (uintptr_t) vector->handler, (uintptr_t)vector->target);
}
}
vector->interruptActive = 0;
}
// if any of the vectors are dissabled, then dissable this controller.
IOSimpleLockLock(controllerLock);
if (vectorsEnabled != vectorsRegistered) {
nub->disableInterrupt(0);
controllerDisabled = 1;
}
IOSimpleLockUnlock(controllerLock);
return kIOReturnSuccess;
}
void IOFilterInterruptEventSource::disableInterruptOccurred
(void */*refcon*/, IOService *prov, int source)
{
bool filterRes;
uint64_t startTime = 0;
uint64_t endTime = 0;
bool trace = (gIOKitTrace & kIOTraceIntEventSource) ? true : false;
if (trace)
IOTimeStampStartConstant(IODBG_INTES(IOINTES_FILTER),
VM_KERNEL_UNSLIDE(filterAction), (uintptr_t) owner, (uintptr_t) this, (uintptr_t) workLoop);
if (IOInterruptEventSource::reserved->statistics) {
if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelTimeIndex)) {
startTime = mach_absolute_time();
}
}
// Call the filter.
filterRes = (*filterAction)(owner, this);
if (IOInterruptEventSource::reserved->statistics) {
if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelCountIndex)) {
IA_ADD_VALUE(&IOInterruptEventSource::reserved->statistics->interruptStatistics[kInterruptAccountingFirstLevelCountIndex], 1);
}
if (IA_GET_STATISTIC_ENABLED(kInterruptAccountingFirstLevelTimeIndex)) {
endTime = mach_absolute_time();
IA_ADD_VALUE(&IOInterruptEventSource::reserved->statistics->interruptStatistics[kInterruptAccountingFirstLevelTimeIndex], endTime - startTime);
}
}
if (trace)
IOTimeStampEndConstant(IODBG_INTES(IOINTES_FILTER),
VM_KERNEL_UNSLIDE(filterAction), (uintptr_t) owner, (uintptr_t) this, (uintptr_t) workLoop);
if (filterRes) {
prov->disableInterrupt(source); /* disable the interrupt */
signalInterrupt();
}
}
IOReturn IOCommandGate::attemptAction(Action inAction,
void *arg0, void *arg1,
void *arg2, void *arg3)
{
IOReturn res;
IOWorkLoop * wl;
if (!inAction)
return kIOReturnBadArgument;
if (!(wl = workLoop))
return kIOReturnNotReady;
// Try to close the gate if can't get return immediately.
if (!wl->tryCloseGate())
return kIOReturnCannotLock;
// If the command gate is disabled then sleep until we get a wakeup
if (!wl->onThread() && !enabled)
res = kIOReturnNotPermitted;
else {
bool trace = ( gIOKitTrace & kIOTraceCommandGates ) ? true : false;
if (trace)
IOTimeStampStartConstant(IODBG_CMDQ(IOCMDQ_ACTION),
VM_KERNEL_UNSLIDE(inAction), (uintptr_t) owner);
IOStatisticsActionCall();
res = (*inAction)(owner, arg0, arg1, arg2, arg3);
if (trace)
IOTimeStampEndConstant(IODBG_CMDQ(IOCMDQ_ACTION),
VM_KERNEL_UNSLIDE(inAction), (uintptr_t) owner);
}
wl->openGate();
return res;
}
// Timeout handler function. This function is called by the kernel when
// the timeout interval expires.
//
void IOTimerEventSource::timeout(void *self)
{
IOTimerEventSource *me = (IOTimerEventSource *) self;
IOStatisticsTimeout();
if (me->enabled && me->action)
{
IOWorkLoop *
wl = me->workLoop;
if (wl)
{
Action doit;
wl->closeGate();
IOStatisticsCloseGate();
doit = (Action) me->action;
if (doit && me->enabled && AbsoluteTime_to_scalar(&me->abstime))
{
bool trace = (gIOKitTrace & kIOTraceTimers) ? true : false;
if (trace)
IOTimeStampStartConstant(IODBG_TIMES(IOTIMES_ACTION),
(uintptr_t) doit, (uintptr_t) me->owner);
(*doit)(me->owner, me);
#if CONFIG_DTRACE
DTRACE_TMR3(iotescallout__expire, Action, doit, OSObject, me->owner, void, me->workLoop);
#endif
if (trace)
IOTimeStampEndConstant(IODBG_TIMES(IOTIMES_ACTION),
(uintptr_t) doit, (uintptr_t) me->owner);
}
IOStatisticsOpenGate();
wl->openGate();
}
IOReturn IOCommandGate::runAction(Action inAction,
void *arg0, void *arg1,
void *arg2, void *arg3)
{
IOWorkLoop * wl;
uintptr_t * sleepersP;
if (!inAction)
return kIOReturnBadArgument;
if (!(wl = workLoop))
return kIOReturnNotReady;
// closeGate is recursive needn't worry if we already hold the lock.
wl->closeGate();
sleepersP = (uintptr_t *) &reserved;
// If the command gate is disabled and we aren't on the workloop thread
// itself then sleep until we get enabled.
IOReturn res;
if (!wl->onThread())
{
while (!enabled)
{
IOReturn sleepResult = kIOReturnSuccess;
if (workLoop)
{
*sleepersP |= kSleepersWaitEnabled;
sleepResult = wl->sleepGate(&enabled, THREAD_ABORTSAFE);
*sleepersP &= ~kSleepersWaitEnabled;
}
bool wakeupTearDown = (!workLoop || (0 != (*sleepersP & kSleepersRemoved)));
if ((kIOReturnSuccess != sleepResult) || wakeupTearDown) {
wl->openGate();
if (wakeupTearDown)
wl->wakeupGate(sleepersP, false); // No further resources used
return kIOReturnAborted;
}
}
}
bool trace = ( gIOKitTrace & kIOTraceCommandGates ) ? true : false;
if (trace) IOTimeStampStartConstant(IODBG_CMDQ(IOCMDQ_ACTION),
VM_KERNEL_UNSLIDE(inAction), (uintptr_t) owner);
IOStatisticsActionCall();
// Must be gated and on the work loop or enabled
*sleepersP += kSleepersActions;
res = (*inAction)(owner, arg0, arg1, arg2, arg3);
*sleepersP -= kSleepersActions;
if (trace) IOTimeStampEndConstant(IODBG_CMDQ(IOCMDQ_ACTION),
VM_KERNEL_UNSLIDE(inAction), (uintptr_t) owner);
if (kSleepersRemoved == ((kSleepersActionsMask|kSleepersRemoved) & *sleepersP))
{
// no actions outstanding
*sleepersP &= ~kSleepersRemoved;
super::setWorkLoop(0);
}
wl->openGate();
return res;
}
