void
NotifyTaskUnixSignal(
PLW_TASK pTask,
siginfo_t* pInfo
)
{
LOCK_THREAD(pTask->pThread);
if (pTask->EventSignal != TASK_COMPLETE_MASK)
{
while (pTask->pUnixSignal->si_signo)
{
pthread_cond_wait(&pTask->pThread->Event, &pTask->pThread->Lock);
if (pTask->EventSignal == TASK_COMPLETE_MASK)
{
goto cleanup;
}
}
*pTask->pUnixSignal = *pInfo;
pTask->EventSignal |= LW_TASK_EVENT_UNIX_SIGNAL;
RingRemove(&pTask->SignalRing);
RingEnqueue(&pTask->pThread->Tasks, &pTask->SignalRing);
SignalThread(pTask->pThread);
}
cleanup:
UNLOCK_THREAD(pTask->pThread);
}
static
VOID
ScheduleTimedTasks(
PRING pTimed,
LONG64 llNow,
PRING pRunnable
)
{
PLW_TASK pTask = NULL;
PRING pRing = NULL;
PRING pNext = NULL;
for (pRing = pTimed->pNext; pRing != pTimed; pRing = pNext)
{
pNext = pRing->pNext;
pTask = LW_STRUCT_FROM_FIELD(pRing, EPOLL_TASK, QueueRing);
/* No more tasks in the queue are past the deadline
since the queue is sorted */
if (pTask->llDeadline > llNow)
{
break;
}
RingRemove(&pTask->QueueRing);
RingEnqueue(pRunnable, &pTask->QueueRing);
pTask->EventArgs |= LW_TASK_EVENT_TIME;
}
}
VOID
LwRtlCancelTaskGroup(
PLW_TASK_GROUP pGroup
)
{
PRING ring = NULL;
PLW_TASK pTask = NULL;
LOCK_GROUP(pGroup);
pGroup->bCancelled = TRUE;
LockAllThreads(pGroup->pPool);
for (ring = pGroup->Tasks.pNext; ring != &pGroup->Tasks; ring = ring->pNext)
{
pTask = LW_STRUCT_FROM_FIELD(ring, EPOLL_TASK, GroupRing);
if (pTask->EventSignal != TASK_COMPLETE_MASK)
{
pTask->EventSignal |= LW_TASK_EVENT_EXPLICIT | LW_TASK_EVENT_CANCEL;
RingRemove(&pTask->SignalRing);
RingEnqueue(&pTask->pThread->Tasks, &pTask->SignalRing);
SignalThread(pTask->pThread);
}
}
UnlockAllThreads(pGroup->pPool);
UNLOCK_GROUP(pGroup);
}
/*
* Updates the event args on tasks from epoll results and
* schedules them to run.
*/
static
VOID
ScheduleWaitingTasks(
struct epoll_event* pEvents,
int eventCount,
LONG64 llNow,
PRING pRunnable,
PBOOLEAN pbSignalled
)
{
PLW_TASK pTask = NULL;
int i = 0;
*pbSignalled = FALSE;
for (i = 0; i < eventCount; i++)
{
pTask = (PLW_TASK) pEvents[i].data.ptr;
if (!pTask)
{
/* Event was the thread signal fd becoming active */
*pbSignalled = TRUE;
continue;
}
if ((pEvents[i].events & (EPOLLIN | EPOLLHUP)) &&
(pTask->EventWait & LW_TASK_EVENT_FD_READABLE))
{
pTask->EventArgs |= LW_TASK_EVENT_FD_READABLE;
}
if (pEvents[i].events & EPOLLOUT)
{
pTask->EventArgs |= LW_TASK_EVENT_FD_WRITABLE;
}
if ((pEvents[i].events & EPOLLERR) &&
(pTask->EventWait & LW_TASK_EVENT_FD_EXCEPTION))
{
pTask->EventArgs |= LW_TASK_EVENT_FD_EXCEPTION;
}
/* If the task's deadline has expired, set the time event bit */
if (pTask->EventWait & LW_TASK_EVENT_TIME &&
pTask->llDeadline != 0 &&
pTask->llDeadline <= llNow)
{
pTask->EventArgs |= LW_TASK_EVENT_TIME;
}
/* Schedule task to run if it has been triggered */
if (pTask->EventWait & pTask->EventArgs)
{
RingRemove(&pTask->QueueRing);
RingEnqueue(pRunnable, &pTask->QueueRing);
}
}
}
/*
* Updates the set events on tasks from kqueue results and
* schedules them to run.
*/
static
VOID
ScheduleWaitingTasks(
struct kevent* pEvents,
int eventCount,
LONG64 llNow,
PRING pRunnable,
PBOOLEAN pbSignalled
)
{
PLW_TASK pTask = NULL;
int i = 0;
*pbSignalled = FALSE;
for (i = 0; i < eventCount; i++)
{
pTask = (PLW_TASK) pEvents[i].udata;
if (!pTask)
{
/* Event was the thread signal fd becoming active */
*pbSignalled = TRUE;
continue;
}
if (pEvents[i].filter == EVFILT_READ &&
(pTask->EventWait & LW_TASK_EVENT_FD_READABLE))
{
pTask->EventArgs |= LW_TASK_EVENT_FD_READABLE;
}
if (pEvents[i].filter == EVFILT_WRITE &&
(pTask->EventWait & LW_TASK_EVENT_FD_WRITABLE))
{
pTask->EventArgs |= LW_TASK_EVENT_FD_WRITABLE;
}
/* FIXME: errors? */
/* If the task's deadline has expired, set the time event bit */
if (pTask->EventWait & LW_TASK_EVENT_TIME &&
pTask->llDeadline != 0 &&
pTask->llDeadline <= llNow)
{
pTask->EventArgs |= LW_TASK_EVENT_TIME;
}
/* Schedule task to run if it has been triggered */
if (pTask->EventWait & pTask->EventArgs)
{
RingRemove(&pTask->QueueRing);
RingEnqueue(pRunnable, &pTask->QueueRing);
}
}
}
static
VOID
DispatchSignal(
siginfo_t* pInfo
)
{
RING dispatch;
PRING pBase = NULL;
PRING pRing = NULL;
PRING pNext = NULL;
PLW_SIGNAL_SUBSCRIPTION pSub = NULL;
if (!gSignal.pSubscribers || pInfo->si_signo > gSignal.maxSig || pInfo->si_signo < 0)
{
return;
}
RingInit(&dispatch);
pBase = &gSignal.pSubscribers[pInfo->si_signo];
for (pRing = pBase->pNext; pRing != pBase; pRing = pRing->pNext)
{
pSub = LW_STRUCT_FROM_FIELD(pRing, LW_SIGNAL_SUBSCRIPTION, Ring);
pSub->ucRefCount++;
RingInit(&pSub->DispatchRing);
RingEnqueue(&dispatch, &pSub->DispatchRing);
}
UNLOCK_SIGNAL();
for (pRing = dispatch.pNext; pRing != &dispatch; pRing = pRing->pNext)
{
pSub = LW_STRUCT_FROM_FIELD(pRing, LW_SIGNAL_SUBSCRIPTION, DispatchRing);
NotifyTaskUnixSignal(pSub->pTask, pInfo);
}
LOCK_SIGNAL();
for (pRing = dispatch.pNext; pRing != &dispatch; pRing = pNext)
{
pNext = pRing->pNext;
pSub = LW_STRUCT_FROM_FIELD(pRing, LW_SIGNAL_SUBSCRIPTION, DispatchRing);
if (--pSub->ucRefCount == 0)
{
RingRemove(&pSub->Ring);
LwRtlReleaseTask(&pSub->pTask);
LwRtlMemoryFree(pSub);
}
}
}
static
VOID
ScheduleSignalled(
PEPOLL_THREAD pThread,
PRING pRunnable,
PBOOLEAN pbShutdown
)
{
PRING pRing = NULL;
PRING pNext = NULL;
PLW_TASK pTask = NULL;
char c = 0;
int res = 0;
LOCK_THREAD(pThread);
if (pThread->bSignalled)
{
pThread->bSignalled = FALSE;
res = read(pThread->SignalFds[0], &c, sizeof(c));
assert(res == sizeof(c));
/* Add all signalled tasks to the runnable list */
for (pRing = pThread->Tasks.pNext; pRing != &pThread->Tasks; pRing = pNext)
{
pNext = pRing->pNext;
pTask = LW_STRUCT_FROM_FIELD(pRing, EPOLL_TASK, SignalRing);
RingRemove(&pTask->SignalRing);
RingRemove(&pTask->QueueRing);
if (pTask->EventSignal != TASK_COMPLETE_MASK)
{
RingEnqueue(pRunnable, &pTask->QueueRing);
/* Transfer the signal bits into the event args */
pTask->EventArgs |= pTask->EventSignal;
pTask->EventSignal = 0;
}
}
if (pThread->bShutdown && !*pbShutdown)
{
*pbShutdown = pThread->bShutdown;
}
}
UNLOCK_THREAD(pThread);
}
VOID
LwRtlWakeTask(
PLW_TASK pTask
)
{
LOCK_THREAD(pTask->pThread);
if (pTask->EventSignal != TASK_COMPLETE_MASK)
{
pTask->EventSignal |= LW_TASK_EVENT_EXPLICIT;
RingRemove(&pTask->SignalRing);
RingEnqueue(&pTask->pThread->Tasks, &pTask->SignalRing);
SignalThread(pTask->pThread);
}
UNLOCK_THREAD(pTask->pThread);
}
static
NTSTATUS
ProcessRunnable(
PEPOLL_THREAD pThread,
PRING pRunnable,
PRING pTimed,
PRING pWaiting,
LONG64 llNow
)
{
NTSTATUS status = STATUS_SUCCESS;
ULONG ulTicks = MAX_TICKS;
PLW_TASK pTask = NULL;
PLW_TASK_GROUP pGroup = NULL;
PRING pRing = NULL;
PRING pNext = NULL;
/* We are guaranteed to run each task at least once. If tasks remain
on the runnable list by yielding, we will continue to run them
all in a round robin until our ticks are depleted. */
while (ulTicks && !RingIsEmpty(pRunnable))
{
for (pRing = pRunnable->pNext; pRing != pRunnable; pRing = pNext)
{
pNext = pRing->pNext;
pTask = LW_STRUCT_FROM_FIELD(pRing, EPOLL_TASK, QueueRing);
RunTask(pTask, llNow);
if (ulTicks)
{
ulTicks--;
}
if (pTask->EventWait != LW_TASK_EVENT_COMPLETE)
{
/* Task is still waiting to be runnable, update events in epoll set */
status = UpdateEventWait(
pTask,
pThread->EpollFd
);
GOTO_ERROR_ON_STATUS(status);
if (pTask->EventWait & LW_TASK_EVENT_YIELD)
{
/* Task is yielding. Set YIELD in its trigger arguments and
and leave it on the runnable list for the next iteration */
pTask->EventArgs |= LW_TASK_EVENT_YIELD;
}
else if (pTask->EventWait & LW_TASK_EVENT_TIME)
{
/* If the task is waiting for a timeout, insert it into the timed queue */
RingRemove(&pTask->QueueRing);
InsertTimedQueue(pTimed, pTask);
}
else
{
/* Otherwise, put it in the generic waiting queue */
RingRemove(&pTask->QueueRing);
RingEnqueue(pWaiting, &pTask->QueueRing);
}
}
else
{
/* Task is complete */
RingRemove(&pTask->QueueRing);
/* Turn off any fd in the epoll set */
if (pTask->Fd >= 0)
{
status = LwRtlSetTaskFd(pTask, pTask->Fd, 0);
GOTO_ERROR_ON_STATUS(status);
}
/* Unsubscribe task from any UNIX signals */
if (pTask->pUnixSignal)
{
RegisterTaskUnixSignal(pTask, 0, FALSE);
}
LOCK_POOL(pThread->pPool);
pThread->ulLoad--;
UNLOCK_POOL(pThread->pPool);
pGroup = pTask->pGroup;
/* If task was in a task group, remove it and notify anyone waiting
on the group */
if (pGroup)
{
LOCK_GROUP(pGroup);
pTask->pGroup = NULL;
RingRemove(&pTask->GroupRing);
pthread_cond_broadcast(&pGroup->Event);
UNLOCK_GROUP(pGroup);
}
LOCK_THREAD(pThread);
if (--pTask->ulRefCount)
{
/* The task still has a reference, so mark it as completed
and notify anyone waiting on it */
pTask->EventSignal = TASK_COMPLETE_MASK;
pthread_cond_broadcast(&pThread->Event);
UNLOCK_THREAD(pThread);
}
else
{
/* We held the last reference to the task, so delete it */
RingRemove(&pTask->SignalRing);
UNLOCK_THREAD(pThread);
TaskDelete(pTask);
}
}
}
}
error:
return status;
}
static
VOID
ProcessRunnable(
PKQUEUE_THREAD pThread,
PKQUEUE_COMMANDS pCommands,
PRING pRunnable,
PRING pTimed,
PRING pWaiting,
LONG64 llNow
)
{
ULONG ulTicks = MAX_TICKS;
PLW_TASK pTask = NULL;
PLW_TASK_GROUP pGroup = NULL;
PRING pRing = NULL;
PRING pNext = NULL;
/* We are guaranteed to run each task at least once. If tasks remain
on the runnable list by yielding, we will continue to run them
all in a round robin until our ticks are depleted. */
while (ulTicks && !RingIsEmpty(pRunnable))
{
for (pRing = pRunnable->pNext; pRing != pRunnable; pRing = pNext)
{
pNext = pRing->pNext;
pTask = LW_STRUCT_FROM_FIELD(pRing, KQUEUE_TASK, QueueRing);
RunTask(pTask, llNow);
if (ulTicks)
{
ulTicks--;
}
if (pTask->EventWait != LW_TASK_EVENT_COMPLETE)
{
if (pTask->EventWait & LW_TASK_EVENT_YIELD)
{
/* Task is yielding. Set the YIELD flag and
leave it on the runnable list for the next iteration. */
pTask->EventArgs |= LW_TASK_EVENT_YIELD;
}
else
{
/* Task is still waiting on events, update kqueue */
UpdateEventWait(pCommands, pTask);
if (pTask->EventWait & LW_TASK_EVENT_TIME)
{
/* If the task is waiting for a timeout,
insert it into the timed queue */
RingRemove(&pTask->QueueRing);
InsertTimedQueue(pTimed, pTask);
}
else
{
/* Otherwise, put it in the generic waiting queue */
RingRemove(&pTask->QueueRing);
RingEnqueue(pWaiting, &pTask->QueueRing);
}
}
}
else
{
/* Task is complete */
RingRemove(&pTask->QueueRing);
/* Remove any associated events from the kqueue */
if (pTask->Fd >= 0)
{
(void) LwRtlSetTaskFd(pTask, pTask->Fd, 0);
}
/* Unsubscribe task from any UNIX signals */
if (pTask->pUnixSignal)
{
RegisterTaskUnixSignal(pTask, 0, FALSE);
}
LOCK_POOL(pThread->pPool);
pThread->ulLoad--;
UNLOCK_POOL(pThread->pPool);
pGroup = pTask->pGroup;
/* If task was in a task group, remove it and notify anyone waiting
on the group */
if (pGroup)
{
LOCK_GROUP(pGroup);
pTask->pGroup = NULL;
RingRemove(&pTask->GroupRing);
pthread_cond_broadcast(&pGroup->Event);
UNLOCK_GROUP(pGroup);
}
LOCK_THREAD(pThread);
if (--pTask->ulRefCount)
{
/* The task still has a reference, so mark it as completed
and notify anyone waiting on it */
pTask->EventSignal = TASK_COMPLETE_MASK;
pthread_cond_broadcast(&pThread->Event);
UNLOCK_THREAD(pThread);
}
else
{
/* We held the last reference to the task, so delete it */
RingRemove(&pTask->SignalRing);
UNLOCK_THREAD(pThread);
TaskDelete(pTask);
}
}
}
}
/* Update kevent commands for yielding tasks */
for (pRing = pRunnable->pNext; pRing != pRunnable; pRing = pRing->pNext)
{
pTask = LW_STRUCT_FROM_FIELD(pRing, KQUEUE_TASK, QueueRing);
if (pTask->EventArgs & LW_TASK_EVENT_YIELD)
{
UpdateEventWait(pCommands, pTask);
}
}
}
NTSTATUS
RegisterTaskUnixSignal(
LW_IN PLW_TASK pTask,
LW_IN int Sig,
LW_IN LW_BOOLEAN bSubscribe
)
{
NTSTATUS status = STATUS_SUCCESS;
size_t i = 0;
PRING pBase = NULL;
PRING pRing = NULL;
PLW_SIGNAL_SUBSCRIPTION pExisting = NULL;
PLW_SIGNAL_SUBSCRIPTION pSub = NULL;
struct sigaction action;
#ifdef SIGRTMAX
int maxSig = SIGRTMAX;
#else
int maxSig = SIGUSR2;
#endif
if (Sig == 0)
{
for (i = 1; i < maxSig + 1; i++)
{
status = RegisterTaskUnixSignal(pTask, (int) i, bSubscribe);
if (status)
{
return status;
}
}
return STATUS_SUCCESS;
}
LOCK_SIGNAL();
if (Sig > maxSig || Sig < 0)
{
status = STATUS_INVALID_PARAMETER;
GOTO_ERROR_ON_STATUS(status);
}
if (!gSignal.pSubscribers)
{
status = LW_RTL_ALLOCATE_ARRAY_AUTO(&gSignal.pSubscribers, maxSig + 1);
GOTO_ERROR_ON_STATUS(status);
for (i = 0; i < maxSig + 1; i++)
{
RingInit(&gSignal.pSubscribers[i]);
}
gSignal.maxSig = maxSig;
}
pBase = &gSignal.pSubscribers[Sig];
for (pRing = pBase->pNext; pRing != pBase; pRing = pRing->pNext)
{
pSub = LW_STRUCT_FROM_FIELD(pRing, LW_SIGNAL_SUBSCRIPTION, Ring);
if (pSub->pTask == pTask)
{
pExisting = pSub;
break;
}
}
if (bSubscribe && !pExisting)
{
if (Sig != SIGINT)
{
memset(&action, 0, sizeof(action));
/* Make sure there is a dummy handler for the signal
so it is actually delivered to the process */
action.sa_handler = DummyHandler;
action.sa_flags = 0;
if (sigaction(Sig, &action, NULL) < 0)
{
status = LwErrnoToNtStatus(errno);
GOTO_ERROR_ON_STATUS(status);
}
}
status = LW_RTL_ALLOCATE_AUTO(&pSub);
GOTO_ERROR_ON_STATUS(status);
pSub->pTask = pTask;
pSub->ucRefCount = 1;
RingInit(&pSub->Ring);
RingInit(&pSub->DispatchRing);
RingEnqueue(pBase, &pSub->Ring);
RetainTask(pTask);
}
else if (!bSubscribe && pExisting)
{
RingRemove(&pExisting->Ring);
if (--pExisting->ucRefCount == 0)
{
LwRtlReleaseTask(&pExisting->pTask);
LwRtlMemoryFree(pExisting);
}
}
error:
UNLOCK_SIGNAL();
return status;
}
VOID
ScheduleWorkItem(
PLW_WORK_THREADS pThreads,
PLW_WORK_ITEM pItem,
LW_SCHEDULE_FLAGS Flags
)
{
size_t i = 0;
if (pThreads == NULL)
{
pThreads = pItem->pThreads;
}
LOCK_THREADS(pThreads);
assert(pThreads->ulStarted > 0);
/* Enqueue work item */
if (Flags & LW_SCHEDULE_HIGH_PRIORITY)
{
RingEnqueueFront(&pThreads->WorkItems, &pItem->Ring);
}
else
{
RingEnqueue(&pThreads->WorkItems, &pItem->Ring);
}
pThreads->ulQueued++;
/*
* If there are more pending work items than there
* are available threads, and not all threads are started,
* try to start another one to handle the additional load
*/
if (pThreads->ulAvailable < pThreads->ulQueued &&
pThreads->ulStarted < pThreads->ulWorkThreadCount)
{
for (i = 0; i < pThreads->ulWorkThreadCount; i++)
{
if (!pThreads->pWorkThreads[i].bStarted)
{
if (StartWorkThread(pThreads, &pThreads->pWorkThreads[i]) != STATUS_SUCCESS)
{
LW_RTL_LOG_WARNING("Could not start work item thread");
/* Signal an existing thread instead */
pthread_cond_signal(&pThreads->Event);
}
break;
}
}
}
else if (pThreads->ulAvailable)
{
/* Signal an existing thread */
pthread_cond_signal(&pThreads->Event);
}
UNLOCK_THREADS(pThreads);
}
