LW_NTSTATUS
LwRtlQueueWorkItem(
LW_IN PLW_THREAD_POOL pPool,
LW_IN LW_WORK_ITEM_FUNCTION_COMPAT pfnFunc,
LW_IN LW_PVOID pContext,
LW_IN LW_WORK_ITEM_FLAGS Flags
)
{
NTSTATUS status = STATUS_SUCCESS;
PCOMPAT_WORK_ITEM pCompat = NULL;
PLW_WORK_ITEM pItem = NULL;
status = LW_RTL_ALLOCATE_AUTO(&pCompat);
GOTO_ERROR_ON_STATUS(status);
pCompat->pfnFunc = pfnFunc;
pCompat->pContext = pContext;
status = LwRtlCreateWorkItem(pPool, &pItem, CompatWorkItem, pCompat);
GOTO_ERROR_ON_STATUS(status);
pCompat = NULL;
LwRtlScheduleWorkItem(pItem, Flags);
pItem = NULL;
error:
RTL_FREE(&pCompat);
LwRtlFreeWorkItem(&pItem);
return status;
}
static
NTSTATUS
WorkWait(
PLW_WORK_THREAD pThread
)
{
NTSTATUS status = STATUS_SUCCESS;
struct timespec ts = {0};
LONG64 llDeadline = 0;
int err = 0;
BOOLEAN bLastThread = FALSE;
PLW_WORK_THREADS pThreads = pThread->pThreads;
while (pThreads->ulQueued == 0)
{
if (pThreads->bShutdown)
{
status = STATUS_CANCELLED;
GOTO_ERROR_ON_STATUS(status);
}
if (pThread->pThreads->ulWorkThreadTimeout && !bLastThread)
{
status = TimeNow(&llDeadline);
GOTO_ERROR_ON_STATUS(status);
llDeadline += (LONG64) 1000000000ll * pThread->pThreads->ulWorkThreadTimeout;
ts.tv_sec = llDeadline / 1000000000ll;
ts.tv_nsec = llDeadline % 1000000000ll;
err = pthread_cond_timedwait(&pThread->pThreads->Event, &pThread->pThreads->Lock, &ts);
}
else
{
err = pthread_cond_wait(&pThread->pThreads->Event, &pThread->pThreads->Lock);
}
bLastThread = pThreads->ulWorkItemCount && pThreads->ulStarted == 1;
switch(err)
{
case ETIMEDOUT:
if (!bLastThread)
{
status = STATUS_TIMEOUT;
GOTO_ERROR_ON_STATUS(status);
}
break;
default:
status = LwErrnoToNtStatus(err);
GOTO_ERROR_ON_STATUS(status);
}
}
error:
return status;
}
LW_NTSTATUS
LwRtlSetTaskFd(
PLW_TASK pTask,
int Fd,
LW_TASK_EVENT_MASK Mask
)
{
NTSTATUS status = STATUS_SUCCESS;
struct epoll_event event;
memset(&event, 0, sizeof(event));
if (Fd < 0)
{
status = STATUS_INVALID_HANDLE;
GOTO_ERROR_ON_STATUS(status);
}
if (Fd == pTask->Fd)
{
if (Mask == 0)
{
pTask->Fd = -1;
if (epoll_ctl(pTask->pThread->EpollFd, EPOLL_CTL_DEL, Fd, &event) < 0)
{
ABORT_ON_FATAL_ERRNO(errno);
status = LwErrnoToNtStatus(errno);
GOTO_ERROR_ON_STATUS(status);
}
}
}
else if (Mask)
{
if (pTask->Fd >= 0)
{
/* Only one fd is supported */
status = STATUS_INSUFFICIENT_RESOURCES;
GOTO_ERROR_ON_STATUS(status);
}
if (epoll_ctl(pTask->pThread->EpollFd, EPOLL_CTL_ADD, Fd, &event) < 0)
{
ABORT_ON_FATAL_ERRNO(errno);
status = LwErrnoToNtStatus(errno);
GOTO_ERROR_ON_STATUS(status);
}
pTask->Fd = Fd;
pTask->EventLastWait = 0;
}
error:
return status;
}
static
NTSTATUS
Poll(
IN PCLOCK pClock,
IN OUT PLONG64 pllNow,
IN int EpollFd,
OUT struct epoll_event* pEvents,
IN int maxEvents,
IN LONG64 llNextDeadline,
OUT int* pReady
)
{
NTSTATUS status = STATUS_SUCCESS;
int ready = 0;
int timeout = 0;
do
{
if (llNextDeadline >= 0)
{
/* Convert to timeout in milliseconds */
timeout = (llNextDeadline - *pllNow) / 1000000ll;
if (timeout < 0)
{
timeout = 0;
}
}
else
{
timeout = -1;
}
ready = epoll_wait(EpollFd, pEvents, maxEvents, timeout);
if (ready < 0 && errno == EINTR)
{
/* Update current time so the next timeout calculation is correct */
status = ClockGetMonotonicTime(pClock, pllNow);
GOTO_ERROR_ON_STATUS(status);
}
} while (ready < 0 && errno == EINTR);
if (ready < 0)
{
ABORT_ON_FATAL_ERRNO(errno);
status = LwErrnoToNtStatus(errno);
GOTO_ERROR_ON_STATUS(status);
}
*pReady = ready;
error:
return status;
}
NTSTATUS
InitWorkThreads(
PLW_WORK_THREADS pThreads,
PLW_THREAD_POOL_ATTRIBUTES pAttrs,
int numCpus
)
{
NTSTATUS status = STATUS_SUCCESS;
size_t i = 0;
RingInit(&pThreads->WorkItems);
status = LwErrnoToNtStatus(pthread_mutex_init(&pThreads->Lock, NULL));
GOTO_ERROR_ON_STATUS(status);
pThreads->bDestroyLock = TRUE;
status = LwErrnoToNtStatus(pthread_cond_init(&pThreads->Event, NULL));
GOTO_ERROR_ON_STATUS(status);
pThreads->bDestroyEvent = TRUE;
pThreads->ulWorkThreadCount = GetWorkThreadsAttr(pAttrs, numCpus);
pThreads->ulWorkThreadStackSize = pAttrs ? pAttrs->ulWorkThreadStackSize : 0;
pThreads->ulWorkThreadTimeout = GetWorkThreadTimeoutAttr(pAttrs);
if (pThreads->ulWorkThreadCount)
{
status = LW_RTL_ALLOCATE_ARRAY_AUTO(
&pThreads->pWorkThreads,
pThreads->ulWorkThreadCount);
GOTO_ERROR_ON_STATUS(status);
for (i = 0; i < pThreads->ulWorkThreadCount; i++)
{
pThreads->pWorkThreads[i].Thread = INVALID_THREAD_HANDLE;
}
}
if (pThreads->ulWorkThreadTimeout == 0)
{
for (i = 0; i < pThreads->ulWorkThreadCount; i++)
{
status = StartWorkThread(pThreads, &pThreads->pWorkThreads[i]);
GOTO_ERROR_ON_STATUS(status);
}
}
error:
return status;
}
/*
* Called with pThreads->Lock held
*/
static
NTSTATUS
StartWorkThread(
PLW_WORK_THREADS pThreads,
PLW_WORK_THREAD pThread
)
{
NTSTATUS status = STATUS_SUCCESS;
pthread_attr_t threadAttr;
pthread_attr_t* pThreadAttr = NULL;
pThread->pThreads = pThreads;
if (pThreads->ulWorkThreadStackSize)
{
status = LwErrnoToNtStatus(pthread_attr_init(&threadAttr));
GOTO_ERROR_ON_STATUS(status);
pThreadAttr = &threadAttr;
status = LwErrnoToNtStatus(
pthread_attr_setstacksize(pThreadAttr, pThreads->ulWorkThreadStackSize));
GOTO_ERROR_ON_STATUS(status);
status = LwRtlResetAffinityThreadAttribute(pThreadAttr);
GOTO_ERROR_ON_STATUS(status);
}
status = LwErrnoToNtStatus(
pthread_create(
&pThread->Thread,
pThreadAttr,
WorkThread,
pThread));
GOTO_ERROR_ON_STATUS(status);
pThread->bStarted = TRUE;
pThreads->ulStarted++;
error:
if (pThreadAttr)
{
pthread_attr_destroy(pThreadAttr);
}
return status;
}
static
NTSTATUS
GetFdLimit(
PULONG pulLimit
)
{
NTSTATUS status = STATUS_SUCCESS;
struct rlimit limit = {0};
if (getrlimit(RLIMIT_NOFILE, &limit) != 0)
{
status = LwErrnoToNtStatus(errno);
GOTO_ERROR_ON_STATUS(status);
}
if (limit.rlim_cur == RLIM_INFINITY)
{
*pulLimit = (ULONG) 0xFFFFFFFF;
}
else
{
*pulLimit = (ULONG) limit.rlim_cur;
}
error:
return status;
}
NTSTATUS
LwRtlQueryTaskFd(
PLW_TASK pTask,
int Fd,
PLW_TASK_EVENT_MASK pMask
)
{
NTSTATUS status = STATUS_SUCCESS;
if (Fd < 0 || Fd != pTask->Fd)
{
status = STATUS_INVALID_HANDLE;
GOTO_ERROR_ON_STATUS(status);
}
*pMask = pTask->FdSetMask;
cleanup:
return status;
error:
*pMask = 0;
goto cleanup;
}
NTSTATUS
LwRtlResetAffinityThreadAttribute(
pthread_attr_t* pAttr
)
{
NTSTATUS status = STATUS_SUCCESS;
CPU_SET_TYPE cpuSet;
ULONG numCpus = 0;
int i = 0;
CPU_ZERO(&cpuSet);
numCpus = LwRtlGetCpuCount();
for (i = 0; i < numCpus; i++)
{
(void) CPU_SET(i, &cpuSet);
}
status = LwErrnoToNtStatus(
pthread_attr_setaffinity_np(pAttr, sizeof(cpuSet), &cpuSet));
GOTO_ERROR_ON_STATUS(status);
error:
return status;
}
LW_NTSTATUS
LwRtlCreateThreadPoolAttributes(
LW_OUT PLW_THREAD_POOL_ATTRIBUTES* ppAttrs
)
{
NTSTATUS status = STATUS_SUCCESS;
PLW_THREAD_POOL_ATTRIBUTES pAttrs = NULL;
status = LW_RTL_ALLOCATE_AUTO(&pAttrs);
GOTO_ERROR_ON_STATUS(status);
pAttrs->bDelegateTasks = TRUE;
pAttrs->lTaskThreads = 0;
pAttrs->lWorkThreads = 0;
pAttrs->ulTaskThreadStackSize = (ULONG) _LW_TASK_THREAD_STACK_SIZE;
pAttrs->ulWorkThreadStackSize = (ULONG) _LW_WORK_THREAD_STACK_SIZE;
pAttrs->ulWorkThreadTimeout = DEFAULT_WORK_THREAD_TIMEOUT;
*ppAttrs = pAttrs;
cleanup:
return status;
error:
*ppAttrs = NULL;
goto cleanup;
}
NTSTATUS
LwRtlQueryTaskFd(
PLW_TASK pTask,
int Fd,
PLW_TASK_EVENT_MASK pMask
)
{
NTSTATUS status = STATUS_SUCCESS;
if (Fd < 0 || Fd != pTask->Fd)
{
status = STATUS_INVALID_HANDLE;
GOTO_ERROR_ON_STATUS(status);
}
*pMask = pTask->EventArgs &
(LW_TASK_EVENT_FD_READABLE |
LW_TASK_EVENT_FD_WRITABLE |
LW_TASK_EVENT_FD_EXCEPTION);
cleanup:
return status;
error:
*pMask = 0;
goto cleanup;
}
LW_NTSTATUS
LwRtlSetTaskFd(
PLW_TASK pTask,
int Fd,
LW_TASK_EVENT_MASK Mask
)
{
NTSTATUS status = STATUS_SUCCESS;
if (Fd < 0 || Fd >= FD_SETSIZE)
{
status = STATUS_INVALID_HANDLE;
GOTO_ERROR_ON_STATUS(status);
}
if (Fd == pTask->Fd)
{
if (Mask == 0)
{
pTask->FdWaitMask = 0;
pTask->FdSetMask = 0;
pTask->Fd = -1;
}
else
{
pTask->FdWaitMask = Mask;
}
}
else
{
if (pTask->Fd >= 0)
{
/* Only one fd is supported per task at the moment */
status = STATUS_INSUFFICIENT_RESOURCES;
GOTO_ERROR_ON_STATUS(status);
}
pTask->Fd = Fd;
pTask->FdWaitMask = Mask;
pTask->FdSetMask = 0;
}
error:
return status;
}
NTSTATUS
LwRtlCreateTaskGroup(
PLW_THREAD_POOL pPool,
PLW_TASK_GROUP* ppGroup
)
{
NTSTATUS status = STATUS_SUCCESS;
PLW_TASK_GROUP pGroup = NULL;
if (pPool->pDelegate)
{
return LwRtlCreateTaskGroup(pPool->pDelegate, ppGroup);
}
status = LW_RTL_ALLOCATE_AUTO(&pGroup);
GOTO_ERROR_ON_STATUS(status);
pGroup->pPool = pPool;
RingInit(&pGroup->Tasks);
status = LwErrnoToNtStatus(pthread_mutex_init(&pGroup->Lock, NULL));
GOTO_ERROR_ON_STATUS(status);
pGroup->bLockInit = TRUE;
status = LwErrnoToNtStatus(pthread_cond_init(&pGroup->Event, NULL));
GOTO_ERROR_ON_STATUS(status);
pGroup->bEventInit = TRUE;
*ppGroup = pGroup;
cleanup:
return status;
error:
LwRtlFreeTaskGroup(&pGroup);
*ppGroup = NULL;
goto cleanup;
}
LW_NTSTATUS
LwRtlSetTaskUnixSignal(
LW_IN PLW_TASK pTask,
LW_IN int Sig,
LW_IN LW_BOOLEAN bSubscribe
)
{
NTSTATUS status = STATUS_SUCCESS;
if (bSubscribe && !pTask->pUnixSignal)
{
status = LW_RTL_ALLOCATE_AUTO(&pTask->pUnixSignal);
GOTO_ERROR_ON_STATUS(status);
}
status = RegisterTaskUnixSignal(pTask, Sig, bSubscribe);
GOTO_ERROR_ON_STATUS(status);
error:
return status;
}
static
NTSTATUS
WorkLoop(
PLW_WORK_THREAD pThread
)
{
NTSTATUS status = STATUS_SUCCESS;
PRING pRing = NULL;
PLW_WORK_ITEM pItem = NULL;
PLW_WORK_THREADS pThreads = pThread->pThreads;
LOCK_THREADS(pThread->pThreads);
for(;;)
{
pThreads->ulAvailable++;
status = WorkWait(pThread);
GOTO_ERROR_ON_STATUS(status);
RingDequeue(&pThreads->WorkItems, &pRing);
pThreads->ulQueued--;
pThreads->ulAvailable--;
UNLOCK_THREADS(pThreads);
pItem = LW_STRUCT_FROM_FIELD(pRing, LW_WORK_ITEM, Ring);
pItem->pfnFunc(pItem, pItem->pContext);
LOCK_THREADS(pThreads);
}
error:
pThreads->ulAvailable--;
pThreads->ulStarted--;
pThread->bStarted = FALSE;
/* If the thread pool is not being shut down, nothing is
going to call pthread_join() on this thread, so call
pthread_detach() now */
if (!pThreads->bShutdown)
{
pthread_detach(pThread->Thread);
pThread->Thread = INVALID_THREAD_HANDLE;
}
UNLOCK_THREADS(pThreads);
return status;
}
NTSTATUS
CreateWorkItem(
LW_IN PLW_WORK_THREADS pThreads,
LW_OUT PLW_WORK_ITEM* ppWorkItem,
LW_WORK_ITEM_FUNCTION pfnFunc,
PVOID pContext
)
{
PLW_WORK_ITEM pItem = NULL;
NTSTATUS status = STATUS_SUCCESS;
LOCK_THREADS(pThreads);
if (pThreads->ulStarted == 0)
{
/* Make sure at least one thread is running */
status = StartWorkThread(pThreads, &pThreads->pWorkThreads[0]);
GOTO_ERROR_ON_STATUS(status);
}
status = LW_RTL_ALLOCATE_AUTO(&pItem);
GOTO_ERROR_ON_STATUS(status);
RingInit(&pItem->Ring);
pItem->pThreads = pThreads;
pItem->pfnFunc = pfnFunc;
pItem->pContext = pContext;
pThreads->ulWorkItemCount++;
error:
UNLOCK_THREADS(pThreads);
*ppWorkItem = pItem;
return status;
}
/*
* Updates the epoll set with the events a task is waiting on.
*/
static
NTSTATUS
UpdateEventWait(
PEPOLL_TASK pTask,
int EpollFd
)
{
NTSTATUS status = STATUS_SUCCESS;
__uint32_t events = 0;
struct epoll_event event;
if ((pTask->EventWait & FD_EVENTS) != (pTask->EventLastWait & FD_EVENTS) && pTask->Fd >= 0)
{
if (pTask->EventWait & LW_TASK_EVENT_FD_READABLE)
{
events |= EPOLLIN;
}
if (pTask->EventWait & LW_TASK_EVENT_FD_WRITABLE)
{
events |= EPOLLOUT;
}
if (pTask->EventWait & LW_TASK_EVENT_FD_EXCEPTION)
{
events |= EPOLLERR;
}
memset(&event, 0, sizeof(event));
event.events = events | EPOLLET;
event.data.ptr = pTask;
if (epoll_ctl(EpollFd, EPOLL_CTL_MOD, pTask->Fd, &event) < 0)
{
ABORT_ON_FATAL_ERRNO(errno);
status = LwErrnoToNtStatus(errno);
GOTO_ERROR_ON_STATUS(status);
}
}
pTask->EventLastWait = pTask->EventWait;
error:
return status;
}
LW_NTSTATUS
LwRtlSetThreadPoolAttribute(
LW_IN LW_OUT PLW_THREAD_POOL_ATTRIBUTES pAttrs,
LW_IN LW_THREAD_POOL_OPTION Option,
...
)
{
NTSTATUS status = STATUS_SUCCESS;
va_list ap;
va_start(ap, Option);
switch(Option)
{
case LW_THREAD_POOL_OPTION_DELEGATE_TASKS:
pAttrs->bDelegateTasks = va_arg(ap, int);
break;
case LW_THREAD_POOL_OPTION_TASK_THREADS:
pAttrs->lTaskThreads = va_arg(ap, LONG);
break;
case LW_THREAD_POOL_OPTION_WORK_THREADS:
pAttrs->lWorkThreads = va_arg(ap, LONG);
break;
case LW_THREAD_POOL_OPTION_TASK_THREAD_STACK_SIZE:
pAttrs->ulTaskThreadStackSize = va_arg(ap, ULONG);
break;
case LW_THREAD_POOL_OPTION_WORK_THREAD_STACK_SIZE:
pAttrs->ulWorkThreadStackSize = va_arg(ap, ULONG);
break;
case LW_THREAD_POOL_OPTION_WORK_THREAD_TIMEOUT:
pAttrs->ulWorkThreadTimeout = va_arg(ap, ULONG);
break;
default:
status = STATUS_NOT_SUPPORTED;
GOTO_ERROR_ON_STATUS(status);
}
cleanup:
va_end(ap);
return status;
error:
goto cleanup;
}
NTSTATUS
AcquireDelegatePool(
PLW_THREAD_POOL* ppPool
)
{
NTSTATUS status = STATUS_SUCCESS;
LW_THREAD_POOL_ATTRIBUTES attrs =
{
.bDelegateTasks = FALSE,
.lTaskThreads = 0,
.lWorkThreads = 0,
.ulTaskThreadStackSize = (ULONG) _LW_TASK_THREAD_STACK_SIZE,
.ulWorkThreadStackSize = (ULONG) _LW_WORK_THREAD_STACK_SIZE
};
if (getenv("LW_GLOBAL_TASK_THREADS"))
{
attrs.lTaskThreads = (LONG) atoi(getenv("LW_GLOBAL_TASK_THREADS"));
}
pthread_mutex_lock(&gpDelegateLock);
if (!gpDelegatePool)
{
status = LwRtlCreateThreadPool(&gpDelegatePool, &attrs);
GOTO_ERROR_ON_STATUS(status);
gpDelegatePoolRefCount = 1;
}
else
{
gpDelegatePoolRefCount++;
}
*ppPool = gpDelegatePool;
cleanup:
pthread_mutex_unlock(&gpDelegateLock);
return status;
error:
goto cleanup;
}
static
NTSTATUS
SelectThreadInit(
PLW_THREAD_POOL_ATTRIBUTES pAttrs,
PSELECT_THREAD pThread
)
{
NTSTATUS status = STATUS_SUCCESS;
pthread_attr_t pthreadAttr;
BOOLEAN bAttrInit = FALSE;
status = LwErrnoToNtStatus(pthread_attr_init(&pthreadAttr));
GOTO_ERROR_ON_STATUS(status);
bAttrInit = TRUE;
GOTO_ERROR_ON_STATUS(status = LwErrnoToNtStatus(pthread_mutex_init(&pThread->Lock, NULL)));
GOTO_ERROR_ON_STATUS(status = LwErrnoToNtStatus(pthread_cond_init(&pThread->Event, NULL)));
if (pipe(pThread->SignalFds) < 0)
{
GOTO_ERROR_ON_STATUS(status = LwErrnoToNtStatus(errno));
}
SetCloseOnExec(pThread->SignalFds[0]);
SetCloseOnExec(pThread->SignalFds[1]);
RingInit(&pThread->Tasks);
if (pAttrs && pAttrs->ulTaskThreadStackSize)
{
GOTO_ERROR_ON_STATUS(status = LwErrnoToNtStatus(
pthread_attr_setstacksize(&pthreadAttr, pAttrs->ulTaskThreadStackSize)));
}
GOTO_ERROR_ON_STATUS(status = LwErrnoToNtStatus(
pthread_create(
&pThread->Thread,
&pthreadAttr,
EventThread,
pThread)));
error:
if (bAttrInit)
{
pthread_attr_destroy(&pthreadAttr);
}
return status;
}
static
NTSTATUS
ExpandCommands(
PKQUEUE_COMMANDS pCommands
)
{
NTSTATUS status = STATUS_SUCCESS;
ULONG ulNewCapacity = 0;
struct kevent* pNewCommands = NULL;
if (pCommands->ulCommandMax + COMMANDS_PER_FD > pCommands->ulCommandCapacity)
{
ulNewCapacity = pCommands->ulCommandCapacity;
if (ulNewCapacity == 0)
{
ulNewCapacity = 8;
}
while (pCommands->ulCommandMax > ulNewCapacity)
{
ulNewCapacity *= 2;
}
pNewCommands = LwRtlMemoryRealloc(
pCommands->pCommands,
sizeof(*pNewCommands) * ulNewCapacity);
if (!pNewCommands)
{
status = STATUS_INSUFFICIENT_RESOURCES;
GOTO_ERROR_ON_STATUS(status);
}
pCommands->ulCommandCapacity = ulNewCapacity;
pCommands->pCommands = pNewCommands;
}
pCommands->ulCommandMax += COMMANDS_PER_FD;
error:
return status;
}
/**
* @brief Set the thread cpu affinity attribute to the
* supplied cpu number.
* Specifying a cpu number that does not exist, or is
* offline will cause pthread_create() to fail with
* EINVAL.
*
* @param pAttr pointer to the pthread_att_t
* @param CpuNumber the cpu number
*/
NTSTATUS
LwRtlSetAffinityThreadAttribute(
pthread_attr_t* pAttr,
ULONG CpuNumber
)
{
NTSTATUS status = STATUS_SUCCESS;
CPU_SET_TYPE cpuSet;
CPU_ZERO(&cpuSet);
(void) CPU_SET(CpuNumber, &cpuSet);
status = LwErrnoToNtStatus(
pthread_attr_setaffinity_np(pAttr, sizeof(cpuSet), &cpuSet));
GOTO_ERROR_ON_STATUS(status);
error:
return status;
}
LW_NTSTATUS
LwRtlBlockSignals(
VOID
)
{
NTSTATUS status = STATUS_SUCCESS;
sigset_t blockSet;
size_t i = 0;
sigfillset(&blockSet);
/* Don't block blacklisted signals */
for (i = 0; SignalBlacklist[i]; i++)
{
sigdelset(&blockSet, SignalBlacklist[i]);
}
status = LwErrnoToNtStatus(pthread_sigmask(SIG_SETMASK, &blockSet, NULL));
GOTO_ERROR_ON_STATUS(status);
error:
return status;
}
NTSTATUS
LwRtlCreateTask(
PLW_THREAD_POOL pPool,
PLW_TASK* ppTask,
PLW_TASK_GROUP pGroup,
LW_TASK_FUNCTION pfnFunc,
PVOID pContext
)
{
NTSTATUS status = STATUS_SUCCESS;
PSELECT_TASK pTask = NULL;
if (pPool->pDelegate)
{
return LwRtlCreateTask(pPool->pDelegate, ppTask, pGroup, pfnFunc, pContext);
}
GOTO_ERROR_ON_STATUS(status = LW_RTL_ALLOCATE_AUTO(&pTask));
RingInit(&pTask->GroupRing);
RingInit(&pTask->EventRing);
pTask->pPool = pPool;
pTask->pGroup = pGroup;
pTask->ulRefCount = 2;
pTask->pfnFunc = pfnFunc;
pTask->pFuncContext = pContext;
pTask->Fd = -1;
pTask->TriggerSet = LW_TASK_EVENT_INIT;
pTask->TriggerWait = LW_TASK_EVENT_EXPLICIT;
pTask->llDeadline = 0;
LOCK_POOL(pPool);
if (pGroup)
{
LOCK_GROUP(pGroup);
if (pGroup->bCancelled)
{
UNLOCK_GROUP(pGroup);
UNLOCK_POOL(pPool);
status = STATUS_CANCELLED;
GOTO_ERROR_ON_STATUS(status);
}
RingInsertBefore(&pGroup->Tasks, &pTask->GroupRing);
}
pTask->pThread = &pPool->pEventThreads[pPool->ulNextEventThread];
pPool->ulNextEventThread = (pPool->ulNextEventThread + 1) % pPool->ulEventThreadCount;
UNLOCK_POOL(pPool);
LOCK_THREAD(pTask->pThread);
RingInsertBefore(&pTask->pThread->Tasks, &pTask->EventRing);
/* It's not necessary to signal the thread about the new task here
since it won't be run anyway */
UNLOCK_THREAD(pTask->pThread);
if (pGroup)
{
UNLOCK_GROUP(pGroup);
}
*ppTask = pTask;
error:
return status;
}
NTSTATUS
LwRtlCreateTask(
PLW_THREAD_POOL pPool,
PLW_TASK* ppTask,
PLW_TASK_GROUP pGroup,
LW_TASK_FUNCTION pfnFunc,
PVOID pContext
)
{
NTSTATUS status = STATUS_SUCCESS;
PEPOLL_TASK pTask = NULL;
PEPOLL_THREAD pThread = NULL;
ULONG ulMinLoad = 0xFFFFFFFF;
ULONG ulIndex = 0;
if (pPool->pDelegate)
{
return LwRtlCreateTask(pPool->pDelegate, ppTask, pGroup, pfnFunc, pContext);
}
status = LW_RTL_ALLOCATE_AUTO(&pTask);
GOTO_ERROR_ON_STATUS(status);
RingInit(&pTask->GroupRing);
RingInit(&pTask->QueueRing);
RingInit(&pTask->SignalRing);
pTask->pGroup = pGroup;
pTask->ulRefCount = 2;
pTask->pfnFunc = pfnFunc;
pTask->pFuncContext = pContext;
pTask->Fd = -1;
pTask->EventArgs = LW_TASK_EVENT_INIT;
pTask->EventWait = LW_TASK_EVENT_EXPLICIT;
pTask->llDeadline = 0;
LOCK_POOL(pPool);
for (ulIndex = 0; ulIndex < pPool->ulEventThreadCount; ulIndex++)
{
if (pPool->pEventThreads[ulIndex].ulLoad < ulMinLoad)
{
pThread = &pPool->pEventThreads[ulIndex];
ulMinLoad = pThread->ulLoad;
}
}
pTask->pThread = pThread;
if (pGroup)
{
LOCK_GROUP(pGroup);
if (pGroup->bCancelled)
{
UNLOCK_GROUP(pGroup);
UNLOCK_POOL(pPool);
status = STATUS_CANCELLED;
GOTO_ERROR_ON_STATUS(status);
}
RingInsertBefore(&pGroup->Tasks, &pTask->GroupRing);
UNLOCK_GROUP(pGroup);
}
pThread->ulLoad++;
UNLOCK_POOL(pPool);
*ppTask = pTask;
cleanup:
return status;
error:
if (pTask)
{
TaskDelete(pTask);
}
*ppTask = NULL;
goto cleanup;
}
static
NTSTATUS
EventLoop(
PEPOLL_THREAD pThread
)
{
NTSTATUS status = STATUS_SUCCESS;
RING timed;
RING runnable;
RING waiting;
CLOCK clock = {0};
LONG64 llNow = 0;
LONG64 llNextDeadline = 0;
struct epoll_event events[MAX_EVENTS];
int ready = 0;
BOOLEAN bShutdown = FALSE;
BOOLEAN bSignalled = FALSE;
RingInit(&runnable);
RingInit(&timed);
RingInit(&waiting);
for (;;)
{
/* Get current time for this iteration */
status = ClockGetMonotonicTime(&clock, &llNow);
GOTO_ERROR_ON_STATUS(status);
/* Schedule any timed tasks that have reached their deadline */
ScheduleTimedTasks(
&timed,
llNow,
&runnable);
/* Schedule any waiting tasks that epoll indicated are ready
and check if the thread received a signal */
ScheduleWaitingTasks(
events,
ready,
llNow,
&runnable,
&bSignalled);
if (bSignalled)
{
/* Schedule explicitly-signalled tasks and check if we
have been told to shut down */
ScheduleSignalled(
pThread,
&runnable,
&bShutdown);
}
/* Process runnable tasks */
status = ProcessRunnable(
pThread,
&runnable,
&timed,
&waiting,
llNow);
GOTO_ERROR_ON_STATUS(status);
if (!RingIsEmpty(&runnable))
{
/* If there are still runnable tasks, set the next deadline
to now so we can check for other tasks becoming runnable but
do not block in Poll() */
llNextDeadline = llNow;
}
else if (!RingIsEmpty(&timed))
{
/* There are timed tasks, so set our next deadline to the
deadline of the first task in the queue */
llNextDeadline = LW_STRUCT_FROM_FIELD(timed.pNext, EPOLL_TASK, QueueRing)->llDeadline;
}
else if (!RingIsEmpty(&waiting) || !bShutdown)
{
/* There are waiting tasks or we are not shutting down, so poll indefinitely */
llNextDeadline = -1;
}
else
{
/* We are shutting down and there are no remaining tasks, so leave */
break;
}
/* Wait (or check) for activity */
status = Poll(
&clock,
&llNow,
pThread->EpollFd,
events,
MAX_EVENTS,
llNextDeadline,
&ready);
GOTO_ERROR_ON_STATUS(status);
}
error:
return status;
}
static
NTSTATUS
ProcessRunnable(
PEPOLL_THREAD pThread,
PRING pRunnable,
PRING pTimed,
PRING pWaiting,
LONG64 llNow
)
{
NTSTATUS status = STATUS_SUCCESS;
ULONG ulTicks = MAX_TICKS;
PLW_TASK pTask = NULL;
PLW_TASK_GROUP pGroup = NULL;
PRING pRing = NULL;
PRING pNext = NULL;
/* We are guaranteed to run each task at least once. If tasks remain
on the runnable list by yielding, we will continue to run them
all in a round robin until our ticks are depleted. */
while (ulTicks && !RingIsEmpty(pRunnable))
{
for (pRing = pRunnable->pNext; pRing != pRunnable; pRing = pNext)
{
pNext = pRing->pNext;
pTask = LW_STRUCT_FROM_FIELD(pRing, EPOLL_TASK, QueueRing);
RunTask(pTask, llNow);
if (ulTicks)
{
ulTicks--;
}
if (pTask->EventWait != LW_TASK_EVENT_COMPLETE)
{
/* Task is still waiting to be runnable, update events in epoll set */
status = UpdateEventWait(
pTask,
pThread->EpollFd
);
GOTO_ERROR_ON_STATUS(status);
if (pTask->EventWait & LW_TASK_EVENT_YIELD)
{
/* Task is yielding. Set YIELD in its trigger arguments and
and leave it on the runnable list for the next iteration */
pTask->EventArgs |= LW_TASK_EVENT_YIELD;
}
else if (pTask->EventWait & LW_TASK_EVENT_TIME)
{
/* If the task is waiting for a timeout, insert it into the timed queue */
RingRemove(&pTask->QueueRing);
InsertTimedQueue(pTimed, pTask);
}
else
{
/* Otherwise, put it in the generic waiting queue */
RingRemove(&pTask->QueueRing);
RingEnqueue(pWaiting, &pTask->QueueRing);
}
}
else
{
/* Task is complete */
RingRemove(&pTask->QueueRing);
/* Turn off any fd in the epoll set */
if (pTask->Fd >= 0)
{
status = LwRtlSetTaskFd(pTask, pTask->Fd, 0);
GOTO_ERROR_ON_STATUS(status);
}
/* Unsubscribe task from any UNIX signals */
if (pTask->pUnixSignal)
{
RegisterTaskUnixSignal(pTask, 0, FALSE);
}
LOCK_POOL(pThread->pPool);
pThread->ulLoad--;
UNLOCK_POOL(pThread->pPool);
pGroup = pTask->pGroup;
/* If task was in a task group, remove it and notify anyone waiting
on the group */
if (pGroup)
{
LOCK_GROUP(pGroup);
pTask->pGroup = NULL;
RingRemove(&pTask->GroupRing);
pthread_cond_broadcast(&pGroup->Event);
UNLOCK_GROUP(pGroup);
}
LOCK_THREAD(pThread);
if (--pTask->ulRefCount)
{
/* The task still has a reference, so mark it as completed
and notify anyone waiting on it */
pTask->EventSignal = TASK_COMPLETE_MASK;
pthread_cond_broadcast(&pThread->Event);
UNLOCK_THREAD(pThread);
}
else
{
/* We held the last reference to the task, so delete it */
RingRemove(&pTask->SignalRing);
UNLOCK_THREAD(pThread);
TaskDelete(pTask);
}
}
}
}
error:
return status;
}
NTSTATUS
LwRtlCreateThreadPool(
PLW_THREAD_POOL* ppPool,
PLW_THREAD_POOL_ATTRIBUTES pAttrs
)
{
NTSTATUS status = STATUS_SUCCESS;
PLW_THREAD_POOL pPool = NULL;
int i = 0;
int numCpus = 0;
status = LW_RTL_ALLOCATE_AUTO(&pPool);
GOTO_ERROR_ON_STATUS(status);
status = LwErrnoToNtStatus(pthread_mutex_init(&pPool->Lock, NULL));
GOTO_ERROR_ON_STATUS(status);
status = LwErrnoToNtStatus(pthread_cond_init(&pPool->Event, NULL));
GOTO_ERROR_ON_STATUS(status);
numCpus = LwRtlGetCpuCount();
if (GetDelegateAttr(pAttrs))
{
status = AcquireDelegatePool(&pPool->pDelegate);
GOTO_ERROR_ON_STATUS(status);
}
else
{
pPool->ulEventThreadCount = GetTaskThreadsAttr(pAttrs, numCpus);
if (pPool->ulEventThreadCount)
{
status = LW_RTL_ALLOCATE_ARRAY_AUTO(
&pPool->pEventThreads,
pPool->ulEventThreadCount);
GOTO_ERROR_ON_STATUS(status);
for (i = 0; i < pPool->ulEventThreadCount; i++)
{
status = InitEventThread(pPool, pAttrs, &pPool->pEventThreads[i], i % numCpus);
GOTO_ERROR_ON_STATUS(status);
}
}
}
status = InitWorkThreads(&pPool->WorkThreads, pAttrs, numCpus);
GOTO_ERROR_ON_STATUS(status);
*ppPool = pPool;
cleanup:
return status;
error:
LwRtlFreeThreadPool(&pPool);
goto cleanup;
}
static
NTSTATUS
InitEventThread(
PEPOLL_POOL pPool,
PLW_THREAD_POOL_ATTRIBUTES pAttrs,
PEPOLL_THREAD pThread,
ULONG ulCpu
)
{
NTSTATUS status = STATUS_SUCCESS;
struct epoll_event event;
pthread_attr_t threadAttr;
BOOLEAN bThreadAttrInit = FALSE;
status = LwErrnoToNtStatus(pthread_attr_init(&threadAttr));
GOTO_ERROR_ON_STATUS(status);
bThreadAttrInit = TRUE;
pThread->pPool = pPool;
status = LwErrnoToNtStatus(pthread_mutex_init(&pThread->Lock, NULL));
GOTO_ERROR_ON_STATUS(status);
status = LwErrnoToNtStatus(pthread_cond_init(&pThread->Event, NULL));
GOTO_ERROR_ON_STATUS(status);
if (pipe(pThread->SignalFds) < 0)
{
status = LwErrnoToNtStatus(errno);
GOTO_ERROR_ON_STATUS(status);
}
SetCloseOnExec(pThread->SignalFds[0]);
SetCloseOnExec(pThread->SignalFds[1]);
if ((pThread->EpollFd = epoll_create(MAX_EVENTS)) < 0)
{
status = LwErrnoToNtStatus(errno);
GOTO_ERROR_ON_STATUS(status);
}
SetCloseOnExec(pThread->EpollFd);
memset(&event, 0, sizeof(event));
/* Add signal fd to epoll set */
event.events = EPOLLIN;
event.data.ptr = NULL;
if (epoll_ctl(pThread->EpollFd, EPOLL_CTL_ADD, pThread->SignalFds[0], &event) < 0)
{
ABORT_ON_FATAL_ERRNO(errno);
status = LwErrnoToNtStatus(errno);
GOTO_ERROR_ON_STATUS(status);
}
RingInit(&pThread->Tasks);
status = LwRtlSetAffinityThreadAttribute(&threadAttr, ulCpu);
GOTO_ERROR_ON_STATUS(status);
if (pAttrs && pAttrs->ulTaskThreadStackSize)
{
status = LwErrnoToNtStatus(
pthread_attr_setstacksize(&threadAttr, pAttrs->ulTaskThreadStackSize));
GOTO_ERROR_ON_STATUS(status);
}
status = LwErrnoToNtStatus(
pthread_create(
&pThread->Thread,
&threadAttr,
EventThread,
pThread));
GOTO_ERROR_ON_STATUS(status);
error:
if (bThreadAttrInit)
{
pthread_attr_destroy(&threadAttr);
}
return status;
}
static
NTSTATUS
EventLoop(
PSELECT_THREAD pThread
)
{
NTSTATUS status = STATUS_SUCCESS;
RING tasks;
RING runnable;
PRING pRing = NULL;
PRING pNext = NULL;
PSELECT_TASK pTask = NULL;
CLOCK clock = {0};
LONG64 llNow;
LONG64 llNextDeadline;
fd_set readSet;
fd_set writeSet;
fd_set exceptSet;
int ready = 0;
int nfds = 0;
char c = 0;
int res = 0;
BOOLEAN bShutdown = FALSE;
PLW_TASK_GROUP pGroup = NULL;
BOOLEAN bYielding = FALSE;
RingInit(&tasks);
RingInit(&runnable);
FD_ZERO(&readSet);
FD_ZERO(&writeSet);
FD_ZERO(&exceptSet);
LOCK_THREAD(pThread);
while (!bShutdown || !RingIsEmpty(&tasks))
{
/* Reset variables */
llNextDeadline = 0;
nfds = 0;
bYielding = FALSE;
/* Get current time for this iteration */
GOTO_ERROR_ON_STATUS(status = ClockGetMonotonicTime(&clock, &llNow));
/* Figure out which tasks are runnable */
for (pRing = tasks.pNext; pRing != &tasks; pRing = pNext)
{
pNext = pRing->pNext;
pTask = LW_STRUCT_FROM_FIELD(pRing, SELECT_TASK, EventRing);
/* Update trigger set with results from select() */
UpdateTriggerSet(
pTask,
&readSet,
&writeSet,
&exceptSet,
llNow);
/* Schedule tasks to run if they have been triggered or were yielding */
if ((pTask->TriggerWait & LW_TASK_EVENT_YIELD) ||
((pTask->TriggerWait | LW_TASK_EVENT_EXPLICIT) & pTask->TriggerSet))
{
/* Put task on a separate list to run its trigger function */
RingRemove(&pTask->EventRing);
RingInsertBefore(&runnable, &pTask->EventRing);
/* Update the trigger args with the trigger set */
pTask->TriggerArgs |= pTask->TriggerSet;
/* Turn off bits (except cancel) now that we have copied them */
pTask->TriggerSet &= (LW_TASK_EVENT_CANCEL);
}
else
{
/* Update select parameters to wait for task to trigger */
UpdateTriggerWait(
pTask,
&nfds,
&readSet,
&writeSet,
&exceptSet,
&llNextDeadline);
}
}
UNLOCK_THREAD(pThread);
for (pRing = runnable.pNext; pRing != &runnable; pRing = pNext)
{
pNext = pRing->pNext;
pTask = LW_STRUCT_FROM_FIELD(pRing, SELECT_TASK, EventRing);
GOTO_ERROR_ON_STATUS(status = TaskProcessTrigger(pTask, llNow));
if (pTask->TriggerWait != 0)
{
/* Task is still waiting to be runnable, update select parameters
and put it back in the task list */
UpdateTriggerWait(
pTask,
&nfds,
&readSet,
&writeSet,
&exceptSet,
&llNextDeadline);
if (pTask->TriggerWait & LW_TASK_EVENT_YIELD)
{
/* Task is yielding temporarily. Set the yield flag on
its trigger arguments. Leave it on the runnable list */
pTask->TriggerArgs |= LW_TASK_EVENT_YIELD;
}
else
{
RingRemove(&pTask->EventRing);
RingInsertBefore(&tasks, &pTask->EventRing);
}
}
else
{
/* Task is complete, notify and remove from task group
if it is in one */
RingRemove(&pTask->EventRing);
/* Unregister task from global signal loop */
if (pTask->pUnixSignal)
{
RegisterTaskUnixSignal(pTask, 0, FALSE);
}
pGroup = pTask->pGroup;
if (pGroup)
{
LOCK_GROUP(pGroup);
pTask->pGroup = NULL;
RingRemove(&pTask->GroupRing);
pthread_cond_broadcast(&pGroup->Event);
UNLOCK_GROUP(pGroup);
}
LOCK_THREAD(pThread);
if (--pTask->ulRefCount)
{
pTask->TriggerSet = TASK_COMPLETE_MASK;
pthread_cond_broadcast(&pThread->Event);
UNLOCK_THREAD(pThread);
}
else
{
UNLOCK_THREAD(pThread);
TaskDelete(pTask);
}
}
}
if (!RingIsEmpty(&runnable))
{
/* We have runnable tasks that are yielding. Move them
back to the event list and note the fact. */
bYielding = TRUE;
RingMove(&runnable, &tasks);
}
if (!bShutdown)
{
/* Also wait for a poke on the thread's signal fd */
FD_SET(pThread->SignalFds[0], &readSet);
if (pThread->SignalFds[0] >= nfds)
{
nfds = pThread->SignalFds[0] + 1;
}
}
if (nfds)
{
/* If there are still runnable tasks due to
LW_TASK_EVENT_YIELD, set the next deadline
to now so we wake immediately. This gives other
tasks the chance to become runnable before we
proceed */
if (bYielding)
{
llNextDeadline = llNow;
}
/* Wait for a task to be runnable */
GOTO_ERROR_ON_STATUS(status = Sleep(
&clock,
&llNow,
nfds,
&readSet,
&writeSet,
&exceptSet,
llNextDeadline,
&ready));
}
LOCK_THREAD(pThread);
/* Check for a signal to the thread */
if (FD_ISSET(pThread->SignalFds[0], &readSet))
{
FD_CLR(pThread->SignalFds[0], &readSet);
pThread->bSignalled = FALSE;
res = read(pThread->SignalFds[0], &c, sizeof(c));
assert(res == sizeof(c));
/* Move all tasks in queue into local task list */
RingMove(&pThread->Tasks, &tasks);
if (pThread->bShutdown && !bShutdown)
{
bShutdown = pThread->bShutdown;
/* Cancel all outstanding tasks */
for (pRing = tasks.pNext; pRing != &tasks; pRing = pRing->pNext)
{
pTask = LW_STRUCT_FROM_FIELD(pRing, SELECT_TASK, EventRing);
pTask->TriggerSet |= LW_TASK_EVENT_CANCEL | LW_TASK_EVENT_EXPLICIT;
}
}
}
}
error:
UNLOCK_THREAD(pThread);
return status;
}