LW_NTSTATUS LwRtlQueueWorkItem( LW_IN PLW_THREAD_POOL pPool, LW_IN LW_WORK_ITEM_FUNCTION_COMPAT pfnFunc, LW_IN LW_PVOID pContext, LW_IN LW_WORK_ITEM_FLAGS Flags ) { NTSTATUS status = STATUS_SUCCESS; PCOMPAT_WORK_ITEM pCompat = NULL; PLW_WORK_ITEM pItem = NULL; status = LW_RTL_ALLOCATE_AUTO(&pCompat); GOTO_ERROR_ON_STATUS(status); pCompat->pfnFunc = pfnFunc; pCompat->pContext = pContext; status = LwRtlCreateWorkItem(pPool, &pItem, CompatWorkItem, pCompat); GOTO_ERROR_ON_STATUS(status); pCompat = NULL; LwRtlScheduleWorkItem(pItem, Flags); pItem = NULL; error: RTL_FREE(&pCompat); LwRtlFreeWorkItem(&pItem); return status; }
static NTSTATUS WorkWait( PLW_WORK_THREAD pThread ) { NTSTATUS status = STATUS_SUCCESS; struct timespec ts = {0}; LONG64 llDeadline = 0; int err = 0; BOOLEAN bLastThread = FALSE; PLW_WORK_THREADS pThreads = pThread->pThreads; while (pThreads->ulQueued == 0) { if (pThreads->bShutdown) { status = STATUS_CANCELLED; GOTO_ERROR_ON_STATUS(status); } if (pThread->pThreads->ulWorkThreadTimeout && !bLastThread) { status = TimeNow(&llDeadline); GOTO_ERROR_ON_STATUS(status); llDeadline += (LONG64) 1000000000ll * pThread->pThreads->ulWorkThreadTimeout; ts.tv_sec = llDeadline / 1000000000ll; ts.tv_nsec = llDeadline % 1000000000ll; err = pthread_cond_timedwait(&pThread->pThreads->Event, &pThread->pThreads->Lock, &ts); } else { err = pthread_cond_wait(&pThread->pThreads->Event, &pThread->pThreads->Lock); } bLastThread = pThreads->ulWorkItemCount && pThreads->ulStarted == 1; switch(err) { case ETIMEDOUT: if (!bLastThread) { status = STATUS_TIMEOUT; GOTO_ERROR_ON_STATUS(status); } break; default: status = LwErrnoToNtStatus(err); GOTO_ERROR_ON_STATUS(status); } } error: return status; }
LW_NTSTATUS LwRtlSetTaskFd( PLW_TASK pTask, int Fd, LW_TASK_EVENT_MASK Mask ) { NTSTATUS status = STATUS_SUCCESS; struct epoll_event event; memset(&event, 0, sizeof(event)); if (Fd < 0) { status = STATUS_INVALID_HANDLE; GOTO_ERROR_ON_STATUS(status); } if (Fd == pTask->Fd) { if (Mask == 0) { pTask->Fd = -1; if (epoll_ctl(pTask->pThread->EpollFd, EPOLL_CTL_DEL, Fd, &event) < 0) { ABORT_ON_FATAL_ERRNO(errno); status = LwErrnoToNtStatus(errno); GOTO_ERROR_ON_STATUS(status); } } } else if (Mask) { if (pTask->Fd >= 0) { /* Only one fd is supported */ status = STATUS_INSUFFICIENT_RESOURCES; GOTO_ERROR_ON_STATUS(status); } if (epoll_ctl(pTask->pThread->EpollFd, EPOLL_CTL_ADD, Fd, &event) < 0) { ABORT_ON_FATAL_ERRNO(errno); status = LwErrnoToNtStatus(errno); GOTO_ERROR_ON_STATUS(status); } pTask->Fd = Fd; pTask->EventLastWait = 0; } error: return status; }
static NTSTATUS Poll( IN PCLOCK pClock, IN OUT PLONG64 pllNow, IN int EpollFd, OUT struct epoll_event* pEvents, IN int maxEvents, IN LONG64 llNextDeadline, OUT int* pReady ) { NTSTATUS status = STATUS_SUCCESS; int ready = 0; int timeout = 0; do { if (llNextDeadline >= 0) { /* Convert to timeout in milliseconds */ timeout = (llNextDeadline - *pllNow) / 1000000ll; if (timeout < 0) { timeout = 0; } } else { timeout = -1; } ready = epoll_wait(EpollFd, pEvents, maxEvents, timeout); if (ready < 0 && errno == EINTR) { /* Update current time so the next timeout calculation is correct */ status = ClockGetMonotonicTime(pClock, pllNow); GOTO_ERROR_ON_STATUS(status); } } while (ready < 0 && errno == EINTR); if (ready < 0) { ABORT_ON_FATAL_ERRNO(errno); status = LwErrnoToNtStatus(errno); GOTO_ERROR_ON_STATUS(status); } *pReady = ready; error: return status; }
NTSTATUS InitWorkThreads( PLW_WORK_THREADS pThreads, PLW_THREAD_POOL_ATTRIBUTES pAttrs, int numCpus ) { NTSTATUS status = STATUS_SUCCESS; size_t i = 0; RingInit(&pThreads->WorkItems); status = LwErrnoToNtStatus(pthread_mutex_init(&pThreads->Lock, NULL)); GOTO_ERROR_ON_STATUS(status); pThreads->bDestroyLock = TRUE; status = LwErrnoToNtStatus(pthread_cond_init(&pThreads->Event, NULL)); GOTO_ERROR_ON_STATUS(status); pThreads->bDestroyEvent = TRUE; pThreads->ulWorkThreadCount = GetWorkThreadsAttr(pAttrs, numCpus); pThreads->ulWorkThreadStackSize = pAttrs ? pAttrs->ulWorkThreadStackSize : 0; pThreads->ulWorkThreadTimeout = GetWorkThreadTimeoutAttr(pAttrs); if (pThreads->ulWorkThreadCount) { status = LW_RTL_ALLOCATE_ARRAY_AUTO( &pThreads->pWorkThreads, pThreads->ulWorkThreadCount); GOTO_ERROR_ON_STATUS(status); for (i = 0; i < pThreads->ulWorkThreadCount; i++) { pThreads->pWorkThreads[i].Thread = INVALID_THREAD_HANDLE; } } if (pThreads->ulWorkThreadTimeout == 0) { for (i = 0; i < pThreads->ulWorkThreadCount; i++) { status = StartWorkThread(pThreads, &pThreads->pWorkThreads[i]); GOTO_ERROR_ON_STATUS(status); } } error: return status; }
/* * Called with pThreads->Lock held */ static NTSTATUS StartWorkThread( PLW_WORK_THREADS pThreads, PLW_WORK_THREAD pThread ) { NTSTATUS status = STATUS_SUCCESS; pthread_attr_t threadAttr; pthread_attr_t* pThreadAttr = NULL; pThread->pThreads = pThreads; if (pThreads->ulWorkThreadStackSize) { status = LwErrnoToNtStatus(pthread_attr_init(&threadAttr)); GOTO_ERROR_ON_STATUS(status); pThreadAttr = &threadAttr; status = LwErrnoToNtStatus( pthread_attr_setstacksize(pThreadAttr, pThreads->ulWorkThreadStackSize)); GOTO_ERROR_ON_STATUS(status); status = LwRtlResetAffinityThreadAttribute(pThreadAttr); GOTO_ERROR_ON_STATUS(status); } status = LwErrnoToNtStatus( pthread_create( &pThread->Thread, pThreadAttr, WorkThread, pThread)); GOTO_ERROR_ON_STATUS(status); pThread->bStarted = TRUE; pThreads->ulStarted++; error: if (pThreadAttr) { pthread_attr_destroy(pThreadAttr); } return status; }
static NTSTATUS GetFdLimit( PULONG pulLimit ) { NTSTATUS status = STATUS_SUCCESS; struct rlimit limit = {0}; if (getrlimit(RLIMIT_NOFILE, &limit) != 0) { status = LwErrnoToNtStatus(errno); GOTO_ERROR_ON_STATUS(status); } if (limit.rlim_cur == RLIM_INFINITY) { *pulLimit = (ULONG) 0xFFFFFFFF; } else { *pulLimit = (ULONG) limit.rlim_cur; } error: return status; }
NTSTATUS LwRtlQueryTaskFd( PLW_TASK pTask, int Fd, PLW_TASK_EVENT_MASK pMask ) { NTSTATUS status = STATUS_SUCCESS; if (Fd < 0 || Fd != pTask->Fd) { status = STATUS_INVALID_HANDLE; GOTO_ERROR_ON_STATUS(status); } *pMask = pTask->FdSetMask; cleanup: return status; error: *pMask = 0; goto cleanup; }
NTSTATUS LwRtlResetAffinityThreadAttribute( pthread_attr_t* pAttr ) { NTSTATUS status = STATUS_SUCCESS; CPU_SET_TYPE cpuSet; ULONG numCpus = 0; int i = 0; CPU_ZERO(&cpuSet); numCpus = LwRtlGetCpuCount(); for (i = 0; i < numCpus; i++) { (void) CPU_SET(i, &cpuSet); } status = LwErrnoToNtStatus( pthread_attr_setaffinity_np(pAttr, sizeof(cpuSet), &cpuSet)); GOTO_ERROR_ON_STATUS(status); error: return status; }
LW_NTSTATUS LwRtlCreateThreadPoolAttributes( LW_OUT PLW_THREAD_POOL_ATTRIBUTES* ppAttrs ) { NTSTATUS status = STATUS_SUCCESS; PLW_THREAD_POOL_ATTRIBUTES pAttrs = NULL; status = LW_RTL_ALLOCATE_AUTO(&pAttrs); GOTO_ERROR_ON_STATUS(status); pAttrs->bDelegateTasks = TRUE; pAttrs->lTaskThreads = 0; pAttrs->lWorkThreads = 0; pAttrs->ulTaskThreadStackSize = (ULONG) _LW_TASK_THREAD_STACK_SIZE; pAttrs->ulWorkThreadStackSize = (ULONG) _LW_WORK_THREAD_STACK_SIZE; pAttrs->ulWorkThreadTimeout = DEFAULT_WORK_THREAD_TIMEOUT; *ppAttrs = pAttrs; cleanup: return status; error: *ppAttrs = NULL; goto cleanup; }
NTSTATUS LwRtlQueryTaskFd( PLW_TASK pTask, int Fd, PLW_TASK_EVENT_MASK pMask ) { NTSTATUS status = STATUS_SUCCESS; if (Fd < 0 || Fd != pTask->Fd) { status = STATUS_INVALID_HANDLE; GOTO_ERROR_ON_STATUS(status); } *pMask = pTask->EventArgs & (LW_TASK_EVENT_FD_READABLE | LW_TASK_EVENT_FD_WRITABLE | LW_TASK_EVENT_FD_EXCEPTION); cleanup: return status; error: *pMask = 0; goto cleanup; }
LW_NTSTATUS LwRtlSetTaskFd( PLW_TASK pTask, int Fd, LW_TASK_EVENT_MASK Mask ) { NTSTATUS status = STATUS_SUCCESS; if (Fd < 0 || Fd >= FD_SETSIZE) { status = STATUS_INVALID_HANDLE; GOTO_ERROR_ON_STATUS(status); } if (Fd == pTask->Fd) { if (Mask == 0) { pTask->FdWaitMask = 0; pTask->FdSetMask = 0; pTask->Fd = -1; } else { pTask->FdWaitMask = Mask; } } else { if (pTask->Fd >= 0) { /* Only one fd is supported per task at the moment */ status = STATUS_INSUFFICIENT_RESOURCES; GOTO_ERROR_ON_STATUS(status); } pTask->Fd = Fd; pTask->FdWaitMask = Mask; pTask->FdSetMask = 0; } error: return status; }
NTSTATUS LwRtlCreateTaskGroup( PLW_THREAD_POOL pPool, PLW_TASK_GROUP* ppGroup ) { NTSTATUS status = STATUS_SUCCESS; PLW_TASK_GROUP pGroup = NULL; if (pPool->pDelegate) { return LwRtlCreateTaskGroup(pPool->pDelegate, ppGroup); } status = LW_RTL_ALLOCATE_AUTO(&pGroup); GOTO_ERROR_ON_STATUS(status); pGroup->pPool = pPool; RingInit(&pGroup->Tasks); status = LwErrnoToNtStatus(pthread_mutex_init(&pGroup->Lock, NULL)); GOTO_ERROR_ON_STATUS(status); pGroup->bLockInit = TRUE; status = LwErrnoToNtStatus(pthread_cond_init(&pGroup->Event, NULL)); GOTO_ERROR_ON_STATUS(status); pGroup->bEventInit = TRUE; *ppGroup = pGroup; cleanup: return status; error: LwRtlFreeTaskGroup(&pGroup); *ppGroup = NULL; goto cleanup; }
LW_NTSTATUS LwRtlSetTaskUnixSignal( LW_IN PLW_TASK pTask, LW_IN int Sig, LW_IN LW_BOOLEAN bSubscribe ) { NTSTATUS status = STATUS_SUCCESS; if (bSubscribe && !pTask->pUnixSignal) { status = LW_RTL_ALLOCATE_AUTO(&pTask->pUnixSignal); GOTO_ERROR_ON_STATUS(status); } status = RegisterTaskUnixSignal(pTask, Sig, bSubscribe); GOTO_ERROR_ON_STATUS(status); error: return status; }
static NTSTATUS WorkLoop( PLW_WORK_THREAD pThread ) { NTSTATUS status = STATUS_SUCCESS; PRING pRing = NULL; PLW_WORK_ITEM pItem = NULL; PLW_WORK_THREADS pThreads = pThread->pThreads; LOCK_THREADS(pThread->pThreads); for(;;) { pThreads->ulAvailable++; status = WorkWait(pThread); GOTO_ERROR_ON_STATUS(status); RingDequeue(&pThreads->WorkItems, &pRing); pThreads->ulQueued--; pThreads->ulAvailable--; UNLOCK_THREADS(pThreads); pItem = LW_STRUCT_FROM_FIELD(pRing, LW_WORK_ITEM, Ring); pItem->pfnFunc(pItem, pItem->pContext); LOCK_THREADS(pThreads); } error: pThreads->ulAvailable--; pThreads->ulStarted--; pThread->bStarted = FALSE; /* If the thread pool is not being shut down, nothing is going to call pthread_join() on this thread, so call pthread_detach() now */ if (!pThreads->bShutdown) { pthread_detach(pThread->Thread); pThread->Thread = INVALID_THREAD_HANDLE; } UNLOCK_THREADS(pThreads); return status; }
NTSTATUS CreateWorkItem( LW_IN PLW_WORK_THREADS pThreads, LW_OUT PLW_WORK_ITEM* ppWorkItem, LW_WORK_ITEM_FUNCTION pfnFunc, PVOID pContext ) { PLW_WORK_ITEM pItem = NULL; NTSTATUS status = STATUS_SUCCESS; LOCK_THREADS(pThreads); if (pThreads->ulStarted == 0) { /* Make sure at least one thread is running */ status = StartWorkThread(pThreads, &pThreads->pWorkThreads[0]); GOTO_ERROR_ON_STATUS(status); } status = LW_RTL_ALLOCATE_AUTO(&pItem); GOTO_ERROR_ON_STATUS(status); RingInit(&pItem->Ring); pItem->pThreads = pThreads; pItem->pfnFunc = pfnFunc; pItem->pContext = pContext; pThreads->ulWorkItemCount++; error: UNLOCK_THREADS(pThreads); *ppWorkItem = pItem; return status; }
/* * Updates the epoll set with the events a task is waiting on. */ static NTSTATUS UpdateEventWait( PEPOLL_TASK pTask, int EpollFd ) { NTSTATUS status = STATUS_SUCCESS; __uint32_t events = 0; struct epoll_event event; if ((pTask->EventWait & FD_EVENTS) != (pTask->EventLastWait & FD_EVENTS) && pTask->Fd >= 0) { if (pTask->EventWait & LW_TASK_EVENT_FD_READABLE) { events |= EPOLLIN; } if (pTask->EventWait & LW_TASK_EVENT_FD_WRITABLE) { events |= EPOLLOUT; } if (pTask->EventWait & LW_TASK_EVENT_FD_EXCEPTION) { events |= EPOLLERR; } memset(&event, 0, sizeof(event)); event.events = events | EPOLLET; event.data.ptr = pTask; if (epoll_ctl(EpollFd, EPOLL_CTL_MOD, pTask->Fd, &event) < 0) { ABORT_ON_FATAL_ERRNO(errno); status = LwErrnoToNtStatus(errno); GOTO_ERROR_ON_STATUS(status); } } pTask->EventLastWait = pTask->EventWait; error: return status; }
LW_NTSTATUS LwRtlSetThreadPoolAttribute( LW_IN LW_OUT PLW_THREAD_POOL_ATTRIBUTES pAttrs, LW_IN LW_THREAD_POOL_OPTION Option, ... ) { NTSTATUS status = STATUS_SUCCESS; va_list ap; va_start(ap, Option); switch(Option) { case LW_THREAD_POOL_OPTION_DELEGATE_TASKS: pAttrs->bDelegateTasks = va_arg(ap, int); break; case LW_THREAD_POOL_OPTION_TASK_THREADS: pAttrs->lTaskThreads = va_arg(ap, LONG); break; case LW_THREAD_POOL_OPTION_WORK_THREADS: pAttrs->lWorkThreads = va_arg(ap, LONG); break; case LW_THREAD_POOL_OPTION_TASK_THREAD_STACK_SIZE: pAttrs->ulTaskThreadStackSize = va_arg(ap, ULONG); break; case LW_THREAD_POOL_OPTION_WORK_THREAD_STACK_SIZE: pAttrs->ulWorkThreadStackSize = va_arg(ap, ULONG); break; case LW_THREAD_POOL_OPTION_WORK_THREAD_TIMEOUT: pAttrs->ulWorkThreadTimeout = va_arg(ap, ULONG); break; default: status = STATUS_NOT_SUPPORTED; GOTO_ERROR_ON_STATUS(status); } cleanup: va_end(ap); return status; error: goto cleanup; }
NTSTATUS AcquireDelegatePool( PLW_THREAD_POOL* ppPool ) { NTSTATUS status = STATUS_SUCCESS; LW_THREAD_POOL_ATTRIBUTES attrs = { .bDelegateTasks = FALSE, .lTaskThreads = 0, .lWorkThreads = 0, .ulTaskThreadStackSize = (ULONG) _LW_TASK_THREAD_STACK_SIZE, .ulWorkThreadStackSize = (ULONG) _LW_WORK_THREAD_STACK_SIZE }; if (getenv("LW_GLOBAL_TASK_THREADS")) { attrs.lTaskThreads = (LONG) atoi(getenv("LW_GLOBAL_TASK_THREADS")); } pthread_mutex_lock(&gpDelegateLock); if (!gpDelegatePool) { status = LwRtlCreateThreadPool(&gpDelegatePool, &attrs); GOTO_ERROR_ON_STATUS(status); gpDelegatePoolRefCount = 1; } else { gpDelegatePoolRefCount++; } *ppPool = gpDelegatePool; cleanup: pthread_mutex_unlock(&gpDelegateLock); return status; error: goto cleanup; }
static NTSTATUS SelectThreadInit( PLW_THREAD_POOL_ATTRIBUTES pAttrs, PSELECT_THREAD pThread ) { NTSTATUS status = STATUS_SUCCESS; pthread_attr_t pthreadAttr; BOOLEAN bAttrInit = FALSE; status = LwErrnoToNtStatus(pthread_attr_init(&pthreadAttr)); GOTO_ERROR_ON_STATUS(status); bAttrInit = TRUE; GOTO_ERROR_ON_STATUS(status = LwErrnoToNtStatus(pthread_mutex_init(&pThread->Lock, NULL))); GOTO_ERROR_ON_STATUS(status = LwErrnoToNtStatus(pthread_cond_init(&pThread->Event, NULL))); if (pipe(pThread->SignalFds) < 0) { GOTO_ERROR_ON_STATUS(status = LwErrnoToNtStatus(errno)); } SetCloseOnExec(pThread->SignalFds[0]); SetCloseOnExec(pThread->SignalFds[1]); RingInit(&pThread->Tasks); if (pAttrs && pAttrs->ulTaskThreadStackSize) { GOTO_ERROR_ON_STATUS(status = LwErrnoToNtStatus( pthread_attr_setstacksize(&pthreadAttr, pAttrs->ulTaskThreadStackSize))); } GOTO_ERROR_ON_STATUS(status = LwErrnoToNtStatus( pthread_create( &pThread->Thread, &pthreadAttr, EventThread, pThread))); error: if (bAttrInit) { pthread_attr_destroy(&pthreadAttr); } return status; }
static NTSTATUS ExpandCommands( PKQUEUE_COMMANDS pCommands ) { NTSTATUS status = STATUS_SUCCESS; ULONG ulNewCapacity = 0; struct kevent* pNewCommands = NULL; if (pCommands->ulCommandMax + COMMANDS_PER_FD > pCommands->ulCommandCapacity) { ulNewCapacity = pCommands->ulCommandCapacity; if (ulNewCapacity == 0) { ulNewCapacity = 8; } while (pCommands->ulCommandMax > ulNewCapacity) { ulNewCapacity *= 2; } pNewCommands = LwRtlMemoryRealloc( pCommands->pCommands, sizeof(*pNewCommands) * ulNewCapacity); if (!pNewCommands) { status = STATUS_INSUFFICIENT_RESOURCES; GOTO_ERROR_ON_STATUS(status); } pCommands->ulCommandCapacity = ulNewCapacity; pCommands->pCommands = pNewCommands; } pCommands->ulCommandMax += COMMANDS_PER_FD; error: return status; }
/** * @brief Set the thread cpu affinity attribute to the * supplied cpu number. * Specifying a cpu number that does not exist, or is * offline will cause pthread_create() to fail with * EINVAL. * * @param pAttr pointer to the pthread_att_t * @param CpuNumber the cpu number */ NTSTATUS LwRtlSetAffinityThreadAttribute( pthread_attr_t* pAttr, ULONG CpuNumber ) { NTSTATUS status = STATUS_SUCCESS; CPU_SET_TYPE cpuSet; CPU_ZERO(&cpuSet); (void) CPU_SET(CpuNumber, &cpuSet); status = LwErrnoToNtStatus( pthread_attr_setaffinity_np(pAttr, sizeof(cpuSet), &cpuSet)); GOTO_ERROR_ON_STATUS(status); error: return status; }
LW_NTSTATUS LwRtlBlockSignals( VOID ) { NTSTATUS status = STATUS_SUCCESS; sigset_t blockSet; size_t i = 0; sigfillset(&blockSet); /* Don't block blacklisted signals */ for (i = 0; SignalBlacklist[i]; i++) { sigdelset(&blockSet, SignalBlacklist[i]); } status = LwErrnoToNtStatus(pthread_sigmask(SIG_SETMASK, &blockSet, NULL)); GOTO_ERROR_ON_STATUS(status); error: return status; }
NTSTATUS LwRtlCreateTask( PLW_THREAD_POOL pPool, PLW_TASK* ppTask, PLW_TASK_GROUP pGroup, LW_TASK_FUNCTION pfnFunc, PVOID pContext ) { NTSTATUS status = STATUS_SUCCESS; PSELECT_TASK pTask = NULL; if (pPool->pDelegate) { return LwRtlCreateTask(pPool->pDelegate, ppTask, pGroup, pfnFunc, pContext); } GOTO_ERROR_ON_STATUS(status = LW_RTL_ALLOCATE_AUTO(&pTask)); RingInit(&pTask->GroupRing); RingInit(&pTask->EventRing); pTask->pPool = pPool; pTask->pGroup = pGroup; pTask->ulRefCount = 2; pTask->pfnFunc = pfnFunc; pTask->pFuncContext = pContext; pTask->Fd = -1; pTask->TriggerSet = LW_TASK_EVENT_INIT; pTask->TriggerWait = LW_TASK_EVENT_EXPLICIT; pTask->llDeadline = 0; LOCK_POOL(pPool); if (pGroup) { LOCK_GROUP(pGroup); if (pGroup->bCancelled) { UNLOCK_GROUP(pGroup); UNLOCK_POOL(pPool); status = STATUS_CANCELLED; GOTO_ERROR_ON_STATUS(status); } RingInsertBefore(&pGroup->Tasks, &pTask->GroupRing); } pTask->pThread = &pPool->pEventThreads[pPool->ulNextEventThread]; pPool->ulNextEventThread = (pPool->ulNextEventThread + 1) % pPool->ulEventThreadCount; UNLOCK_POOL(pPool); LOCK_THREAD(pTask->pThread); RingInsertBefore(&pTask->pThread->Tasks, &pTask->EventRing); /* It's not necessary to signal the thread about the new task here since it won't be run anyway */ UNLOCK_THREAD(pTask->pThread); if (pGroup) { UNLOCK_GROUP(pGroup); } *ppTask = pTask; error: return status; }
NTSTATUS LwRtlCreateTask( PLW_THREAD_POOL pPool, PLW_TASK* ppTask, PLW_TASK_GROUP pGroup, LW_TASK_FUNCTION pfnFunc, PVOID pContext ) { NTSTATUS status = STATUS_SUCCESS; PEPOLL_TASK pTask = NULL; PEPOLL_THREAD pThread = NULL; ULONG ulMinLoad = 0xFFFFFFFF; ULONG ulIndex = 0; if (pPool->pDelegate) { return LwRtlCreateTask(pPool->pDelegate, ppTask, pGroup, pfnFunc, pContext); } status = LW_RTL_ALLOCATE_AUTO(&pTask); GOTO_ERROR_ON_STATUS(status); RingInit(&pTask->GroupRing); RingInit(&pTask->QueueRing); RingInit(&pTask->SignalRing); pTask->pGroup = pGroup; pTask->ulRefCount = 2; pTask->pfnFunc = pfnFunc; pTask->pFuncContext = pContext; pTask->Fd = -1; pTask->EventArgs = LW_TASK_EVENT_INIT; pTask->EventWait = LW_TASK_EVENT_EXPLICIT; pTask->llDeadline = 0; LOCK_POOL(pPool); for (ulIndex = 0; ulIndex < pPool->ulEventThreadCount; ulIndex++) { if (pPool->pEventThreads[ulIndex].ulLoad < ulMinLoad) { pThread = &pPool->pEventThreads[ulIndex]; ulMinLoad = pThread->ulLoad; } } pTask->pThread = pThread; if (pGroup) { LOCK_GROUP(pGroup); if (pGroup->bCancelled) { UNLOCK_GROUP(pGroup); UNLOCK_POOL(pPool); status = STATUS_CANCELLED; GOTO_ERROR_ON_STATUS(status); } RingInsertBefore(&pGroup->Tasks, &pTask->GroupRing); UNLOCK_GROUP(pGroup); } pThread->ulLoad++; UNLOCK_POOL(pPool); *ppTask = pTask; cleanup: return status; error: if (pTask) { TaskDelete(pTask); } *ppTask = NULL; goto cleanup; }
static NTSTATUS EventLoop( PEPOLL_THREAD pThread ) { NTSTATUS status = STATUS_SUCCESS; RING timed; RING runnable; RING waiting; CLOCK clock = {0}; LONG64 llNow = 0; LONG64 llNextDeadline = 0; struct epoll_event events[MAX_EVENTS]; int ready = 0; BOOLEAN bShutdown = FALSE; BOOLEAN bSignalled = FALSE; RingInit(&runnable); RingInit(&timed); RingInit(&waiting); for (;;) { /* Get current time for this iteration */ status = ClockGetMonotonicTime(&clock, &llNow); GOTO_ERROR_ON_STATUS(status); /* Schedule any timed tasks that have reached their deadline */ ScheduleTimedTasks( &timed, llNow, &runnable); /* Schedule any waiting tasks that epoll indicated are ready and check if the thread received a signal */ ScheduleWaitingTasks( events, ready, llNow, &runnable, &bSignalled); if (bSignalled) { /* Schedule explicitly-signalled tasks and check if we have been told to shut down */ ScheduleSignalled( pThread, &runnable, &bShutdown); } /* Process runnable tasks */ status = ProcessRunnable( pThread, &runnable, &timed, &waiting, llNow); GOTO_ERROR_ON_STATUS(status); if (!RingIsEmpty(&runnable)) { /* If there are still runnable tasks, set the next deadline to now so we can check for other tasks becoming runnable but do not block in Poll() */ llNextDeadline = llNow; } else if (!RingIsEmpty(&timed)) { /* There are timed tasks, so set our next deadline to the deadline of the first task in the queue */ llNextDeadline = LW_STRUCT_FROM_FIELD(timed.pNext, EPOLL_TASK, QueueRing)->llDeadline; } else if (!RingIsEmpty(&waiting) || !bShutdown) { /* There are waiting tasks or we are not shutting down, so poll indefinitely */ llNextDeadline = -1; } else { /* We are shutting down and there are no remaining tasks, so leave */ break; } /* Wait (or check) for activity */ status = Poll( &clock, &llNow, pThread->EpollFd, events, MAX_EVENTS, llNextDeadline, &ready); GOTO_ERROR_ON_STATUS(status); } error: return status; }
static NTSTATUS ProcessRunnable( PEPOLL_THREAD pThread, PRING pRunnable, PRING pTimed, PRING pWaiting, LONG64 llNow ) { NTSTATUS status = STATUS_SUCCESS; ULONG ulTicks = MAX_TICKS; PLW_TASK pTask = NULL; PLW_TASK_GROUP pGroup = NULL; PRING pRing = NULL; PRING pNext = NULL; /* We are guaranteed to run each task at least once. If tasks remain on the runnable list by yielding, we will continue to run them all in a round robin until our ticks are depleted. */ while (ulTicks && !RingIsEmpty(pRunnable)) { for (pRing = pRunnable->pNext; pRing != pRunnable; pRing = pNext) { pNext = pRing->pNext; pTask = LW_STRUCT_FROM_FIELD(pRing, EPOLL_TASK, QueueRing); RunTask(pTask, llNow); if (ulTicks) { ulTicks--; } if (pTask->EventWait != LW_TASK_EVENT_COMPLETE) { /* Task is still waiting to be runnable, update events in epoll set */ status = UpdateEventWait( pTask, pThread->EpollFd ); GOTO_ERROR_ON_STATUS(status); if (pTask->EventWait & LW_TASK_EVENT_YIELD) { /* Task is yielding. Set YIELD in its trigger arguments and and leave it on the runnable list for the next iteration */ pTask->EventArgs |= LW_TASK_EVENT_YIELD; } else if (pTask->EventWait & LW_TASK_EVENT_TIME) { /* If the task is waiting for a timeout, insert it into the timed queue */ RingRemove(&pTask->QueueRing); InsertTimedQueue(pTimed, pTask); } else { /* Otherwise, put it in the generic waiting queue */ RingRemove(&pTask->QueueRing); RingEnqueue(pWaiting, &pTask->QueueRing); } } else { /* Task is complete */ RingRemove(&pTask->QueueRing); /* Turn off any fd in the epoll set */ if (pTask->Fd >= 0) { status = LwRtlSetTaskFd(pTask, pTask->Fd, 0); GOTO_ERROR_ON_STATUS(status); } /* Unsubscribe task from any UNIX signals */ if (pTask->pUnixSignal) { RegisterTaskUnixSignal(pTask, 0, FALSE); } LOCK_POOL(pThread->pPool); pThread->ulLoad--; UNLOCK_POOL(pThread->pPool); pGroup = pTask->pGroup; /* If task was in a task group, remove it and notify anyone waiting on the group */ if (pGroup) { LOCK_GROUP(pGroup); pTask->pGroup = NULL; RingRemove(&pTask->GroupRing); pthread_cond_broadcast(&pGroup->Event); UNLOCK_GROUP(pGroup); } LOCK_THREAD(pThread); if (--pTask->ulRefCount) { /* The task still has a reference, so mark it as completed and notify anyone waiting on it */ pTask->EventSignal = TASK_COMPLETE_MASK; pthread_cond_broadcast(&pThread->Event); UNLOCK_THREAD(pThread); } else { /* We held the last reference to the task, so delete it */ RingRemove(&pTask->SignalRing); UNLOCK_THREAD(pThread); TaskDelete(pTask); } } } } error: return status; }
NTSTATUS LwRtlCreateThreadPool( PLW_THREAD_POOL* ppPool, PLW_THREAD_POOL_ATTRIBUTES pAttrs ) { NTSTATUS status = STATUS_SUCCESS; PLW_THREAD_POOL pPool = NULL; int i = 0; int numCpus = 0; status = LW_RTL_ALLOCATE_AUTO(&pPool); GOTO_ERROR_ON_STATUS(status); status = LwErrnoToNtStatus(pthread_mutex_init(&pPool->Lock, NULL)); GOTO_ERROR_ON_STATUS(status); status = LwErrnoToNtStatus(pthread_cond_init(&pPool->Event, NULL)); GOTO_ERROR_ON_STATUS(status); numCpus = LwRtlGetCpuCount(); if (GetDelegateAttr(pAttrs)) { status = AcquireDelegatePool(&pPool->pDelegate); GOTO_ERROR_ON_STATUS(status); } else { pPool->ulEventThreadCount = GetTaskThreadsAttr(pAttrs, numCpus); if (pPool->ulEventThreadCount) { status = LW_RTL_ALLOCATE_ARRAY_AUTO( &pPool->pEventThreads, pPool->ulEventThreadCount); GOTO_ERROR_ON_STATUS(status); for (i = 0; i < pPool->ulEventThreadCount; i++) { status = InitEventThread(pPool, pAttrs, &pPool->pEventThreads[i], i % numCpus); GOTO_ERROR_ON_STATUS(status); } } } status = InitWorkThreads(&pPool->WorkThreads, pAttrs, numCpus); GOTO_ERROR_ON_STATUS(status); *ppPool = pPool; cleanup: return status; error: LwRtlFreeThreadPool(&pPool); goto cleanup; }
static NTSTATUS InitEventThread( PEPOLL_POOL pPool, PLW_THREAD_POOL_ATTRIBUTES pAttrs, PEPOLL_THREAD pThread, ULONG ulCpu ) { NTSTATUS status = STATUS_SUCCESS; struct epoll_event event; pthread_attr_t threadAttr; BOOLEAN bThreadAttrInit = FALSE; status = LwErrnoToNtStatus(pthread_attr_init(&threadAttr)); GOTO_ERROR_ON_STATUS(status); bThreadAttrInit = TRUE; pThread->pPool = pPool; status = LwErrnoToNtStatus(pthread_mutex_init(&pThread->Lock, NULL)); GOTO_ERROR_ON_STATUS(status); status = LwErrnoToNtStatus(pthread_cond_init(&pThread->Event, NULL)); GOTO_ERROR_ON_STATUS(status); if (pipe(pThread->SignalFds) < 0) { status = LwErrnoToNtStatus(errno); GOTO_ERROR_ON_STATUS(status); } SetCloseOnExec(pThread->SignalFds[0]); SetCloseOnExec(pThread->SignalFds[1]); if ((pThread->EpollFd = epoll_create(MAX_EVENTS)) < 0) { status = LwErrnoToNtStatus(errno); GOTO_ERROR_ON_STATUS(status); } SetCloseOnExec(pThread->EpollFd); memset(&event, 0, sizeof(event)); /* Add signal fd to epoll set */ event.events = EPOLLIN; event.data.ptr = NULL; if (epoll_ctl(pThread->EpollFd, EPOLL_CTL_ADD, pThread->SignalFds[0], &event) < 0) { ABORT_ON_FATAL_ERRNO(errno); status = LwErrnoToNtStatus(errno); GOTO_ERROR_ON_STATUS(status); } RingInit(&pThread->Tasks); status = LwRtlSetAffinityThreadAttribute(&threadAttr, ulCpu); GOTO_ERROR_ON_STATUS(status); if (pAttrs && pAttrs->ulTaskThreadStackSize) { status = LwErrnoToNtStatus( pthread_attr_setstacksize(&threadAttr, pAttrs->ulTaskThreadStackSize)); GOTO_ERROR_ON_STATUS(status); } status = LwErrnoToNtStatus( pthread_create( &pThread->Thread, &threadAttr, EventThread, pThread)); GOTO_ERROR_ON_STATUS(status); error: if (bThreadAttrInit) { pthread_attr_destroy(&threadAttr); } return status; }
static NTSTATUS EventLoop( PSELECT_THREAD pThread ) { NTSTATUS status = STATUS_SUCCESS; RING tasks; RING runnable; PRING pRing = NULL; PRING pNext = NULL; PSELECT_TASK pTask = NULL; CLOCK clock = {0}; LONG64 llNow; LONG64 llNextDeadline; fd_set readSet; fd_set writeSet; fd_set exceptSet; int ready = 0; int nfds = 0; char c = 0; int res = 0; BOOLEAN bShutdown = FALSE; PLW_TASK_GROUP pGroup = NULL; BOOLEAN bYielding = FALSE; RingInit(&tasks); RingInit(&runnable); FD_ZERO(&readSet); FD_ZERO(&writeSet); FD_ZERO(&exceptSet); LOCK_THREAD(pThread); while (!bShutdown || !RingIsEmpty(&tasks)) { /* Reset variables */ llNextDeadline = 0; nfds = 0; bYielding = FALSE; /* Get current time for this iteration */ GOTO_ERROR_ON_STATUS(status = ClockGetMonotonicTime(&clock, &llNow)); /* Figure out which tasks are runnable */ for (pRing = tasks.pNext; pRing != &tasks; pRing = pNext) { pNext = pRing->pNext; pTask = LW_STRUCT_FROM_FIELD(pRing, SELECT_TASK, EventRing); /* Update trigger set with results from select() */ UpdateTriggerSet( pTask, &readSet, &writeSet, &exceptSet, llNow); /* Schedule tasks to run if they have been triggered or were yielding */ if ((pTask->TriggerWait & LW_TASK_EVENT_YIELD) || ((pTask->TriggerWait | LW_TASK_EVENT_EXPLICIT) & pTask->TriggerSet)) { /* Put task on a separate list to run its trigger function */ RingRemove(&pTask->EventRing); RingInsertBefore(&runnable, &pTask->EventRing); /* Update the trigger args with the trigger set */ pTask->TriggerArgs |= pTask->TriggerSet; /* Turn off bits (except cancel) now that we have copied them */ pTask->TriggerSet &= (LW_TASK_EVENT_CANCEL); } else { /* Update select parameters to wait for task to trigger */ UpdateTriggerWait( pTask, &nfds, &readSet, &writeSet, &exceptSet, &llNextDeadline); } } UNLOCK_THREAD(pThread); for (pRing = runnable.pNext; pRing != &runnable; pRing = pNext) { pNext = pRing->pNext; pTask = LW_STRUCT_FROM_FIELD(pRing, SELECT_TASK, EventRing); GOTO_ERROR_ON_STATUS(status = TaskProcessTrigger(pTask, llNow)); if (pTask->TriggerWait != 0) { /* Task is still waiting to be runnable, update select parameters and put it back in the task list */ UpdateTriggerWait( pTask, &nfds, &readSet, &writeSet, &exceptSet, &llNextDeadline); if (pTask->TriggerWait & LW_TASK_EVENT_YIELD) { /* Task is yielding temporarily. Set the yield flag on its trigger arguments. Leave it on the runnable list */ pTask->TriggerArgs |= LW_TASK_EVENT_YIELD; } else { RingRemove(&pTask->EventRing); RingInsertBefore(&tasks, &pTask->EventRing); } } else { /* Task is complete, notify and remove from task group if it is in one */ RingRemove(&pTask->EventRing); /* Unregister task from global signal loop */ if (pTask->pUnixSignal) { RegisterTaskUnixSignal(pTask, 0, FALSE); } pGroup = pTask->pGroup; if (pGroup) { LOCK_GROUP(pGroup); pTask->pGroup = NULL; RingRemove(&pTask->GroupRing); pthread_cond_broadcast(&pGroup->Event); UNLOCK_GROUP(pGroup); } LOCK_THREAD(pThread); if (--pTask->ulRefCount) { pTask->TriggerSet = TASK_COMPLETE_MASK; pthread_cond_broadcast(&pThread->Event); UNLOCK_THREAD(pThread); } else { UNLOCK_THREAD(pThread); TaskDelete(pTask); } } } if (!RingIsEmpty(&runnable)) { /* We have runnable tasks that are yielding. Move them back to the event list and note the fact. */ bYielding = TRUE; RingMove(&runnable, &tasks); } if (!bShutdown) { /* Also wait for a poke on the thread's signal fd */ FD_SET(pThread->SignalFds[0], &readSet); if (pThread->SignalFds[0] >= nfds) { nfds = pThread->SignalFds[0] + 1; } } if (nfds) { /* If there are still runnable tasks due to LW_TASK_EVENT_YIELD, set the next deadline to now so we wake immediately. This gives other tasks the chance to become runnable before we proceed */ if (bYielding) { llNextDeadline = llNow; } /* Wait for a task to be runnable */ GOTO_ERROR_ON_STATUS(status = Sleep( &clock, &llNow, nfds, &readSet, &writeSet, &exceptSet, llNextDeadline, &ready)); } LOCK_THREAD(pThread); /* Check for a signal to the thread */ if (FD_ISSET(pThread->SignalFds[0], &readSet)) { FD_CLR(pThread->SignalFds[0], &readSet); pThread->bSignalled = FALSE; res = read(pThread->SignalFds[0], &c, sizeof(c)); assert(res == sizeof(c)); /* Move all tasks in queue into local task list */ RingMove(&pThread->Tasks, &tasks); if (pThread->bShutdown && !bShutdown) { bShutdown = pThread->bShutdown; /* Cancel all outstanding tasks */ for (pRing = tasks.pNext; pRing != &tasks; pRing = pRing->pNext) { pTask = LW_STRUCT_FROM_FIELD(pRing, SELECT_TASK, EventRing); pTask->TriggerSet |= LW_TASK_EVENT_CANCEL | LW_TASK_EVENT_EXPLICIT; } } } } error: UNLOCK_THREAD(pThread); return status; }