void Schedule(const char * name, task_fn fn, uint32_t threadid)
{
BASIS_ASSERT(Scheduler != nullptr);
if (threadid >=0 && threadid < ThreadCount)
{
scheduler_data * s = SchedulerList + threadid;
s->privateTasks.push_back<task_entry>({ fn, basis::stralloc(name) });
s->privateTaskCount.fetch_add(1, std::memory_order_relaxed);
if (s != Scheduler)
{
SignalScheduler(s);
}
}
else
{
BASIS_ASSERT(threadid == TACO_INVALID_THREAD_ID);
Scheduler->sharedTasks.push_back<task_entry>({ fn, basis::stralloc(name) });
uint32_t count = GlobalSharedTaskCount.fetch_add(1, std::memory_order_relaxed) + 1;
if (count > 1 || !Scheduler->isActive)
{
AskForHelp(count);
}
}
}
void Suspend(std::function<void()> on_suspend)
{
fiber_base * f = (fiber_base *) FiberCurrent();
BASIS_ASSERT(!f->onExit);
f->onExit = on_suspend;
FiberSwitch(GetNextFiber());
}
void EnterMain()
{
BASIS_ASSERT(Scheduler == SchedulerList);
BASIS_ASSERT(!Scheduler->isActive);
fiber * f = FiberCreate(&WorkerLoop);
TACO_PROFILER_EMIT(profiler::event_object::thread, profiler::event_action::start);
Scheduler->isActive = true;
FiberInvoke(f);
Scheduler->isActive = false;
TACO_PROFILER_EMIT(profiler::event_object::thread, profiler::event_action::complete);
Scheduler->exitRequested = false;
}
static void FiberSwitch(fiber * to)
{
BASIS_ASSERT(!((fiber_base *)to)->isBlocking);
TACO_PROFILER_EMIT(profiler::event_object::fiber, profiler::event_action::suspend);
FiberInvoke(to);
FiberSwitchPoint();
TACO_PROFILER_EMIT(profiler::event_object::fiber, profiler::event_action::resume);
}
void condition::_wait(std::function<void()> on_suspend)
{
TACO_PROFILER_LOG("condition::wait <%p>", this);
fiber * cur = FiberCurrent();
BASIS_ASSERT(cur);
Suspend([&]() -> void {
std::unique_lock<mutex> lock(m_mutex);
m_waiting.push_back(cur);
on_suspend();
});
}
void Resume(fiber * f)
{
fiber_base * base = (fiber_base *) f;
if (base->threadId < 0)
{
Scheduler->sharedFibers.push_back(f);
}
else
{
BASIS_ASSERT(base->threadId >= 0 && (unsigned)base->threadId < ThreadCount);
SchedulerList[base->threadId].privateFibers.push_back(f);
SignalScheduler(SchedulerList + base->threadId);
}
}
void EndBlocking()
{
fiber * f = FiberCurrent();
fiber_base * base = (fiber_base *) f;
BASIS_ASSERT(!base->onExit);
base->onExit = [=]() -> void {
base->isBlocking = false;
if (base->threadId < 0)
{
SchedulerList[-(base->threadId + 1)].sharedFibers.push_back(f);
SignalScheduler(SchedulerList - (base->threadId + 1));
}
else
{
BASIS_ASSERT((unsigned)base->threadId < Scheduler->threadId);
Scheduler->privateFibers.push_back(f);
SignalScheduler(Scheduler);
}
};
FiberInvoke(FiberRoot());
}
void Initialize(int nthreads)
{
BASIS_ASSERT(ThreadCount == 0);
ThreadCount = (nthreads <= 0) ? std::thread::hardware_concurrency() : nthreads;
GlobalSharedTaskCount = 0;
SchedulerList = new scheduler_data[ThreadCount];
for (unsigned i=0; i<ThreadCount; i++)
{
SchedulerList[i].exitRequested = false;
SchedulerList[i].threadId = i;
SchedulerList[i].isActive = false;
SchedulerList[i].isSignaled = false;
}
for (unsigned i=1; i<ThreadCount; i++)
{
SchedulerList[i].thread = std::thread([=]() -> void {
Scheduler = SchedulerList + i;
TACO_PROFILER_EMIT(profiler::event_object::thread, profiler::event_action::start);
FiberInitializeThread();
fiber * f = FiberCreate(&WorkerLoop);
Scheduler->isActive = true;
FiberInvoke(f);
Scheduler->isActive = false;
TACO_PROFILER_EMIT(profiler::event_object::thread, profiler::event_action::complete);
ShutdownScheduler();
FiberShutdownThread();
});
}
Scheduler = SchedulerList;
FiberInitializeThread();
}
void BeginBlocking()
{
fiber * f = FiberCurrent();
fiber_base * base = (fiber_base *) f;
BASIS_ASSERT(!base->onExit);
blocking_thread * thread = nullptr;
while (!thread && !BlockingThreads.pop_front(thread))
{
int cur = BlockingThreadCount.fetch_add(1);
if (cur == BLOCKING_THREAD_LIMIT)
{
BlockingThreadCount.store(cur);
Switch();
}
else
{
thread = new blocking_thread;
thread->exitRequested = false;
thread->current = nullptr;
thread->thread = std::thread([=]() -> void {
BlockingThread(thread);
});
}
}
base->onExit = [=]() -> void {
base->isBlocking = true;
{
std::unique_lock<std::mutex> lock(thread->workMutex);
thread->current = f;
}
thread->workCondition.notify_one();
};
FiberInvoke(GetNextFiber());
}
void Shutdown()
{
// Must be called from main thread and main thread must not
// be in the scheduler loop
BASIS_ASSERT(Scheduler == SchedulerList && !Scheduler->isActive);
for (unsigned i=1; i<ThreadCount; i++)
{
SchedulerList[i].exitRequested = true;
SignalScheduler(SchedulerList + i);
}
for (unsigned i=1; i<ThreadCount; i++)
{
SchedulerList[i].thread.join();
}
blocking_thread * thread;
while (BlockingThreads.pop_front(thread))
{
{
std::unique_lock<std::mutex> lock(thread->workMutex);
thread->exitRequested = true;
}
thread->workCondition.notify_one();
thread->thread.join();
delete thread;
}
ShutdownScheduler();
delete [] SchedulerList;
SchedulerList = nullptr;
ThreadCount = 0;
BlockingThreadCount = 0;
FiberShutdownThread();
}
void BlockingThread(blocking_thread * self)
{
FiberInitializeThread();
std::unique_lock<std::mutex> lock(self->workMutex);
for (;;)
{
while (!self->current && !self->exitRequested)
{
self->workCondition.wait(lock);
}
if (self->exitRequested)
{
break;
}
BASIS_ASSERT(self->current);
FiberInvoke(self->current);
self->current = nullptr;
BlockingThreads.push_back(self);
}
FiberShutdownThread();
}
