__LINK_C void scheduler_run() { while(1) { while(NG(current_priority) < NUM_PRIORITIES) { check_structs_are_valid(); for(uint8_t id = pop_task((NG(current_priority))); id != NO_TASK; id = pop_task(NG(current_priority))) { check_structs_are_valid(); NG(m_info)[id].task(); } //this needs to be done atomically since otherwise we risk decrementing the current priority //while a higher priority task is waiting in the queue start_atomic(); if (!tasks_waiting(NG(current_priority))) NG(current_priority)++; #ifndef NDEBUG for(int i = 0; i < NG(current_priority); i++) assert(!tasks_waiting(i)); #endif end_atomic(); } hw_enter_lowpower_mode(FRAMEWORK_SCHEDULER_LP_MODE); } }
static void handle_remaining_tasks(ErtsRunQueue *runq, Port *pp) { int i; ErtsPortTask *ptp; ErtsPortTaskQueue *ptqps[] = {pp->sched.exe_taskq, pp->sched.taskq}; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp)); for (i = 0; i < sizeof(ptqps)/sizeof(ErtsPortTaskQueue *); i++) { if (!ptqps[i]) continue; ptp = pop_task(ptqps[i]); while (ptp) { reset_handle(ptp); erts_smp_runq_unlock(runq); switch (ptp->type) { case ERTS_PORT_TASK_FREE: case ERTS_PORT_TASK_TIMEOUT: break; case ERTS_PORT_TASK_INPUT: erts_stale_drv_select(pp->id, ptp->event, DO_READ, 1); break; case ERTS_PORT_TASK_OUTPUT: erts_stale_drv_select(pp->id, ptp->event, DO_WRITE, 1); break; case ERTS_PORT_TASK_EVENT: erts_stale_drv_select(pp->id, ptp->event, 0, 1); break; case ERTS_PORT_TASK_DIST_CMD: break; default: erl_exit(ERTS_ABORT_EXIT, "Invalid port task type: %d\n", (int) ptp->type); } port_task_free(ptp); erts_smp_runq_lock(runq); ptp = pop_task(ptqps[i]); } } ASSERT(!pp->sched.taskq || !pp->sched.taskq->first); }
threadpool::~threadpool() { // signal that threads should not perform any new work shutdown_flag.store(true); #ifndef USE_YIELD { std::lock_guard<std::mutex> lock(wakeup_mutex); wakeup_flag = true; wakeup_signal.notify_all(); } #endif // wait for work to complete then destroy thread for (auto && thread : threads) { thread.join(); } auto current_task_package = std::unique_ptr<task_package>{nullptr}; // signal to each uncomplete task that it will not complete due to // threadpool destruction while (pop_task(current_task_package)) { try { auto except = std::runtime_error("Could not perform task before threadpool destruction"); current_task_package->completion_promise.set_exception(std::make_exception_ptr(except)); } catch (...) { } } };
void gpu_worker(void *arg) { hs_worker *worker_arg = (hs_worker *) arg; bind_to_cpu(worker_arg); init_cuda(worker_arg->device_id); // printf("GPU I am id:%d\n", worker_arg->worker_id); pthread_mutex_lock(&worker_arg->mutex); worker_arg->initialized = 1; pthread_cond_signal(&worker_arg->ready); pthread_mutex_unlock(&worker_arg->mutex); _task_t task; while (is_running()) { lock_queue(worker_arg->task_queue); task = pop_task(worker_arg->task_queue); if (task == NULL) { if (is_running()) sleep_worker(worker_arg); unlock_queue(worker_arg->task_queue); continue; } unlock_queue(worker_arg->task_queue); if ((task->task->arch_type & worker_arg->arch) != worker_arg->arch) { push_task(worker_arg->task_queue, task); continue; } execute_task(worker_arg, task); } deinit_cuda(); pthread_exit((void*) 0); }
int erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) { int port_was_enqueued = 0; Port *pp; ErtsPortTaskQueue *ptqp; ErtsPortTask *ptp; int res = 0; int reds = ERTS_PORT_REDS_EXECUTE; erts_aint_t io_tasks_executed = 0; int fpe_was_unmasked; ErtsPortTaskExeBlockData blk_data = {runq, NULL}; ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); ERTS_PT_CHK_PORTQ(runq); pp = pop_port(runq); if (!pp) { res = 0; goto done; } ERTS_PORT_NOT_IN_RUNQ(pp); *curr_port_pp = pp; ASSERT(pp->sched.taskq); ASSERT(pp->sched.taskq->first); ptqp = pp->sched.taskq; pp->sched.taskq = NULL; ASSERT(!pp->sched.exe_taskq); pp->sched.exe_taskq = ptqp; if (erts_smp_port_trylock(pp) == EBUSY) { erts_smp_runq_unlock(runq); erts_smp_port_lock(pp); erts_smp_runq_lock(runq); } if (erts_sched_stat.enabled) { ErtsSchedulerData *esdp = erts_get_scheduler_data(); Uint old = ERTS_PORT_SCHED_ID(pp, esdp->no); int migrated = old && old != esdp->no; erts_smp_spin_lock(&erts_sched_stat.lock); erts_sched_stat.prio[ERTS_PORT_PRIO_LEVEL].total_executed++; erts_sched_stat.prio[ERTS_PORT_PRIO_LEVEL].executed++; if (migrated) { erts_sched_stat.prio[ERTS_PORT_PRIO_LEVEL].total_migrated++; erts_sched_stat.prio[ERTS_PORT_PRIO_LEVEL].migrated++; } erts_smp_spin_unlock(&erts_sched_stat.lock); } /* trace port scheduling, in */ if (IS_TRACED_FL(pp, F_TRACE_SCHED_PORTS)) { trace_sched_ports(pp, am_in); } ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp)); ERTS_PT_CHK_PRES_PORTQ(runq, pp); ptp = pop_task(ptqp); fpe_was_unmasked = erts_block_fpe(); while (ptp) { ASSERT(pp->sched.taskq != pp->sched.exe_taskq); reset_handle(ptp); erts_smp_runq_unlock(runq); ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp)); ERTS_SMP_CHK_NO_PROC_LOCKS; ASSERT(pp->drv_ptr); switch (ptp->type) { case ERTS_PORT_TASK_FREE: /* May be pushed in q at any time */ reds += ERTS_PORT_REDS_FREE; erts_smp_runq_lock(runq); erts_unblock_fpe(fpe_was_unmasked); ASSERT(pp->status & ERTS_PORT_SFLG_FREE_SCHEDULED); if (ptqp->first || (pp->sched.taskq && pp->sched.taskq->first)) handle_remaining_tasks(runq, pp); ASSERT(!ptqp->first && (!pp->sched.taskq || !pp->sched.taskq->first)); #ifdef ERTS_SMP erts_smp_atomic_dec_nob(&pp->refc); /* Not alive */ ERTS_SMP_LC_ASSERT(erts_smp_atomic_read_nob(&pp->refc) > 0); /* Lock */ #else erts_port_status_bor_set(pp, ERTS_PORT_SFLG_FREE); #endif port_task_free(ptp); if (pp->sched.taskq) port_taskq_free(pp->sched.taskq); pp->sched.taskq = NULL; goto tasks_done; case ERTS_PORT_TASK_TIMEOUT: reds += ERTS_PORT_REDS_TIMEOUT; if (!(pp->status & ERTS_PORT_SFLGS_DEAD)) (*pp->drv_ptr->timeout)((ErlDrvData) pp->drv_data); break; case ERTS_PORT_TASK_INPUT: reds += ERTS_PORT_REDS_INPUT; ASSERT((pp->status & ERTS_PORT_SFLGS_DEAD) == 0); /* NOTE some windows drivers use ->ready_input for input and output */ (*pp->drv_ptr->ready_input)((ErlDrvData) pp->drv_data, ptp->event); io_tasks_executed++; break; case ERTS_PORT_TASK_OUTPUT: reds += ERTS_PORT_REDS_OUTPUT; ASSERT((pp->status & ERTS_PORT_SFLGS_DEAD) == 0); (*pp->drv_ptr->ready_output)((ErlDrvData) pp->drv_data, ptp->event); io_tasks_executed++; break; case ERTS_PORT_TASK_EVENT: reds += ERTS_PORT_REDS_EVENT; ASSERT((pp->status & ERTS_PORT_SFLGS_DEAD) == 0); (*pp->drv_ptr->event)((ErlDrvData) pp->drv_data, ptp->event, ptp->event_data); io_tasks_executed++; break; case ERTS_PORT_TASK_DIST_CMD: reds += erts_dist_command(pp, CONTEXT_REDS-reds); break; default: erl_exit(ERTS_ABORT_EXIT, "Invalid port task type: %d\n", (int) ptp->type); break; } if ((pp->status & ERTS_PORT_SFLG_CLOSING) && erts_is_port_ioq_empty(pp)) { reds += ERTS_PORT_REDS_TERMINATE; erts_terminate_port(pp); } ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp)); #ifdef ERTS_SMP if (pp->xports) erts_smp_xports_unlock(pp); ASSERT(!pp->xports); #endif ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp)); port_task_free(ptp); erts_smp_runq_lock(runq); ptp = pop_task(ptqp); } tasks_done: erts_unblock_fpe(fpe_was_unmasked); if (io_tasks_executed) { ASSERT(erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks) >= io_tasks_executed); erts_smp_atomic_add_relb(&erts_port_task_outstanding_io_tasks, -1*io_tasks_executed); } *curr_port_pp = NULL; #ifdef ERTS_SMP ASSERT(runq == (ErtsRunQueue *) erts_smp_atomic_read_nob(&pp->run_queue)); #endif if (!pp->sched.taskq) { ASSERT(pp->sched.exe_taskq); pp->sched.exe_taskq = NULL; } else { #ifdef ERTS_SMP ErtsRunQueue *xrunq; #endif ASSERT(!(pp->status & ERTS_PORT_SFLGS_DEAD)); ASSERT(pp->sched.taskq->first); #ifdef ERTS_SMP xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL); if (!xrunq) { #endif enqueue_port(runq, pp); ASSERT(pp->sched.exe_taskq); pp->sched.exe_taskq = NULL; /* No need to notify ourselves about inc in runq. */ #ifdef ERTS_SMP } else { /* Port emigrated ... */ erts_smp_atomic_set_nob(&pp->run_queue, (erts_aint_t) xrunq); enqueue_port(xrunq, pp); ASSERT(pp->sched.exe_taskq); pp->sched.exe_taskq = NULL; erts_smp_runq_unlock(xrunq); erts_smp_notify_inc_runq(xrunq); } #endif port_was_enqueued = 1; } res = (erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks) != (erts_aint_t) 0); ERTS_PT_CHK_PRES_PORTQ(runq, pp); port_taskq_free(ptqp); if (erts_system_profile_flags.runnable_ports && (port_was_enqueued != 1)) { profile_runnable_port(pp, am_inactive); } /* trace port scheduling, out */ if (IS_TRACED_FL(pp, F_TRACE_SCHED_PORTS)) { trace_sched_ports(pp, am_out); } #ifndef ERTS_SMP erts_port_release(pp); #else { erts_aint_t refc; erts_smp_mtx_unlock(pp->lock); refc = erts_smp_atomic_dec_read_nob(&pp->refc); ASSERT(refc >= 0); if (refc == 0) { erts_smp_runq_unlock(runq); erts_port_cleanup(pp); /* Might aquire runq lock */ erts_smp_runq_lock(runq); res = (erts_smp_atomic_read_nob(&erts_port_task_outstanding_io_tasks) != (erts_aint_t) 0); } } #endif done: blk_data.resp = &res; ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); ERTS_PORT_REDUCTIONS_EXECUTED(runq, reds); return res; }
/* * The worker thread function. Take a task from the queue and perform it if * there is any. Otherwise, put itself into the idle thread list and waiting * for signal to wake up. * The thread terminate directly by detach and exit when it is asked to stop * after finishing a task. Otherwise, the thread should be in idle thread list * and should be joined. */ static void *APR_THREAD_FUNC thread_pool_func(apr_thread_t * t, void *param) { apr_thread_pool_t *me = param; apr_thread_pool_task_t *task = NULL; apr_interval_time_t wait; struct apr_thread_list_elt *elt; apr_thread_mutex_lock(me->lock); --me->spawning_cnt; elt = elt_new(me, t); if (!elt) { apr_thread_mutex_unlock(me->lock); apr_thread_exit(t, APR_ENOMEM); } while (!me->terminated && elt->state != TH_STOP) { /* Test if not new element, it is awakened from idle */ if (APR_RING_NEXT(elt, link) != elt) { --me->idle_cnt; APR_RING_REMOVE(elt, link); } APR_RING_INSERT_TAIL(me->busy_thds, elt, apr_thread_list_elt, link); task = pop_task(me); while (NULL != task && !me->terminated) { ++me->tasks_run; elt->current_owner = task->owner; apr_thread_mutex_unlock(me->lock); apr_thread_data_set(task, "apr_thread_pool_task", NULL, t); task->func(t, task->param); apr_thread_mutex_lock(me->lock); APR_RING_INSERT_TAIL(me->recycled_tasks, task, apr_thread_pool_task, link); elt->current_owner = NULL; if (TH_STOP == elt->state) { break; } task = pop_task(me); } assert(NULL == elt->current_owner); if (TH_STOP != elt->state) APR_RING_REMOVE(elt, link); /* Test if a busy thread been asked to stop, which is not joinable */ if ((me->idle_cnt >= me->idle_max && !(me->scheduled_task_cnt && 0 >= me->idle_max) && !me->idle_wait) || me->terminated || elt->state != TH_RUN) { --me->thd_cnt; if ((TH_PROBATION == elt->state) && me->idle_wait) ++me->thd_timed_out; APR_RING_INSERT_TAIL(me->recycled_thds, elt, apr_thread_list_elt, link); apr_thread_mutex_unlock(me->lock); apr_thread_detach(t); apr_thread_exit(t, APR_SUCCESS); return NULL; /* should not be here, safe net */ } /* busy thread become idle */ ++me->idle_cnt; APR_RING_INSERT_TAIL(me->idle_thds, elt, apr_thread_list_elt, link); /* * If there is a scheduled task, always scheduled to perform that task. * Since there is no guarantee that current idle threads are scheduled * for next scheduled task. */ if (me->scheduled_task_cnt) wait = waiting_time(me); else if (me->idle_cnt > me->idle_max) { wait = me->idle_wait; elt->state = TH_PROBATION; } else wait = -1; if (wait >= 0) { apr_thread_cond_timedwait(me->cond, me->lock, wait); } else { apr_thread_cond_wait(me->cond, me->lock); } } /* idle thread been asked to stop, will be joined */ --me->thd_cnt; apr_thread_mutex_unlock(me->lock); apr_thread_exit(t, APR_SUCCESS); return NULL; /* should not be here, safe net */ }
threadpool::threadpool(size_t concurrency, size_t queue_size) : tasks(queue_size), shutdown_flag(false), threads() #ifndef USE_YIELD ,wakeup_flag(false), wakeup_signal(), wakeup_mutex() #endif { // This is more efficient than creating the 'threads' vector with // size constructor and populating with std::generate since // std::thread objects will be constructed only to be replaced threads.reserve(concurrency); for (auto a = zero(concurrency); a < concurrency; ++a) { // emplace_back so thread is constructed in place threads.emplace_back([this]() { // checks whether parent threadpool is being destroyed, // if it is, stop running. while (!shutdown_flag.load()) { auto current_task_package = std::unique_ptr<task_package>{nullptr}; // use pop_task so we only ever have one reference to the // task_package if (pop_task(current_task_package)) { try { current_task_package->task(); current_task_package->completion_promise.set_value(); } catch (...) { // try and tell the owner that something bad has happened... try { // ...but this can also throw, so stay protected current_task_package->completion_promise.set_exception(std::current_exception()); } catch (...) { } } } else { // rather than spinning, give up thread time to other things #ifdef USE_YIELD std::this_thread::yield(); #else auto lock = std::unique_lock<std::mutex>(wakeup_mutex); wakeup_flag = false; wakeup_signal.wait(lock, [this](){ return wakeup_flag; }); #endif } } }); } };