static void thread_body(unsigned my_idx, const ::std::vector<Entry>* list_p, Queue* queue_p, const Builder* builder) { const auto& list = *list_p; auto& queue = *queue_p; for(;;) { DEBUG("Thread " << my_idx << ": waiting"); queue.avaliable_tasks.wait(); if( queue.complete || queue.failure ) { DEBUG("Thread " << my_idx << ": Terminating"); break; } unsigned cur; { ::std::lock_guard<::std::mutex> sl { queue.mutex }; cur = queue.state.get_next(); queue.num_active ++; } DEBUG("Thread " << my_idx << ": Starting " << cur << " - " << list[cur].package->name()); if( ! builder->build_library(*list[cur].package, list[cur].is_host) ) { queue.failure = true; queue.signal_all(); } else { ::std::lock_guard<::std::mutex> sl { queue.mutex }; queue.num_active --; int v = queue.state.complete_package(cur, list); while(v--) { queue.avaliable_tasks.notify(); } // If the queue is empty, and there's no active jobs, stop. if( queue.state.build_queue.empty() && queue.num_active == 0 ) { queue.complete = true; queue.signal_all(); } } } queue.dead_threads.notify(); }
bool BuildList::build(BuildOptions opts, unsigned num_jobs) { bool include_build = !opts.build_script_overrides.is_valid(); Builder builder { ::std::move(opts) }; // Pre-count how many dependencies are remaining for each package struct BuildState { ::std::vector<unsigned> num_deps_remaining; ::std::vector<unsigned> build_queue; int complete_package(unsigned index, const ::std::vector<Entry>& list) { int rv = 0; DEBUG("Completed " << list[index].package->name() << " (" << list[index].dependents.size() << " dependents)"); for(auto d : list[index].dependents) { assert(this->num_deps_remaining[d] > 0); this->num_deps_remaining[d] --; DEBUG("- " << list[d].package->name() << " has " << this->num_deps_remaining[d] << " deps remaining"); if( this->num_deps_remaining[d] == 0 ) { rv ++; this->build_queue.push_back(d); } } return rv; } unsigned get_next() { assert(!this->build_queue.empty()); unsigned rv = this->build_queue.back(); this->build_queue.pop_back(); return rv; } }; BuildState state; state.num_deps_remaining.reserve(m_list.size()); for(const auto& e : m_list) { auto idx = static_cast<unsigned>(state.num_deps_remaining.size()); const auto& p = *e.package; unsigned n_deps = 0; for (const auto& dep : p.dependencies()) { if( dep.is_disabled() ) { continue ; } n_deps ++; } if( p.build_script() != "" && include_build ) { for(const auto& dep : p.build_dependencies()) { if( dep.is_disabled() ) { continue ; } n_deps ++; } } // If there's no dependencies for this package, add it to the build queue if( n_deps == 0 ) { state.build_queue.push_back(idx); } DEBUG("Package '" << p.name() << "' has " << n_deps << " dependencies and " << m_list[idx].dependents.size() << " dependents"); state.num_deps_remaining.push_back( n_deps ); } // Actually do the build if( num_jobs > 1 ) { #ifndef DISABLE_MULTITHREAD class Semaphore { ::std::mutex mutex; ::std::condition_variable condvar; unsigned count = 0; public: void notify_max() { ::std::lock_guard<::std::mutex> lh { this->mutex }; count = UINT_MAX; condvar.notify_all(); } void notify() { ::std::lock_guard<::std::mutex> lh { this->mutex }; if( count == UINT_MAX ) return; ++ count; condvar.notify_one(); } void wait() { ::std::unique_lock<::std::mutex> lh { this->mutex }; while( count == 0 ) { condvar.wait(lh); } if( count == UINT_MAX ) return; -- count; } }; struct Queue { Semaphore dead_threads; Semaphore avaliable_tasks; ::std::mutex mutex; BuildState state; unsigned num_active; bool failure; bool complete; // Set if num_active==0 and tasks.empty() Queue(BuildState x): state(::std::move(x)), num_active(0), failure(false), complete(false) { } void signal_all() { avaliable_tasks.notify_max(); } }; struct H { static void thread_body(unsigned my_idx, const ::std::vector<Entry>* list_p, Queue* queue_p, const Builder* builder) { const auto& list = *list_p; auto& queue = *queue_p; for(;;) { DEBUG("Thread " << my_idx << ": waiting"); queue.avaliable_tasks.wait(); if( queue.complete || queue.failure ) { DEBUG("Thread " << my_idx << ": Terminating"); break; } unsigned cur; { ::std::lock_guard<::std::mutex> sl { queue.mutex }; cur = queue.state.get_next(); queue.num_active ++; } DEBUG("Thread " << my_idx << ": Starting " << cur << " - " << list[cur].package->name()); if( ! builder->build_library(*list[cur].package, list[cur].is_host) ) { queue.failure = true; queue.signal_all(); } else { ::std::lock_guard<::std::mutex> sl { queue.mutex }; queue.num_active --; int v = queue.state.complete_package(cur, list); while(v--) { queue.avaliable_tasks.notify(); } // If the queue is empty, and there's no active jobs, stop. if( queue.state.build_queue.empty() && queue.num_active == 0 ) { queue.complete = true; queue.signal_all(); } } } queue.dead_threads.notify(); } }; Queue queue { state }; ::std::vector<::std::thread> threads; threads.reserve(num_jobs); DEBUG("Spawning " << num_jobs << " worker threads"); for(unsigned i = 0; i < num_jobs; i++) { threads.push_back(::std::thread(H::thread_body, i, &this->m_list, &queue, &builder)); } DEBUG("Poking jobs"); for(auto i = queue.state.build_queue.size(); i --;) { queue.avaliable_tasks.notify(); } // All jobs are done, wait for each thread to complete for(unsigned i = 0; i < num_jobs; i++) { threads[i].join(); DEBUG("> Thread " << i << " complete"); } if( queue.failure ) { return false; } state = ::std::move(queue.state); #else while( !state.build_queue.empty() ) { auto cur = state.get_next(); if( ! builder.build_library(*m_list[cur].package, m_list[cur].is_host) ) { return false; } state.complete_package(cur, m_list); } #endif } else if( num_jobs == 1 ) { while( !state.build_queue.empty() ) { auto cur = state.get_next(); if( ! builder.build_library(*m_list[cur].package, m_list[cur].is_host) ) { return false; } state.complete_package(cur, m_list); } } else { ::std::cout << "DRY RUN BUILD" << ::std::endl; unsigned pass = 0; while( !state.build_queue.empty() ) { auto queue = ::std::move(state.build_queue); for(auto idx : queue) { ::std::cout << pass << ": " << m_list[idx].package->name() << ::std::endl; } for(auto idx : queue) { state.complete_package(idx, m_list); } pass ++; } // TODO: Binaries? return false; } // DEBUG ASSERT { bool any_incomplete = false; for(size_t i = 0; i < state.num_deps_remaining.size(); i ++) { if( state.num_deps_remaining[i] != 0 ) { ::std::cerr << "BUG: Package '" << m_list[i].package->name() << "' still had " << state.num_deps_remaining[i] << " dependecies still to be built" << ::std::endl; any_incomplete = true; } } if( any_incomplete ) { throw ::std::runtime_error("Incomplete packages (still have dependencies remaining)"); } } // Now that all libraries are done, build the binaries (if present) return this->m_root_manifest.foreach_binaries([&](const auto& bin_target) { return builder.build_target(this->m_root_manifest, bin_target, /*is_for_host=*/false); }); }