static void thread_body(unsigned my_idx, const ::std::vector<Entry>* list_p, Queue* queue_p, const Builder* builder)
{
const auto& list = *list_p;
auto& queue = *queue_p;
for(;;)
{
DEBUG("Thread " << my_idx << ": waiting");
queue.avaliable_tasks.wait();
if( queue.complete || queue.failure )
{
DEBUG("Thread " << my_idx << ": Terminating");
break;
}
unsigned cur;
{
::std::lock_guard<::std::mutex> sl { queue.mutex };
cur = queue.state.get_next();
queue.num_active ++;
}
DEBUG("Thread " << my_idx << ": Starting " << cur << " - " << list[cur].package->name());
if( ! builder->build_library(*list[cur].package, list[cur].is_host) )
{
queue.failure = true;
queue.signal_all();
}
else
{
::std::lock_guard<::std::mutex> sl { queue.mutex };
queue.num_active --;
int v = queue.state.complete_package(cur, list);
while(v--)
{
queue.avaliable_tasks.notify();
}
// If the queue is empty, and there's no active jobs, stop.
if( queue.state.build_queue.empty() && queue.num_active == 0 )
{
queue.complete = true;
queue.signal_all();
}
}
}
queue.dead_threads.notify();
}
bool BuildList::build(BuildOptions opts, unsigned num_jobs)
{
bool include_build = !opts.build_script_overrides.is_valid();
Builder builder { ::std::move(opts) };
// Pre-count how many dependencies are remaining for each package
struct BuildState
{
::std::vector<unsigned> num_deps_remaining;
::std::vector<unsigned> build_queue;
int complete_package(unsigned index, const ::std::vector<Entry>& list)
{
int rv = 0;
DEBUG("Completed " << list[index].package->name() << " (" << list[index].dependents.size() << " dependents)");
for(auto d : list[index].dependents)
{
assert(this->num_deps_remaining[d] > 0);
this->num_deps_remaining[d] --;
DEBUG("- " << list[d].package->name() << " has " << this->num_deps_remaining[d] << " deps remaining");
if( this->num_deps_remaining[d] == 0 )
{
rv ++;
this->build_queue.push_back(d);
}
}
return rv;
}
unsigned get_next()
{
assert(!this->build_queue.empty());
unsigned rv = this->build_queue.back();
this->build_queue.pop_back();
return rv;
}
};
BuildState state;
state.num_deps_remaining.reserve(m_list.size());
for(const auto& e : m_list)
{
auto idx = static_cast<unsigned>(state.num_deps_remaining.size());
const auto& p = *e.package;
unsigned n_deps = 0;
for (const auto& dep : p.dependencies())
{
if( dep.is_disabled() )
{
continue ;
}
n_deps ++;
}
if( p.build_script() != "" && include_build )
{
for(const auto& dep : p.build_dependencies())
{
if( dep.is_disabled() )
{
continue ;
}
n_deps ++;
}
}
// If there's no dependencies for this package, add it to the build queue
if( n_deps == 0 )
{
state.build_queue.push_back(idx);
}
DEBUG("Package '" << p.name() << "' has " << n_deps << " dependencies and " << m_list[idx].dependents.size() << " dependents");
state.num_deps_remaining.push_back( n_deps );
}
// Actually do the build
if( num_jobs > 1 )
{
#ifndef DISABLE_MULTITHREAD
class Semaphore
{
::std::mutex mutex;
::std::condition_variable condvar;
unsigned count = 0;
public:
void notify_max() {
::std::lock_guard<::std::mutex> lh { this->mutex };
count = UINT_MAX;
condvar.notify_all();
}
void notify() {
::std::lock_guard<::std::mutex> lh { this->mutex };
if( count == UINT_MAX )
return;
++ count;
condvar.notify_one();
}
void wait() {
::std::unique_lock<::std::mutex> lh { this->mutex };
while( count == 0 )
{
condvar.wait(lh);
}
if( count == UINT_MAX )
return;
-- count;
}
};
struct Queue
{
Semaphore dead_threads;
Semaphore avaliable_tasks;
::std::mutex mutex;
BuildState state;
unsigned num_active;
bool failure;
bool complete; // Set if num_active==0 and tasks.empty()
Queue(BuildState x):
state(::std::move(x)),
num_active(0),
failure(false),
complete(false)
{
}
void signal_all() {
avaliable_tasks.notify_max();
}
};
struct H {
static void thread_body(unsigned my_idx, const ::std::vector<Entry>* list_p, Queue* queue_p, const Builder* builder)
{
const auto& list = *list_p;
auto& queue = *queue_p;
for(;;)
{
DEBUG("Thread " << my_idx << ": waiting");
queue.avaliable_tasks.wait();
if( queue.complete || queue.failure )
{
DEBUG("Thread " << my_idx << ": Terminating");
break;
}
unsigned cur;
{
::std::lock_guard<::std::mutex> sl { queue.mutex };
cur = queue.state.get_next();
queue.num_active ++;
}
DEBUG("Thread " << my_idx << ": Starting " << cur << " - " << list[cur].package->name());
if( ! builder->build_library(*list[cur].package, list[cur].is_host) )
{
queue.failure = true;
queue.signal_all();
}
else
{
::std::lock_guard<::std::mutex> sl { queue.mutex };
queue.num_active --;
int v = queue.state.complete_package(cur, list);
while(v--)
{
queue.avaliable_tasks.notify();
}
// If the queue is empty, and there's no active jobs, stop.
if( queue.state.build_queue.empty() && queue.num_active == 0 )
{
queue.complete = true;
queue.signal_all();
}
}
}
queue.dead_threads.notify();
}
};
Queue queue { state };
::std::vector<::std::thread> threads;
threads.reserve(num_jobs);
DEBUG("Spawning " << num_jobs << " worker threads");
for(unsigned i = 0; i < num_jobs; i++)
{
threads.push_back(::std::thread(H::thread_body, i, &this->m_list, &queue, &builder));
}
DEBUG("Poking jobs");
for(auto i = queue.state.build_queue.size(); i --;)
{
queue.avaliable_tasks.notify();
}
// All jobs are done, wait for each thread to complete
for(unsigned i = 0; i < num_jobs; i++)
{
threads[i].join();
DEBUG("> Thread " << i << " complete");
}
if( queue.failure )
{
return false;
}
state = ::std::move(queue.state);
#else
while( !state.build_queue.empty() )
{
auto cur = state.get_next();
if( ! builder.build_library(*m_list[cur].package, m_list[cur].is_host) )
{
return false;
}
state.complete_package(cur, m_list);
}
#endif
}
else if( num_jobs == 1 )
{
while( !state.build_queue.empty() )
{
auto cur = state.get_next();
if( ! builder.build_library(*m_list[cur].package, m_list[cur].is_host) )
{
return false;
}
state.complete_package(cur, m_list);
}
}
else
{
::std::cout << "DRY RUN BUILD" << ::std::endl;
unsigned pass = 0;
while( !state.build_queue.empty() )
{
auto queue = ::std::move(state.build_queue);
for(auto idx : queue)
{
::std::cout << pass << ": " << m_list[idx].package->name() << ::std::endl;
}
for(auto idx : queue)
{
state.complete_package(idx, m_list);
}
pass ++;
}
// TODO: Binaries?
return false;
}
// DEBUG ASSERT
{
bool any_incomplete = false;
for(size_t i = 0; i < state.num_deps_remaining.size(); i ++)
{
if( state.num_deps_remaining[i] != 0 ) {
::std::cerr << "BUG: Package '" << m_list[i].package->name() << "' still had " << state.num_deps_remaining[i] << " dependecies still to be built" << ::std::endl;
any_incomplete = true;
}
}
if( any_incomplete ) {
throw ::std::runtime_error("Incomplete packages (still have dependencies remaining)");
}
}
// Now that all libraries are done, build the binaries (if present)
return this->m_root_manifest.foreach_binaries([&](const auto& bin_target) {
return builder.build_target(this->m_root_manifest, bin_target, /*is_for_host=*/false);
});
}
