/* the generic interface that call the proper underlying implementation */
int _starpu_push_task(starpu_job_t j, unsigned job_is_already_locked)
{
struct starpu_task *task = j->task;
task->status = STARPU_TASK_READY;
/* in case there is no codelet associated to the task (that's a control
* task), we directly execute its callback and enforce the
* corresponding dependencies */
if (task->cl == NULL)
{
_starpu_handle_job_termination(j, job_is_already_locked);
return 0;
}
if (STARPU_UNLIKELY(task->execute_on_a_specific_worker))
{
unsigned workerid = task->workerid;
struct starpu_worker_s *worker = _starpu_get_worker_struct(workerid);
if (use_prefetch)
{
uint32_t memory_node = starpu_worker_get_memory_node(workerid);
_starpu_prefetch_task_input_on_node(task, memory_node);
}
return _starpu_push_local_task(worker, j);
}
else {
STARPU_ASSERT(policy.push_task);
return policy.push_task(task);
}
}
/* return true if workerid can execute task, and fill task->predicted and task->predicted_transfer
* according to best implementation predictions
*/
static int find_best_impl(unsigned sched_ctx_id, struct starpu_task * task, int workerid)
{
double len = DBL_MAX;
int best_impl = -1;
int impl;
for(impl = 0; impl < STARPU_MAXIMPLEMENTATIONS; impl++)
{
if(starpu_worker_can_execute_task(workerid, task, impl))
{
struct starpu_perfmodel_arch* archtype = starpu_worker_get_perf_archtype(workerid, sched_ctx_id);
double d = starpu_task_expected_length(task, archtype, impl);
if(isnan(d))
{
best_impl = impl;
len = 0.0;
break;
}
if(d < len)
{
len = d;
best_impl = impl;
}
}
}
if(best_impl == -1)
return 0;
int memory_node = starpu_worker_get_memory_node(workerid);
task->predicted = len;
task->predicted_transfer = starpu_task_expected_data_transfer_time(memory_node, task);
starpu_task_set_implementation(task, best_impl);
return 1;
}
int main(int argc, char **argv)
{
starpu_init(NULL);
starpu_data_malloc_pinned_if_possible((void **)&v, VECTORSIZE*sizeof(unsigned));
starpu_vector_data_register(&v_handle, 0, (uintptr_t)v, VECTORSIZE, sizeof(unsigned));
unsigned nworker = starpu_worker_get_count();
cnt = nworker*N;
unsigned iter, worker;
for (iter = 0; iter < N; iter++)
{
for (worker = 0; worker < nworker; worker++)
{
/* synchronous prefetch */
unsigned node = starpu_worker_get_memory_node(worker);
starpu_data_prefetch_on_node(v_handle, node, 0);
/* execute a task */
struct starpu_task *task = starpu_task_create();
task->cl = &cl;
task->buffers[0].handle = v_handle;
task->buffers[0].mode = select_random_mode();
task->callback_func = callback;
task->callback_arg = NULL;
task->synchronous = 1;
int ret = starpu_task_submit(task, NULL);
if (ret == -ENODEV)
goto enodev;
}
}
pthread_mutex_lock(&mutex);
if (!finished)
pthread_cond_wait(&cond, &mutex);
pthread_mutex_unlock(&mutex);
starpu_shutdown();
return 0;
enodev:
fprintf(stderr, "WARNING: No one can execute this task\n");
/* yes, we do not perform the computation but we did detect that no one
* could perform the kernel, so this is not an error from StarPU */
return 0;
}
static void common_data_cpy_func(void *descr[], void *cl_arg)
{
unsigned interface_id = *(unsigned *)cl_arg;
const struct starpu_data_interface_ops *interface_ops = _starpu_data_interface_get_ops(interface_id);
const struct starpu_data_copy_methods *copy_methods = interface_ops->copy_methods;
int workerid = starpu_worker_get_id();
enum starpu_worker_archtype type = starpu_worker_get_type(workerid);
unsigned memory_node = starpu_worker_get_memory_node(workerid);
void *dst_interface = descr[0];
void *src_interface = descr[1];
switch (type)
{
case STARPU_CPU_WORKER:
if (copy_methods->ram_to_ram)
{
copy_methods->ram_to_ram(src_interface, memory_node, dst_interface, memory_node);
return;
}
break;
case STARPU_CUDA_WORKER:
if (copy_methods->cuda_to_cuda)
{
copy_methods->cuda_to_cuda(src_interface, memory_node, dst_interface, memory_node);
return;
}
break;
case STARPU_OPENCL_WORKER:
if (copy_methods->opencl_to_opencl)
{
copy_methods->opencl_to_opencl(src_interface, memory_node, dst_interface, memory_node);
return;
}
break;
default:
/* unknown architecture */
STARPU_ABORT();
}
STARPU_ASSERT(copy_methods->any_to_any);
copy_methods->any_to_any(src_interface, memory_node, dst_interface, memory_node, NULL);
}
void starpu_data_unpartition(starpu_data_handle_t root_handle, unsigned gathering_node)
{
unsigned child;
unsigned worker;
unsigned nworkers = starpu_worker_get_count();
unsigned node;
unsigned sizes[root_handle->nchildren];
_STARPU_TRACE_START_UNPARTITION(root_handle, gathering_node);
_starpu_spin_lock(&root_handle->header_lock);
STARPU_ASSERT_MSG(root_handle->nchildren != 0, "data %p is not partitioned, can not unpartition it", root_handle);
/* first take all the children lock (in order !) */
for (child = 0; child < root_handle->nchildren; child++)
{
starpu_data_handle_t child_handle = starpu_data_get_child(root_handle, child);
/* make sure the intermediate children is unpartitionned as well */
if (child_handle->nchildren > 0)
starpu_data_unpartition(child_handle, gathering_node);
/* If this is a multiformat handle, we must convert the data now */
#ifdef STARPU_DEVEL
#warning TODO: _starpu_fetch_data_on_node should be doing it
#endif
if (_starpu_data_is_multiformat_handle(child_handle) &&
starpu_node_get_kind(child_handle->mf_node) != STARPU_CPU_RAM)
{
struct starpu_codelet cl =
{
.where = STARPU_CPU,
.cpu_funcs = { _starpu_empty_codelet_function },
.modes = { STARPU_RW },
.nbuffers = 1
};
struct starpu_task *task = starpu_task_create();
task->name = "convert_data";
STARPU_TASK_SET_HANDLE(task, child_handle, 0);
task->cl = &cl;
task->synchronous = 1;
if (_starpu_task_submit_internally(task) != 0)
_STARPU_ERROR("Could not submit the conversion task while unpartitionning\n");
}
int ret;
/* for now we pretend that the RAM is almost unlimited and that gathering
* data should be possible from the node that does the unpartionning ... we
* don't want to have the programming deal with memory shortage at that time,
* really */
/* Acquire the child data on the gathering node. This will trigger collapsing any reduction */
ret = starpu_data_acquire_on_node(child_handle, gathering_node, STARPU_RW);
STARPU_ASSERT(ret == 0);
starpu_data_release_on_node(child_handle, gathering_node);
_starpu_spin_lock(&child_handle->header_lock);
child_handle->busy_waiting = 1;
_starpu_spin_unlock(&child_handle->header_lock);
/* Wait for all requests to finish (notably WT requests) */
STARPU_PTHREAD_MUTEX_LOCK(&child_handle->busy_mutex);
while (1)
{
/* Here helgrind would shout that this an unprotected access,
* but this is actually fine: all threads who do busy_count--
* are supposed to call _starpu_data_check_not_busy, which will
* wake us up through the busy_mutex/busy_cond. */
if (!child_handle->busy_count)
break;
/* This is woken by _starpu_data_check_not_busy, always called
* after decrementing busy_count */
STARPU_PTHREAD_COND_WAIT(&child_handle->busy_cond, &child_handle->busy_mutex);
}
STARPU_PTHREAD_MUTEX_UNLOCK(&child_handle->busy_mutex);
_starpu_spin_lock(&child_handle->header_lock);
sizes[child] = _starpu_data_get_size(child_handle);
_starpu_data_unregister_ram_pointer(child_handle);
for (worker = 0; worker < nworkers; worker++)
{
struct _starpu_data_replicate *local = &child_handle->per_worker[worker];
STARPU_ASSERT(local->state == STARPU_INVALID);
if (local->allocated && local->automatically_allocated)
_starpu_request_mem_chunk_removal(child_handle, local, starpu_worker_get_memory_node(worker), sizes[child]);
}
_starpu_memory_stats_free(child_handle);
}
static void _starpu_data_partition(starpu_data_handle_t initial_handle, starpu_data_handle_t *childrenp, unsigned nparts, struct starpu_data_filter *f, int inherit_state)
{
unsigned i;
unsigned node;
/* first take care to properly lock the data header */
_starpu_spin_lock(&initial_handle->header_lock);
initial_handle->nplans++;
STARPU_ASSERT_MSG(nparts > 0, "Partitioning data %p in 0 piece does not make sense", initial_handle);
/* allocate the children */
if (inherit_state)
{
initial_handle->children = (struct _starpu_data_state *) calloc(nparts, sizeof(struct _starpu_data_state));
STARPU_ASSERT(initial_handle->children);
/* this handle now has children */
initial_handle->nchildren = nparts;
}
unsigned nworkers = starpu_worker_get_count();
for (node = 0; node < STARPU_MAXNODES; node++)
{
if (initial_handle->per_node[node].state != STARPU_INVALID)
break;
}
if (node == STARPU_MAXNODES)
{
/* This is lazy allocation, allocate it now in main RAM, so as
* to have somewhere to gather pieces later */
/* FIXME: mark as unevictable! */
int ret = _starpu_allocate_memory_on_node(initial_handle, &initial_handle->per_node[STARPU_MAIN_RAM], 0);
#ifdef STARPU_DEVEL
#warning we should reclaim memory if allocation failed
#endif
STARPU_ASSERT(!ret);
}
for (i = 0; i < nparts; i++)
{
starpu_data_handle_t child;
if (inherit_state)
child = &initial_handle->children[i];
else
child = childrenp[i];
STARPU_ASSERT(child);
struct starpu_data_interface_ops *ops;
/* each child may have his own interface type */
/* what's this child's interface ? */
if (f->get_child_ops)
ops = f->get_child_ops(f, i);
else
ops = initial_handle->ops;
_starpu_data_handle_init(child, ops, initial_handle->mf_node);
child->nchildren = 0;
child->nplans = 0;
child->switch_cl = NULL;
child->partitioned = 0;
child->readonly = 0;
child->mpi_data = initial_handle->mpi_data;
child->root_handle = initial_handle->root_handle;
child->father_handle = initial_handle;
child->sibling_index = i;
child->depth = initial_handle->depth + 1;
child->is_not_important = initial_handle->is_not_important;
child->wt_mask = initial_handle->wt_mask;
child->home_node = initial_handle->home_node;
child->is_readonly = initial_handle->is_readonly;
/* initialize the chunk lock */
_starpu_data_requester_list_init(&child->req_list);
_starpu_data_requester_list_init(&child->reduction_req_list);
child->reduction_tmp_handles = NULL;
child->write_invalidation_req = NULL;
child->refcnt = 0;
child->unlocking_reqs = 0;
child->busy_count = 0;
child->busy_waiting = 0;
STARPU_PTHREAD_MUTEX_INIT(&child->busy_mutex, NULL);
STARPU_PTHREAD_COND_INIT(&child->busy_cond, NULL);
child->reduction_refcnt = 0;
_starpu_spin_init(&child->header_lock);
child->sequential_consistency = initial_handle->sequential_consistency;
STARPU_PTHREAD_MUTEX_INIT(&child->sequential_consistency_mutex, NULL);
child->last_submitted_mode = STARPU_R;
child->last_sync_task = NULL;
child->last_submitted_accessors.task = NULL;
child->last_submitted_accessors.next = &child->last_submitted_accessors;
child->last_submitted_accessors.prev = &child->last_submitted_accessors;
child->post_sync_tasks = NULL;
/* Tell helgrind that the race in _starpu_unlock_post_sync_tasks is fine */
STARPU_HG_DISABLE_CHECKING(child->post_sync_tasks_cnt);
child->post_sync_tasks_cnt = 0;
/* The methods used for reduction are propagated to the
* children. */
child->redux_cl = initial_handle->redux_cl;
child->init_cl = initial_handle->init_cl;
#ifdef STARPU_USE_FXT
child->last_submitted_ghost_sync_id_is_valid = 0;
child->last_submitted_ghost_sync_id = 0;
child->last_submitted_ghost_accessors_id = NULL;
#endif
if (_starpu_global_arbiter)
/* Just for testing purpose */
starpu_data_assign_arbiter(child, _starpu_global_arbiter);
else
child->arbiter = NULL;
_starpu_data_requester_list_init(&child->arbitered_req_list);
for (node = 0; node < STARPU_MAXNODES; node++)
{
struct _starpu_data_replicate *initial_replicate;
struct _starpu_data_replicate *child_replicate;
initial_replicate = &initial_handle->per_node[node];
child_replicate = &child->per_node[node];
if (inherit_state)
child_replicate->state = initial_replicate->state;
else
child_replicate->state = STARPU_INVALID;
if (inherit_state || !initial_replicate->automatically_allocated)
child_replicate->allocated = initial_replicate->allocated;
else
child_replicate->allocated = 0;
/* Do not allow memory reclaiming within the child for parent bits */
child_replicate->automatically_allocated = 0;
child_replicate->refcnt = 0;
child_replicate->memory_node = node;
child_replicate->relaxed_coherency = 0;
if (inherit_state)
child_replicate->initialized = initial_replicate->initialized;
else
child_replicate->initialized = 0;
/* update the interface */
void *initial_interface = starpu_data_get_interface_on_node(initial_handle, node);
void *child_interface = starpu_data_get_interface_on_node(child, node);
STARPU_ASSERT_MSG(!(!inherit_state && child_replicate->automatically_allocated && child_replicate->allocated), "partition planning is currently not supported when handle has some automatically allocated buffers");
f->filter_func(initial_interface, child_interface, f, i, nparts);
}
unsigned worker;
for (worker = 0; worker < nworkers; worker++)
{
struct _starpu_data_replicate *child_replicate;
child_replicate = &child->per_worker[worker];
child_replicate->state = STARPU_INVALID;
child_replicate->allocated = 0;
child_replicate->automatically_allocated = 0;
child_replicate->refcnt = 0;
child_replicate->memory_node = starpu_worker_get_memory_node(worker);
child_replicate->requested = 0;
for (node = 0; node < STARPU_MAXNODES; node++)
{
child_replicate->request[node] = NULL;
}
child_replicate->relaxed_coherency = 1;
child_replicate->initialized = 0;
/* duplicate the content of the interface on node 0 */
memcpy(child_replicate->data_interface, child->per_node[0].data_interface, child->ops->interface_size);
}
/* We compute the size and the footprint of the child once and
* store it in the handle */
child->footprint = _starpu_compute_data_footprint(child);
void *ptr;
ptr = starpu_data_handle_to_pointer(child, STARPU_MAIN_RAM);
if (ptr != NULL)
_starpu_data_register_ram_pointer(child, ptr);
}
/* now let the header */
_starpu_spin_unlock(&initial_handle->header_lock);
}
int main(int argc, char **argv)
{
unsigned *foo;
starpu_data_handle_t handle;
int ret;
unsigned n, i, size;
ret = starpu_initialize(NULL, &argc, &argv);
if (ret == -ENODEV) return STARPU_TEST_SKIPPED;
STARPU_CHECK_RETURN_VALUE(ret, "starpu_init");
#ifdef STARPU_USE_OPENCL
ret = starpu_opencl_load_opencl_from_file("tests/datawizard/scal_opencl.cl", &opencl_program, NULL);
STARPU_CHECK_RETURN_VALUE(ret, "starpu_opencl_load_opencl_from_file");
#endif
n = starpu_worker_get_count();
if (n == 1)
{
starpu_shutdown();
return STARPU_TEST_SKIPPED;
}
size = 10 * n;
foo = (unsigned *) calloc(size, sizeof(*foo));
for (i = 0; i < size; i++)
foo[i] = i;
starpu_vector_data_register(&handle, STARPU_MAIN_RAM, (uintptr_t)foo, size, sizeof(*foo));
/* Broadcast the data to force in-place partitioning */
for (i = 0; i < n; i++)
starpu_data_prefetch_on_node(handle, starpu_worker_get_memory_node(i), 0);
struct starpu_data_filter f =
{
.filter_func = starpu_vector_filter_block,
.nchildren = n,
};
starpu_data_partition(handle, &f);
for (i = 0; i < f.nchildren; i++)
{
struct starpu_task *task = starpu_task_create();
task->handles[0] = starpu_data_get_sub_data(handle, 1, i);
task->cl = &scal_codelet;
task->execute_on_a_specific_worker = 1;
task->workerid = i;
ret = starpu_task_submit(task);
if (ret == -ENODEV) goto enodev;
STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_submit");
}
ret = starpu_task_wait_for_all();
STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_wait_for_all");
starpu_data_unpartition(handle, STARPU_MAIN_RAM);
starpu_data_unregister(handle);
starpu_shutdown();
ret = EXIT_SUCCESS;
for (i = 0; i < size; i++)
{
if (foo[i] != i*2)
{
FPRINTF(stderr,"value %u is %u instead of %u\n", i, foo[i], 2*i);
ret = EXIT_FAILURE;
}
}
return ret;
enodev:
starpu_data_unregister(handle);
fprintf(stderr, "WARNING: No one can execute this task\n");
/* yes, we do not perform the computation but we did detect that no one
* could perform the kernel, so this is not an error from StarPU */
starpu_shutdown();
return STARPU_TEST_SKIPPED;
}
struct starpu_task *_starpu_pop_task(struct _starpu_worker *worker)
{
struct starpu_task *task;
int worker_id;
unsigned node;
/* We can't tell in advance which task will be picked up, so we measure
* a timestamp, and will attribute it afterwards to the task. */
int profiling = starpu_profiling_status_get();
struct timespec pop_start_time;
if (profiling)
_starpu_clock_gettime(&pop_start_time);
pick:
/* perhaps there is some local task to be executed first */
task = _starpu_pop_local_task(worker);
/* get tasks from the stacks of the strategy */
if(!task)
{
struct _starpu_sched_ctx *sched_ctx ;
#ifndef STARPU_NON_BLOCKING_DRIVERS
int been_here[STARPU_NMAX_SCHED_CTXS];
int i;
for(i = 0; i < STARPU_NMAX_SCHED_CTXS; i++)
been_here[i] = 0;
while(!task)
#endif
{
if(worker->nsched_ctxs == 1)
sched_ctx = _starpu_get_initial_sched_ctx();
else
{
while(1)
{
sched_ctx = _get_next_sched_ctx_to_pop_into(worker);
if(worker->removed_from_ctx[sched_ctx->id] == 1 && worker->shares_tasks_lists[sched_ctx->id] == 1)
{
_starpu_worker_gets_out_of_ctx(sched_ctx->id, worker);
worker->removed_from_ctx[sched_ctx->id] = 0;
sched_ctx = NULL;
}
else
break;
}
}
if(sched_ctx && sched_ctx->id != STARPU_NMAX_SCHED_CTXS)
{
if (sched_ctx->sched_policy && sched_ctx->sched_policy->pop_task)
{
task = sched_ctx->sched_policy->pop_task(sched_ctx->id);
_starpu_pop_task_end(task);
}
}
if(!task)
{
/* it doesn't matter if it shares tasks list or not in the scheduler,
if it does not have any task to pop just get it out of here */
/* however if it shares a task list it will be removed as soon as he
finishes this job (in handle_job_termination) */
if(worker->removed_from_ctx[sched_ctx->id])
{
_starpu_worker_gets_out_of_ctx(sched_ctx->id, worker);
worker->removed_from_ctx[sched_ctx->id] = 0;
}
#ifdef STARPU_USE_SC_HYPERVISOR
if(worker->pop_ctx_priority)
{
struct starpu_sched_ctx_performance_counters *perf_counters = sched_ctx->perf_counters;
if(sched_ctx->id != 0 && perf_counters != NULL && perf_counters->notify_idle_cycle && _starpu_sched_ctx_allow_hypervisor(sched_ctx->id))
{
// _STARPU_TRACE_HYPERVISOR_BEGIN();
perf_counters->notify_idle_cycle(sched_ctx->id, worker->workerid, 1.0);
// _STARPU_TRACE_HYPERVISOR_END();
}
}
#endif //STARPU_USE_SC_HYPERVISOR
#ifndef STARPU_NON_BLOCKING_DRIVERS
if(been_here[sched_ctx->id] || worker->nsched_ctxs == 1)
break;
been_here[sched_ctx->id] = 1;
#endif
}
}
}
if (!task)
{
idle_start[worker->workerid] = starpu_timing_now();
return NULL;
}
if(idle_start[worker->workerid] != 0.0)
{
double idle_end = starpu_timing_now();
idle[worker->workerid] += (idle_end - idle_start[worker->workerid]);
idle_start[worker->workerid] = 0.0;
}
#ifdef STARPU_USE_SC_HYPERVISOR
struct _starpu_sched_ctx *sched_ctx = _starpu_get_sched_ctx_struct(task->sched_ctx);
struct starpu_sched_ctx_performance_counters *perf_counters = sched_ctx->perf_counters;
if(sched_ctx->id != 0 && perf_counters != NULL && perf_counters->notify_poped_task && _starpu_sched_ctx_allow_hypervisor(sched_ctx->id))
{
// _STARPU_TRACE_HYPERVISOR_BEGIN();
perf_counters->notify_poped_task(task->sched_ctx, worker->workerid);
// _STARPU_TRACE_HYPERVISOR_END();
}
#endif //STARPU_USE_SC_HYPERVISOR
/* Make sure we do not bother with all the multiformat-specific code if
* it is not necessary. */
if (!_starpu_task_uses_multiformat_handles(task))
goto profiling;
/* This is either a conversion task, or a regular task for which the
* conversion tasks have already been created and submitted */
if (task->mf_skip)
goto profiling;
/*
* This worker may not be able to execute this task. In this case, we
* should return the task anyway. It will be pushed back almost immediatly.
* This way, we avoid computing and executing the conversions tasks.
* Here, we do not care about what implementation is used.
*/
worker_id = starpu_worker_get_id();
if (!starpu_worker_can_execute_task_first_impl(worker_id, task, NULL))
return task;
node = starpu_worker_get_memory_node(worker_id);
/*
* We do have a task that uses multiformat handles. Let's create the
* required conversion tasks.
*/
STARPU_PTHREAD_MUTEX_UNLOCK(&worker->sched_mutex);
unsigned i;
unsigned nbuffers = STARPU_TASK_GET_NBUFFERS(task);
for (i = 0; i < nbuffers; i++)
{
struct starpu_task *conversion_task;
starpu_data_handle_t handle;
handle = STARPU_TASK_GET_HANDLE(task, i);
if (!_starpu_handle_needs_conversion_task(handle, node))
continue;
conversion_task = _starpu_create_conversion_task(handle, node);
conversion_task->mf_skip = 1;
conversion_task->execute_on_a_specific_worker = 1;
conversion_task->workerid = worker_id;
/*
* Next tasks will need to know where these handles have gone.
*/
handle->mf_node = node;
_starpu_task_submit_conversion_task(conversion_task, worker_id);
}
task->mf_skip = 1;
starpu_task_list_push_back(&worker->local_tasks, task);
STARPU_PTHREAD_MUTEX_LOCK(&worker->sched_mutex);
goto pick;
profiling:
if (profiling)
{
struct starpu_profiling_task_info *profiling_info;
profiling_info = task->profiling_info;
/* The task may have been created before profiling was enabled,
* so we check if the profiling_info structure is available
* even though we already tested if profiling is enabled. */
if (profiling_info)
{
memcpy(&profiling_info->pop_start_time,
&pop_start_time, sizeof(struct timespec));
_starpu_clock_gettime(&profiling_info->pop_end_time);
}
}
if(task->prologue_callback_pop_func)
task->prologue_callback_pop_func(task->prologue_callback_pop_arg);
return task;
}
int _starpu_push_task_to_workers(struct starpu_task *task)
{
struct _starpu_sched_ctx *sched_ctx = _starpu_get_sched_ctx_struct(task->sched_ctx);
unsigned nworkers = 0;
_STARPU_TRACE_JOB_PUSH(task, task->priority > 0);
/* if the contexts still does not have workers put the task back to its place in
the empty ctx list */
if(!sched_ctx->is_initial_sched)
{
/*if there are workers in the ctx that are not able to execute tasks
we consider the ctx empty */
nworkers = _starpu_nworkers_able_to_execute_task(task, sched_ctx);
if (nworkers == 0)
{
STARPU_PTHREAD_MUTEX_LOCK(&sched_ctx->empty_ctx_mutex);
starpu_task_list_push_back(&sched_ctx->empty_ctx_tasks, task);
STARPU_PTHREAD_MUTEX_UNLOCK(&sched_ctx->empty_ctx_mutex);
#ifdef STARPU_USE_SC_HYPERVISOR
if(sched_ctx != NULL && sched_ctx->id != 0 && sched_ctx->perf_counters != NULL
&& sched_ctx->perf_counters->notify_empty_ctx)
{
_STARPU_TRACE_HYPERVISOR_BEGIN();
sched_ctx->perf_counters->notify_empty_ctx(sched_ctx->id, task);
_STARPU_TRACE_HYPERVISOR_END();
}
#endif
return -EAGAIN;
}
}
_starpu_profiling_set_task_push_start_time(task);
int ret = 0;
if (STARPU_UNLIKELY(task->execute_on_a_specific_worker))
{
unsigned node = starpu_worker_get_memory_node(task->workerid);
if (starpu_get_prefetch_flag())
starpu_prefetch_task_input_on_node(task, node);
ret = _starpu_push_task_on_specific_worker(task, task->workerid);
}
else
{
struct _starpu_machine_config *config = _starpu_get_machine_config();
/* When a task can only be executed on a given arch and we have
* only one memory node for that arch, we can systematically
* prefetch before the scheduling decision. */
if (starpu_get_prefetch_flag())
{
if (task->cl->where == STARPU_CPU && config->cpus_nodeid >= 0)
starpu_prefetch_task_input_on_node(task, config->cpus_nodeid);
else if (task->cl->where == STARPU_CUDA && config->cuda_nodeid >= 0)
starpu_prefetch_task_input_on_node(task, config->cuda_nodeid);
else if (task->cl->where == STARPU_OPENCL && config->opencl_nodeid >= 0)
starpu_prefetch_task_input_on_node(task, config->opencl_nodeid);
else if (task->cl->where == STARPU_MIC && config->mic_nodeid >= 0)
starpu_prefetch_task_input_on_node(task, config->mic_nodeid);
else if (task->cl->where == STARPU_SCC && config->scc_nodeid >= 0)
starpu_prefetch_task_input_on_node(task, config->scc_nodeid);
}
if(!sched_ctx->sched_policy)
{
/* Note: we have to call that early, or else the task may have
* disappeared already */
starpu_push_task_end(task);
if(!sched_ctx->awake_workers)
ret = _starpu_push_task_on_specific_worker(task, sched_ctx->main_master);
else
{
struct starpu_worker_collection *workers = sched_ctx->workers;
struct _starpu_job *job = _starpu_get_job_associated_to_task(task);
job->task_size = workers->nworkers;
job->combined_workerid = -1; // workerid; its a ctx not combined worker
job->active_task_alias_count = 0;
STARPU_PTHREAD_BARRIER_INIT(&job->before_work_barrier, NULL, workers->nworkers);
STARPU_PTHREAD_BARRIER_INIT(&job->after_work_barrier, NULL, workers->nworkers);
job->after_work_busy_barrier = workers->nworkers;
unsigned workerid;
struct starpu_sched_ctx_iterator it;
if(workers->init_iterator)
workers->init_iterator(workers, &it);
while(workers->has_next(workers, &it))
{
workerid = workers->get_next(workers, &it);
struct starpu_task *alias = starpu_task_dup(task);
alias->destroy = 1;
ret |= _starpu_push_task_on_specific_worker(alias, workerid);
}
}
}
else
{
STARPU_ASSERT(sched_ctx->sched_policy->push_task);
/* check out if there are any workers in the context */
starpu_pthread_rwlock_t *changing_ctx_mutex = _starpu_sched_ctx_get_changing_ctx_mutex(sched_ctx->id);
STARPU_PTHREAD_RWLOCK_RDLOCK(changing_ctx_mutex);
nworkers = starpu_sched_ctx_get_nworkers(sched_ctx->id);
if (nworkers == 0)
ret = -1;
else
{
_STARPU_TRACE_WORKER_SCHEDULING_PUSH;
ret = sched_ctx->sched_policy->push_task(task);
_STARPU_TRACE_WORKER_SCHEDULING_POP;
}
STARPU_PTHREAD_RWLOCK_UNLOCK(changing_ctx_mutex);
}
if(ret == -1)
{
fprintf(stderr, "repush task \n");
_STARPU_TRACE_JOB_POP(task, task->priority > 0);
ret = _starpu_push_task_to_workers(task);
}
}
/* Note: from here, the task might have been destroyed already! */
_STARPU_LOG_OUT();
return ret;
}
/* Enqueue a task into the list of tasks explicitely attached to a worker. In
* case workerid identifies a combined worker, a task will be enqueued into
* each worker of the combination. */
static int _starpu_push_task_on_specific_worker(struct starpu_task *task, int workerid)
{
int nbasic_workers = (int)starpu_worker_get_count();
/* Is this a basic worker or a combined worker ? */
int is_basic_worker = (workerid < nbasic_workers);
unsigned memory_node;
struct _starpu_worker *worker = NULL;
struct _starpu_combined_worker *combined_worker = NULL;
if (is_basic_worker)
{
worker = _starpu_get_worker_struct(workerid);
memory_node = worker->memory_node;
}
else
{
combined_worker = _starpu_get_combined_worker_struct(workerid);
memory_node = combined_worker->memory_node;
}
if (use_prefetch)
starpu_prefetch_task_input_on_node(task, memory_node);
if (is_basic_worker)
_starpu_push_task_on_specific_worker_notify_sched(task, worker, workerid, workerid);
else
{
/* Notify all workers of the combined worker */
int worker_size = combined_worker->worker_size;
int *combined_workerid = combined_worker->combined_workerid;
int j;
for (j = 0; j < worker_size; j++)
{
int subworkerid = combined_workerid[j];
_starpu_push_task_on_specific_worker_notify_sched(task, _starpu_get_worker_struct(subworkerid), subworkerid, workerid);
}
}
#ifdef STARPU_USE_SC_HYPERVISOR
starpu_sched_ctx_call_pushed_task_cb(workerid, task->sched_ctx);
#endif //STARPU_USE_SC_HYPERVISOR
unsigned i;
if (is_basic_worker)
{
unsigned node = starpu_worker_get_memory_node(workerid);
if (_starpu_task_uses_multiformat_handles(task))
{
unsigned nbuffers = STARPU_TASK_GET_NBUFFERS(task);
for (i = 0; i < nbuffers; i++)
{
struct starpu_task *conversion_task;
starpu_data_handle_t handle;
handle = STARPU_TASK_GET_HANDLE(task, i);
if (!_starpu_handle_needs_conversion_task(handle, node))
continue;
conversion_task = _starpu_create_conversion_task(handle, node);
conversion_task->mf_skip = 1;
conversion_task->execute_on_a_specific_worker = 1;
conversion_task->workerid = workerid;
_starpu_task_submit_conversion_task(conversion_task, workerid);
//_STARPU_DEBUG("Pushing a conversion task\n");
}
for (i = 0; i < nbuffers; i++)
{
starpu_data_handle_t handle = STARPU_TASK_GET_HANDLE(task, i);
handle->mf_node = node;
}
}
// if(task->sched_ctx != _starpu_get_initial_sched_ctx()->id)
if(task->priority > 0)
return _starpu_push_local_task(worker, task, 1);
else
return _starpu_push_local_task(worker, task, 0);
}
else
{
/* This is a combined worker so we create task aliases */
int worker_size = combined_worker->worker_size;
int *combined_workerid = combined_worker->combined_workerid;
int ret = 0;
struct _starpu_job *job = _starpu_get_job_associated_to_task(task);
job->task_size = worker_size;
job->combined_workerid = workerid;
job->active_task_alias_count = 0;
STARPU_PTHREAD_BARRIER_INIT(&job->before_work_barrier, NULL, worker_size);
STARPU_PTHREAD_BARRIER_INIT(&job->after_work_barrier, NULL, worker_size);
job->after_work_busy_barrier = worker_size;
/* Note: we have to call that early, or else the task may have
* disappeared already */
starpu_push_task_end(task);
int j;
for (j = 0; j < worker_size; j++)
{
struct starpu_task *alias = starpu_task_dup(task);
alias->destroy = 1;
worker = _starpu_get_worker_struct(combined_workerid[j]);
ret |= _starpu_push_local_task(worker, alias, 0);
}
return ret;
}
}
