/* No lock is held */
unsigned _starpu_submit_job_enforce_data_deps(struct _starpu_job *j)
{
struct starpu_codelet *cl = j->task->cl;
if ((cl == NULL) || (STARPU_TASK_GET_NBUFFERS(j->task) == 0))
return 0;
_starpu_job_set_ordered_buffers(j);
return _submit_job_enforce_data_deps(j, 0);
}
/* No lock is held */
static unsigned unlock_one_requester(struct _starpu_data_requester *r)
{
struct _starpu_job *j = r->j;
unsigned nbuffers = STARPU_TASK_GET_NBUFFERS(j->task);
unsigned buffer_index = r->buffer_index;
if (buffer_index + 1 < nbuffers)
/* not all buffers are protected yet */
return _submit_job_enforce_data_deps(j, buffer_index + 1);
else
return 0;
}
void _starpu_job_set_ordered_buffers(struct _starpu_job *j)
{
/* Compute an ordered list of the different pieces of data so that we
* grab then according to a total order, thus avoiding a deadlock
* condition */
unsigned i;
unsigned nbuffers = STARPU_TASK_GET_NBUFFERS(j->task);
struct starpu_task *task = j->task;
for (i=0 ; i<nbuffers; i++)
{
starpu_data_handle_t handle = STARPU_TASK_GET_HANDLE(task, i);
_STARPU_JOB_SET_ORDERED_BUFFER_HANDLE(j, handle, i);
enum starpu_data_access_mode mode = STARPU_TASK_GET_MODE(task, i);
_STARPU_JOB_SET_ORDERED_BUFFER_MODE(j, mode, i);
int node = -1;
if (task->cl->specific_nodes)
node = STARPU_CODELET_GET_NODE(task->cl, i);
_STARPU_JOB_SET_ORDERED_BUFFER_NODE(j, node, i);
}
_starpu_sort_task_handles(_STARPU_JOB_GET_ORDERED_BUFFERS(j), nbuffers);
}
/* No lock is held */
static unsigned _submit_job_enforce_data_deps(struct _starpu_job *j, unsigned start_buffer_index)
{
unsigned buf;
unsigned nbuffers = STARPU_TASK_GET_NBUFFERS(j->task);
for (buf = start_buffer_index; buf < nbuffers; buf++)
{
starpu_data_handle_t handle = _STARPU_JOB_GET_ORDERED_BUFFER_HANDLE(j, buf);
if (buf)
{
starpu_data_handle_t handle_m1 = _STARPU_JOB_GET_ORDERED_BUFFER_HANDLE(j, buf-1);
if (handle_m1 == handle)
/* We have already requested this data, skip it. This
* depends on ordering putting writes before reads, see
* _starpu_compar_handles. */
continue;
}
j->task->status = STARPU_TASK_BLOCKED_ON_DATA;
if(handle->arbiter)
{
/* We arrived on an arbitered data, we stop and proceed
* with the arbiter second step. */
_starpu_submit_job_enforce_arbitered_deps(j, buf, nbuffers);
return 1;
}
if (attempt_to_submit_data_request_from_job(j, buf))
{
return 1;
}
}
return 0;
}
struct starpu_task *_starpu_pop_task(struct _starpu_worker *worker)
{
struct starpu_task *task;
int worker_id;
unsigned node;
/* We can't tell in advance which task will be picked up, so we measure
* a timestamp, and will attribute it afterwards to the task. */
int profiling = starpu_profiling_status_get();
struct timespec pop_start_time;
if (profiling)
_starpu_clock_gettime(&pop_start_time);
pick:
/* perhaps there is some local task to be executed first */
task = _starpu_pop_local_task(worker);
/* get tasks from the stacks of the strategy */
if(!task)
{
struct _starpu_sched_ctx *sched_ctx ;
#ifndef STARPU_NON_BLOCKING_DRIVERS
int been_here[STARPU_NMAX_SCHED_CTXS];
int i;
for(i = 0; i < STARPU_NMAX_SCHED_CTXS; i++)
been_here[i] = 0;
while(!task)
#endif
{
if(worker->nsched_ctxs == 1)
sched_ctx = _starpu_get_initial_sched_ctx();
else
{
while(1)
{
sched_ctx = _get_next_sched_ctx_to_pop_into(worker);
if(worker->removed_from_ctx[sched_ctx->id] == 1 && worker->shares_tasks_lists[sched_ctx->id] == 1)
{
_starpu_worker_gets_out_of_ctx(sched_ctx->id, worker);
worker->removed_from_ctx[sched_ctx->id] = 0;
sched_ctx = NULL;
}
else
break;
}
}
if(sched_ctx && sched_ctx->id != STARPU_NMAX_SCHED_CTXS)
{
if (sched_ctx->sched_policy && sched_ctx->sched_policy->pop_task)
{
task = sched_ctx->sched_policy->pop_task(sched_ctx->id);
_starpu_pop_task_end(task);
}
}
if(!task)
{
/* it doesn't matter if it shares tasks list or not in the scheduler,
if it does not have any task to pop just get it out of here */
/* however if it shares a task list it will be removed as soon as he
finishes this job (in handle_job_termination) */
if(worker->removed_from_ctx[sched_ctx->id])
{
_starpu_worker_gets_out_of_ctx(sched_ctx->id, worker);
worker->removed_from_ctx[sched_ctx->id] = 0;
}
#ifdef STARPU_USE_SC_HYPERVISOR
if(worker->pop_ctx_priority)
{
struct starpu_sched_ctx_performance_counters *perf_counters = sched_ctx->perf_counters;
if(sched_ctx->id != 0 && perf_counters != NULL && perf_counters->notify_idle_cycle && _starpu_sched_ctx_allow_hypervisor(sched_ctx->id))
{
// _STARPU_TRACE_HYPERVISOR_BEGIN();
perf_counters->notify_idle_cycle(sched_ctx->id, worker->workerid, 1.0);
// _STARPU_TRACE_HYPERVISOR_END();
}
}
#endif //STARPU_USE_SC_HYPERVISOR
#ifndef STARPU_NON_BLOCKING_DRIVERS
if(been_here[sched_ctx->id] || worker->nsched_ctxs == 1)
break;
been_here[sched_ctx->id] = 1;
#endif
}
}
}
if (!task)
{
idle_start[worker->workerid] = starpu_timing_now();
return NULL;
}
if(idle_start[worker->workerid] != 0.0)
{
double idle_end = starpu_timing_now();
idle[worker->workerid] += (idle_end - idle_start[worker->workerid]);
idle_start[worker->workerid] = 0.0;
}
#ifdef STARPU_USE_SC_HYPERVISOR
struct _starpu_sched_ctx *sched_ctx = _starpu_get_sched_ctx_struct(task->sched_ctx);
struct starpu_sched_ctx_performance_counters *perf_counters = sched_ctx->perf_counters;
if(sched_ctx->id != 0 && perf_counters != NULL && perf_counters->notify_poped_task && _starpu_sched_ctx_allow_hypervisor(sched_ctx->id))
{
// _STARPU_TRACE_HYPERVISOR_BEGIN();
perf_counters->notify_poped_task(task->sched_ctx, worker->workerid);
// _STARPU_TRACE_HYPERVISOR_END();
}
#endif //STARPU_USE_SC_HYPERVISOR
/* Make sure we do not bother with all the multiformat-specific code if
* it is not necessary. */
if (!_starpu_task_uses_multiformat_handles(task))
goto profiling;
/* This is either a conversion task, or a regular task for which the
* conversion tasks have already been created and submitted */
if (task->mf_skip)
goto profiling;
/*
* This worker may not be able to execute this task. In this case, we
* should return the task anyway. It will be pushed back almost immediatly.
* This way, we avoid computing and executing the conversions tasks.
* Here, we do not care about what implementation is used.
*/
worker_id = starpu_worker_get_id();
if (!starpu_worker_can_execute_task_first_impl(worker_id, task, NULL))
return task;
node = starpu_worker_get_memory_node(worker_id);
/*
* We do have a task that uses multiformat handles. Let's create the
* required conversion tasks.
*/
STARPU_PTHREAD_MUTEX_UNLOCK(&worker->sched_mutex);
unsigned i;
unsigned nbuffers = STARPU_TASK_GET_NBUFFERS(task);
for (i = 0; i < nbuffers; i++)
{
struct starpu_task *conversion_task;
starpu_data_handle_t handle;
handle = STARPU_TASK_GET_HANDLE(task, i);
if (!_starpu_handle_needs_conversion_task(handle, node))
continue;
conversion_task = _starpu_create_conversion_task(handle, node);
conversion_task->mf_skip = 1;
conversion_task->execute_on_a_specific_worker = 1;
conversion_task->workerid = worker_id;
/*
* Next tasks will need to know where these handles have gone.
*/
handle->mf_node = node;
_starpu_task_submit_conversion_task(conversion_task, worker_id);
}
task->mf_skip = 1;
starpu_task_list_push_back(&worker->local_tasks, task);
STARPU_PTHREAD_MUTEX_LOCK(&worker->sched_mutex);
goto pick;
profiling:
if (profiling)
{
struct starpu_profiling_task_info *profiling_info;
profiling_info = task->profiling_info;
/* The task may have been created before profiling was enabled,
* so we check if the profiling_info structure is available
* even though we already tested if profiling is enabled. */
if (profiling_info)
{
memcpy(&profiling_info->pop_start_time,
&pop_start_time, sizeof(struct timespec));
_starpu_clock_gettime(&profiling_info->pop_end_time);
}
}
if(task->prologue_callback_pop_func)
task->prologue_callback_pop_func(task->prologue_callback_pop_arg);
return task;
}
/* Enqueue a task into the list of tasks explicitely attached to a worker. In
* case workerid identifies a combined worker, a task will be enqueued into
* each worker of the combination. */
static int _starpu_push_task_on_specific_worker(struct starpu_task *task, int workerid)
{
int nbasic_workers = (int)starpu_worker_get_count();
/* Is this a basic worker or a combined worker ? */
int is_basic_worker = (workerid < nbasic_workers);
unsigned memory_node;
struct _starpu_worker *worker = NULL;
struct _starpu_combined_worker *combined_worker = NULL;
if (is_basic_worker)
{
worker = _starpu_get_worker_struct(workerid);
memory_node = worker->memory_node;
}
else
{
combined_worker = _starpu_get_combined_worker_struct(workerid);
memory_node = combined_worker->memory_node;
}
if (use_prefetch)
starpu_prefetch_task_input_on_node(task, memory_node);
if (is_basic_worker)
_starpu_push_task_on_specific_worker_notify_sched(task, worker, workerid, workerid);
else
{
/* Notify all workers of the combined worker */
int worker_size = combined_worker->worker_size;
int *combined_workerid = combined_worker->combined_workerid;
int j;
for (j = 0; j < worker_size; j++)
{
int subworkerid = combined_workerid[j];
_starpu_push_task_on_specific_worker_notify_sched(task, _starpu_get_worker_struct(subworkerid), subworkerid, workerid);
}
}
#ifdef STARPU_USE_SC_HYPERVISOR
starpu_sched_ctx_call_pushed_task_cb(workerid, task->sched_ctx);
#endif //STARPU_USE_SC_HYPERVISOR
unsigned i;
if (is_basic_worker)
{
unsigned node = starpu_worker_get_memory_node(workerid);
if (_starpu_task_uses_multiformat_handles(task))
{
unsigned nbuffers = STARPU_TASK_GET_NBUFFERS(task);
for (i = 0; i < nbuffers; i++)
{
struct starpu_task *conversion_task;
starpu_data_handle_t handle;
handle = STARPU_TASK_GET_HANDLE(task, i);
if (!_starpu_handle_needs_conversion_task(handle, node))
continue;
conversion_task = _starpu_create_conversion_task(handle, node);
conversion_task->mf_skip = 1;
conversion_task->execute_on_a_specific_worker = 1;
conversion_task->workerid = workerid;
_starpu_task_submit_conversion_task(conversion_task, workerid);
//_STARPU_DEBUG("Pushing a conversion task\n");
}
for (i = 0; i < nbuffers; i++)
{
starpu_data_handle_t handle = STARPU_TASK_GET_HANDLE(task, i);
handle->mf_node = node;
}
}
// if(task->sched_ctx != _starpu_get_initial_sched_ctx()->id)
if(task->priority > 0)
return _starpu_push_local_task(worker, task, 1);
else
return _starpu_push_local_task(worker, task, 0);
}
else
{
/* This is a combined worker so we create task aliases */
int worker_size = combined_worker->worker_size;
int *combined_workerid = combined_worker->combined_workerid;
int ret = 0;
struct _starpu_job *job = _starpu_get_job_associated_to_task(task);
job->task_size = worker_size;
job->combined_workerid = workerid;
job->active_task_alias_count = 0;
STARPU_PTHREAD_BARRIER_INIT(&job->before_work_barrier, NULL, worker_size);
STARPU_PTHREAD_BARRIER_INIT(&job->after_work_barrier, NULL, worker_size);
job->after_work_busy_barrier = worker_size;
/* Note: we have to call that early, or else the task may have
* disappeared already */
starpu_push_task_end(task);
int j;
for (j = 0; j < worker_size; j++)
{
struct starpu_task *alias = starpu_task_dup(task);
alias->destroy = 1;
worker = _starpu_get_worker_struct(combined_workerid[j]);
ret |= _starpu_push_local_task(worker, alias, 0);
}
return ret;
}
}
