int _starpu_data_cpy(starpu_data_handle_t dst_handle, starpu_data_handle_t src_handle,
int asynchronous, void (*callback_func)(void*), void *callback_arg,
int reduction, struct starpu_task *reduction_dep_task)
{
struct starpu_task *task = starpu_task_create();
STARPU_ASSERT(task);
task->name = "data_cpy";
struct _starpu_job *j = _starpu_get_job_associated_to_task(task);
if (reduction)
{
j->reduction_task = reduction;
if (reduction_dep_task)
starpu_task_declare_deps_array(task, 1, &reduction_dep_task);
}
task->cl = ©_cl;
unsigned *interface_id = malloc(sizeof(*interface_id));
*interface_id = dst_handle->ops->interfaceid;
task->cl_arg = interface_id;
task->cl_arg_size = sizeof(*interface_id);
task->cl_arg_free = 1;
task->callback_func = callback_func;
task->callback_arg = callback_arg;
STARPU_TASK_SET_HANDLE(task, dst_handle, 0);
STARPU_TASK_SET_HANDLE(task, src_handle, 1);
task->synchronous = !asynchronous;
int ret = _starpu_task_submit_internally(task);
STARPU_ASSERT(!ret);
return 0;
}
void starpu_data_unpartition(starpu_data_handle_t root_handle, unsigned gathering_node)
{
unsigned child;
unsigned worker;
unsigned nworkers = starpu_worker_get_count();
unsigned node;
unsigned sizes[root_handle->nchildren];
_STARPU_TRACE_START_UNPARTITION(root_handle, gathering_node);
_starpu_spin_lock(&root_handle->header_lock);
STARPU_ASSERT_MSG(root_handle->nchildren != 0, "data %p is not partitioned, can not unpartition it", root_handle);
/* first take all the children lock (in order !) */
for (child = 0; child < root_handle->nchildren; child++)
{
starpu_data_handle_t child_handle = starpu_data_get_child(root_handle, child);
/* make sure the intermediate children is unpartitionned as well */
if (child_handle->nchildren > 0)
starpu_data_unpartition(child_handle, gathering_node);
/* If this is a multiformat handle, we must convert the data now */
#ifdef STARPU_DEVEL
#warning TODO: _starpu_fetch_data_on_node should be doing it
#endif
if (_starpu_data_is_multiformat_handle(child_handle) &&
starpu_node_get_kind(child_handle->mf_node) != STARPU_CPU_RAM)
{
struct starpu_codelet cl =
{
.where = STARPU_CPU,
.cpu_funcs = { _starpu_empty_codelet_function },
.modes = { STARPU_RW },
.nbuffers = 1
};
struct starpu_task *task = starpu_task_create();
task->name = "convert_data";
STARPU_TASK_SET_HANDLE(task, child_handle, 0);
task->cl = &cl;
task->synchronous = 1;
if (_starpu_task_submit_internally(task) != 0)
_STARPU_ERROR("Could not submit the conversion task while unpartitionning\n");
}
int ret;
/* for now we pretend that the RAM is almost unlimited and that gathering
* data should be possible from the node that does the unpartionning ... we
* don't want to have the programming deal with memory shortage at that time,
* really */
/* Acquire the child data on the gathering node. This will trigger collapsing any reduction */
ret = starpu_data_acquire_on_node(child_handle, gathering_node, STARPU_RW);
STARPU_ASSERT(ret == 0);
starpu_data_release_on_node(child_handle, gathering_node);
_starpu_spin_lock(&child_handle->header_lock);
child_handle->busy_waiting = 1;
_starpu_spin_unlock(&child_handle->header_lock);
/* Wait for all requests to finish (notably WT requests) */
STARPU_PTHREAD_MUTEX_LOCK(&child_handle->busy_mutex);
while (1)
{
/* Here helgrind would shout that this an unprotected access,
* but this is actually fine: all threads who do busy_count--
* are supposed to call _starpu_data_check_not_busy, which will
* wake us up through the busy_mutex/busy_cond. */
if (!child_handle->busy_count)
break;
/* This is woken by _starpu_data_check_not_busy, always called
* after decrementing busy_count */
STARPU_PTHREAD_COND_WAIT(&child_handle->busy_cond, &child_handle->busy_mutex);
}
STARPU_PTHREAD_MUTEX_UNLOCK(&child_handle->busy_mutex);
_starpu_spin_lock(&child_handle->header_lock);
sizes[child] = _starpu_data_get_size(child_handle);
_starpu_data_unregister_ram_pointer(child_handle);
for (worker = 0; worker < nworkers; worker++)
{
struct _starpu_data_replicate *local = &child_handle->per_worker[worker];
STARPU_ASSERT(local->state == STARPU_INVALID);
if (local->allocated && local->automatically_allocated)
_starpu_request_mem_chunk_removal(child_handle, local, starpu_worker_get_memory_node(worker), sizes[child]);
}
_starpu_memory_stats_free(child_handle);
}
struct starpu_task *_starpu_create_conversion_task_for_arch(starpu_data_handle_t handle,
enum starpu_node_kind node_kind)
{
struct starpu_task *conversion_task;
#if defined(STARPU_USE_OPENCL) || defined(STARPU_USE_CUDA) || defined(STARPU_USE_MIC) || defined(STARPU_USE_SCC) || defined(STARPU_SIMGRID)
struct starpu_multiformat_interface *format_interface;
#endif
conversion_task = starpu_task_create();
conversion_task->name = "conversion_task";
conversion_task->synchronous = 0;
STARPU_TASK_SET_HANDLE(conversion_task, handle, 0);
#if defined(STARPU_USE_OPENCL) || defined(STARPU_USE_CUDA) || defined(STARPU_USE_MIC) || defined(STARPU_USE_SCC) || defined(STARPU_SIMGRID)
/* The node does not really matter here */
format_interface = (struct starpu_multiformat_interface *) starpu_data_get_interface_on_node(handle, STARPU_MAIN_RAM);
#endif
_starpu_spin_lock(&handle->header_lock);
handle->refcnt++;
handle->busy_count++;
_starpu_spin_unlock(&handle->header_lock);
switch(node_kind)
{
case STARPU_CPU_RAM:
case STARPU_SCC_RAM:
case STARPU_SCC_SHM:
switch (starpu_node_get_kind(handle->mf_node))
{
case STARPU_CPU_RAM:
case STARPU_SCC_RAM:
case STARPU_SCC_SHM:
STARPU_ABORT();
#if defined(STARPU_USE_CUDA) || defined(STARPU_SIMGRID)
case STARPU_CUDA_RAM:
{
struct starpu_multiformat_data_interface_ops *mf_ops;
mf_ops = (struct starpu_multiformat_data_interface_ops *) handle->ops->get_mf_ops(format_interface);
conversion_task->cl = mf_ops->cuda_to_cpu_cl;
break;
}
#endif
#if defined(STARPU_USE_OPENCL) || defined(STARPU_SIMGRID)
case STARPU_OPENCL_RAM:
{
struct starpu_multiformat_data_interface_ops *mf_ops;
mf_ops = (struct starpu_multiformat_data_interface_ops *) handle->ops->get_mf_ops(format_interface);
conversion_task->cl = mf_ops->opencl_to_cpu_cl;
break;
}
#endif
#ifdef STARPU_USE_MIC
case STARPU_MIC_RAM:
{
struct starpu_multiformat_data_interface_ops *mf_ops;
mf_ops = (struct starpu_multiformat_data_interface_ops *) handle->ops->get_mf_ops(format_interface);
conversion_task->cl = mf_ops->mic_to_cpu_cl;
break;
}
#endif
default:
_STARPU_ERROR("Oops : %u\n", handle->mf_node);
}
break;
#if defined(STARPU_USE_CUDA) || defined(STARPU_SIMGRID)
case STARPU_CUDA_RAM:
{
struct starpu_multiformat_data_interface_ops *mf_ops;
mf_ops = (struct starpu_multiformat_data_interface_ops *) handle->ops->get_mf_ops(format_interface);
conversion_task->cl = mf_ops->cpu_to_cuda_cl;
break;
}
#endif
#if defined(STARPU_USE_OPENCL) || defined(STARPU_SIMGRID)
case STARPU_OPENCL_RAM:
{
struct starpu_multiformat_data_interface_ops *mf_ops;
mf_ops = (struct starpu_multiformat_data_interface_ops *) handle->ops->get_mf_ops(format_interface);
conversion_task->cl = mf_ops->cpu_to_opencl_cl;
break;
}
#endif
#ifdef STARPU_USE_MIC
case STARPU_MIC_RAM:
{
struct starpu_multiformat_data_interface_ops *mf_ops;
mf_ops = (struct starpu_multiformat_data_interface_ops *) handle->ops->get_mf_ops(format_interface);
conversion_task->cl = mf_ops->cpu_to_mic_cl;
break;
}
#endif
default:
STARPU_ABORT();
}
STARPU_TASK_SET_MODE(conversion_task, STARPU_RW, 0);
return conversion_task;
}
