static int copy_data_1_to_1_generic(starpu_data_handle_t handle,
struct _starpu_data_replicate *src_replicate,
struct _starpu_data_replicate *dst_replicate,
struct _starpu_data_request *req)
{
unsigned src_node = src_replicate->memory_node;
unsigned dst_node = dst_replicate->memory_node;
STARPU_ASSERT(src_replicate->refcnt);
STARPU_ASSERT(dst_replicate->refcnt);
STARPU_ASSERT(src_replicate->allocated);
STARPU_ASSERT(dst_replicate->allocated);
_starpu_comm_amounts_inc(src_node, dst_node, handle->ops->get_size(handle));
#ifdef STARPU_SIMGRID
return _starpu_simgrid_transfer(handle->ops->get_size(handle), src_node, dst_node, req);
#else /* !SIMGRID */
int ret = 0;
const struct starpu_data_copy_methods *copy_methods = handle->ops->copy_methods;
enum starpu_node_kind src_kind = starpu_node_get_kind(src_node);
enum starpu_node_kind dst_kind = starpu_node_get_kind(dst_node);
#ifdef STARPU_USE_CUDA
cudaError_t cures;
cudaStream_t stream;
#endif
void *src_interface = src_replicate->data_interface;
void *dst_interface = dst_replicate->data_interface;
#if defined(STARPU_USE_CUDA) && defined(HAVE_CUDA_MEMCPY_PEER) && !defined(STARPU_SIMGRID)
if ((src_kind == STARPU_CUDA_RAM) || (dst_kind == STARPU_CUDA_RAM))
{
unsigned devid;
if ((src_kind == STARPU_CUDA_RAM) && (dst_kind == STARPU_CUDA_RAM))
{
/* GPU-GPU transfer, issue it from the device we are supposed to drive */
int worker = starpu_worker_get_id();
devid = starpu_worker_get_devid(worker);
}
else
{
unsigned node = (dst_kind == STARPU_CUDA_RAM)?dst_node:src_node;
devid = _starpu_memory_node_get_devid(node);
}
starpu_cuda_set_device(devid);
}
#endif
switch (_STARPU_MEMORY_NODE_TUPLE(src_kind,dst_kind))
{
case _STARPU_MEMORY_NODE_TUPLE(STARPU_CPU_RAM,STARPU_CPU_RAM):
/* STARPU_CPU_RAM -> STARPU_CPU_RAM */
if (copy_methods->ram_to_ram)
copy_methods->ram_to_ram(src_interface, src_node, dst_interface, dst_node);
else
copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, req ? &req->async_channel : NULL);
break;
#ifdef STARPU_USE_CUDA
case _STARPU_MEMORY_NODE_TUPLE(STARPU_CUDA_RAM,STARPU_CPU_RAM):
/* only the proper CUBLAS thread can initiate this directly ! */
#if !defined(HAVE_CUDA_MEMCPY_PEER)
STARPU_ASSERT(_starpu_memory_node_get_local_key() == src_node);
#endif
if (!req || starpu_asynchronous_copy_disabled() || starpu_asynchronous_cuda_copy_disabled() ||
!(copy_methods->cuda_to_ram_async || copy_methods->any_to_any))
{
/* this is not associated to a request so it's synchronous */
STARPU_ASSERT(copy_methods->cuda_to_ram || copy_methods->any_to_any);
if (copy_methods->cuda_to_ram)
copy_methods->cuda_to_ram(src_interface, src_node, dst_interface, dst_node);
else
copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL);
}
else
{
req->async_channel.type = STARPU_CUDA_RAM;
cures = cudaEventCreateWithFlags(&req->async_channel.event.cuda_event, cudaEventDisableTiming);
if (STARPU_UNLIKELY(cures != cudaSuccess)) STARPU_CUDA_REPORT_ERROR(cures);
stream = starpu_cuda_get_local_out_transfer_stream();
if (copy_methods->cuda_to_ram_async)
ret = copy_methods->cuda_to_ram_async(src_interface, src_node, dst_interface, dst_node, stream);
else
{
STARPU_ASSERT(copy_methods->any_to_any);
ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, &req->async_channel);
}
cures = cudaEventRecord(req->async_channel.event.cuda_event, stream);
if (STARPU_UNLIKELY(cures != cudaSuccess)) STARPU_CUDA_REPORT_ERROR(cures);
}
break;
case _STARPU_MEMORY_NODE_TUPLE(STARPU_CPU_RAM,STARPU_CUDA_RAM):
/* STARPU_CPU_RAM -> CUBLAS_RAM */
/* only the proper CUBLAS thread can initiate this ! */
#if !defined(HAVE_CUDA_MEMCPY_PEER)
STARPU_ASSERT(_starpu_memory_node_get_local_key() == dst_node);
#endif
if (!req || starpu_asynchronous_copy_disabled() || starpu_asynchronous_cuda_copy_disabled() ||
!(copy_methods->ram_to_cuda_async || copy_methods->any_to_any))
{
/* this is not associated to a request so it's synchronous */
STARPU_ASSERT(copy_methods->ram_to_cuda || copy_methods->any_to_any);
if (copy_methods->ram_to_cuda)
copy_methods->ram_to_cuda(src_interface, src_node, dst_interface, dst_node);
else
copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL);
}
else
{
req->async_channel.type = STARPU_CUDA_RAM;
cures = cudaEventCreateWithFlags(&req->async_channel.event.cuda_event, cudaEventDisableTiming);
if (STARPU_UNLIKELY(cures != cudaSuccess))
STARPU_CUDA_REPORT_ERROR(cures);
stream = starpu_cuda_get_local_in_transfer_stream();
if (copy_methods->ram_to_cuda_async)
ret = copy_methods->ram_to_cuda_async(src_interface, src_node, dst_interface, dst_node, stream);
else
{
STARPU_ASSERT(copy_methods->any_to_any);
ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, &req->async_channel);
}
cures = cudaEventRecord(req->async_channel.event.cuda_event, stream);
if (STARPU_UNLIKELY(cures != cudaSuccess))
STARPU_CUDA_REPORT_ERROR(cures);
}
break;
case _STARPU_MEMORY_NODE_TUPLE(STARPU_CUDA_RAM,STARPU_CUDA_RAM):
/* CUDA - CUDA transfer */
if (!req || starpu_asynchronous_copy_disabled() || starpu_asynchronous_cuda_copy_disabled() ||
!(copy_methods->cuda_to_cuda_async || copy_methods->any_to_any))
{
STARPU_ASSERT(copy_methods->cuda_to_cuda || copy_methods->any_to_any);
/* this is not associated to a request so it's synchronous */
if (copy_methods->cuda_to_cuda)
copy_methods->cuda_to_cuda(src_interface, src_node, dst_interface, dst_node);
else
copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL);
}
else
{
req->async_channel.type = STARPU_CUDA_RAM;
cures = cudaEventCreateWithFlags(&req->async_channel.event.cuda_event, cudaEventDisableTiming);
if (STARPU_UNLIKELY(cures != cudaSuccess)) STARPU_CUDA_REPORT_ERROR(cures);
stream = starpu_cuda_get_peer_transfer_stream(src_node, dst_node);
if (copy_methods->cuda_to_cuda_async)
ret = copy_methods->cuda_to_cuda_async(src_interface, src_node, dst_interface, dst_node, stream);
else
{
STARPU_ASSERT(copy_methods->any_to_any);
ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, &req->async_channel);
}
cures = cudaEventRecord(req->async_channel.event.cuda_event, stream);
if (STARPU_UNLIKELY(cures != cudaSuccess)) STARPU_CUDA_REPORT_ERROR(cures);
}
break;
#endif
#ifdef STARPU_USE_OPENCL
case _STARPU_MEMORY_NODE_TUPLE(STARPU_OPENCL_RAM,STARPU_CPU_RAM):
/* OpenCL -> RAM */
STARPU_ASSERT(_starpu_memory_node_get_local_key() == src_node);
if (!req || starpu_asynchronous_copy_disabled() || starpu_asynchronous_opencl_copy_disabled() ||
!(copy_methods->opencl_to_ram_async || copy_methods->any_to_any))
{
STARPU_ASSERT(copy_methods->opencl_to_ram || copy_methods->any_to_any);
/* this is not associated to a request so it's synchronous */
if (copy_methods->opencl_to_ram)
copy_methods->opencl_to_ram(src_interface, src_node, dst_interface, dst_node);
else
copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL);
}
else
{
req->async_channel.type = STARPU_OPENCL_RAM;
if (copy_methods->opencl_to_ram_async)
ret = copy_methods->opencl_to_ram_async(src_interface, src_node, dst_interface, dst_node, &(req->async_channel.event.opencl_event));
else
{
STARPU_ASSERT(copy_methods->any_to_any);
ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, &req->async_channel);
}
}
break;
case _STARPU_MEMORY_NODE_TUPLE(STARPU_CPU_RAM,STARPU_OPENCL_RAM):
/* STARPU_CPU_RAM -> STARPU_OPENCL_RAM */
STARPU_ASSERT(_starpu_memory_node_get_local_key() == dst_node);
if (!req || starpu_asynchronous_copy_disabled() || starpu_asynchronous_opencl_copy_disabled() ||
!(copy_methods->ram_to_opencl_async || copy_methods->any_to_any))
{
STARPU_ASSERT(copy_methods->ram_to_opencl || copy_methods->any_to_any);
/* this is not associated to a request so it's synchronous */
if (copy_methods->ram_to_opencl)
copy_methods->ram_to_opencl(src_interface, src_node, dst_interface, dst_node);
else
copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL);
}
else
{
req->async_channel.type = STARPU_OPENCL_RAM;
if (copy_methods->ram_to_opencl_async)
ret = copy_methods->ram_to_opencl_async(src_interface, src_node, dst_interface, dst_node, &(req->async_channel.event.opencl_event));
else
{
STARPU_ASSERT(copy_methods->any_to_any);
ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, &req->async_channel);
}
}
break;
case _STARPU_MEMORY_NODE_TUPLE(STARPU_OPENCL_RAM,STARPU_OPENCL_RAM):
/* STARPU_OPENCL_RAM -> STARPU_OPENCL_RAM */
STARPU_ASSERT(_starpu_memory_node_get_local_key() == dst_node || _starpu_memory_node_get_local_key() == src_node);
if (!req || starpu_asynchronous_copy_disabled() || starpu_asynchronous_opencl_copy_disabled() ||
!(copy_methods->opencl_to_opencl_async || copy_methods->any_to_any))
{
STARPU_ASSERT(copy_methods->opencl_to_opencl || copy_methods->any_to_any);
/* this is not associated to a request so it's synchronous */
if (copy_methods->opencl_to_opencl)
copy_methods->opencl_to_opencl(src_interface, src_node, dst_interface, dst_node);
else
copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL);
}
else
{
req->async_channel.type = STARPU_OPENCL_RAM;
if (copy_methods->opencl_to_opencl_async)
ret = copy_methods->opencl_to_opencl_async(src_interface, src_node, dst_interface, dst_node, &(req->async_channel.event.opencl_event));
else
{
STARPU_ASSERT(copy_methods->any_to_any);
ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, &req->async_channel);
}
}
break;
#endif
#ifdef STARPU_USE_MIC
case _STARPU_MEMORY_NODE_TUPLE(STARPU_CPU_RAM,STARPU_MIC_RAM):
/* RAM -> MIC */
if (!req || starpu_asynchronous_copy_disabled() || starpu_asynchronous_mic_copy_disabled() ||
!(copy_methods->ram_to_mic_async || copy_methods->any_to_any))
{
/* this is not associated to a request so it's synchronous */
STARPU_ASSERT(copy_methods->ram_to_mic || copy_methods->any_to_any);
if (copy_methods->ram_to_mic)
copy_methods->ram_to_mic(src_interface, src_node, dst_interface, dst_node);
else
copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL);
}
else
{
req->async_channel.type = STARPU_MIC_RAM;
if (copy_methods->ram_to_mic_async)
ret = copy_methods->ram_to_mic_async(src_interface, src_node, dst_interface, dst_node);
else
{
STARPU_ASSERT(copy_methods->any_to_any);
ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, &req->async_channel);
}
_starpu_mic_init_event(&(req->async_channel.event.mic_event), dst_node);
}
break;
case _STARPU_MEMORY_NODE_TUPLE(STARPU_MIC_RAM,STARPU_CPU_RAM):
/* MIC -> RAM */
if (!req || starpu_asynchronous_copy_disabled() || starpu_asynchronous_mic_copy_disabled() ||
!(copy_methods->mic_to_ram_async || copy_methods->any_to_any))
{
/* this is not associated to a request so it's synchronous */
STARPU_ASSERT(copy_methods->mic_to_ram || copy_methods->any_to_any);
if (copy_methods->mic_to_ram)
copy_methods->mic_to_ram(src_interface, src_node, dst_interface, dst_node);
else
copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL);
}
else
{
req->async_channel.type = STARPU_MIC_RAM;
if (copy_methods->mic_to_ram_async)
ret = copy_methods->mic_to_ram_async(src_interface, src_node, dst_interface, dst_node);
else
{
STARPU_ASSERT(copy_methods->any_to_any);
ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, &req->async_channel);
}
_starpu_mic_init_event(&(req->async_channel.event.mic_event), src_node);
}
break;
#endif
#ifdef STARPU_USE_SCC
/* SCC RAM associated to the master process is considered as
* the main memory node. */
case _STARPU_MEMORY_NODE_TUPLE(STARPU_CPU_RAM,STARPU_SCC_RAM):
/* master private SCC RAM -> slave private SCC RAM */
if (copy_methods->scc_src_to_sink)
copy_methods->scc_src_to_sink(src_interface, src_node, dst_interface, dst_node);
else
copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL);
break;
case _STARPU_MEMORY_NODE_TUPLE(STARPU_SCC_RAM,STARPU_CPU_RAM):
/* slave private SCC RAM -> master private SCC RAM */
if (copy_methods->scc_sink_to_src)
copy_methods->scc_sink_to_src(src_interface, src_node, dst_interface, dst_node);
else
copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL);
break;
case _STARPU_MEMORY_NODE_TUPLE(STARPU_SCC_RAM,STARPU_SCC_RAM):
/* slave private SCC RAM -> slave private SCC RAM */
if (copy_methods->scc_sink_to_sink)
copy_methods->scc_sink_to_sink(src_interface, src_node, dst_interface, dst_node);
else
copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL);
break;
#endif
case _STARPU_MEMORY_NODE_TUPLE(STARPU_CPU_RAM,STARPU_DISK_RAM):
if(copy_methods->any_to_any)
ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, req && !starpu_asynchronous_copy_disabled() ? &req->async_channel : NULL);
else
{
void *obj = starpu_data_handle_to_pointer(handle, dst_node);
void * ptr = NULL;
starpu_ssize_t size = 0;
handle->ops->pack_data(handle, src_node, &ptr, &size);
ret = _starpu_disk_full_write(src_node, dst_node, obj, ptr, size, &req->async_channel);
if (ret == 0)
/* write is already finished, ptr was allocated in pack_data */
free(ptr);
/* For now, asynchronous is not supported */
STARPU_ASSERT(ret == 0);
}
break;
case _STARPU_MEMORY_NODE_TUPLE(STARPU_DISK_RAM,STARPU_CPU_RAM):
if(copy_methods->any_to_any)
ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, req && !starpu_asynchronous_copy_disabled() ? &req->async_channel : NULL);
else
{
void *obj = starpu_data_handle_to_pointer(handle, src_node);
void * ptr = NULL;
size_t size = 0;
ret = _starpu_disk_full_read(src_node, dst_node, obj, &ptr, &size, &req->async_channel);
if (ret == 0)
{
/* read is already finished, we can already unpack */
handle->ops->unpack_data(handle, dst_node, ptr, size);
/* ptr is allocated in full_read */
free(ptr);
}
/* For now, asynchronous is not supported */
STARPU_ASSERT(ret == 0);
}
break;
case _STARPU_MEMORY_NODE_TUPLE(STARPU_DISK_RAM,STARPU_DISK_RAM):
ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, req ? &req->async_channel : NULL);
break;
default:
STARPU_ABORT();
break;
}
return ret;
#endif /* !SIMGRID */
}
static void _starpu_data_partition(starpu_data_handle_t initial_handle, starpu_data_handle_t *childrenp, unsigned nparts, struct starpu_data_filter *f, int inherit_state)
{
unsigned i;
unsigned node;
/* first take care to properly lock the data header */
_starpu_spin_lock(&initial_handle->header_lock);
initial_handle->nplans++;
STARPU_ASSERT_MSG(nparts > 0, "Partitioning data %p in 0 piece does not make sense", initial_handle);
/* allocate the children */
if (inherit_state)
{
initial_handle->children = (struct _starpu_data_state *) calloc(nparts, sizeof(struct _starpu_data_state));
STARPU_ASSERT(initial_handle->children);
/* this handle now has children */
initial_handle->nchildren = nparts;
}
unsigned nworkers = starpu_worker_get_count();
for (node = 0; node < STARPU_MAXNODES; node++)
{
if (initial_handle->per_node[node].state != STARPU_INVALID)
break;
}
if (node == STARPU_MAXNODES)
{
/* This is lazy allocation, allocate it now in main RAM, so as
* to have somewhere to gather pieces later */
/* FIXME: mark as unevictable! */
int ret = _starpu_allocate_memory_on_node(initial_handle, &initial_handle->per_node[STARPU_MAIN_RAM], 0);
#ifdef STARPU_DEVEL
#warning we should reclaim memory if allocation failed
#endif
STARPU_ASSERT(!ret);
}
for (i = 0; i < nparts; i++)
{
starpu_data_handle_t child;
if (inherit_state)
child = &initial_handle->children[i];
else
child = childrenp[i];
STARPU_ASSERT(child);
struct starpu_data_interface_ops *ops;
/* each child may have his own interface type */
/* what's this child's interface ? */
if (f->get_child_ops)
ops = f->get_child_ops(f, i);
else
ops = initial_handle->ops;
_starpu_data_handle_init(child, ops, initial_handle->mf_node);
child->nchildren = 0;
child->nplans = 0;
child->switch_cl = NULL;
child->partitioned = 0;
child->readonly = 0;
child->mpi_data = initial_handle->mpi_data;
child->root_handle = initial_handle->root_handle;
child->father_handle = initial_handle;
child->sibling_index = i;
child->depth = initial_handle->depth + 1;
child->is_not_important = initial_handle->is_not_important;
child->wt_mask = initial_handle->wt_mask;
child->home_node = initial_handle->home_node;
child->is_readonly = initial_handle->is_readonly;
/* initialize the chunk lock */
_starpu_data_requester_list_init(&child->req_list);
_starpu_data_requester_list_init(&child->reduction_req_list);
child->reduction_tmp_handles = NULL;
child->write_invalidation_req = NULL;
child->refcnt = 0;
child->unlocking_reqs = 0;
child->busy_count = 0;
child->busy_waiting = 0;
STARPU_PTHREAD_MUTEX_INIT(&child->busy_mutex, NULL);
STARPU_PTHREAD_COND_INIT(&child->busy_cond, NULL);
child->reduction_refcnt = 0;
_starpu_spin_init(&child->header_lock);
child->sequential_consistency = initial_handle->sequential_consistency;
STARPU_PTHREAD_MUTEX_INIT(&child->sequential_consistency_mutex, NULL);
child->last_submitted_mode = STARPU_R;
child->last_sync_task = NULL;
child->last_submitted_accessors.task = NULL;
child->last_submitted_accessors.next = &child->last_submitted_accessors;
child->last_submitted_accessors.prev = &child->last_submitted_accessors;
child->post_sync_tasks = NULL;
/* Tell helgrind that the race in _starpu_unlock_post_sync_tasks is fine */
STARPU_HG_DISABLE_CHECKING(child->post_sync_tasks_cnt);
child->post_sync_tasks_cnt = 0;
/* The methods used for reduction are propagated to the
* children. */
child->redux_cl = initial_handle->redux_cl;
child->init_cl = initial_handle->init_cl;
#ifdef STARPU_USE_FXT
child->last_submitted_ghost_sync_id_is_valid = 0;
child->last_submitted_ghost_sync_id = 0;
child->last_submitted_ghost_accessors_id = NULL;
#endif
if (_starpu_global_arbiter)
/* Just for testing purpose */
starpu_data_assign_arbiter(child, _starpu_global_arbiter);
else
child->arbiter = NULL;
_starpu_data_requester_list_init(&child->arbitered_req_list);
for (node = 0; node < STARPU_MAXNODES; node++)
{
struct _starpu_data_replicate *initial_replicate;
struct _starpu_data_replicate *child_replicate;
initial_replicate = &initial_handle->per_node[node];
child_replicate = &child->per_node[node];
if (inherit_state)
child_replicate->state = initial_replicate->state;
else
child_replicate->state = STARPU_INVALID;
if (inherit_state || !initial_replicate->automatically_allocated)
child_replicate->allocated = initial_replicate->allocated;
else
child_replicate->allocated = 0;
/* Do not allow memory reclaiming within the child for parent bits */
child_replicate->automatically_allocated = 0;
child_replicate->refcnt = 0;
child_replicate->memory_node = node;
child_replicate->relaxed_coherency = 0;
if (inherit_state)
child_replicate->initialized = initial_replicate->initialized;
else
child_replicate->initialized = 0;
/* update the interface */
void *initial_interface = starpu_data_get_interface_on_node(initial_handle, node);
void *child_interface = starpu_data_get_interface_on_node(child, node);
STARPU_ASSERT_MSG(!(!inherit_state && child_replicate->automatically_allocated && child_replicate->allocated), "partition planning is currently not supported when handle has some automatically allocated buffers");
f->filter_func(initial_interface, child_interface, f, i, nparts);
}
unsigned worker;
for (worker = 0; worker < nworkers; worker++)
{
struct _starpu_data_replicate *child_replicate;
child_replicate = &child->per_worker[worker];
child_replicate->state = STARPU_INVALID;
child_replicate->allocated = 0;
child_replicate->automatically_allocated = 0;
child_replicate->refcnt = 0;
child_replicate->memory_node = starpu_worker_get_memory_node(worker);
child_replicate->requested = 0;
for (node = 0; node < STARPU_MAXNODES; node++)
{
child_replicate->request[node] = NULL;
}
child_replicate->relaxed_coherency = 1;
child_replicate->initialized = 0;
/* duplicate the content of the interface on node 0 */
memcpy(child_replicate->data_interface, child->per_node[0].data_interface, child->ops->interface_size);
}
/* We compute the size and the footprint of the child once and
* store it in the handle */
child->footprint = _starpu_compute_data_footprint(child);
void *ptr;
ptr = starpu_data_handle_to_pointer(child, STARPU_MAIN_RAM);
if (ptr != NULL)
_starpu_data_register_ram_pointer(child, ptr);
}
/* now let the header */
_starpu_spin_unlock(&initial_handle->header_lock);
}