int _starpu_is_same_kind_disk(unsigned node1, unsigned node2) { if(starpu_node_get_kind(node1) == STARPU_DISK_RAM && starpu_node_get_kind(node2) == STARPU_DISK_RAM) { int pos1 = get_location_with_node(node1); int pos2 = get_location_with_node(node2); if(disk_register_list[pos1]->functions == disk_register_list[pos2]->functions) /* they must have a copy function */ if(disk_register_list[pos1]->functions->copy != NULL) return 1; } return 0; }
static int copy_data_1_to_1_generic(starpu_data_handle_t handle, struct _starpu_data_replicate *src_replicate, struct _starpu_data_replicate *dst_replicate, struct _starpu_data_request *req) { unsigned src_node = src_replicate->memory_node; unsigned dst_node = dst_replicate->memory_node; STARPU_ASSERT(src_replicate->refcnt); STARPU_ASSERT(dst_replicate->refcnt); STARPU_ASSERT(src_replicate->allocated); STARPU_ASSERT(dst_replicate->allocated); _starpu_comm_amounts_inc(src_node, dst_node, handle->ops->get_size(handle)); #ifdef STARPU_SIMGRID return _starpu_simgrid_transfer(handle->ops->get_size(handle), src_node, dst_node, req); #else /* !SIMGRID */ int ret = 0; const struct starpu_data_copy_methods *copy_methods = handle->ops->copy_methods; enum starpu_node_kind src_kind = starpu_node_get_kind(src_node); enum starpu_node_kind dst_kind = starpu_node_get_kind(dst_node); #ifdef STARPU_USE_CUDA cudaError_t cures; cudaStream_t stream; #endif void *src_interface = src_replicate->data_interface; void *dst_interface = dst_replicate->data_interface; #if defined(STARPU_USE_CUDA) && defined(HAVE_CUDA_MEMCPY_PEER) && !defined(STARPU_SIMGRID) if ((src_kind == STARPU_CUDA_RAM) || (dst_kind == STARPU_CUDA_RAM)) { unsigned devid; if ((src_kind == STARPU_CUDA_RAM) && (dst_kind == STARPU_CUDA_RAM)) { /* GPU-GPU transfer, issue it from the device we are supposed to drive */ int worker = starpu_worker_get_id(); devid = starpu_worker_get_devid(worker); } else { unsigned node = (dst_kind == STARPU_CUDA_RAM)?dst_node:src_node; devid = _starpu_memory_node_get_devid(node); } starpu_cuda_set_device(devid); } #endif switch (_STARPU_MEMORY_NODE_TUPLE(src_kind,dst_kind)) { case _STARPU_MEMORY_NODE_TUPLE(STARPU_CPU_RAM,STARPU_CPU_RAM): /* STARPU_CPU_RAM -> STARPU_CPU_RAM */ if (copy_methods->ram_to_ram) copy_methods->ram_to_ram(src_interface, src_node, dst_interface, dst_node); else copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, req ? &req->async_channel : NULL); break; #ifdef STARPU_USE_CUDA case _STARPU_MEMORY_NODE_TUPLE(STARPU_CUDA_RAM,STARPU_CPU_RAM): /* only the proper CUBLAS thread can initiate this directly ! */ #if !defined(HAVE_CUDA_MEMCPY_PEER) STARPU_ASSERT(_starpu_memory_node_get_local_key() == src_node); #endif if (!req || starpu_asynchronous_copy_disabled() || starpu_asynchronous_cuda_copy_disabled() || !(copy_methods->cuda_to_ram_async || copy_methods->any_to_any)) { /* this is not associated to a request so it's synchronous */ STARPU_ASSERT(copy_methods->cuda_to_ram || copy_methods->any_to_any); if (copy_methods->cuda_to_ram) copy_methods->cuda_to_ram(src_interface, src_node, dst_interface, dst_node); else copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL); } else { req->async_channel.type = STARPU_CUDA_RAM; cures = cudaEventCreateWithFlags(&req->async_channel.event.cuda_event, cudaEventDisableTiming); if (STARPU_UNLIKELY(cures != cudaSuccess)) STARPU_CUDA_REPORT_ERROR(cures); stream = starpu_cuda_get_local_out_transfer_stream(); if (copy_methods->cuda_to_ram_async) ret = copy_methods->cuda_to_ram_async(src_interface, src_node, dst_interface, dst_node, stream); else { STARPU_ASSERT(copy_methods->any_to_any); ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, &req->async_channel); } cures = cudaEventRecord(req->async_channel.event.cuda_event, stream); if (STARPU_UNLIKELY(cures != cudaSuccess)) STARPU_CUDA_REPORT_ERROR(cures); } break; case _STARPU_MEMORY_NODE_TUPLE(STARPU_CPU_RAM,STARPU_CUDA_RAM): /* STARPU_CPU_RAM -> CUBLAS_RAM */ /* only the proper CUBLAS thread can initiate this ! */ #if !defined(HAVE_CUDA_MEMCPY_PEER) STARPU_ASSERT(_starpu_memory_node_get_local_key() == dst_node); #endif if (!req || starpu_asynchronous_copy_disabled() || starpu_asynchronous_cuda_copy_disabled() || !(copy_methods->ram_to_cuda_async || copy_methods->any_to_any)) { /* this is not associated to a request so it's synchronous */ STARPU_ASSERT(copy_methods->ram_to_cuda || copy_methods->any_to_any); if (copy_methods->ram_to_cuda) copy_methods->ram_to_cuda(src_interface, src_node, dst_interface, dst_node); else copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL); } else { req->async_channel.type = STARPU_CUDA_RAM; cures = cudaEventCreateWithFlags(&req->async_channel.event.cuda_event, cudaEventDisableTiming); if (STARPU_UNLIKELY(cures != cudaSuccess)) STARPU_CUDA_REPORT_ERROR(cures); stream = starpu_cuda_get_local_in_transfer_stream(); if (copy_methods->ram_to_cuda_async) ret = copy_methods->ram_to_cuda_async(src_interface, src_node, dst_interface, dst_node, stream); else { STARPU_ASSERT(copy_methods->any_to_any); ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, &req->async_channel); } cures = cudaEventRecord(req->async_channel.event.cuda_event, stream); if (STARPU_UNLIKELY(cures != cudaSuccess)) STARPU_CUDA_REPORT_ERROR(cures); } break; case _STARPU_MEMORY_NODE_TUPLE(STARPU_CUDA_RAM,STARPU_CUDA_RAM): /* CUDA - CUDA transfer */ if (!req || starpu_asynchronous_copy_disabled() || starpu_asynchronous_cuda_copy_disabled() || !(copy_methods->cuda_to_cuda_async || copy_methods->any_to_any)) { STARPU_ASSERT(copy_methods->cuda_to_cuda || copy_methods->any_to_any); /* this is not associated to a request so it's synchronous */ if (copy_methods->cuda_to_cuda) copy_methods->cuda_to_cuda(src_interface, src_node, dst_interface, dst_node); else copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL); } else { req->async_channel.type = STARPU_CUDA_RAM; cures = cudaEventCreateWithFlags(&req->async_channel.event.cuda_event, cudaEventDisableTiming); if (STARPU_UNLIKELY(cures != cudaSuccess)) STARPU_CUDA_REPORT_ERROR(cures); stream = starpu_cuda_get_peer_transfer_stream(src_node, dst_node); if (copy_methods->cuda_to_cuda_async) ret = copy_methods->cuda_to_cuda_async(src_interface, src_node, dst_interface, dst_node, stream); else { STARPU_ASSERT(copy_methods->any_to_any); ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, &req->async_channel); } cures = cudaEventRecord(req->async_channel.event.cuda_event, stream); if (STARPU_UNLIKELY(cures != cudaSuccess)) STARPU_CUDA_REPORT_ERROR(cures); } break; #endif #ifdef STARPU_USE_OPENCL case _STARPU_MEMORY_NODE_TUPLE(STARPU_OPENCL_RAM,STARPU_CPU_RAM): /* OpenCL -> RAM */ STARPU_ASSERT(_starpu_memory_node_get_local_key() == src_node); if (!req || starpu_asynchronous_copy_disabled() || starpu_asynchronous_opencl_copy_disabled() || !(copy_methods->opencl_to_ram_async || copy_methods->any_to_any)) { STARPU_ASSERT(copy_methods->opencl_to_ram || copy_methods->any_to_any); /* this is not associated to a request so it's synchronous */ if (copy_methods->opencl_to_ram) copy_methods->opencl_to_ram(src_interface, src_node, dst_interface, dst_node); else copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL); } else { req->async_channel.type = STARPU_OPENCL_RAM; if (copy_methods->opencl_to_ram_async) ret = copy_methods->opencl_to_ram_async(src_interface, src_node, dst_interface, dst_node, &(req->async_channel.event.opencl_event)); else { STARPU_ASSERT(copy_methods->any_to_any); ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, &req->async_channel); } } break; case _STARPU_MEMORY_NODE_TUPLE(STARPU_CPU_RAM,STARPU_OPENCL_RAM): /* STARPU_CPU_RAM -> STARPU_OPENCL_RAM */ STARPU_ASSERT(_starpu_memory_node_get_local_key() == dst_node); if (!req || starpu_asynchronous_copy_disabled() || starpu_asynchronous_opencl_copy_disabled() || !(copy_methods->ram_to_opencl_async || copy_methods->any_to_any)) { STARPU_ASSERT(copy_methods->ram_to_opencl || copy_methods->any_to_any); /* this is not associated to a request so it's synchronous */ if (copy_methods->ram_to_opencl) copy_methods->ram_to_opencl(src_interface, src_node, dst_interface, dst_node); else copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL); } else { req->async_channel.type = STARPU_OPENCL_RAM; if (copy_methods->ram_to_opencl_async) ret = copy_methods->ram_to_opencl_async(src_interface, src_node, dst_interface, dst_node, &(req->async_channel.event.opencl_event)); else { STARPU_ASSERT(copy_methods->any_to_any); ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, &req->async_channel); } } break; case _STARPU_MEMORY_NODE_TUPLE(STARPU_OPENCL_RAM,STARPU_OPENCL_RAM): /* STARPU_OPENCL_RAM -> STARPU_OPENCL_RAM */ STARPU_ASSERT(_starpu_memory_node_get_local_key() == dst_node || _starpu_memory_node_get_local_key() == src_node); if (!req || starpu_asynchronous_copy_disabled() || starpu_asynchronous_opencl_copy_disabled() || !(copy_methods->opencl_to_opencl_async || copy_methods->any_to_any)) { STARPU_ASSERT(copy_methods->opencl_to_opencl || copy_methods->any_to_any); /* this is not associated to a request so it's synchronous */ if (copy_methods->opencl_to_opencl) copy_methods->opencl_to_opencl(src_interface, src_node, dst_interface, dst_node); else copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL); } else { req->async_channel.type = STARPU_OPENCL_RAM; if (copy_methods->opencl_to_opencl_async) ret = copy_methods->opencl_to_opencl_async(src_interface, src_node, dst_interface, dst_node, &(req->async_channel.event.opencl_event)); else { STARPU_ASSERT(copy_methods->any_to_any); ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, &req->async_channel); } } break; #endif #ifdef STARPU_USE_MIC case _STARPU_MEMORY_NODE_TUPLE(STARPU_CPU_RAM,STARPU_MIC_RAM): /* RAM -> MIC */ if (!req || starpu_asynchronous_copy_disabled() || starpu_asynchronous_mic_copy_disabled() || !(copy_methods->ram_to_mic_async || copy_methods->any_to_any)) { /* this is not associated to a request so it's synchronous */ STARPU_ASSERT(copy_methods->ram_to_mic || copy_methods->any_to_any); if (copy_methods->ram_to_mic) copy_methods->ram_to_mic(src_interface, src_node, dst_interface, dst_node); else copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL); } else { req->async_channel.type = STARPU_MIC_RAM; if (copy_methods->ram_to_mic_async) ret = copy_methods->ram_to_mic_async(src_interface, src_node, dst_interface, dst_node); else { STARPU_ASSERT(copy_methods->any_to_any); ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, &req->async_channel); } _starpu_mic_init_event(&(req->async_channel.event.mic_event), dst_node); } break; case _STARPU_MEMORY_NODE_TUPLE(STARPU_MIC_RAM,STARPU_CPU_RAM): /* MIC -> RAM */ if (!req || starpu_asynchronous_copy_disabled() || starpu_asynchronous_mic_copy_disabled() || !(copy_methods->mic_to_ram_async || copy_methods->any_to_any)) { /* this is not associated to a request so it's synchronous */ STARPU_ASSERT(copy_methods->mic_to_ram || copy_methods->any_to_any); if (copy_methods->mic_to_ram) copy_methods->mic_to_ram(src_interface, src_node, dst_interface, dst_node); else copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL); } else { req->async_channel.type = STARPU_MIC_RAM; if (copy_methods->mic_to_ram_async) ret = copy_methods->mic_to_ram_async(src_interface, src_node, dst_interface, dst_node); else { STARPU_ASSERT(copy_methods->any_to_any); ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, &req->async_channel); } _starpu_mic_init_event(&(req->async_channel.event.mic_event), src_node); } break; #endif #ifdef STARPU_USE_SCC /* SCC RAM associated to the master process is considered as * the main memory node. */ case _STARPU_MEMORY_NODE_TUPLE(STARPU_CPU_RAM,STARPU_SCC_RAM): /* master private SCC RAM -> slave private SCC RAM */ if (copy_methods->scc_src_to_sink) copy_methods->scc_src_to_sink(src_interface, src_node, dst_interface, dst_node); else copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL); break; case _STARPU_MEMORY_NODE_TUPLE(STARPU_SCC_RAM,STARPU_CPU_RAM): /* slave private SCC RAM -> master private SCC RAM */ if (copy_methods->scc_sink_to_src) copy_methods->scc_sink_to_src(src_interface, src_node, dst_interface, dst_node); else copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL); break; case _STARPU_MEMORY_NODE_TUPLE(STARPU_SCC_RAM,STARPU_SCC_RAM): /* slave private SCC RAM -> slave private SCC RAM */ if (copy_methods->scc_sink_to_sink) copy_methods->scc_sink_to_sink(src_interface, src_node, dst_interface, dst_node); else copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL); break; #endif case _STARPU_MEMORY_NODE_TUPLE(STARPU_CPU_RAM,STARPU_DISK_RAM): if(copy_methods->any_to_any) ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, req && !starpu_asynchronous_copy_disabled() ? &req->async_channel : NULL); else { void *obj = starpu_data_handle_to_pointer(handle, dst_node); void * ptr = NULL; starpu_ssize_t size = 0; handle->ops->pack_data(handle, src_node, &ptr, &size); ret = _starpu_disk_full_write(src_node, dst_node, obj, ptr, size, &req->async_channel); if (ret == 0) /* write is already finished, ptr was allocated in pack_data */ free(ptr); /* For now, asynchronous is not supported */ STARPU_ASSERT(ret == 0); } break; case _STARPU_MEMORY_NODE_TUPLE(STARPU_DISK_RAM,STARPU_CPU_RAM): if(copy_methods->any_to_any) ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, req && !starpu_asynchronous_copy_disabled() ? &req->async_channel : NULL); else { void *obj = starpu_data_handle_to_pointer(handle, src_node); void * ptr = NULL; size_t size = 0; ret = _starpu_disk_full_read(src_node, dst_node, obj, &ptr, &size, &req->async_channel); if (ret == 0) { /* read is already finished, we can already unpack */ handle->ops->unpack_data(handle, dst_node, ptr, size); /* ptr is allocated in full_read */ free(ptr); } /* For now, asynchronous is not supported */ STARPU_ASSERT(ret == 0); } break; case _STARPU_MEMORY_NODE_TUPLE(STARPU_DISK_RAM,STARPU_DISK_RAM): ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, req ? &req->async_channel : NULL); break; default: STARPU_ABORT(); break; } return ret; #endif /* !SIMGRID */ }
/* This should either return 0 if the transfer is complete, or -EAGAIN if the * transfer is still pending, and will have to be waited for by * _starpu_driver_test_request_completion/_starpu_driver_wait_request_completion */ int starpu_interface_copy(uintptr_t src, size_t src_offset, unsigned src_node, uintptr_t dst, size_t dst_offset, unsigned dst_node, size_t size, void *async_data) { struct _starpu_async_channel *async_channel = async_data; enum starpu_node_kind src_kind = starpu_node_get_kind(src_node); enum starpu_node_kind dst_kind = starpu_node_get_kind(dst_node); switch (_STARPU_MEMORY_NODE_TUPLE(src_kind,dst_kind)) { case _STARPU_MEMORY_NODE_TUPLE(STARPU_CPU_RAM,STARPU_CPU_RAM): memcpy((void *) (dst + dst_offset), (void *) (src + src_offset), size); return 0; #ifdef STARPU_USE_CUDA case _STARPU_MEMORY_NODE_TUPLE(STARPU_CUDA_RAM,STARPU_CPU_RAM): return starpu_cuda_copy_async_sync( (void*) (src + src_offset), src_node, (void*) (dst + dst_offset), dst_node, size, async_channel?starpu_cuda_get_local_out_transfer_stream():NULL, cudaMemcpyDeviceToHost); case _STARPU_MEMORY_NODE_TUPLE(STARPU_CPU_RAM,STARPU_CUDA_RAM): return starpu_cuda_copy_async_sync( (void*) (src + src_offset), src_node, (void*) (dst + dst_offset), dst_node, size, async_channel?starpu_cuda_get_local_in_transfer_stream():NULL, cudaMemcpyHostToDevice); case _STARPU_MEMORY_NODE_TUPLE(STARPU_CUDA_RAM,STARPU_CUDA_RAM): return starpu_cuda_copy_async_sync( (void*) (src + src_offset), src_node, (void*) (dst + dst_offset), dst_node, size, async_channel?starpu_cuda_get_peer_transfer_stream(src_node, dst_node):NULL, cudaMemcpyDeviceToDevice); #endif #ifdef STARPU_USE_OPENCL case _STARPU_MEMORY_NODE_TUPLE(STARPU_OPENCL_RAM,STARPU_CPU_RAM): case _STARPU_MEMORY_NODE_TUPLE(STARPU_CPU_RAM,STARPU_OPENCL_RAM): case _STARPU_MEMORY_NODE_TUPLE(STARPU_OPENCL_RAM,STARPU_OPENCL_RAM): return starpu_opencl_copy_async_sync( src, src_offset, src_node, dst, dst_offset, dst_node, size, &async_channel->event.opencl_event); #endif #ifdef STARPU_USE_MIC case _STARPU_MEMORY_NODE_TUPLE(STARPU_MIC_RAM,STARPU_CPU_RAM): if (async_data) return _starpu_mic_copy_mic_to_ram_async( (void*) (src + src_offset), src_node, (void*) (dst + dst_offset), dst_node, size); else return _starpu_mic_copy_mic_to_ram( (void*) (src + src_offset), src_node, (void*) (dst + dst_offset), dst_node, size); case _STARPU_MEMORY_NODE_TUPLE(STARPU_CPU_RAM,STARPU_MIC_RAM): if (async_data) return _starpu_mic_copy_ram_to_mic_async( (void*) (src + src_offset), src_node, (void*) (dst + dst_offset), dst_node, size); else return _starpu_mic_copy_ram_to_mic( (void*) (src + src_offset), src_node, (void*) (dst + dst_offset), dst_node, size); #endif #ifdef STARPU_USE_SCC case _STARPU_MEMORY_NODE_TUPLE(STARPU_SCC_RAM,STARPU_CPU_RAM): return _starpu_scc_copy_sink_to_src( (void*) (src + src_offset), src_node, (void*) (dst + dst_offset), dst_node, size); case _STARPU_MEMORY_NODE_TUPLE(STARPU_CPU_RAM,STARPU_SCC_RAM): return _starpu_scc_copy_src_to_sink( (void*) (src + src_offset), src_node, (void*) (dst + dst_offset), dst_node, size); case _STARPU_MEMORY_NODE_TUPLE(STARPU_SCC_RAM,STARPU_SCC_RAM): return _starpu_scc_copy_sink_to_sink( (void*) (src + src_offset), src_node, (void*) (dst + dst_offset), dst_node, size); #endif case _STARPU_MEMORY_NODE_TUPLE(STARPU_CPU_RAM, STARPU_DISK_RAM): { return _starpu_disk_copy_src_to_disk( (void*) (src + src_offset), src_node, (void*) dst, dst_offset, dst_node, size, async_channel); } case _STARPU_MEMORY_NODE_TUPLE(STARPU_DISK_RAM, STARPU_CPU_RAM): return _starpu_disk_copy_disk_to_src( (void*) src, src_offset, src_node, (void*) (dst + dst_offset), dst_node, size, async_channel); case _STARPU_MEMORY_NODE_TUPLE(STARPU_DISK_RAM, STARPU_DISK_RAM): return _starpu_disk_copy_disk_to_disk( (void*) src, src_offset, src_node, (void*) dst, dst_offset, dst_node, size, async_channel); default: STARPU_ABORT(); return -1; } return 0; }
void starpu_data_unpartition(starpu_data_handle_t root_handle, unsigned gathering_node) { unsigned child; unsigned worker; unsigned nworkers = starpu_worker_get_count(); unsigned node; unsigned sizes[root_handle->nchildren]; _STARPU_TRACE_START_UNPARTITION(root_handle, gathering_node); _starpu_spin_lock(&root_handle->header_lock); STARPU_ASSERT_MSG(root_handle->nchildren != 0, "data %p is not partitioned, can not unpartition it", root_handle); /* first take all the children lock (in order !) */ for (child = 0; child < root_handle->nchildren; child++) { starpu_data_handle_t child_handle = starpu_data_get_child(root_handle, child); /* make sure the intermediate children is unpartitionned as well */ if (child_handle->nchildren > 0) starpu_data_unpartition(child_handle, gathering_node); /* If this is a multiformat handle, we must convert the data now */ #ifdef STARPU_DEVEL #warning TODO: _starpu_fetch_data_on_node should be doing it #endif if (_starpu_data_is_multiformat_handle(child_handle) && starpu_node_get_kind(child_handle->mf_node) != STARPU_CPU_RAM) { struct starpu_codelet cl = { .where = STARPU_CPU, .cpu_funcs = { _starpu_empty_codelet_function }, .modes = { STARPU_RW }, .nbuffers = 1 }; struct starpu_task *task = starpu_task_create(); task->name = "convert_data"; STARPU_TASK_SET_HANDLE(task, child_handle, 0); task->cl = &cl; task->synchronous = 1; if (_starpu_task_submit_internally(task) != 0) _STARPU_ERROR("Could not submit the conversion task while unpartitionning\n"); } int ret; /* for now we pretend that the RAM is almost unlimited and that gathering * data should be possible from the node that does the unpartionning ... we * don't want to have the programming deal with memory shortage at that time, * really */ /* Acquire the child data on the gathering node. This will trigger collapsing any reduction */ ret = starpu_data_acquire_on_node(child_handle, gathering_node, STARPU_RW); STARPU_ASSERT(ret == 0); starpu_data_release_on_node(child_handle, gathering_node); _starpu_spin_lock(&child_handle->header_lock); child_handle->busy_waiting = 1; _starpu_spin_unlock(&child_handle->header_lock); /* Wait for all requests to finish (notably WT requests) */ STARPU_PTHREAD_MUTEX_LOCK(&child_handle->busy_mutex); while (1) { /* Here helgrind would shout that this an unprotected access, * but this is actually fine: all threads who do busy_count-- * are supposed to call _starpu_data_check_not_busy, which will * wake us up through the busy_mutex/busy_cond. */ if (!child_handle->busy_count) break; /* This is woken by _starpu_data_check_not_busy, always called * after decrementing busy_count */ STARPU_PTHREAD_COND_WAIT(&child_handle->busy_cond, &child_handle->busy_mutex); } STARPU_PTHREAD_MUTEX_UNLOCK(&child_handle->busy_mutex); _starpu_spin_lock(&child_handle->header_lock); sizes[child] = _starpu_data_get_size(child_handle); _starpu_data_unregister_ram_pointer(child_handle); for (worker = 0; worker < nworkers; worker++) { struct _starpu_data_replicate *local = &child_handle->per_worker[worker]; STARPU_ASSERT(local->state == STARPU_INVALID); if (local->allocated && local->automatically_allocated) _starpu_request_mem_chunk_removal(child_handle, local, starpu_worker_get_memory_node(worker), sizes[child]); } _starpu_memory_stats_free(child_handle); }
struct starpu_task *_starpu_create_conversion_task_for_arch(starpu_data_handle_t handle, enum starpu_node_kind node_kind) { struct starpu_task *conversion_task; #if defined(STARPU_USE_OPENCL) || defined(STARPU_USE_CUDA) || defined(STARPU_USE_MIC) || defined(STARPU_USE_SCC) || defined(STARPU_SIMGRID) struct starpu_multiformat_interface *format_interface; #endif conversion_task = starpu_task_create(); conversion_task->name = "conversion_task"; conversion_task->synchronous = 0; STARPU_TASK_SET_HANDLE(conversion_task, handle, 0); #if defined(STARPU_USE_OPENCL) || defined(STARPU_USE_CUDA) || defined(STARPU_USE_MIC) || defined(STARPU_USE_SCC) || defined(STARPU_SIMGRID) /* The node does not really matter here */ format_interface = (struct starpu_multiformat_interface *) starpu_data_get_interface_on_node(handle, STARPU_MAIN_RAM); #endif _starpu_spin_lock(&handle->header_lock); handle->refcnt++; handle->busy_count++; _starpu_spin_unlock(&handle->header_lock); switch(node_kind) { case STARPU_CPU_RAM: case STARPU_SCC_RAM: case STARPU_SCC_SHM: switch (starpu_node_get_kind(handle->mf_node)) { case STARPU_CPU_RAM: case STARPU_SCC_RAM: case STARPU_SCC_SHM: STARPU_ABORT(); #if defined(STARPU_USE_CUDA) || defined(STARPU_SIMGRID) case STARPU_CUDA_RAM: { struct starpu_multiformat_data_interface_ops *mf_ops; mf_ops = (struct starpu_multiformat_data_interface_ops *) handle->ops->get_mf_ops(format_interface); conversion_task->cl = mf_ops->cuda_to_cpu_cl; break; } #endif #if defined(STARPU_USE_OPENCL) || defined(STARPU_SIMGRID) case STARPU_OPENCL_RAM: { struct starpu_multiformat_data_interface_ops *mf_ops; mf_ops = (struct starpu_multiformat_data_interface_ops *) handle->ops->get_mf_ops(format_interface); conversion_task->cl = mf_ops->opencl_to_cpu_cl; break; } #endif #ifdef STARPU_USE_MIC case STARPU_MIC_RAM: { struct starpu_multiformat_data_interface_ops *mf_ops; mf_ops = (struct starpu_multiformat_data_interface_ops *) handle->ops->get_mf_ops(format_interface); conversion_task->cl = mf_ops->mic_to_cpu_cl; break; } #endif default: _STARPU_ERROR("Oops : %u\n", handle->mf_node); } break; #if defined(STARPU_USE_CUDA) || defined(STARPU_SIMGRID) case STARPU_CUDA_RAM: { struct starpu_multiformat_data_interface_ops *mf_ops; mf_ops = (struct starpu_multiformat_data_interface_ops *) handle->ops->get_mf_ops(format_interface); conversion_task->cl = mf_ops->cpu_to_cuda_cl; break; } #endif #if defined(STARPU_USE_OPENCL) || defined(STARPU_SIMGRID) case STARPU_OPENCL_RAM: { struct starpu_multiformat_data_interface_ops *mf_ops; mf_ops = (struct starpu_multiformat_data_interface_ops *) handle->ops->get_mf_ops(format_interface); conversion_task->cl = mf_ops->cpu_to_opencl_cl; break; } #endif #ifdef STARPU_USE_MIC case STARPU_MIC_RAM: { struct starpu_multiformat_data_interface_ops *mf_ops; mf_ops = (struct starpu_multiformat_data_interface_ops *) handle->ops->get_mf_ops(format_interface); conversion_task->cl = mf_ops->cpu_to_mic_cl; break; } #endif default: STARPU_ABORT(); } STARPU_TASK_SET_MODE(conversion_task, STARPU_RW, 0); return conversion_task; }
/* * Given a handle that needs to be converted in order to be used on the given * node, returns a task that takes care of the conversion. */ struct starpu_task *_starpu_create_conversion_task(starpu_data_handle_t handle, unsigned int node) { return _starpu_create_conversion_task_for_arch(handle, starpu_node_get_kind(node)); }