void _starpu_driver_wait_request_completion(struct _starpu_async_channel *async_channel) { #ifdef STARPU_SIMGRID STARPU_PTHREAD_MUTEX_LOCK(&async_channel->event.mutex); while (!async_channel->event.finished) STARPU_PTHREAD_COND_WAIT(&async_channel->event.cond, &async_channel->event.mutex); STARPU_PTHREAD_MUTEX_UNLOCK(&async_channel->event.mutex); #else /* !SIMGRID */ enum starpu_node_kind kind = async_channel->type; #ifdef STARPU_USE_CUDA cudaEvent_t event; cudaError_t cures; #endif switch (kind) { #ifdef STARPU_USE_CUDA case STARPU_CUDA_RAM: event = (*async_channel).event.cuda_event; cures = cudaEventSynchronize(event); if (STARPU_UNLIKELY(cures)) STARPU_CUDA_REPORT_ERROR(cures); cures = cudaEventDestroy(event); if (STARPU_UNLIKELY(cures)) STARPU_CUDA_REPORT_ERROR(cures); break; #endif #ifdef STARPU_USE_OPENCL case STARPU_OPENCL_RAM: { cl_int err; if ((*async_channel).event.opencl_event == NULL) STARPU_ABORT(); err = clWaitForEvents(1, &((*async_channel).event.opencl_event)); if (STARPU_UNLIKELY(err != CL_SUCCESS)) STARPU_OPENCL_REPORT_ERROR(err); err = clReleaseEvent((*async_channel).event.opencl_event); if (STARPU_UNLIKELY(err != CL_SUCCESS)) STARPU_OPENCL_REPORT_ERROR(err); break; } #endif #ifdef STARPU_USE_MIC case STARPU_MIC_RAM: _starpu_mic_wait_request_completion(&(async_channel->event.mic_event)); break; #endif case STARPU_MAIN_RAM: starpu_disk_wait_request(async_channel); case STARPU_CPU_RAM: default: STARPU_ABORT(); } #endif /* !SIMGRID */ }
unsigned _starpu_driver_test_request_completion(struct _starpu_async_channel *async_channel) { #ifdef STARPU_SIMGRID unsigned ret; STARPU_PTHREAD_MUTEX_LOCK(&async_channel->event.mutex); ret = async_channel->event.finished; STARPU_PTHREAD_MUTEX_UNLOCK(&async_channel->event.mutex); return ret; #else /* !SIMGRID */ enum starpu_node_kind kind = async_channel->type; unsigned success = 0; #ifdef STARPU_USE_CUDA cudaEvent_t event; #endif switch (kind) { #ifdef STARPU_USE_CUDA case STARPU_CUDA_RAM: event = (*async_channel).event.cuda_event; cudaError_t cures = cudaEventQuery(event); success = (cures == cudaSuccess); if (success) cudaEventDestroy(event); else if (cures != cudaErrorNotReady) STARPU_CUDA_REPORT_ERROR(cures); break; #endif #ifdef STARPU_USE_OPENCL case STARPU_OPENCL_RAM: { cl_int event_status; cl_event opencl_event = (*async_channel).event.opencl_event; if (opencl_event == NULL) STARPU_ABORT(); cl_int err = clGetEventInfo(opencl_event, CL_EVENT_COMMAND_EXECUTION_STATUS, sizeof(event_status), &event_status, NULL); if (STARPU_UNLIKELY(err != CL_SUCCESS)) STARPU_OPENCL_REPORT_ERROR(err); if (event_status < 0) STARPU_OPENCL_REPORT_ERROR(event_status); success = (event_status == CL_COMPLETE); break; } #endif #ifdef STARPU_USE_MIC case STARPU_MIC_RAM: success = _starpu_mic_request_is_complete(&(async_channel->event.mic_event)); break; #endif case STARPU_DISK_RAM: success = starpu_disk_test_request(async_channel); break; case STARPU_CPU_RAM: default: STARPU_ABORT(); } return success; #endif /* !SIMGRID */ }
int main(int argc, char *argv[]) { int i; struct timeval begin, end; int size; size_t bytes; int n = 0, m = 0; STARPUFFT(plan) plan; #ifdef STARPU_HAVE_FFTW _FFTW(plan) fftw_plan; #endif #ifdef STARPU_USE_CUDA cufftHandle cuda_plan; cudaError_t cures; #endif double timing; if (argc < 2 || argc > 3) { fprintf(stderr,"need one or two size of vector\n"); exit(EXIT_FAILURE); } starpu_init(NULL); if (argc == 2) { n = atoi(argv[1]); /* 1D */ size = n; } else if (argc == 3) { n = atoi(argv[1]); m = atoi(argv[2]); /* 2D */ size = n * m; } else { assert(0); } bytes = size * sizeof(STARPUFFT(complex)); STARPUFFT(complex) *in = STARPUFFT(malloc)(size * sizeof(*in)); starpu_srand48(0); for (i = 0; i < size; i++) in[i] = starpu_drand48() + I * starpu_drand48(); STARPUFFT(complex) *out = STARPUFFT(malloc)(size * sizeof(*out)); #ifdef STARPU_HAVE_FFTW STARPUFFT(complex) *out_fftw = STARPUFFT(malloc)(size * sizeof(*out_fftw)); #endif #ifdef STARPU_USE_CUDA STARPUFFT(complex) *out_cuda = malloc(size * sizeof(*out_cuda)); #endif if (argc == 2) { plan = STARPUFFT(plan_dft_1d)(n, SIGN, 0); #ifdef STARPU_HAVE_FFTW fftw_plan = _FFTW(plan_dft_1d)(n, in, out_fftw, SIGN, FFTW_ESTIMATE); #endif #ifdef STARPU_USE_CUDA if (cufftPlan1d(&cuda_plan, n, _CUFFT_C2C, 1) != CUFFT_SUCCESS) printf("erf\n"); #endif } else if (argc == 3) { plan = STARPUFFT(plan_dft_2d)(n, m, SIGN, 0); #ifdef STARPU_HAVE_FFTW fftw_plan = _FFTW(plan_dft_2d)(n, m, in, out_fftw, SIGN, FFTW_ESTIMATE); #endif #ifdef STARPU_USE_CUDA STARPU_ASSERT(cufftPlan2d(&cuda_plan, n, m, _CUFFT_C2C) == CUFFT_SUCCESS); #endif } else { assert(0); } #ifdef STARPU_HAVE_FFTW gettimeofday(&begin, NULL); _FFTW(execute)(fftw_plan); gettimeofday(&end, NULL); _FFTW(destroy_plan)(fftw_plan); timing = (double)((end.tv_sec - begin.tv_sec)*1000000 + (end.tv_usec - begin.tv_usec)); printf("FFTW took %2.2f ms (%2.2f MB/s)\n\n", timing/1000, bytes/timing); #endif #ifdef STARPU_USE_CUDA gettimeofday(&begin, NULL); if (cufftExecC2C(cuda_plan, (cufftComplex*) in, (cufftComplex*) out_cuda, CUFFT_FORWARD) != CUFFT_SUCCESS) printf("erf2\n"); if ((cures = cudaThreadSynchronize()) != cudaSuccess) STARPU_CUDA_REPORT_ERROR(cures); gettimeofday(&end, NULL); cufftDestroy(cuda_plan); timing = (double)((end.tv_sec - begin.tv_sec)*1000000 + (end.tv_usec - begin.tv_usec)); printf("CUDA took %2.2f ms (%2.2f MB/s)\n\n", timing/1000, bytes/timing); #endif STARPUFFT(execute)(plan, in, out); STARPUFFT(showstats)(stdout); STARPUFFT(destroy_plan)(plan); printf("\n"); #if 0 for (i = 0; i < 16; i++) printf("(%f,%f) ", cimag(in[i]), creal(in[i])); printf("\n\n"); for (i = 0; i < 16; i++) printf("(%f,%f) ", cimag(out[i]), creal(out[i])); printf("\n\n"); #ifdef STARPU_HAVE_FFTW for (i = 0; i < 16; i++) printf("(%f,%f) ", cimag(out_fftw[i]), creal(out_fftw[i])); printf("\n\n"); #endif #endif #ifdef STARPU_HAVE_FFTW { double max = 0., tot = 0., norm = 0., normdiff = 0.; for (i = 0; i < size; i++) { double diff = cabs(out[i]-out_fftw[i]); double diff2 = diff * diff; double size = cabs(out_fftw[i]); double size2 = size * size; if (diff > max) max = diff; tot += diff; normdiff += diff2; norm += size2; } fprintf(stderr, "\nmaximum difference %g\n", max); fprintf(stderr, "average difference %g\n", tot / size); fprintf(stderr, "difference norm %g\n", sqrt(normdiff)); double relmaxdiff = max / sqrt(norm); fprintf(stderr, "relative maximum difference %g\n", relmaxdiff); double relavgdiff = (tot / size) / sqrt(norm); fprintf(stderr, "relative average difference %g\n", relavgdiff); if (!strcmp(TYPE, "f") && (relmaxdiff > 1e-8 || relavgdiff > 1e-8)) return EXIT_FAILURE; if (!strcmp(TYPE, "") && (relmaxdiff > 1e-16 || relavgdiff > 1e-16)) return EXIT_FAILURE; } #endif #ifdef STARPU_USE_CUDA { double max = 0., tot = 0., norm = 0., normdiff = 0.; for (i = 0; i < size; i++) { double diff = cabs(out_cuda[i]-out_fftw[i]); double diff2 = diff * diff; double size = cabs(out_fftw[i]); double size2 = size * size; if (diff > max) max = diff; tot += diff; normdiff += diff2; norm += size2; } fprintf(stderr, "\nmaximum difference %g\n", max); fprintf(stderr, "average difference %g\n", tot / size); fprintf(stderr, "difference norm %g\n", sqrt(normdiff)); double relmaxdiff = max / sqrt(norm); fprintf(stderr, "relative maximum difference %g\n", relmaxdiff); double relavgdiff = (tot / size) / sqrt(norm); fprintf(stderr, "relative average difference %g\n", relavgdiff); if (!strcmp(TYPE, "f") && (relmaxdiff > 1e-8 || relavgdiff > 1e-8)) return EXIT_FAILURE; if (!strcmp(TYPE, "") && (relmaxdiff > 1e-16 || relavgdiff > 1e-16)) return EXIT_FAILURE; } #endif STARPUFFT(free)(in); STARPUFFT(free)(out); #ifdef STARPU_HAVE_FFTW STARPUFFT(free)(out_fftw); #endif #ifdef STARPU_USE_CUDA free(out_cuda); #endif starpu_shutdown(); return EXIT_SUCCESS; }
static int copy_data_1_to_1_generic(starpu_data_handle_t handle, struct _starpu_data_replicate *src_replicate, struct _starpu_data_replicate *dst_replicate, struct _starpu_data_request *req) { unsigned src_node = src_replicate->memory_node; unsigned dst_node = dst_replicate->memory_node; STARPU_ASSERT(src_replicate->refcnt); STARPU_ASSERT(dst_replicate->refcnt); STARPU_ASSERT(src_replicate->allocated); STARPU_ASSERT(dst_replicate->allocated); _starpu_comm_amounts_inc(src_node, dst_node, handle->ops->get_size(handle)); #ifdef STARPU_SIMGRID return _starpu_simgrid_transfer(handle->ops->get_size(handle), src_node, dst_node, req); #else /* !SIMGRID */ int ret = 0; const struct starpu_data_copy_methods *copy_methods = handle->ops->copy_methods; enum starpu_node_kind src_kind = starpu_node_get_kind(src_node); enum starpu_node_kind dst_kind = starpu_node_get_kind(dst_node); #ifdef STARPU_USE_CUDA cudaError_t cures; cudaStream_t stream; #endif void *src_interface = src_replicate->data_interface; void *dst_interface = dst_replicate->data_interface; #if defined(STARPU_USE_CUDA) && defined(HAVE_CUDA_MEMCPY_PEER) && !defined(STARPU_SIMGRID) if ((src_kind == STARPU_CUDA_RAM) || (dst_kind == STARPU_CUDA_RAM)) { unsigned devid; if ((src_kind == STARPU_CUDA_RAM) && (dst_kind == STARPU_CUDA_RAM)) { /* GPU-GPU transfer, issue it from the device we are supposed to drive */ int worker = starpu_worker_get_id(); devid = starpu_worker_get_devid(worker); } else { unsigned node = (dst_kind == STARPU_CUDA_RAM)?dst_node:src_node; devid = _starpu_memory_node_get_devid(node); } starpu_cuda_set_device(devid); } #endif switch (_STARPU_MEMORY_NODE_TUPLE(src_kind,dst_kind)) { case _STARPU_MEMORY_NODE_TUPLE(STARPU_CPU_RAM,STARPU_CPU_RAM): /* STARPU_CPU_RAM -> STARPU_CPU_RAM */ if (copy_methods->ram_to_ram) copy_methods->ram_to_ram(src_interface, src_node, dst_interface, dst_node); else copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, req ? &req->async_channel : NULL); break; #ifdef STARPU_USE_CUDA case _STARPU_MEMORY_NODE_TUPLE(STARPU_CUDA_RAM,STARPU_CPU_RAM): /* only the proper CUBLAS thread can initiate this directly ! */ #if !defined(HAVE_CUDA_MEMCPY_PEER) STARPU_ASSERT(_starpu_memory_node_get_local_key() == src_node); #endif if (!req || starpu_asynchronous_copy_disabled() || starpu_asynchronous_cuda_copy_disabled() || !(copy_methods->cuda_to_ram_async || copy_methods->any_to_any)) { /* this is not associated to a request so it's synchronous */ STARPU_ASSERT(copy_methods->cuda_to_ram || copy_methods->any_to_any); if (copy_methods->cuda_to_ram) copy_methods->cuda_to_ram(src_interface, src_node, dst_interface, dst_node); else copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL); } else { req->async_channel.type = STARPU_CUDA_RAM; cures = cudaEventCreateWithFlags(&req->async_channel.event.cuda_event, cudaEventDisableTiming); if (STARPU_UNLIKELY(cures != cudaSuccess)) STARPU_CUDA_REPORT_ERROR(cures); stream = starpu_cuda_get_local_out_transfer_stream(); if (copy_methods->cuda_to_ram_async) ret = copy_methods->cuda_to_ram_async(src_interface, src_node, dst_interface, dst_node, stream); else { STARPU_ASSERT(copy_methods->any_to_any); ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, &req->async_channel); } cures = cudaEventRecord(req->async_channel.event.cuda_event, stream); if (STARPU_UNLIKELY(cures != cudaSuccess)) STARPU_CUDA_REPORT_ERROR(cures); } break; case _STARPU_MEMORY_NODE_TUPLE(STARPU_CPU_RAM,STARPU_CUDA_RAM): /* STARPU_CPU_RAM -> CUBLAS_RAM */ /* only the proper CUBLAS thread can initiate this ! */ #if !defined(HAVE_CUDA_MEMCPY_PEER) STARPU_ASSERT(_starpu_memory_node_get_local_key() == dst_node); #endif if (!req || starpu_asynchronous_copy_disabled() || starpu_asynchronous_cuda_copy_disabled() || !(copy_methods->ram_to_cuda_async || copy_methods->any_to_any)) { /* this is not associated to a request so it's synchronous */ STARPU_ASSERT(copy_methods->ram_to_cuda || copy_methods->any_to_any); if (copy_methods->ram_to_cuda) copy_methods->ram_to_cuda(src_interface, src_node, dst_interface, dst_node); else copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL); } else { req->async_channel.type = STARPU_CUDA_RAM; cures = cudaEventCreateWithFlags(&req->async_channel.event.cuda_event, cudaEventDisableTiming); if (STARPU_UNLIKELY(cures != cudaSuccess)) STARPU_CUDA_REPORT_ERROR(cures); stream = starpu_cuda_get_local_in_transfer_stream(); if (copy_methods->ram_to_cuda_async) ret = copy_methods->ram_to_cuda_async(src_interface, src_node, dst_interface, dst_node, stream); else { STARPU_ASSERT(copy_methods->any_to_any); ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, &req->async_channel); } cures = cudaEventRecord(req->async_channel.event.cuda_event, stream); if (STARPU_UNLIKELY(cures != cudaSuccess)) STARPU_CUDA_REPORT_ERROR(cures); } break; case _STARPU_MEMORY_NODE_TUPLE(STARPU_CUDA_RAM,STARPU_CUDA_RAM): /* CUDA - CUDA transfer */ if (!req || starpu_asynchronous_copy_disabled() || starpu_asynchronous_cuda_copy_disabled() || !(copy_methods->cuda_to_cuda_async || copy_methods->any_to_any)) { STARPU_ASSERT(copy_methods->cuda_to_cuda || copy_methods->any_to_any); /* this is not associated to a request so it's synchronous */ if (copy_methods->cuda_to_cuda) copy_methods->cuda_to_cuda(src_interface, src_node, dst_interface, dst_node); else copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL); } else { req->async_channel.type = STARPU_CUDA_RAM; cures = cudaEventCreateWithFlags(&req->async_channel.event.cuda_event, cudaEventDisableTiming); if (STARPU_UNLIKELY(cures != cudaSuccess)) STARPU_CUDA_REPORT_ERROR(cures); stream = starpu_cuda_get_peer_transfer_stream(src_node, dst_node); if (copy_methods->cuda_to_cuda_async) ret = copy_methods->cuda_to_cuda_async(src_interface, src_node, dst_interface, dst_node, stream); else { STARPU_ASSERT(copy_methods->any_to_any); ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, &req->async_channel); } cures = cudaEventRecord(req->async_channel.event.cuda_event, stream); if (STARPU_UNLIKELY(cures != cudaSuccess)) STARPU_CUDA_REPORT_ERROR(cures); } break; #endif #ifdef STARPU_USE_OPENCL case _STARPU_MEMORY_NODE_TUPLE(STARPU_OPENCL_RAM,STARPU_CPU_RAM): /* OpenCL -> RAM */ STARPU_ASSERT(_starpu_memory_node_get_local_key() == src_node); if (!req || starpu_asynchronous_copy_disabled() || starpu_asynchronous_opencl_copy_disabled() || !(copy_methods->opencl_to_ram_async || copy_methods->any_to_any)) { STARPU_ASSERT(copy_methods->opencl_to_ram || copy_methods->any_to_any); /* this is not associated to a request so it's synchronous */ if (copy_methods->opencl_to_ram) copy_methods->opencl_to_ram(src_interface, src_node, dst_interface, dst_node); else copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL); } else { req->async_channel.type = STARPU_OPENCL_RAM; if (copy_methods->opencl_to_ram_async) ret = copy_methods->opencl_to_ram_async(src_interface, src_node, dst_interface, dst_node, &(req->async_channel.event.opencl_event)); else { STARPU_ASSERT(copy_methods->any_to_any); ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, &req->async_channel); } } break; case _STARPU_MEMORY_NODE_TUPLE(STARPU_CPU_RAM,STARPU_OPENCL_RAM): /* STARPU_CPU_RAM -> STARPU_OPENCL_RAM */ STARPU_ASSERT(_starpu_memory_node_get_local_key() == dst_node); if (!req || starpu_asynchronous_copy_disabled() || starpu_asynchronous_opencl_copy_disabled() || !(copy_methods->ram_to_opencl_async || copy_methods->any_to_any)) { STARPU_ASSERT(copy_methods->ram_to_opencl || copy_methods->any_to_any); /* this is not associated to a request so it's synchronous */ if (copy_methods->ram_to_opencl) copy_methods->ram_to_opencl(src_interface, src_node, dst_interface, dst_node); else copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL); } else { req->async_channel.type = STARPU_OPENCL_RAM; if (copy_methods->ram_to_opencl_async) ret = copy_methods->ram_to_opencl_async(src_interface, src_node, dst_interface, dst_node, &(req->async_channel.event.opencl_event)); else { STARPU_ASSERT(copy_methods->any_to_any); ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, &req->async_channel); } } break; case _STARPU_MEMORY_NODE_TUPLE(STARPU_OPENCL_RAM,STARPU_OPENCL_RAM): /* STARPU_OPENCL_RAM -> STARPU_OPENCL_RAM */ STARPU_ASSERT(_starpu_memory_node_get_local_key() == dst_node || _starpu_memory_node_get_local_key() == src_node); if (!req || starpu_asynchronous_copy_disabled() || starpu_asynchronous_opencl_copy_disabled() || !(copy_methods->opencl_to_opencl_async || copy_methods->any_to_any)) { STARPU_ASSERT(copy_methods->opencl_to_opencl || copy_methods->any_to_any); /* this is not associated to a request so it's synchronous */ if (copy_methods->opencl_to_opencl) copy_methods->opencl_to_opencl(src_interface, src_node, dst_interface, dst_node); else copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL); } else { req->async_channel.type = STARPU_OPENCL_RAM; if (copy_methods->opencl_to_opencl_async) ret = copy_methods->opencl_to_opencl_async(src_interface, src_node, dst_interface, dst_node, &(req->async_channel.event.opencl_event)); else { STARPU_ASSERT(copy_methods->any_to_any); ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, &req->async_channel); } } break; #endif #ifdef STARPU_USE_MIC case _STARPU_MEMORY_NODE_TUPLE(STARPU_CPU_RAM,STARPU_MIC_RAM): /* RAM -> MIC */ if (!req || starpu_asynchronous_copy_disabled() || starpu_asynchronous_mic_copy_disabled() || !(copy_methods->ram_to_mic_async || copy_methods->any_to_any)) { /* this is not associated to a request so it's synchronous */ STARPU_ASSERT(copy_methods->ram_to_mic || copy_methods->any_to_any); if (copy_methods->ram_to_mic) copy_methods->ram_to_mic(src_interface, src_node, dst_interface, dst_node); else copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL); } else { req->async_channel.type = STARPU_MIC_RAM; if (copy_methods->ram_to_mic_async) ret = copy_methods->ram_to_mic_async(src_interface, src_node, dst_interface, dst_node); else { STARPU_ASSERT(copy_methods->any_to_any); ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, &req->async_channel); } _starpu_mic_init_event(&(req->async_channel.event.mic_event), dst_node); } break; case _STARPU_MEMORY_NODE_TUPLE(STARPU_MIC_RAM,STARPU_CPU_RAM): /* MIC -> RAM */ if (!req || starpu_asynchronous_copy_disabled() || starpu_asynchronous_mic_copy_disabled() || !(copy_methods->mic_to_ram_async || copy_methods->any_to_any)) { /* this is not associated to a request so it's synchronous */ STARPU_ASSERT(copy_methods->mic_to_ram || copy_methods->any_to_any); if (copy_methods->mic_to_ram) copy_methods->mic_to_ram(src_interface, src_node, dst_interface, dst_node); else copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL); } else { req->async_channel.type = STARPU_MIC_RAM; if (copy_methods->mic_to_ram_async) ret = copy_methods->mic_to_ram_async(src_interface, src_node, dst_interface, dst_node); else { STARPU_ASSERT(copy_methods->any_to_any); ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, &req->async_channel); } _starpu_mic_init_event(&(req->async_channel.event.mic_event), src_node); } break; #endif #ifdef STARPU_USE_SCC /* SCC RAM associated to the master process is considered as * the main memory node. */ case _STARPU_MEMORY_NODE_TUPLE(STARPU_CPU_RAM,STARPU_SCC_RAM): /* master private SCC RAM -> slave private SCC RAM */ if (copy_methods->scc_src_to_sink) copy_methods->scc_src_to_sink(src_interface, src_node, dst_interface, dst_node); else copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL); break; case _STARPU_MEMORY_NODE_TUPLE(STARPU_SCC_RAM,STARPU_CPU_RAM): /* slave private SCC RAM -> master private SCC RAM */ if (copy_methods->scc_sink_to_src) copy_methods->scc_sink_to_src(src_interface, src_node, dst_interface, dst_node); else copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL); break; case _STARPU_MEMORY_NODE_TUPLE(STARPU_SCC_RAM,STARPU_SCC_RAM): /* slave private SCC RAM -> slave private SCC RAM */ if (copy_methods->scc_sink_to_sink) copy_methods->scc_sink_to_sink(src_interface, src_node, dst_interface, dst_node); else copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, NULL); break; #endif case _STARPU_MEMORY_NODE_TUPLE(STARPU_CPU_RAM,STARPU_DISK_RAM): if(copy_methods->any_to_any) ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, req && !starpu_asynchronous_copy_disabled() ? &req->async_channel : NULL); else { void *obj = starpu_data_handle_to_pointer(handle, dst_node); void * ptr = NULL; starpu_ssize_t size = 0; handle->ops->pack_data(handle, src_node, &ptr, &size); ret = _starpu_disk_full_write(src_node, dst_node, obj, ptr, size, &req->async_channel); if (ret == 0) /* write is already finished, ptr was allocated in pack_data */ free(ptr); /* For now, asynchronous is not supported */ STARPU_ASSERT(ret == 0); } break; case _STARPU_MEMORY_NODE_TUPLE(STARPU_DISK_RAM,STARPU_CPU_RAM): if(copy_methods->any_to_any) ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, req && !starpu_asynchronous_copy_disabled() ? &req->async_channel : NULL); else { void *obj = starpu_data_handle_to_pointer(handle, src_node); void * ptr = NULL; size_t size = 0; ret = _starpu_disk_full_read(src_node, dst_node, obj, &ptr, &size, &req->async_channel); if (ret == 0) { /* read is already finished, we can already unpack */ handle->ops->unpack_data(handle, dst_node, ptr, size); /* ptr is allocated in full_read */ free(ptr); } /* For now, asynchronous is not supported */ STARPU_ASSERT(ret == 0); } break; case _STARPU_MEMORY_NODE_TUPLE(STARPU_DISK_RAM,STARPU_DISK_RAM): ret = copy_methods->any_to_any(src_interface, src_node, dst_interface, dst_node, req ? &req->async_channel : NULL); break; default: STARPU_ABORT(); break; } return ret; #endif /* !SIMGRID */ }
/* * cl_update (CUDA version) */ static void update_func_cuda(void *descr[], void *arg) { struct block_description *block = arg; int workerid = starpu_worker_get_id(); DEBUG( "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"); if (block->bz == 0) fprintf(stderr,"!!! DO update_func_cuda z %d CUDA%d !!!\n", block->bz, workerid); else DEBUG( "!!! DO update_func_cuda z %d CUDA%d !!!\n", block->bz, workerid); #ifdef STARPU_USE_MPI int rank = 0; MPI_Comm_rank(MPI_COMM_WORLD, &rank); DEBUG( "!!! RANK %d !!!\n", rank); #endif DEBUG( "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"); unsigned block_size_z = get_block_size(block->bz); unsigned i; update_per_worker[workerid]++; struct timeval tv, tv2, diff, delta = {.tv_sec = 0, .tv_usec = get_ticks()*1000}; gettimeofday(&tv, NULL); timersub(&tv, &start, &tv2); timersub(&tv2, &last_tick[block->bz], &diff); while (timercmp(&diff, &delta, >=)) { timeradd(&last_tick[block->bz], &delta, &last_tick[block->bz]); timersub(&tv2, &last_tick[block->bz], &diff); if (who_runs_what_index[block->bz] < who_runs_what_len) who_runs_what[block->bz + (who_runs_what_index[block->bz]++) * get_nbz()] = -1; } if (who_runs_what_index[block->bz] < who_runs_what_len) who_runs_what[block->bz + (who_runs_what_index[block->bz]++) * get_nbz()] = global_workerid(workerid); /* * Load neighbours' boundaries : TOP */ /* The offset along the z axis is (block_size_z + K) */ load_subblock_from_buffer_cuda(descr[0], descr[2], block_size_z+K); load_subblock_from_buffer_cuda(descr[1], descr[3], block_size_z+K); /* * Load neighbours' boundaries : BOTTOM */ load_subblock_from_buffer_cuda(descr[0], descr[4], 0); load_subblock_from_buffer_cuda(descr[1], descr[5], 0); /* * Stencils ... do the actual work here :) TODO */ for (i=1; i<=K; i++) { starpu_block_interface_t *oldb = descr[i%2], *newb = descr[(i+1)%2]; TYPE *old = (void*) oldb->ptr, *new = (void*) newb->ptr; /* Shadow data */ cuda_shadow_host(block->bz, old, oldb->nx, oldb->ny, oldb->nz, oldb->ldy, oldb->ldz, i); /* And perform actual computation */ #ifdef LIFE cuda_life_update_host(block->bz, old, new, oldb->nx, oldb->ny, oldb->nz, oldb->ldy, oldb->ldz, i); #else cudaMemcpy(new, old, oldb->nx * oldb->ny * oldb->nz * sizeof(*new), cudaMemcpyDeviceToDevice); #endif /* LIFE */ } cudaError_t cures; if ((cures = cudaThreadSynchronize()) != cudaSuccess) STARPU_CUDA_REPORT_ERROR(cures); } #endif /* STARPU_USE_CUDA */ /* * cl_update (CPU version) */ static void update_func_cpu(void *descr[], void *arg) { struct block_description *block = arg; int workerid = starpu_worker_get_id(); DEBUG( "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"); if (block->bz == 0) fprintf(stderr,"!!! DO update_func_cpu z %d CPU%d !!!\n", block->bz, workerid); else DEBUG( "!!! DO update_func_cpu z %d CPU%d !!!\n", block->bz, workerid); #ifdef STARPU_USE_MPI int rank = 0; MPI_Comm_rank(MPI_COMM_WORLD, &rank); DEBUG( "!!! RANK %d !!!\n", rank); #endif DEBUG( "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"); unsigned block_size_z = get_block_size(block->bz); unsigned i; update_per_worker[workerid]++; struct timeval tv, tv2, diff, delta = {.tv_sec = 0, .tv_usec = get_ticks() * 1000}; gettimeofday(&tv, NULL); timersub(&tv, &start, &tv2); timersub(&tv2, &last_tick[block->bz], &diff); while (timercmp(&diff, &delta, >=)) { timeradd(&last_tick[block->bz], &delta, &last_tick[block->bz]); timersub(&tv2, &last_tick[block->bz], &diff); if (who_runs_what_index[block->bz] < who_runs_what_len) who_runs_what[block->bz + (who_runs_what_index[block->bz]++) * get_nbz()] = -1; } if (who_runs_what_index[block->bz] < who_runs_what_len) who_runs_what[block->bz + (who_runs_what_index[block->bz]++) * get_nbz()] = global_workerid(workerid); /* * Load neighbours' boundaries : TOP */ /* The offset along the z axis is (block_size_z + K) */ load_subblock_from_buffer_cpu(descr[0], descr[2], block_size_z+K); load_subblock_from_buffer_cpu(descr[1], descr[3], block_size_z+K); /* * Load neighbours' boundaries : BOTTOM */ load_subblock_from_buffer_cpu(descr[0], descr[4], 0); load_subblock_from_buffer_cpu(descr[1], descr[5], 0); /* * Stencils ... do the actual work here :) TODO */ for (i=1; i<=K; i++) { starpu_block_interface_t *oldb = descr[i%2], *newb = descr[(i+1)%2]; TYPE *old = (void*) oldb->ptr, *new = (void*) newb->ptr; /* Shadow data */ unsigned ldy = oldb->ldy, ldz = oldb->ldz; unsigned nx = oldb->nx, ny = oldb->ny, nz = oldb->nz; unsigned x, y, z; unsigned stepx = 1; unsigned stepy = 1; unsigned stepz = 1; unsigned idx = 0; unsigned idy = 0; unsigned idz = 0; TYPE *ptr = old; # include "shadow.h" /* And perform actual computation */ #ifdef LIFE life_update(block->bz, old, new, oldb->nx, oldb->ny, oldb->nz, oldb->ldy, oldb->ldz, i); #else memcpy(new, old, oldb->nx * oldb->ny * oldb->nz * sizeof(*new)); #endif /* LIFE */ } } /* Performance model and codelet structure */ static struct starpu_perfmodel_t cl_update_model = { .type = STARPU_HISTORY_BASED, .symbol = "cl_update" }; starpu_codelet cl_update = { .where = #ifdef STARPU_USE_CUDA STARPU_CUDA| #endif STARPU_CPU, .cpu_func = update_func_cpu, #ifdef STARPU_USE_CUDA .cuda_func = update_func_cuda, #endif .model = &cl_update_model, .nbuffers = 6 }; /* * Save the block internal boundaries to give them to our neighbours. */ /* CPU version */ static void load_subblock_into_buffer_cpu(starpu_block_interface_t *block, starpu_block_interface_t *boundary, unsigned firstz) { /* Sanity checks */ STARPU_ASSERT(block->nx == boundary->nx); STARPU_ASSERT(block->ny == boundary->ny); STARPU_ASSERT(boundary->nz == K); /* NB: this is not fully garanteed ... but it's *very* likely and that * makes our life much simpler */ STARPU_ASSERT(block->ldy == boundary->ldy); STARPU_ASSERT(block->ldz == boundary->ldz); /* We do a contiguous memory transfer */ size_t boundary_size = K*block->ldz*block->elemsize; unsigned offset = firstz*block->ldz; TYPE *block_data = (TYPE *)block->ptr; TYPE *boundary_data = (TYPE *)boundary->ptr; memcpy(boundary_data, &block_data[offset], boundary_size); } /* CUDA version */ #ifdef STARPU_USE_CUDA static void load_subblock_into_buffer_cuda(starpu_block_interface_t *block, starpu_block_interface_t *boundary, unsigned firstz) { /* Sanity checks */ STARPU_ASSERT(block->nx == boundary->nx); STARPU_ASSERT(block->ny == boundary->ny); STARPU_ASSERT(boundary->nz == K); /* NB: this is not fully garanteed ... but it's *very* likely and that * makes our life much simpler */ STARPU_ASSERT(block->ldy == boundary->ldy); STARPU_ASSERT(block->ldz == boundary->ldz); /* We do a contiguous memory transfer */ size_t boundary_size = K*block->ldz*block->elemsize; unsigned offset = firstz*block->ldz; TYPE *block_data = (TYPE *)block->ptr; TYPE *boundary_data = (TYPE *)boundary->ptr; cudaMemcpy(boundary_data, &block_data[offset], boundary_size, cudaMemcpyDeviceToDevice); } #endif /* STARPU_USE_CUDA */ /* Record how many top/bottom saves each worker performed */ unsigned top_per_worker[STARPU_NMAXWORKERS]; unsigned bottom_per_worker[STARPU_NMAXWORKERS]; /* top save, CPU version */ static void dummy_func_top_cpu(void *descr[] __attribute__((unused)), void *arg) { struct block_description *block = arg; int workerid = starpu_worker_get_id(); top_per_worker[workerid]++; DEBUG( "DO SAVE Bottom block %d\n", block->bz); /* The offset along the z axis is (block_size_z + K)- K */ unsigned block_size_z = get_block_size(block->bz); load_subblock_into_buffer_cpu(descr[0], descr[2], block_size_z); load_subblock_into_buffer_cpu(descr[1], descr[3], block_size_z); } /* bottom save, CPU version */ static void dummy_func_bottom_cpu(void *descr[] __attribute__((unused)), void *arg) { struct block_description *block = arg; int workerid = starpu_worker_get_id(); bottom_per_worker[workerid]++; DEBUG( "DO SAVE Top block %d\n", block->bz); load_subblock_into_buffer_cpu(descr[0], descr[2], K); load_subblock_into_buffer_cpu(descr[1], descr[3], K); } /* top save, CUDA version */ #ifdef STARPU_USE_CUDA static void dummy_func_top_cuda(void *descr[] __attribute__((unused)), void *arg) { struct block_description *block = arg; int workerid = starpu_worker_get_id(); top_per_worker[workerid]++; DEBUG( "DO SAVE Top block %d\n", block->bz); /* The offset along the z axis is (block_size_z + K)- K */ unsigned block_size_z = get_block_size(block->bz); load_subblock_into_buffer_cuda(descr[0], descr[2], block_size_z); load_subblock_into_buffer_cuda(descr[1], descr[3], block_size_z); cudaThreadSynchronize(); } /* bottom save, CUDA version */ static void dummy_func_bottom_cuda(void *descr[] __attribute__((unused)), void *arg) { struct block_description *block = arg; int workerid = starpu_worker_get_id(); bottom_per_worker[workerid]++; DEBUG( "DO SAVE Bottom block %d on CUDA\n", block->bz); load_subblock_into_buffer_cuda(descr[0], descr[2], K); load_subblock_into_buffer_cuda(descr[1], descr[3], K); cudaThreadSynchronize(); } #endif /* STARPU_USE_CUDA */ /* Performance models and codelet for save */ static struct starpu_perfmodel_t save_cl_bottom_model = { .type = STARPU_HISTORY_BASED, .symbol = "save_cl_bottom" }; static struct starpu_perfmodel_t save_cl_top_model = { .type = STARPU_HISTORY_BASED, .symbol = "save_cl_top" }; starpu_codelet save_cl_bottom = { .where = #ifdef STARPU_USE_CUDA STARPU_CUDA| #endif STARPU_CPU, .cpu_func = dummy_func_bottom_cpu, #ifdef STARPU_USE_CUDA .cuda_func = dummy_func_bottom_cuda, #endif .model = &save_cl_bottom_model, .nbuffers = 4 }; starpu_codelet save_cl_top = { .where = #ifdef STARPU_USE_CUDA STARPU_CUDA| #endif STARPU_CPU, .cpu_func = dummy_func_top_cpu, #ifdef STARPU_USE_CUDA .cuda_func = dummy_func_top_cuda, #endif .model = &save_cl_top_model, .nbuffers = 4 };