static bool nvptx_init (void) { CUresult r; int ndevs; if (instantiated_devices != 0) return true; r = cuInit (0); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuInit error: %s", cuda_error (r)); ptx_events = NULL; pthread_mutex_init (&ptx_event_lock, NULL); r = cuDeviceGetCount (&ndevs); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuDeviceGetCount error: %s", cuda_error (r)); ptx_devices = GOMP_PLUGIN_malloc_cleared (sizeof (struct ptx_device *) * ndevs); return true; }
static void nvptx_wait_async (int async1, int async2) { CUevent *e; struct ptx_stream *s1, *s2; pthread_t self = pthread_self (); /* The stream that is waiting (rather than being waited for) doesn't necessarily have to exist already. */ s2 = select_stream_for_async (async2, self, true, NULL); s1 = select_stream_for_async (async1, self, false, NULL); if (!s1) GOMP_PLUGIN_fatal ("invalid async 1\n"); if (s1 == s2) GOMP_PLUGIN_fatal ("identical parameters"); e = (CUevent *) GOMP_PLUGIN_malloc (sizeof (CUevent)); CUDA_CALL_ASSERT (cuEventCreate, e, CU_EVENT_DISABLE_TIMING); event_gc (true); CUDA_CALL_ASSERT (cuEventRecord, *e, s1->stream); event_add (PTX_EVT_SYNC, e, NULL, 0); CUDA_CALL_ASSERT (cuStreamWaitEvent, s2->stream, *e, 0); }
static void map_init (struct ptx_stream *s) { CUresult r; int size = getpagesize (); assert (s); assert (!s->d); assert (!s->h); r = cuMemAllocHost (&s->h, size); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuMemAllocHost error: %s", cuda_error (r)); r = cuMemHostGetDevicePointer (&s->d, s->h, 0); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuMemHostGetDevicePointer error: %s", cuda_error (r)); assert (s->h); s->h_begin = s->h; s->h_end = s->h_begin + size; s->h_next = s->h_prev = s->h_tail = s->h_begin; assert (s->h_next); assert (s->h_end); }
static void nvptx_wait_all (void) { CUresult r; struct ptx_stream *s; pthread_t self = pthread_self (); struct nvptx_thread *nvthd = nvptx_thread (); pthread_mutex_lock (&nvthd->ptx_dev->stream_lock); /* Wait for active streams initiated by this thread (or by multiple threads) to complete. */ for (s = nvthd->ptx_dev->active_streams; s != NULL; s = s->next) { if (s->multithreaded || pthread_equal (s->host_thread, self)) { r = cuStreamQuery (s->stream); if (r == CUDA_SUCCESS) continue; else if (r != CUDA_ERROR_NOT_READY) GOMP_PLUGIN_fatal ("cuStreamQuery error: %s", cuda_error (r)); r = cuStreamSynchronize (s->stream); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuStreamSynchronize error: %s", cuda_error (r)); } } pthread_mutex_unlock (&nvthd->ptx_dev->stream_lock); event_gc (true); }
static int nvptx_async_test (int async) { CUresult r; struct ptx_stream *s; s = select_stream_for_async (async, pthread_self (), false, NULL); if (!s) GOMP_PLUGIN_fatal ("unknown async %d", async); r = cuStreamQuery (s->stream); if (r == CUDA_SUCCESS) { /* The oacc-parallel.c:goacc_wait function calls this hook to determine whether all work has completed on this stream, and if so omits the call to the wait hook. If that happens, event_gc might not get called (which prevents variables from getting unmapped and their associated device storage freed), so call it here. */ event_gc (true); return 1; } else if (r == CUDA_ERROR_NOT_READY) return 0; GOMP_PLUGIN_fatal ("cuStreamQuery error: %s", cuda_error (r)); return 0; }
void * GOMP_OFFLOAD_openacc_create_thread_data (int ord) { struct ptx_device *ptx_dev; struct nvptx_thread *nvthd = GOMP_PLUGIN_malloc (sizeof (struct nvptx_thread)); CUresult r; CUcontext thd_ctx; ptx_dev = ptx_devices[ord]; assert (ptx_dev); r = cuCtxGetCurrent (&thd_ctx); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuCtxGetCurrent error: %s", cuda_error (r)); assert (ptx_dev->ctx); if (!thd_ctx) { r = cuCtxPushCurrent (ptx_dev->ctx); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuCtxPushCurrent error: %s", cuda_error (r)); } nvthd->current_stream = ptx_dev->null_stream; nvthd->ptx_dev = ptx_dev; return (void *) nvthd; }
static void nvptx_wait_all_async (int async) { CUresult r; struct ptx_stream *waiting_stream, *other_stream; CUevent *e; struct nvptx_thread *nvthd = nvptx_thread (); pthread_t self = pthread_self (); /* The stream doing the waiting. This could be the first mention of the stream, so create it if necessary. */ waiting_stream = select_stream_for_async (async, pthread_self (), true, NULL); /* Launches on the null stream already block on other streams in the context. */ if (!waiting_stream || waiting_stream == nvthd->ptx_dev->null_stream) return; event_gc (true); pthread_mutex_lock (&nvthd->ptx_dev->stream_lock); for (other_stream = nvthd->ptx_dev->active_streams; other_stream != NULL; other_stream = other_stream->next) { if (!other_stream->multithreaded && !pthread_equal (other_stream->host_thread, self)) continue; e = (CUevent *) GOMP_PLUGIN_malloc (sizeof (CUevent)); r = cuEventCreate (e, CU_EVENT_DISABLE_TIMING); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuEventCreate error: %s", cuda_error (r)); /* Record an event on the waited-for stream. */ r = cuEventRecord (*e, other_stream->stream); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuEventRecord error: %s", cuda_error (r)); event_add (PTX_EVT_SYNC, e, NULL); r = cuStreamWaitEvent (waiting_stream->stream, *e, 0); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuStreamWaitEvent error: %s", cuda_error (r)); } pthread_mutex_unlock (&nvthd->ptx_dev->stream_lock); }
static int nvptx_set_cuda_stream (int async, void *stream) { struct ptx_stream *oldstream; pthread_t self = pthread_self (); struct nvptx_thread *nvthd = nvptx_thread (); if (async < 0) GOMP_PLUGIN_fatal ("bad async %d", async); pthread_mutex_lock (&nvthd->ptx_dev->stream_lock); /* We have a list of active streams and an array mapping async values to entries of that list. We need to take "ownership" of the passed-in stream, and add it to our list, removing the previous entry also (if there was one) in order to prevent resource leaks. Note the potential for surprise here: maybe we should keep track of passed-in streams and leave it up to the user to tidy those up, but that doesn't work for stream handles returned from acc_get_cuda_stream above... */ oldstream = select_stream_for_async (async, self, false, NULL); if (oldstream) { if (nvthd->ptx_dev->active_streams == oldstream) nvthd->ptx_dev->active_streams = nvthd->ptx_dev->active_streams->next; else { struct ptx_stream *s = nvthd->ptx_dev->active_streams; while (s->next != oldstream) s = s->next; s->next = s->next->next; } CUDA_CALL_ASSERT (cuStreamDestroy, oldstream->stream); if (!map_fini (oldstream)) GOMP_PLUGIN_fatal ("error when freeing host memory"); free (oldstream); } pthread_mutex_unlock (&nvthd->ptx_dev->stream_lock); (void) select_stream_for_async (async, self, true, (CUstream) stream); return 1; }
static int nvptx_get_num_devices (void) { int n; CUresult r; /* PR libgomp/65099: Currently, we only support offloading in 64-bit configurations. */ if (sizeof (void *) != 8) return 0; /* This function will be called before the plugin has been initialized in order to enumerate available devices, but CUDA API routines can't be used until cuInit has been called. Just call it now (but don't yet do any further initialization). */ if (instantiated_devices == 0) { r = cuInit (0); /* This is not an error: e.g. we may have CUDA libraries installed but no devices available. */ if (r != CUDA_SUCCESS) return 0; } r = cuDeviceGetCount (&n); if (r!= CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuDeviceGetCount error: %s", cuda_error (r)); return n; }
static void nvptx_wait (int async) { CUresult r; struct ptx_stream *s; s = select_stream_for_async (async, pthread_self (), false, NULL); if (!s) GOMP_PLUGIN_fatal ("unknown async %d", async); r = cuStreamSynchronize (s->stream); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuStreamSynchronize error: %s", cuda_error (r)); event_gc (true); }
static void map_fini (struct ptx_stream *s) { CUresult r; r = cuMemFreeHost (s->h); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuMemFreeHost error: %s", cuda_error (r)); }
static void nvptx_free (void *p) { CUresult r; CUdeviceptr pb; size_t ps; r = cuMemGetAddressRange (&pb, &ps, (CUdeviceptr)p); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuMemGetAddressRange error: %s", cuda_error (r)); if ((CUdeviceptr)p != pb) GOMP_PLUGIN_fatal ("invalid device address"); r = cuMemFree ((CUdeviceptr)p); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuMemFree error: %s", cuda_error (r)); }
void GOMP_OFFLOAD_openacc_register_async_cleanup (void *targ_mem_desc) { CUevent *e; CUresult r; struct nvptx_thread *nvthd = nvptx_thread (); e = (CUevent *) GOMP_PLUGIN_malloc (sizeof (CUevent)); r = cuEventCreate (e, CU_EVENT_DISABLE_TIMING); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuEventCreate error: %s", cuda_error (r)); r = cuEventRecord (*e, nvthd->current_stream->stream); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuEventRecord error: %s", cuda_error (r)); event_add (PTX_EVT_ASYNC_CLEANUP, e, targ_mem_desc); }
static void * nvptx_alloc (size_t s) { CUdeviceptr d; CUresult r; r = cuMemAlloc (&d, s); if (r == CUDA_ERROR_OUT_OF_MEMORY) return 0; if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuMemAlloc error: %s", cuda_error (r)); return (void *)d; }
static void nvptx_wait (int async) { struct ptx_stream *s; s = select_stream_for_async (async, pthread_self (), false, NULL); if (!s) GOMP_PLUGIN_fatal ("unknown async %d", async); CUDA_CALL_ASSERT (cuStreamSynchronize, s->stream); event_gc (true); }
static void nvptx_attach_host_thread_to_device (int n) { CUdevice dev; CUresult r; struct ptx_device *ptx_dev; CUcontext thd_ctx; r = cuCtxGetDevice (&dev); if (r != CUDA_SUCCESS && r != CUDA_ERROR_INVALID_CONTEXT) GOMP_PLUGIN_fatal ("cuCtxGetDevice error: %s", cuda_error (r)); if (r != CUDA_ERROR_INVALID_CONTEXT && dev == n) return; else { CUcontext old_ctx; ptx_dev = ptx_devices[n]; assert (ptx_dev); r = cuCtxGetCurrent (&thd_ctx); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuCtxGetCurrent error: %s", cuda_error (r)); /* We don't necessarily have a current context (e.g. if it has been destroyed. Pop it if we do though. */ if (thd_ctx != NULL) { r = cuCtxPopCurrent (&old_ctx); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuCtxPopCurrent error: %s", cuda_error (r)); } r = cuCtxPushCurrent (ptx_dev->ctx); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuCtxPushCurrent error: %s", cuda_error (r)); } }
static void map_push (struct ptx_stream *s, int async, size_t size, void **h, void **d) { int left; int offset; struct map *m; assert (s != NULL); left = s->h_end - s->h_next; size += sizeof (struct map); assert (s->h_prev); assert (s->h_next); if (size >= left) { m = s->h_prev; m->size += left; s->h_next = s->h_begin; if (s->h_next + size > s->h_end) GOMP_PLUGIN_fatal ("unable to push map"); } assert (s->h_next); m = s->h_next; m->async = async; m->size = size; offset = (void *)&m->mappings[0] - s->h; *d = (void *)(s->d + offset); *h = (void *)(s->h + offset); s->h_prev = s->h_next; s->h_next += size; assert (s->h_prev); assert (s->h_next); assert (s->h_next >= s->h_begin); assert (s->h_tail >= s->h_begin); assert (s->h_prev >= s->h_begin); assert (s->h_next <= s->h_end); assert (s->h_tail <= s->h_end); assert (s->h_prev <= s->h_end); return; }
static void nvptx_close_device (struct ptx_device *ptx_dev) { CUresult r; if (!ptx_dev) return; fini_streams_for_device (ptx_dev); pthread_mutex_destroy (&ptx_dev->image_lock); if (!ptx_dev->ctx_shared) { r = cuCtxDestroy (ptx_dev->ctx); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuCtxDestroy error: %s", cuda_error (r)); } free (ptx_dev); }
void nvptx_exec (void (*fn), size_t mapnum, void **hostaddrs, void **devaddrs, int async, unsigned *dims, void *targ_mem_desc) { struct targ_fn_descriptor *targ_fn = (struct targ_fn_descriptor *) fn; CUfunction function; CUresult r; int i; struct ptx_stream *dev_str; void *kargs[1]; void *hp, *dp; struct nvptx_thread *nvthd = nvptx_thread (); const char *maybe_abort_msg = "(perhaps abort was called)"; function = targ_fn->fn; dev_str = select_stream_for_async (async, pthread_self (), false, NULL); assert (dev_str == nvthd->current_stream); /* Initialize the launch dimensions. Typically this is constant, provided by the device compiler, but we must permit runtime values. */ for (i = 0; i != 3; i++) if (targ_fn->launch->dim[i]) dims[i] = targ_fn->launch->dim[i]; /* This reserves a chunk of a pre-allocated page of memory mapped on both the host and the device. HP is a host pointer to the new chunk, and DP is the corresponding device pointer. */ map_push (dev_str, async, mapnum * sizeof (void *), &hp, &dp); GOMP_PLUGIN_debug (0, " %s: prepare mappings\n", __FUNCTION__); /* Copy the array of arguments to the mapped page. */ for (i = 0; i < mapnum; i++) ((void **) hp)[i] = devaddrs[i]; /* Copy the (device) pointers to arguments to the device (dp and hp might in fact have the same value on a unified-memory system). */ r = cuMemcpy ((CUdeviceptr)dp, (CUdeviceptr)hp, mapnum * sizeof (void *)); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuMemcpy failed: %s", cuda_error (r)); GOMP_PLUGIN_debug (0, " %s: kernel %s: launch" " gangs=%u, workers=%u, vectors=%u\n", __FUNCTION__, targ_fn->launch->fn, dims[0], dims[1], dims[2]); // OpenACC CUDA // // num_gangs nctaid.x // num_workers ntid.y // vector length ntid.x kargs[0] = &dp; r = cuLaunchKernel (function, dims[GOMP_DIM_GANG], 1, 1, dims[GOMP_DIM_VECTOR], dims[GOMP_DIM_WORKER], 1, 0, dev_str->stream, kargs, 0); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuLaunchKernel error: %s", cuda_error (r)); #ifndef DISABLE_ASYNC if (async < acc_async_noval) { r = cuStreamSynchronize (dev_str->stream); if (r == CUDA_ERROR_LAUNCH_FAILED) GOMP_PLUGIN_fatal ("cuStreamSynchronize error: %s %s\n", cuda_error (r), maybe_abort_msg); else if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuStreamSynchronize error: %s", cuda_error (r)); } else { CUevent *e; e = (CUevent *)GOMP_PLUGIN_malloc (sizeof (CUevent)); r = cuEventCreate (e, CU_EVENT_DISABLE_TIMING); if (r == CUDA_ERROR_LAUNCH_FAILED) GOMP_PLUGIN_fatal ("cuEventCreate error: %s %s\n", cuda_error (r), maybe_abort_msg); else if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuEventCreate error: %s", cuda_error (r)); event_gc (true); r = cuEventRecord (*e, dev_str->stream); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuEventRecord error: %s", cuda_error (r)); event_add (PTX_EVT_KNL, e, (void *)dev_str); } #else r = cuCtxSynchronize (); if (r == CUDA_ERROR_LAUNCH_FAILED) GOMP_PLUGIN_fatal ("cuCtxSynchronize error: %s %s\n", cuda_error (r), maybe_abort_msg); else if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuCtxSynchronize error: %s", cuda_error (r)); #endif GOMP_PLUGIN_debug (0, " %s: kernel %s: finished\n", __FUNCTION__, targ_fn->launch->fn); #ifndef DISABLE_ASYNC if (async < acc_async_noval) #endif map_pop (dev_str); }
static void link_ptx (CUmodule *module, const struct targ_ptx_obj *ptx_objs, unsigned num_objs) { CUjit_option opts[7]; void *optvals[7]; float elapsed = 0.0; #define LOGSIZE 8192 char elog[LOGSIZE]; char ilog[LOGSIZE]; unsigned long logsize = LOGSIZE; CUlinkState linkstate; CUresult r; void *linkout; size_t linkoutsize __attribute__ ((unused)); opts[0] = CU_JIT_WALL_TIME; optvals[0] = &elapsed; opts[1] = CU_JIT_INFO_LOG_BUFFER; optvals[1] = &ilog[0]; opts[2] = CU_JIT_INFO_LOG_BUFFER_SIZE_BYTES; optvals[2] = (void *) logsize; opts[3] = CU_JIT_ERROR_LOG_BUFFER; optvals[3] = &elog[0]; opts[4] = CU_JIT_ERROR_LOG_BUFFER_SIZE_BYTES; optvals[4] = (void *) logsize; opts[5] = CU_JIT_LOG_VERBOSE; optvals[5] = (void *) 1; opts[6] = CU_JIT_TARGET; optvals[6] = (void *) CU_TARGET_COMPUTE_30; r = cuLinkCreate (7, opts, optvals, &linkstate); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuLinkCreate error: %s", cuda_error (r)); for (; num_objs--; ptx_objs++) { /* cuLinkAddData's 'data' argument erroneously omits the const qualifier. */ GOMP_PLUGIN_debug (0, "Loading:\n---\n%s\n---\n", ptx_objs->code); r = cuLinkAddData (linkstate, CU_JIT_INPUT_PTX, (char*)ptx_objs->code, ptx_objs->size, 0, 0, 0, 0); if (r != CUDA_SUCCESS) { GOMP_PLUGIN_error ("Link error log %s\n", &elog[0]); GOMP_PLUGIN_fatal ("cuLinkAddData (ptx_code) error: %s", cuda_error (r)); } } GOMP_PLUGIN_debug (0, "Linking\n"); r = cuLinkComplete (linkstate, &linkout, &linkoutsize); GOMP_PLUGIN_debug (0, "Link complete: %fms\n", elapsed); GOMP_PLUGIN_debug (0, "Link log %s\n", &ilog[0]); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuLinkComplete error: %s", cuda_error (r)); r = cuModuleLoadData (module, linkout); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuModuleLoadData error: %s", cuda_error (r)); r = cuLinkDestroy (linkstate); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuLinkDestory error: %s", cuda_error (r)); }
static void * nvptx_dev2host (void *h, const void *d, size_t s) { CUresult r; CUdeviceptr pb; size_t ps; struct nvptx_thread *nvthd = nvptx_thread (); if (!s) return 0; if (!d) GOMP_PLUGIN_fatal ("invalid device address"); r = cuMemGetAddressRange (&pb, &ps, (CUdeviceptr)d); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuMemGetAddressRange error: %s", cuda_error (r)); if (!pb) GOMP_PLUGIN_fatal ("invalid device address"); if (!h) GOMP_PLUGIN_fatal ("invalid host address"); if (d == h) GOMP_PLUGIN_fatal ("invalid host or device address"); if ((void *)(d + s) > (void *)(pb + ps)) GOMP_PLUGIN_fatal ("invalid size"); #ifndef DISABLE_ASYNC if (nvthd->current_stream != nvthd->ptx_dev->null_stream) { CUevent *e; e = (CUevent *)GOMP_PLUGIN_malloc (sizeof (CUevent)); r = cuEventCreate (e, CU_EVENT_DISABLE_TIMING); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuEventCreate error: %s\n", cuda_error (r)); event_gc (false); r = cuMemcpyDtoHAsync (h, (CUdeviceptr)d, s, nvthd->current_stream->stream); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuMemcpyDtoHAsync error: %s", cuda_error (r)); r = cuEventRecord (*e, nvthd->current_stream->stream); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuEventRecord error: %s", cuda_error (r)); event_add (PTX_EVT_MEM, e, (void *)h); } else #endif { r = cuMemcpyDtoH (h, (CUdeviceptr)d, s); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuMemcpyDtoH error: %s", cuda_error (r)); } return 0; }
static struct ptx_device * nvptx_open_device (int n) { struct ptx_device *ptx_dev; CUdevice dev, ctx_dev; CUresult r; int async_engines, pi; r = cuDeviceGet (&dev, n); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuDeviceGet error: %s", cuda_error (r)); ptx_dev = GOMP_PLUGIN_malloc (sizeof (struct ptx_device)); ptx_dev->ord = n; ptx_dev->dev = dev; ptx_dev->ctx_shared = false; r = cuCtxGetDevice (&ctx_dev); if (r != CUDA_SUCCESS && r != CUDA_ERROR_INVALID_CONTEXT) GOMP_PLUGIN_fatal ("cuCtxGetDevice error: %s", cuda_error (r)); if (r != CUDA_ERROR_INVALID_CONTEXT && ctx_dev != dev) { /* The current host thread has an active context for a different device. Detach it. */ CUcontext old_ctx; r = cuCtxPopCurrent (&old_ctx); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuCtxPopCurrent error: %s", cuda_error (r)); } r = cuCtxGetCurrent (&ptx_dev->ctx); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuCtxGetCurrent error: %s", cuda_error (r)); if (!ptx_dev->ctx) { r = cuCtxCreate (&ptx_dev->ctx, CU_CTX_SCHED_AUTO, dev); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuCtxCreate error: %s", cuda_error (r)); } else ptx_dev->ctx_shared = true; r = cuDeviceGetAttribute (&pi, CU_DEVICE_ATTRIBUTE_GPU_OVERLAP, dev); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuDeviceGetAttribute error: %s", cuda_error (r)); ptx_dev->overlap = pi; r = cuDeviceGetAttribute (&pi, CU_DEVICE_ATTRIBUTE_CAN_MAP_HOST_MEMORY, dev); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuDeviceGetAttribute error: %s", cuda_error (r)); ptx_dev->map = pi; r = cuDeviceGetAttribute (&pi, CU_DEVICE_ATTRIBUTE_CONCURRENT_KERNELS, dev); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuDeviceGetAttribute error: %s", cuda_error (r)); ptx_dev->concur = pi; r = cuDeviceGetAttribute (&pi, CU_DEVICE_ATTRIBUTE_COMPUTE_MODE, dev); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuDeviceGetAttribute error: %s", cuda_error (r)); ptx_dev->mode = pi; r = cuDeviceGetAttribute (&pi, CU_DEVICE_ATTRIBUTE_INTEGRATED, dev); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuDeviceGetAttribute error: %s", cuda_error (r)); ptx_dev->mkern = pi; r = cuDeviceGetAttribute (&async_engines, CU_DEVICE_ATTRIBUTE_ASYNC_ENGINE_COUNT, dev); if (r != CUDA_SUCCESS) async_engines = 1; ptx_dev->images = NULL; pthread_mutex_init (&ptx_dev->image_lock, NULL); init_streams_for_device (ptx_dev, async_engines); return ptx_dev; }
int GOMP_OFFLOAD_load_image (int ord, unsigned version, const void *target_data, struct addr_pair **target_table) { CUmodule module; const char *const *var_names; const struct targ_fn_launch *fn_descs; unsigned int fn_entries, var_entries, i, j; CUresult r; struct targ_fn_descriptor *targ_fns; struct addr_pair *targ_tbl; const nvptx_tdata_t *img_header = (const nvptx_tdata_t *) target_data; struct ptx_image_data *new_image; struct ptx_device *dev; if (GOMP_VERSION_DEV (version) > GOMP_VERSION_NVIDIA_PTX) GOMP_PLUGIN_fatal ("Offload data incompatible with PTX plugin" " (expected %u, received %u)", GOMP_VERSION_NVIDIA_PTX, GOMP_VERSION_DEV (version)); GOMP_OFFLOAD_init_device (ord); dev = ptx_devices[ord]; nvptx_attach_host_thread_to_device (ord); link_ptx (&module, img_header->ptx_objs, img_header->ptx_num); /* The mkoffload utility emits a struct of pointers/integers at the start of each offload image. The array of kernel names and the functions addresses form a one-to-one correspondence. */ var_entries = img_header->var_num; var_names = img_header->var_names; fn_entries = img_header->fn_num; fn_descs = img_header->fn_descs; targ_tbl = GOMP_PLUGIN_malloc (sizeof (struct addr_pair) * (fn_entries + var_entries)); targ_fns = GOMP_PLUGIN_malloc (sizeof (struct targ_fn_descriptor) * fn_entries); *target_table = targ_tbl; new_image = GOMP_PLUGIN_malloc (sizeof (struct ptx_image_data)); new_image->target_data = target_data; new_image->module = module; new_image->fns = targ_fns; pthread_mutex_lock (&dev->image_lock); new_image->next = dev->images; dev->images = new_image; pthread_mutex_unlock (&dev->image_lock); for (i = 0; i < fn_entries; i++, targ_fns++, targ_tbl++) { CUfunction function; r = cuModuleGetFunction (&function, module, fn_descs[i].fn); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuModuleGetFunction error: %s", cuda_error (r)); targ_fns->fn = function; targ_fns->launch = &fn_descs[i]; targ_tbl->start = (uintptr_t) targ_fns; targ_tbl->end = targ_tbl->start + 1; } for (j = 0; j < var_entries; j++, targ_tbl++) { CUdeviceptr var; size_t bytes; r = cuModuleGetGlobal (&var, &bytes, module, var_names[j]); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuModuleGetGlobal error: %s", cuda_error (r)); targ_tbl->start = (uintptr_t) var; targ_tbl->end = targ_tbl->start + bytes; } return fn_entries + var_entries; }
static struct ptx_stream * select_stream_for_async (int async, pthread_t thread, bool create, CUstream existing) { struct nvptx_thread *nvthd = nvptx_thread (); /* Local copy of TLS variable. */ struct ptx_device *ptx_dev = nvthd->ptx_dev; struct ptx_stream *stream = NULL; int orig_async = async; /* The special value acc_async_noval (-1) maps (for now) to an implicitly-created stream, which is then handled the same as any other numbered async stream. Other options are available, e.g. using the null stream for anonymous async operations, or choosing an idle stream from an active set. But, stick with this for now. */ if (async > acc_async_sync) async++; if (create) pthread_mutex_lock (&ptx_dev->stream_lock); /* NOTE: AFAICT there's no particular need for acc_async_sync to map to the null stream, and in fact better performance may be obtainable if it doesn't (because the null stream enforces overly-strict synchronisation with respect to other streams for legacy reasons, and that's probably not needed with OpenACC). Maybe investigate later. */ if (async == acc_async_sync) stream = ptx_dev->null_stream; else if (async >= 0 && async < ptx_dev->async_streams.size && ptx_dev->async_streams.arr[async] && !(create && existing)) stream = ptx_dev->async_streams.arr[async]; else if (async >= 0 && create) { if (async >= ptx_dev->async_streams.size) { int i, newsize = ptx_dev->async_streams.size * 2; if (async >= newsize) newsize = async + 1; ptx_dev->async_streams.arr = GOMP_PLUGIN_realloc (ptx_dev->async_streams.arr, newsize * sizeof (struct ptx_stream *)); for (i = ptx_dev->async_streams.size; i < newsize; i++) ptx_dev->async_streams.arr[i] = NULL; ptx_dev->async_streams.size = newsize; } /* Create a new stream on-demand if there isn't one already, or if we're setting a particular async value to an existing (externally-provided) stream. */ if (!ptx_dev->async_streams.arr[async] || existing) { CUresult r; struct ptx_stream *s = GOMP_PLUGIN_malloc (sizeof (struct ptx_stream)); if (existing) s->stream = existing; else { r = cuStreamCreate (&s->stream, CU_STREAM_DEFAULT); if (r != CUDA_SUCCESS) GOMP_PLUGIN_fatal ("cuStreamCreate error: %s", cuda_error (r)); } /* If CREATE is true, we're going to be queueing some work on this stream. Associate it with the current host thread. */ s->host_thread = thread; s->multithreaded = false; s->d = (CUdeviceptr) NULL; s->h = NULL; map_init (s); s->next = ptx_dev->active_streams; ptx_dev->active_streams = s; ptx_dev->async_streams.arr[async] = s; } stream = ptx_dev->async_streams.arr[async]; } else if (async < 0) GOMP_PLUGIN_fatal ("bad async %d", async); if (create) { assert (stream != NULL); /* If we're trying to use the same stream from different threads simultaneously, set stream->multithreaded to true. This affects the behaviour of acc_async_test_all and acc_wait_all, which are supposed to only wait for asynchronous launches from the same host thread they are invoked on. If multiple threads use the same async value, we make note of that here and fall back to testing/waiting for all threads in those functions. */ if (thread != stream->host_thread) stream->multithreaded = true; pthread_mutex_unlock (&ptx_dev->stream_lock); } else if (stream && !stream->multithreaded && !pthread_equal (stream->host_thread, thread)) GOMP_PLUGIN_fatal ("async %d used on wrong thread", orig_async); return stream; }