static void
map_init (struct ptx_stream *s)
{
CUresult r;
int size = getpagesize ();
assert (s);
assert (!s->d);
assert (!s->h);
r = cuMemAllocHost (&s->h, size);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuMemAllocHost error: %s", cuda_error (r));
r = cuMemHostGetDevicePointer (&s->d, s->h, 0);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuMemHostGetDevicePointer error: %s", cuda_error (r));
assert (s->h);
s->h_begin = s->h;
s->h_end = s->h_begin + size;
s->h_next = s->h_prev = s->h_tail = s->h_begin;
assert (s->h_next);
assert (s->h_end);
}
static bool
nvptx_init (void)
{
CUresult r;
int ndevs;
if (instantiated_devices != 0)
return true;
r = cuInit (0);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuInit error: %s", cuda_error (r));
ptx_events = NULL;
pthread_mutex_init (&ptx_event_lock, NULL);
r = cuDeviceGetCount (&ndevs);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuDeviceGetCount error: %s", cuda_error (r));
ptx_devices = GOMP_PLUGIN_malloc_cleared (sizeof (struct ptx_device *)
* ndevs);
return true;
}
void *
GOMP_OFFLOAD_openacc_create_thread_data (int ord)
{
struct ptx_device *ptx_dev;
struct nvptx_thread *nvthd
= GOMP_PLUGIN_malloc (sizeof (struct nvptx_thread));
CUresult r;
CUcontext thd_ctx;
ptx_dev = ptx_devices[ord];
assert (ptx_dev);
r = cuCtxGetCurrent (&thd_ctx);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuCtxGetCurrent error: %s", cuda_error (r));
assert (ptx_dev->ctx);
if (!thd_ctx)
{
r = cuCtxPushCurrent (ptx_dev->ctx);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuCtxPushCurrent error: %s", cuda_error (r));
}
nvthd->current_stream = ptx_dev->null_stream;
nvthd->ptx_dev = ptx_dev;
return (void *) nvthd;
}
CUDADevice(DeviceInfo& info, Stats &stats, bool background_) : Device(stats)
{
background = background_;
cuDevId = info.num;
cuDevice = 0;
cuContext = 0;
/* intialize */
if(cuda_error(cuInit(0)))
return;
/* setup device and context */
if(cuda_error(cuDeviceGet(&cuDevice, cuDevId)))
return;
CUresult result;
if(background) {
result = cuCtxCreate(&cuContext, 0, cuDevice);
}
else {
result = cuGLCtxCreate(&cuContext, 0, cuDevice);
if(result != CUDA_SUCCESS) {
result = cuCtxCreate(&cuContext, 0, cuDevice);
background = true;
}
}
if(cuda_error_(result, "cuCtxCreate"))
return;
cuda_pop_context();
}
bool load_kernels(bool experimental)
{
/* check if cuda init succeeded */
if(cuContext == 0)
return false;
if(!support_device(experimental))
return false;
/* get kernel */
string cubin = compile_kernel();
if(cubin == "")
return false;
/* open module */
cuda_push_context();
CUresult result = cuModuleLoad(&cuModule, cubin.c_str());
if(cuda_error(result))
cuda_error(string_printf("Failed loading CUDA kernel %s.", cubin.c_str()));
cuda_pop_context();
return (result == CUDA_SUCCESS);
}
static void
nvptx_wait_all (void)
{
CUresult r;
struct ptx_stream *s;
pthread_t self = pthread_self ();
struct nvptx_thread *nvthd = nvptx_thread ();
pthread_mutex_lock (&nvthd->ptx_dev->stream_lock);
/* Wait for active streams initiated by this thread (or by multiple threads)
to complete. */
for (s = nvthd->ptx_dev->active_streams; s != NULL; s = s->next)
{
if (s->multithreaded || pthread_equal (s->host_thread, self))
{
r = cuStreamQuery (s->stream);
if (r == CUDA_SUCCESS)
continue;
else if (r != CUDA_ERROR_NOT_READY)
GOMP_PLUGIN_fatal ("cuStreamQuery error: %s", cuda_error (r));
r = cuStreamSynchronize (s->stream);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuStreamSynchronize error: %s", cuda_error (r));
}
}
pthread_mutex_unlock (&nvthd->ptx_dev->stream_lock);
event_gc (true);
}
static void
link_device_libraries(void *ptx_image, size_t ptx_image_len,
void **p_bin_image, size_t *p_bin_image_len,
long target_capability)
{
CUlinkState lstate;
CUjit_option jit_options[10];
void *jit_option_values[10];
int jit_index = 0;
CUresult rc;
char pathname[1024];
/*
* JIT options
*/
jit_options[jit_index] = CU_JIT_TARGET;
jit_option_values[jit_index] = (void *)target_capability;
jit_index++;
#ifdef PGSTROM_DEBUG
jit_options[jit_index] = CU_JIT_GENERATE_DEBUG_INFO;
jit_option_values[jit_index] = (void *)1UL;
jit_index++;
jit_options[jit_index] = CU_JIT_GENERATE_LINE_INFO;
jit_option_values[jit_index] = (void *)1UL;
jit_index++;
#endif
/* makes a linkage object */
rc = cuLinkCreate(jit_index, jit_options, jit_option_values, &lstate);
if (rc != CUDA_SUCCESS)
cuda_error(rc, "cuLinkCreate");
/* add the base PTX image */
rc = cuLinkAddData(lstate, CU_JIT_INPUT_PTX,
ptx_image, ptx_image_len,
"PG-Strom", 0, NULL, NULL);
if (rc != CUDA_SUCCESS)
cuda_error(rc, "cuLinkAddData");
/* libcudart.a, if any */
snprintf(pathname, sizeof(pathname), "%s/libcudadevrt.a",
CUDA_LIBRARY_PATH);
rc = cuLinkAddFile(lstate, CU_JIT_INPUT_LIBRARY, pathname,
0, NULL, NULL);
if (rc != CUDA_SUCCESS)
cuda_error(rc, "cuLinkAddFile");
/* do the linkage */
rc = cuLinkComplete(lstate, p_bin_image, p_bin_image_len);
if (rc != CUDA_SUCCESS)
cuda_error(rc, "cuLinkComplete");
}
static void
nvptx_wait_all_async (int async)
{
CUresult r;
struct ptx_stream *waiting_stream, *other_stream;
CUevent *e;
struct nvptx_thread *nvthd = nvptx_thread ();
pthread_t self = pthread_self ();
/* The stream doing the waiting. This could be the first mention of the
stream, so create it if necessary. */
waiting_stream
= select_stream_for_async (async, pthread_self (), true, NULL);
/* Launches on the null stream already block on other streams in the
context. */
if (!waiting_stream || waiting_stream == nvthd->ptx_dev->null_stream)
return;
event_gc (true);
pthread_mutex_lock (&nvthd->ptx_dev->stream_lock);
for (other_stream = nvthd->ptx_dev->active_streams;
other_stream != NULL;
other_stream = other_stream->next)
{
if (!other_stream->multithreaded
&& !pthread_equal (other_stream->host_thread, self))
continue;
e = (CUevent *) GOMP_PLUGIN_malloc (sizeof (CUevent));
r = cuEventCreate (e, CU_EVENT_DISABLE_TIMING);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuEventCreate error: %s", cuda_error (r));
/* Record an event on the waited-for stream. */
r = cuEventRecord (*e, other_stream->stream);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuEventRecord error: %s", cuda_error (r));
event_add (PTX_EVT_SYNC, e, NULL);
r = cuStreamWaitEvent (waiting_stream->stream, *e, 0);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuStreamWaitEvent error: %s", cuda_error (r));
}
pthread_mutex_unlock (&nvthd->ptx_dev->stream_lock);
}
static int
nvptx_async_test (int async)
{
CUresult r;
struct ptx_stream *s;
s = select_stream_for_async (async, pthread_self (), false, NULL);
if (!s)
GOMP_PLUGIN_fatal ("unknown async %d", async);
r = cuStreamQuery (s->stream);
if (r == CUDA_SUCCESS)
{
/* The oacc-parallel.c:goacc_wait function calls this hook to determine
whether all work has completed on this stream, and if so omits the call
to the wait hook. If that happens, event_gc might not get called
(which prevents variables from getting unmapped and their associated
device storage freed), so call it here. */
event_gc (true);
return 1;
}
else if (r == CUDA_ERROR_NOT_READY)
return 0;
GOMP_PLUGIN_fatal ("cuStreamQuery error: %s", cuda_error (r));
return 0;
}
static bool
fini_streams_for_device (struct ptx_device *ptx_dev)
{
free (ptx_dev->async_streams.arr);
bool ret = true;
while (ptx_dev->active_streams != NULL)
{
struct ptx_stream *s = ptx_dev->active_streams;
ptx_dev->active_streams = ptx_dev->active_streams->next;
ret &= map_fini (s);
CUresult r = cuStreamDestroy (s->stream);
if (r != CUDA_SUCCESS)
{
GOMP_PLUGIN_error ("cuStreamDestroy error: %s", cuda_error (r));
ret = false;
}
free (s);
}
ret &= map_fini (ptx_dev->null_stream);
free (ptx_dev->null_stream);
return ret;
}
static int
nvptx_get_num_devices (void)
{
int n;
CUresult r;
/* PR libgomp/65099: Currently, we only support offloading in 64-bit
configurations. */
if (sizeof (void *) != 8)
return 0;
/* This function will be called before the plugin has been initialized in
order to enumerate available devices, but CUDA API routines can't be used
until cuInit has been called. Just call it now (but don't yet do any
further initialization). */
if (instantiated_devices == 0)
{
r = cuInit (0);
/* This is not an error: e.g. we may have CUDA libraries installed but
no devices available. */
if (r != CUDA_SUCCESS)
return 0;
}
r = cuDeviceGetCount (&n);
if (r!= CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuDeviceGetCount error: %s", cuda_error (r));
return n;
}
static void
map_fini (struct ptx_stream *s)
{
CUresult r;
r = cuMemFreeHost (s->h);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuMemFreeHost error: %s", cuda_error (r));
}
static void
nvptx_free (void *p)
{
CUresult r;
CUdeviceptr pb;
size_t ps;
r = cuMemGetAddressRange (&pb, &ps, (CUdeviceptr)p);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuMemGetAddressRange error: %s", cuda_error (r));
if ((CUdeviceptr)p != pb)
GOMP_PLUGIN_fatal ("invalid device address");
r = cuMemFree ((CUdeviceptr)p);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuMemFree error: %s", cuda_error (r));
}
void
GOMP_OFFLOAD_openacc_register_async_cleanup (void *targ_mem_desc)
{
CUevent *e;
CUresult r;
struct nvptx_thread *nvthd = nvptx_thread ();
e = (CUevent *) GOMP_PLUGIN_malloc (sizeof (CUevent));
r = cuEventCreate (e, CU_EVENT_DISABLE_TIMING);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuEventCreate error: %s", cuda_error (r));
r = cuEventRecord (*e, nvthd->current_stream->stream);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuEventRecord error: %s", cuda_error (r));
event_add (PTX_EVT_ASYNC_CLEANUP, e, targ_mem_desc);
}
static void *
nvptx_alloc (size_t s)
{
CUdeviceptr d;
CUresult r;
r = cuMemAlloc (&d, s);
if (r == CUDA_ERROR_OUT_OF_MEMORY)
return 0;
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuMemAlloc error: %s", cuda_error (r));
return (void *)d;
}
static void
nvptx_attach_host_thread_to_device (int n)
{
CUdevice dev;
CUresult r;
struct ptx_device *ptx_dev;
CUcontext thd_ctx;
r = cuCtxGetDevice (&dev);
if (r != CUDA_SUCCESS && r != CUDA_ERROR_INVALID_CONTEXT)
GOMP_PLUGIN_fatal ("cuCtxGetDevice error: %s", cuda_error (r));
if (r != CUDA_ERROR_INVALID_CONTEXT && dev == n)
return;
else
{
CUcontext old_ctx;
ptx_dev = ptx_devices[n];
assert (ptx_dev);
r = cuCtxGetCurrent (&thd_ctx);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuCtxGetCurrent error: %s", cuda_error (r));
/* We don't necessarily have a current context (e.g. if it has been
destroyed. Pop it if we do though. */
if (thd_ctx != NULL)
{
r = cuCtxPopCurrent (&old_ctx);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuCtxPopCurrent error: %s", cuda_error (r));
}
r = cuCtxPushCurrent (ptx_dev->ctx);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuCtxPushCurrent error: %s", cuda_error (r));
}
}
bool support_device(bool experimental)
{
if(!experimental) {
int major, minor;
cuDeviceComputeCapability(&major, &minor, cuDevId);
if(major <= 1 && minor <= 2) {
cuda_error(string_printf("CUDA device supported only with compute capability 1.3 or up, found %d.%d.", major, minor));
return false;
}
}
return true;
}
static void
nvptx_wait_async (int async1, int async2)
{
CUresult r;
CUevent *e;
struct ptx_stream *s1, *s2;
pthread_t self = pthread_self ();
/* The stream that is waiting (rather than being waited for) doesn't
necessarily have to exist already. */
s2 = select_stream_for_async (async2, self, true, NULL);
s1 = select_stream_for_async (async1, self, false, NULL);
if (!s1)
GOMP_PLUGIN_fatal ("invalid async 1\n");
if (s1 == s2)
GOMP_PLUGIN_fatal ("identical parameters");
e = (CUevent *)GOMP_PLUGIN_malloc (sizeof (CUevent));
r = cuEventCreate (e, CU_EVENT_DISABLE_TIMING);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuEventCreate error: %s", cuda_error (r));
event_gc (true);
r = cuEventRecord (*e, s1->stream);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuEventRecord error: %s", cuda_error (r));
event_add (PTX_EVT_SYNC, e, NULL);
r = cuStreamWaitEvent (s2->stream, *e, 0);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuStreamWaitEvent error: %s", cuda_error (r));
}
static void
nvptx_wait (int async)
{
CUresult r;
struct ptx_stream *s;
s = select_stream_for_async (async, pthread_self (), false, NULL);
if (!s)
GOMP_PLUGIN_fatal ("unknown async %d", async);
r = cuStreamSynchronize (s->stream);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuStreamSynchronize error: %s", cuda_error (r));
event_gc (true);
}
static void
nvptx_close_device (struct ptx_device *ptx_dev)
{
CUresult r;
if (!ptx_dev)
return;
fini_streams_for_device (ptx_dev);
pthread_mutex_destroy (&ptx_dev->image_lock);
if (!ptx_dev->ctx_shared)
{
r = cuCtxDestroy (ptx_dev->ctx);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuCtxDestroy error: %s", cuda_error (r));
}
free (ptx_dev);
}
static bool
nvptx_attach_host_thread_to_device (int n)
{
CUdevice dev;
CUresult r;
struct ptx_device *ptx_dev;
CUcontext thd_ctx;
r = cuCtxGetDevice (&dev);
if (r != CUDA_SUCCESS && r != CUDA_ERROR_INVALID_CONTEXT)
{
GOMP_PLUGIN_error ("cuCtxGetDevice error: %s", cuda_error (r));
return false;
}
if (r != CUDA_ERROR_INVALID_CONTEXT && dev == n)
return true;
else
{
CUcontext old_ctx;
ptx_dev = ptx_devices[n];
if (!ptx_dev)
{
GOMP_PLUGIN_error ("device %d not found", n);
return false;
}
CUDA_CALL (cuCtxGetCurrent, &thd_ctx);
/* We don't necessarily have a current context (e.g. if it has been
destroyed. Pop it if we do though. */
if (thd_ctx != NULL)
CUDA_CALL (cuCtxPopCurrent, &old_ctx);
CUDA_CALL (cuCtxPushCurrent, ptx_dev->ctx);
}
return true;
}
void
nvptx_exec (void (*fn), size_t mapnum, void **hostaddrs, void **devaddrs,
int async, unsigned *dims, void *targ_mem_desc)
{
struct targ_fn_descriptor *targ_fn = (struct targ_fn_descriptor *) fn;
CUfunction function;
CUresult r;
int i;
struct ptx_stream *dev_str;
void *kargs[1];
void *hp, *dp;
struct nvptx_thread *nvthd = nvptx_thread ();
const char *maybe_abort_msg = "(perhaps abort was called)";
function = targ_fn->fn;
dev_str = select_stream_for_async (async, pthread_self (), false, NULL);
assert (dev_str == nvthd->current_stream);
/* Initialize the launch dimensions. Typically this is constant,
provided by the device compiler, but we must permit runtime
values. */
for (i = 0; i != 3; i++)
if (targ_fn->launch->dim[i])
dims[i] = targ_fn->launch->dim[i];
/* This reserves a chunk of a pre-allocated page of memory mapped on both
the host and the device. HP is a host pointer to the new chunk, and DP is
the corresponding device pointer. */
map_push (dev_str, async, mapnum * sizeof (void *), &hp, &dp);
GOMP_PLUGIN_debug (0, " %s: prepare mappings\n", __FUNCTION__);
/* Copy the array of arguments to the mapped page. */
for (i = 0; i < mapnum; i++)
((void **) hp)[i] = devaddrs[i];
/* Copy the (device) pointers to arguments to the device (dp and hp might in
fact have the same value on a unified-memory system). */
r = cuMemcpy ((CUdeviceptr)dp, (CUdeviceptr)hp, mapnum * sizeof (void *));
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuMemcpy failed: %s", cuda_error (r));
GOMP_PLUGIN_debug (0, " %s: kernel %s: launch"
" gangs=%u, workers=%u, vectors=%u\n",
__FUNCTION__, targ_fn->launch->fn,
dims[0], dims[1], dims[2]);
// OpenACC CUDA
//
// num_gangs nctaid.x
// num_workers ntid.y
// vector length ntid.x
kargs[0] = &dp;
r = cuLaunchKernel (function,
dims[GOMP_DIM_GANG], 1, 1,
dims[GOMP_DIM_VECTOR], dims[GOMP_DIM_WORKER], 1,
0, dev_str->stream, kargs, 0);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuLaunchKernel error: %s", cuda_error (r));
#ifndef DISABLE_ASYNC
if (async < acc_async_noval)
{
r = cuStreamSynchronize (dev_str->stream);
if (r == CUDA_ERROR_LAUNCH_FAILED)
GOMP_PLUGIN_fatal ("cuStreamSynchronize error: %s %s\n", cuda_error (r),
maybe_abort_msg);
else if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuStreamSynchronize error: %s", cuda_error (r));
}
else
{
CUevent *e;
e = (CUevent *)GOMP_PLUGIN_malloc (sizeof (CUevent));
r = cuEventCreate (e, CU_EVENT_DISABLE_TIMING);
if (r == CUDA_ERROR_LAUNCH_FAILED)
GOMP_PLUGIN_fatal ("cuEventCreate error: %s %s\n", cuda_error (r),
maybe_abort_msg);
else if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuEventCreate error: %s", cuda_error (r));
event_gc (true);
r = cuEventRecord (*e, dev_str->stream);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuEventRecord error: %s", cuda_error (r));
event_add (PTX_EVT_KNL, e, (void *)dev_str);
}
#else
r = cuCtxSynchronize ();
if (r == CUDA_ERROR_LAUNCH_FAILED)
GOMP_PLUGIN_fatal ("cuCtxSynchronize error: %s %s\n", cuda_error (r),
maybe_abort_msg);
else if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuCtxSynchronize error: %s", cuda_error (r));
#endif
GOMP_PLUGIN_debug (0, " %s: kernel %s: finished\n", __FUNCTION__,
targ_fn->launch->fn);
#ifndef DISABLE_ASYNC
if (async < acc_async_noval)
#endif
map_pop (dev_str);
}
static void
link_ptx (CUmodule *module, const struct targ_ptx_obj *ptx_objs,
unsigned num_objs)
{
CUjit_option opts[7];
void *optvals[7];
float elapsed = 0.0;
#define LOGSIZE 8192
char elog[LOGSIZE];
char ilog[LOGSIZE];
unsigned long logsize = LOGSIZE;
CUlinkState linkstate;
CUresult r;
void *linkout;
size_t linkoutsize __attribute__ ((unused));
opts[0] = CU_JIT_WALL_TIME;
optvals[0] = &elapsed;
opts[1] = CU_JIT_INFO_LOG_BUFFER;
optvals[1] = &ilog[0];
opts[2] = CU_JIT_INFO_LOG_BUFFER_SIZE_BYTES;
optvals[2] = (void *) logsize;
opts[3] = CU_JIT_ERROR_LOG_BUFFER;
optvals[3] = &elog[0];
opts[4] = CU_JIT_ERROR_LOG_BUFFER_SIZE_BYTES;
optvals[4] = (void *) logsize;
opts[5] = CU_JIT_LOG_VERBOSE;
optvals[5] = (void *) 1;
opts[6] = CU_JIT_TARGET;
optvals[6] = (void *) CU_TARGET_COMPUTE_30;
r = cuLinkCreate (7, opts, optvals, &linkstate);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuLinkCreate error: %s", cuda_error (r));
for (; num_objs--; ptx_objs++)
{
/* cuLinkAddData's 'data' argument erroneously omits the const
qualifier. */
GOMP_PLUGIN_debug (0, "Loading:\n---\n%s\n---\n", ptx_objs->code);
r = cuLinkAddData (linkstate, CU_JIT_INPUT_PTX, (char*)ptx_objs->code,
ptx_objs->size, 0, 0, 0, 0);
if (r != CUDA_SUCCESS)
{
GOMP_PLUGIN_error ("Link error log %s\n", &elog[0]);
GOMP_PLUGIN_fatal ("cuLinkAddData (ptx_code) error: %s",
cuda_error (r));
}
}
GOMP_PLUGIN_debug (0, "Linking\n");
r = cuLinkComplete (linkstate, &linkout, &linkoutsize);
GOMP_PLUGIN_debug (0, "Link complete: %fms\n", elapsed);
GOMP_PLUGIN_debug (0, "Link log %s\n", &ilog[0]);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuLinkComplete error: %s", cuda_error (r));
r = cuModuleLoadData (module, linkout);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuModuleLoadData error: %s", cuda_error (r));
r = cuLinkDestroy (linkstate);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuLinkDestory error: %s", cuda_error (r));
}
static CUmodule
build_kernel_source(const char *source_file, long target_capability)
{
char *source;
int link_dev_runtime;
nvrtcProgram program;
nvrtcResult rc;
char arch_buf[128];
const char *options[10];
int opt_index = 0;
int build_failure = 0;
char *build_log;
size_t build_log_len;
char *ptx_image;
size_t ptx_image_len;
void *bin_image;
size_t bin_image_len;
CUmodule cuda_module;
CUresult cuda_rc;
source = load_kernel_source(source_file, &link_dev_runtime);
rc = nvrtcCreateProgram(&program,
source,
NULL,
0,
NULL,
NULL);
if (rc != NVRTC_SUCCESS)
nvrtc_error(rc, "nvrtcCreateProgram");
/*
* Put command line options as cuda_program.c doing
*/
options[opt_index++] = "-I " CUDA_INCLUDE_PATH;
snprintf(arch_buf, sizeof(arch_buf),
"--gpu-architecture=compute_%ld", target_capability);
options[opt_index++] = arch_buf;
#ifdef PGSTROM_DEBUG
options[opt_index++] = "--device-debug";
options[opt_index++] = "--generate-line-info";
#endif
options[opt_index++] = "--use_fast_math";
if (link_dev_runtime)
options[opt_index++] = "--relocatable-device-code=true";
/*
* Kick runtime compiler
*/
rc = nvrtcCompileProgram(program, opt_index, options);
if (rc != NVRTC_SUCCESS)
{
if (rc == NVRTC_ERROR_COMPILATION)
build_failure = 1;
else
nvrtc_error(rc, "nvrtcCompileProgram");
}
/*
* Print build log
*/
rc = nvrtcGetProgramLogSize(program, &build_log_len);
if (rc != NVRTC_SUCCESS)
nvrtc_error(rc, "nvrtcGetProgramLogSize");
build_log = malloc(build_log_len + 1);
if (!build_log)
{
fputs("out of memory", stderr);
exit(1);
}
rc = nvrtcGetProgramLog(program, build_log);
if (rc != NVRTC_SUCCESS)
nvrtc_error(rc, "nvrtcGetProgramLog");
if (build_log_len > 1)
printf("build log:\n%s\n", build_log);
if (build_failure)
exit(1);
/*
* Get PTX Image
*/
rc = nvrtcGetPTXSize(program, &ptx_image_len);
if (rc != NVRTC_SUCCESS)
nvrtc_error(rc, "nvrtcGetPTXSize");
ptx_image = malloc(ptx_image_len + 1);
if (!ptx_image)
{
fputs("out of memory", stderr);
exit(1);
}
rc = nvrtcGetPTX(program, ptx_image);
if (rc != NVRTC_SUCCESS)
nvrtc_error(rc, "nvrtcGetPTX");
ptx_image[ptx_image_len] = '\0';
/*
* Link device runtime if needed
*/
if (link_dev_runtime)
{
link_device_libraries(ptx_image, ptx_image_len,
&bin_image, &bin_image_len,
target_capability);
}
else
{
bin_image = ptx_image;
bin_image_len = ptx_image_len;
}
cuda_rc = cuModuleLoadData(&cuda_module, bin_image);
if (cuda_rc != CUDA_SUCCESS)
cuda_error(rc, "cuModuleLoadData");
return cuda_module;
}
static struct ptx_device *
nvptx_open_device (int n)
{
struct ptx_device *ptx_dev;
CUdevice dev, ctx_dev;
CUresult r;
int async_engines, pi;
r = cuDeviceGet (&dev, n);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuDeviceGet error: %s", cuda_error (r));
ptx_dev = GOMP_PLUGIN_malloc (sizeof (struct ptx_device));
ptx_dev->ord = n;
ptx_dev->dev = dev;
ptx_dev->ctx_shared = false;
r = cuCtxGetDevice (&ctx_dev);
if (r != CUDA_SUCCESS && r != CUDA_ERROR_INVALID_CONTEXT)
GOMP_PLUGIN_fatal ("cuCtxGetDevice error: %s", cuda_error (r));
if (r != CUDA_ERROR_INVALID_CONTEXT && ctx_dev != dev)
{
/* The current host thread has an active context for a different device.
Detach it. */
CUcontext old_ctx;
r = cuCtxPopCurrent (&old_ctx);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuCtxPopCurrent error: %s", cuda_error (r));
}
r = cuCtxGetCurrent (&ptx_dev->ctx);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuCtxGetCurrent error: %s", cuda_error (r));
if (!ptx_dev->ctx)
{
r = cuCtxCreate (&ptx_dev->ctx, CU_CTX_SCHED_AUTO, dev);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuCtxCreate error: %s", cuda_error (r));
}
else
ptx_dev->ctx_shared = true;
r = cuDeviceGetAttribute (&pi, CU_DEVICE_ATTRIBUTE_GPU_OVERLAP, dev);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuDeviceGetAttribute error: %s", cuda_error (r));
ptx_dev->overlap = pi;
r = cuDeviceGetAttribute (&pi, CU_DEVICE_ATTRIBUTE_CAN_MAP_HOST_MEMORY, dev);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuDeviceGetAttribute error: %s", cuda_error (r));
ptx_dev->map = pi;
r = cuDeviceGetAttribute (&pi, CU_DEVICE_ATTRIBUTE_CONCURRENT_KERNELS, dev);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuDeviceGetAttribute error: %s", cuda_error (r));
ptx_dev->concur = pi;
r = cuDeviceGetAttribute (&pi, CU_DEVICE_ATTRIBUTE_COMPUTE_MODE, dev);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuDeviceGetAttribute error: %s", cuda_error (r));
ptx_dev->mode = pi;
r = cuDeviceGetAttribute (&pi, CU_DEVICE_ATTRIBUTE_INTEGRATED, dev);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuDeviceGetAttribute error: %s", cuda_error (r));
ptx_dev->mkern = pi;
r = cuDeviceGetAttribute (&async_engines,
CU_DEVICE_ATTRIBUTE_ASYNC_ENGINE_COUNT, dev);
if (r != CUDA_SUCCESS)
async_engines = 1;
ptx_dev->images = NULL;
pthread_mutex_init (&ptx_dev->image_lock, NULL);
init_streams_for_device (ptx_dev, async_engines);
return ptx_dev;
}
static void print_function_attrs(CUmodule cuda_module, const char *func_name)
{
CUfunction kernel;
CUresult rc;
int max_threads_per_block;
int shared_mem_sz;
int const_mem_sz;
int local_mem_sz;
int num_regs;
int ptx_version;
int binary_version;
int cache_mode_ca;
int min_grid_sz;
int max_block_sz;
int i;
struct {
CUfunction_attribute attr;
int *vptr;
} catalog[] = {
{ CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, &max_threads_per_block },
{ CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, &shared_mem_sz },
{ CU_FUNC_ATTRIBUTE_CONST_SIZE_BYTES, &const_mem_sz },
{ CU_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES, &local_mem_sz },
{ CU_FUNC_ATTRIBUTE_NUM_REGS, &num_regs },
{ CU_FUNC_ATTRIBUTE_PTX_VERSION, &ptx_version },
{ CU_FUNC_ATTRIBUTE_BINARY_VERSION, &binary_version },
{ CU_FUNC_ATTRIBUTE_CACHE_MODE_CA, &cache_mode_ca },
};
rc = cuModuleGetFunction(&kernel, cuda_module, func_name);
if (rc != CUDA_SUCCESS)
cuda_error(rc, "cuModuleGetFunction");
for (i=0; i < lengthof(catalog); i++)
{
rc = cuFuncGetAttribute(catalog[i].vptr,
catalog[i].attr,
kernel);
if (rc != CUDA_SUCCESS)
cuda_error(rc, "cuFuncGetAttribute");
}
rc = cuOccupancyMaxPotentialBlockSize(&min_grid_sz,
&max_block_sz,
kernel,
cb_occupancy_shmem_size,
dynamic_shmem_per_block,
1024 * 1024);
if (rc != CUDA_SUCCESS)
cuda_error(rc, "cuOccupancyMaxPotentialBlockSize");
printf("Kernel Function: %s\n"
" Max threads per block: %d\n"
" Shared memory usage: %d\n"
" Constant memory usage: %d\n"
" Local memory usage: %d\n"
" Number of registers: %d\n"
" PTX version: %d\n"
" Binary version: %d\n"
" Global memory caching: %s\n"
" Max potential block size: %u\n"
" (shmem usage: %ld/thread + %ld/block)\n",
func_name,
max_threads_per_block,
shared_mem_sz,
const_mem_sz,
local_mem_sz,
num_regs,
ptx_version,
binary_version,
cache_mode_ca ? "enabled" : "disabled",
max_block_sz,
dynamic_shmem_per_thread,
dynamic_shmem_per_block);
}
static struct ptx_stream *
select_stream_for_async (int async, pthread_t thread, bool create,
CUstream existing)
{
struct nvptx_thread *nvthd = nvptx_thread ();
/* Local copy of TLS variable. */
struct ptx_device *ptx_dev = nvthd->ptx_dev;
struct ptx_stream *stream = NULL;
int orig_async = async;
/* The special value acc_async_noval (-1) maps (for now) to an
implicitly-created stream, which is then handled the same as any other
numbered async stream. Other options are available, e.g. using the null
stream for anonymous async operations, or choosing an idle stream from an
active set. But, stick with this for now. */
if (async > acc_async_sync)
async++;
if (create)
pthread_mutex_lock (&ptx_dev->stream_lock);
/* NOTE: AFAICT there's no particular need for acc_async_sync to map to the
null stream, and in fact better performance may be obtainable if it doesn't
(because the null stream enforces overly-strict synchronisation with
respect to other streams for legacy reasons, and that's probably not
needed with OpenACC). Maybe investigate later. */
if (async == acc_async_sync)
stream = ptx_dev->null_stream;
else if (async >= 0 && async < ptx_dev->async_streams.size
&& ptx_dev->async_streams.arr[async] && !(create && existing))
stream = ptx_dev->async_streams.arr[async];
else if (async >= 0 && create)
{
if (async >= ptx_dev->async_streams.size)
{
int i, newsize = ptx_dev->async_streams.size * 2;
if (async >= newsize)
newsize = async + 1;
ptx_dev->async_streams.arr
= GOMP_PLUGIN_realloc (ptx_dev->async_streams.arr,
newsize * sizeof (struct ptx_stream *));
for (i = ptx_dev->async_streams.size; i < newsize; i++)
ptx_dev->async_streams.arr[i] = NULL;
ptx_dev->async_streams.size = newsize;
}
/* Create a new stream on-demand if there isn't one already, or if we're
setting a particular async value to an existing (externally-provided)
stream. */
if (!ptx_dev->async_streams.arr[async] || existing)
{
CUresult r;
struct ptx_stream *s
= GOMP_PLUGIN_malloc (sizeof (struct ptx_stream));
if (existing)
s->stream = existing;
else
{
r = cuStreamCreate (&s->stream, CU_STREAM_DEFAULT);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuStreamCreate error: %s", cuda_error (r));
}
/* If CREATE is true, we're going to be queueing some work on this
stream. Associate it with the current host thread. */
s->host_thread = thread;
s->multithreaded = false;
s->d = (CUdeviceptr) NULL;
s->h = NULL;
map_init (s);
s->next = ptx_dev->active_streams;
ptx_dev->active_streams = s;
ptx_dev->async_streams.arr[async] = s;
}
stream = ptx_dev->async_streams.arr[async];
}
else if (async < 0)
GOMP_PLUGIN_fatal ("bad async %d", async);
if (create)
{
assert (stream != NULL);
/* If we're trying to use the same stream from different threads
simultaneously, set stream->multithreaded to true. This affects the
behaviour of acc_async_test_all and acc_wait_all, which are supposed to
only wait for asynchronous launches from the same host thread they are
invoked on. If multiple threads use the same async value, we make note
of that here and fall back to testing/waiting for all threads in those
functions. */
if (thread != stream->host_thread)
stream->multithreaded = true;
pthread_mutex_unlock (&ptx_dev->stream_lock);
}
else if (stream && !stream->multithreaded
&& !pthread_equal (stream->host_thread, thread))
GOMP_PLUGIN_fatal ("async %d used on wrong thread", orig_async);
return stream;
}
int main(int argc, char *argv[])
{
char *kernel_source;
char *kfunc_names[MAX_KERNEL_FUNCTIONS];
int kfunc_index = 0;
int target_device = -1;
long target_capability = -1;
int print_version = 0;
int print_devices = 0;
int num_devices;
int i, opt;
int major;
int minor;
CUdevice device;
CUcontext context;
CUmodule cuda_module;
CUresult rc;
/* misc initialization */
cmdname = basename(strdup(argv[0]));
cuInit(0);
rc = cuDeviceGetCount(&num_devices);
if (rc != CUDA_SUCCESS)
cuda_error(rc, "cuDeviceGetCount");
while ((opt = getopt(argc, argv, "k:d:c:s:S:vlh")) >= 0)
{
switch (opt)
{
case 'k':
if (kfunc_index == MAX_KERNEL_FUNCTIONS)
{
fputs("Too much kernel function specified", stderr);
return 1;
}
kfunc_names[kfunc_index++] = strdup(optarg);
break;
case 'd':
if (target_device >= 0)
{
fputs("-d is specified twice or more", stderr);
usage();
}
if (target_capability >= 0)
{
fputs("-d and -c are exclusive option", stderr);
usage();
}
target_device = atoi(optarg);
if (target_device < 0 || target_device >= num_devices)
{
fprintf(stderr, "invalid device: -d %d\n", target_device);
usage();
}
break;
case 'c':
if (target_capability >= 0)
{
fputs("-c is specified twice or more", stderr);
usage();
}
if (target_device >= 0)
{
fputs("-d and -c are exclusive option", stderr);
usage();
}
if (sscanf(optarg, "%d.%d", &major, &minor) != 2)
{
fprintf(stderr, "invalid capability format: -c %s\n",
optarg);
usage();
}
target_capability = major * 10 + minor;
break;
case 's':
dynamic_shmem_per_thread = atol(optarg);
if (dynamic_shmem_per_thread < 0)
{
fprintf(stderr, "invalid dynamic shmem per thread: %ld\n",
dynamic_shmem_per_thread);
usage();
}
break;
case 'S':
dynamic_shmem_per_block = atol(optarg);
if (dynamic_shmem_per_block < 0)
{
fprintf(stderr, "invalid dynamic shmem per block: %ld",
dynamic_shmem_per_block);
usage();
}
break;
case 'v':
print_version = 1;
break;
case 'l':
print_devices = 1;
break;
case 'h':
default:
usage();
break;
}
}
if (optind + 1 != argc)
{
if (print_version || print_devices)
{
if (print_version)
print_nvrtc_version();
if (print_devices)
print_cuda_devices(num_devices);
return 0;
}
fputs("no kernel source is specified", stderr);
usage();
}
kernel_source = argv[optind];
if (target_capability < 0)
{
CUdevice dev;
if (target_device < 0)
target_device = 0; /* default device */
rc = cuDeviceGet(&dev, target_device);
if (rc != CUDA_SUCCESS)
cuda_error(rc, "cuDeviceGet");
rc = cuDeviceGetAttribute(&major,
CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, dev);
if (rc != CUDA_SUCCESS)
cuda_error(rc, "cuDeviceGetAttribute");
rc = cuDeviceGetAttribute(&minor,
CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, dev);
if (rc != CUDA_SUCCESS)
cuda_error(rc, "cuDeviceGetAttribute");
target_capability = 10 * major + minor;
}
if (print_version)
print_nvrtc_version();
if (print_devices)
print_cuda_devices(num_devices);
/* make a dummy context */
rc = cuDeviceGet(&device, 0);
if (rc != CUDA_SUCCESS)
cuda_error(rc, "cuDeviceGet");
rc = cuCtxCreate(&context, 0, device);
if (rc != CUDA_SUCCESS)
cuda_error(rc, "cuCtxCreate");
cuda_module = build_kernel_source(kernel_source, target_capability);
for (i=0; i < kfunc_index; i++)
{
if (i > 0)
putchar('\n');
print_function_attrs(cuda_module, kfunc_names[i]);
}
/* drop a cuda context */
rc = cuCtxDestroy(context);
if (rc != CUDA_SUCCESS)
cuda_error(rc, "cuCtxDestroy");
return 0;
}
int
GOMP_OFFLOAD_load_image (int ord, unsigned version, const void *target_data,
struct addr_pair **target_table)
{
CUmodule module;
const char *const *var_names;
const struct targ_fn_launch *fn_descs;
unsigned int fn_entries, var_entries, i, j;
CUresult r;
struct targ_fn_descriptor *targ_fns;
struct addr_pair *targ_tbl;
const nvptx_tdata_t *img_header = (const nvptx_tdata_t *) target_data;
struct ptx_image_data *new_image;
struct ptx_device *dev;
if (GOMP_VERSION_DEV (version) > GOMP_VERSION_NVIDIA_PTX)
GOMP_PLUGIN_fatal ("Offload data incompatible with PTX plugin"
" (expected %u, received %u)",
GOMP_VERSION_NVIDIA_PTX, GOMP_VERSION_DEV (version));
GOMP_OFFLOAD_init_device (ord);
dev = ptx_devices[ord];
nvptx_attach_host_thread_to_device (ord);
link_ptx (&module, img_header->ptx_objs, img_header->ptx_num);
/* The mkoffload utility emits a struct of pointers/integers at the
start of each offload image. The array of kernel names and the
functions addresses form a one-to-one correspondence. */
var_entries = img_header->var_num;
var_names = img_header->var_names;
fn_entries = img_header->fn_num;
fn_descs = img_header->fn_descs;
targ_tbl = GOMP_PLUGIN_malloc (sizeof (struct addr_pair)
* (fn_entries + var_entries));
targ_fns = GOMP_PLUGIN_malloc (sizeof (struct targ_fn_descriptor)
* fn_entries);
*target_table = targ_tbl;
new_image = GOMP_PLUGIN_malloc (sizeof (struct ptx_image_data));
new_image->target_data = target_data;
new_image->module = module;
new_image->fns = targ_fns;
pthread_mutex_lock (&dev->image_lock);
new_image->next = dev->images;
dev->images = new_image;
pthread_mutex_unlock (&dev->image_lock);
for (i = 0; i < fn_entries; i++, targ_fns++, targ_tbl++)
{
CUfunction function;
r = cuModuleGetFunction (&function, module, fn_descs[i].fn);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuModuleGetFunction error: %s", cuda_error (r));
targ_fns->fn = function;
targ_fns->launch = &fn_descs[i];
targ_tbl->start = (uintptr_t) targ_fns;
targ_tbl->end = targ_tbl->start + 1;
}
for (j = 0; j < var_entries; j++, targ_tbl++)
{
CUdeviceptr var;
size_t bytes;
r = cuModuleGetGlobal (&var, &bytes, module, var_names[j]);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuModuleGetGlobal error: %s", cuda_error (r));
targ_tbl->start = (uintptr_t) var;
targ_tbl->end = targ_tbl->start + bytes;
}
return fn_entries + var_entries;
}
static void *
nvptx_dev2host (void *h, const void *d, size_t s)
{
CUresult r;
CUdeviceptr pb;
size_t ps;
struct nvptx_thread *nvthd = nvptx_thread ();
if (!s)
return 0;
if (!d)
GOMP_PLUGIN_fatal ("invalid device address");
r = cuMemGetAddressRange (&pb, &ps, (CUdeviceptr)d);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuMemGetAddressRange error: %s", cuda_error (r));
if (!pb)
GOMP_PLUGIN_fatal ("invalid device address");
if (!h)
GOMP_PLUGIN_fatal ("invalid host address");
if (d == h)
GOMP_PLUGIN_fatal ("invalid host or device address");
if ((void *)(d + s) > (void *)(pb + ps))
GOMP_PLUGIN_fatal ("invalid size");
#ifndef DISABLE_ASYNC
if (nvthd->current_stream != nvthd->ptx_dev->null_stream)
{
CUevent *e;
e = (CUevent *)GOMP_PLUGIN_malloc (sizeof (CUevent));
r = cuEventCreate (e, CU_EVENT_DISABLE_TIMING);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuEventCreate error: %s\n", cuda_error (r));
event_gc (false);
r = cuMemcpyDtoHAsync (h, (CUdeviceptr)d, s,
nvthd->current_stream->stream);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuMemcpyDtoHAsync error: %s", cuda_error (r));
r = cuEventRecord (*e, nvthd->current_stream->stream);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuEventRecord error: %s", cuda_error (r));
event_add (PTX_EVT_MEM, e, (void *)h);
}
else
#endif
{
r = cuMemcpyDtoH (h, (CUdeviceptr)d, s);
if (r != CUDA_SUCCESS)
GOMP_PLUGIN_fatal ("cuMemcpyDtoH error: %s", cuda_error (r));
}
return 0;
}