/* creates either a program with binaries, or an empty program. The latter * is useful for clLinkProgram() which needs an empty program to put the * compiled results in. */ cl_program create_program_skeleton (cl_context context, cl_uint num_devices, const cl_device_id *device_list, const size_t *lengths, const unsigned char **binaries, cl_int *binary_status, cl_int *errcode_ret, int allow_empty_binaries) { cl_program program; unsigned i,j; int errcode, is_spirv_opencl; cl_device_id *unique_devlist = NULL; POCL_GOTO_ERROR_COND((context == NULL), CL_INVALID_CONTEXT); POCL_GOTO_ERROR_COND((device_list == NULL), CL_INVALID_VALUE); POCL_GOTO_ERROR_COND((num_devices == 0), CL_INVALID_VALUE); if (!allow_empty_binaries) { POCL_GOTO_ERROR_COND ((lengths == NULL), CL_INVALID_VALUE); for (i = 0; i < num_devices; ++i) { POCL_GOTO_ERROR_ON ((lengths[i] == 0 || binaries[i] == NULL), CL_INVALID_VALUE, "%i-th binary is NULL or its length==0\n", i); } } // check for duplicates in device_list[]. for (i = 0; i < context->num_devices; i++) { int count = 0; for (j = 0; j < num_devices; j++) { count += context->devices[i] == device_list[j]; } // duplicate devices POCL_GOTO_ERROR_ON((count > 1), CL_INVALID_DEVICE, "device %s specified multiple times\n", context->devices[i]->long_name); } // convert subdevices to devices and remove duplicates cl_uint real_num_devices = 0; unique_devlist = pocl_unique_device_list(device_list, num_devices, &real_num_devices); num_devices = real_num_devices; device_list = unique_devlist; // check for invalid devices in device_list[]. for (i = 0; i < num_devices; i++) { int found = 0; for (j = 0; j < context->num_devices; j++) { found |= context->devices[j] == device_list[i]; } POCL_GOTO_ERROR_ON((!found), CL_INVALID_DEVICE, "device not found in the device list of the context\n"); } if ((program = (cl_program) calloc (1, sizeof (struct _cl_program))) == NULL) { errcode = CL_OUT_OF_HOST_MEMORY; goto ERROR; } POCL_INIT_OBJECT(program); if ((program->binary_sizes = (size_t*) calloc (num_devices, sizeof(size_t))) == NULL || (program->binaries = (unsigned char**) calloc (num_devices, sizeof(unsigned char*))) == NULL || (program->pocl_binaries = (unsigned char**) calloc (num_devices, sizeof(unsigned char*))) == NULL || (program->pocl_binary_sizes = (size_t*) calloc (num_devices, sizeof(size_t))) == NULL || (program->build_log = (char**) calloc (num_devices, sizeof(char*))) == NULL || ((program->llvm_irs = (void**) calloc (num_devices, sizeof(void*))) == NULL) || ((program->build_hash = (SHA1_digest_t*) calloc (num_devices, sizeof(SHA1_digest_t))) == NULL)) { errcode = CL_OUT_OF_HOST_MEMORY; goto ERROR_CLEAN_PROGRAM_AND_BINARIES; } program->context = context; program->num_devices = num_devices; program->devices = unique_devlist; program->build_status = CL_BUILD_NONE; program->binary_type = CL_PROGRAM_BINARY_TYPE_NONE; char program_bc_path[POCL_FILENAME_LENGTH]; if (allow_empty_binaries && (lengths == NULL) && (binaries == NULL)) goto SUCCESS; for (i = 0; i < num_devices; ++i) { /* LLVM IR */ if (!strncmp((const char *)binaries[i], "BC", 2)) { program->binary_sizes[i] = lengths[i]; program->binaries[i] = (unsigned char*) malloc(lengths[i]); memcpy (program->binaries[i], binaries[i], lengths[i]); if (binary_status != NULL) binary_status[i] = CL_SUCCESS; } /* SPIR-V binary needs to be converted, and requires * linking of the converted BC */ #ifdef OCS_AVAILABLE else if (bitcode_is_spirv ((const char *)binaries[i], lengths[i], &is_spirv_opencl)) { if (is_spirv_opencl == 0) { // SPIR-V but not OpenCL-type. POCL_GOTO_ERROR_ON ( 1, CL_BUILD_PROGRAM_FAILURE, "SPIR-V binary provided, but is not using Kernel mode." "Pocl can't process this binary.\n"); } int no_spir = strstr (device_list[i]->extensions, "cl_khr_spir") == NULL; POCL_GOTO_ERROR_ON ( no_spir, CL_BUILD_PROGRAM_FAILURE, "SPIR binary provided, but device has no SPIR support"); #ifdef ENABLE_SPIRV POCL_MSG_PRINT_LLVM ( "SPIR-V binary detected, converting to LLVM SPIR\n"); char program_bc_spirv[POCL_FILENAME_LENGTH]; char program_bc_temp[POCL_FILENAME_LENGTH]; pocl_cache_write_spirv (program_bc_spirv, (const char *)binaries[i], (uint64_t)lengths[i]); pocl_cache_tempname (program_bc_temp, ".bc", NULL); char *args[] = { LLVM_SPIRV, "-r", "-o", program_bc_temp, program_bc_spirv, NULL }; errcode = pocl_run_command (args); assert (errcode == 0); /* load LLVM SPIR binary. */ uint64_t fsize; char *content; pocl_read_file (program_bc_temp, &content, &fsize); program->binary_sizes[i] = fsize; program->binaries[i] = (unsigned char *)content; pocl_remove (program_bc_temp); #else POCL_GOTO_ERROR_ON ( 1, CL_BUILD_PROGRAM_FAILURE, "SPIR binary provided, but this pocl has no SPIR-V support." "SPIR-V support requires llvm-spirv converter binary.\n"); #endif } #endif /* Poclcc binary */ else if (pocl_binary_check_binary(device_list[i], binaries[i])) { program->pocl_binary_sizes[i] = lengths[i]; program->pocl_binaries[i] = (unsigned char*) malloc (lengths[i]); memcpy (program->pocl_binaries[i], binaries[i], lengths[i]); pocl_binary_set_program_buildhash (program, i, binaries[i]); int error = pocl_cache_create_program_cachedir (program, i, NULL, 0, program_bc_path); POCL_GOTO_ERROR_ON((error != 0), CL_BUILD_PROGRAM_FAILURE, "Could not create program cachedir"); POCL_GOTO_ERROR_ON(pocl_binary_deserialize (program, i), CL_INVALID_BINARY, "Could not unpack a pocl binary\n"); /* read program.bc, can be useful later */ if (pocl_exists (program_bc_path)) { pocl_read_file (program_bc_path, (char **)(&program->binaries[i]), (uint64_t *)(&program->binary_sizes[i])); } if (binary_status != NULL) binary_status[i] = CL_SUCCESS; } /* Unknown binary */ else { POCL_MSG_WARN ("Could not recognize binary\n"); if (binary_status != NULL) binary_status[i] = CL_INVALID_BINARY; errcode = CL_INVALID_BINARY; goto ERROR_CLEAN_PROGRAM_AND_BINARIES; } } SUCCESS: POCL_RETAIN_OBJECT(context); if (errcode_ret != NULL) *errcode_ret = CL_SUCCESS; return program; ERROR_CLEAN_PROGRAM_AND_BINARIES: if (program->binaries) for (i = 0; i < num_devices; ++i) POCL_MEM_FREE(program->binaries[i]); POCL_MEM_FREE(program->binaries); POCL_MEM_FREE(program->binary_sizes); if (program->pocl_binaries) for (i = 0; i < num_devices; ++i) POCL_MEM_FREE(program->pocl_binaries[i]); POCL_MEM_FREE(program->pocl_binaries); POCL_MEM_FREE(program->pocl_binary_sizes); /*ERROR_CLEAN_PROGRAM:*/ POCL_MEM_FREE(program); ERROR: POCL_MEM_FREE(unique_devlist); if(errcode_ret != NULL) { *errcode_ret = errcode; } return NULL; }
cl_int compile_and_link_program(int compile_program, int link_program, cl_program program, cl_uint num_devices, const cl_device_id *device_list, const char *options, cl_uint num_input_headers, const cl_program *input_headers, const char **header_include_names, cl_uint num_input_programs, const cl_program *input_programs, void (CL_CALLBACK *pfn_notify) (cl_program program, void *user_data), void *user_data) { char program_bc_path[POCL_FILENAME_LENGTH]; char link_options[512]; int errcode, error; int create_library = 0; int requires_cr_sqrt_div = 0; int spir_build = 0; unsigned flush_denorms = 0; uint64_t fsize; cl_device_id *unique_devlist = NULL; char *binary = NULL; unsigned device_i = 0, actually_built = 0; size_t i, j; char *temp_options = NULL; const char *extra_build_options = pocl_get_string_option ("POCL_EXTRA_BUILD_FLAGS", NULL); int build_error_code = (link_program ? CL_BUILD_PROGRAM_FAILURE : CL_COMPILE_PROGRAM_FAILURE); POCL_GOTO_LABEL_COND (PFN_NOTIFY, (program == NULL), CL_INVALID_PROGRAM); POCL_GOTO_LABEL_COND (PFN_NOTIFY, (num_devices > 0 && device_list == NULL), CL_INVALID_VALUE); POCL_GOTO_LABEL_COND (PFN_NOTIFY, (num_devices == 0 && device_list != NULL), CL_INVALID_VALUE); POCL_GOTO_LABEL_COND (PFN_NOTIFY, (pfn_notify == NULL && user_data != NULL), CL_INVALID_VALUE); POCL_GOTO_LABEL_ON (PFN_NOTIFY, program->kernels, CL_INVALID_OPERATION, "Program already has kernels\n"); POCL_GOTO_LABEL_ON (PFN_NOTIFY, (program->source == NULL && program->binaries == NULL), CL_INVALID_PROGRAM, "Program doesn't have sources or binaries! You need " "to call clCreateProgramWith{Binary|Source} first\n"); POCL_GOTO_LABEL_ON (PFN_NOTIFY, ((program->source == NULL) && (link_program == 0)), CL_INVALID_OPERATION, "Cannot clCompileProgram when program has no source\n"); POCL_LOCK_OBJ (program); program->main_build_log[0] = 0; /* TODO this should be somehow utilized at linking */ POCL_MEM_FREE (program->compiler_options); if (extra_build_options) { size_t len = (options != NULL) ? strlen (options) : 0; len += strlen (extra_build_options) + 2; temp_options = (char *)malloc (len); temp_options[0] = 0; if (options != NULL) { strcpy (temp_options, options); strcat (temp_options, " "); } strcat (temp_options, extra_build_options); } else temp_options = (char*) options; if (temp_options) { i = strlen (temp_options); size_t size = i + 512; /* add some space for pocl-added options */ program->compiler_options = (char *)malloc (size); errcode = process_options (temp_options, program->compiler_options, link_options, program, compile_program, link_program, &create_library, &flush_denorms, &requires_cr_sqrt_div, &spir_build, size); if (errcode != CL_SUCCESS) goto ERROR_CLEAN_OPTIONS; } POCL_MSG_PRINT_LLVM ("building program with options %s\n", program->compiler_options); program->flush_denorms = flush_denorms; #if !(defined(__x86_64__) && defined(__GNUC__)) if (flush_denorms) { POCL_MSG_WARN ("flush to zero is currently only implemented for " "x86-64 & gcc/clang, ignoring flag\n"); } #endif /* DEVICE LIST */ if (num_devices == 0) { num_devices = program->num_devices; device_list = program->devices; } else { // convert subdevices to devices and remove duplicates cl_uint real_num_devices = 0; unique_devlist = pocl_unique_device_list (device_list, num_devices, &real_num_devices); num_devices = real_num_devices; device_list = unique_devlist; } clean_program_on_rebuild (program); /* Build the fully linked non-parallel bitcode for all devices. */ for (device_i = 0; device_i < program->num_devices; ++device_i) { cl_device_id device = program->devices[device_i]; /* find the device in the supplied devices-to-build-for list */ int found = 0; for (i = 0; i < num_devices; ++i) if (device_list[i] == device) found = 1; if (!found) continue; if (requires_cr_sqrt_div && !(device->single_fp_config & CL_FP_CORRECTLY_ROUNDED_DIVIDE_SQRT)) { APPEND_TO_MAIN_BUILD_LOG (REQUIRES_CR_SQRT_DIV_ERR); POCL_GOTO_ERROR_ON (1, build_error_code, REQUIRES_CR_SQRT_DIV_ERR " %s\n", device->short_name); } actually_built++; /* clCreateProgramWithSource */ if (program->source) { #ifdef OCS_AVAILABLE if (device->compiler_available == CL_TRUE) { POCL_MSG_PRINT_INFO ("building from sources for device %d\n", device_i); error = pocl_llvm_build_program ( program, device_i, program->compiler_options, program_bc_path, num_input_headers, input_headers, header_include_names, (create_library ? 0 : link_program)); POCL_GOTO_ERROR_ON ((error != 0), build_error_code, "pocl_llvm_build_program() failed\n"); } else #endif { APPEND_TO_MAIN_BUILD_LOG ( "Cannot build a program from sources with pocl " "that does not have online compiler support\n"); POCL_GOTO_ERROR_ON (1, CL_COMPILER_NOT_AVAILABLE, "%s", program->main_build_log); } } /* clCreateProgramWithBinaries */ else if (program->binaries[device_i] && (program->pocl_binaries[device_i] == NULL)) { #ifdef OCS_AVAILABLE /* bitcode is now either plain LLVM IR or SPIR IR */ int spir_binary = bitcode_is_spir ((char*)program->binaries[device_i], program->binary_sizes[device_i]); if (spir_binary) POCL_MSG_PRINT_LLVM ("LLVM-SPIR binary detected\n"); else POCL_MSG_PRINT_LLVM ("building from a BC binary for device %d\n", device_i); if (spir_binary) { #ifdef ENABLE_SPIR if (!strstr (device->extensions, "cl_khr_spir")) { APPEND_TO_MAIN_BUILD_LOG (REQUIRES_SPIR_SUPPORT); POCL_GOTO_ERROR_ON (1, build_error_code, REQUIRES_SPIR_SUPPORT " %s\n", device->short_name); } if (!spir_build) POCL_MSG_WARN ( "SPIR binary provided, but no spir in build options\n"); /* SPIR binaries need to be explicitly linked to the kernel * library. for non-SPIR binaries this happens as part of build * process when program.bc is generated. */ error = pocl_llvm_link_program (program, device_i, program_bc_path, 0, NULL, NULL, NULL, 0, 1); POCL_GOTO_ERROR_ON (error, CL_LINK_PROGRAM_FAILURE, "Failed to link SPIR program.bc\n"); #else APPEND_TO_MAIN_BUILD_LOG (REQUIRES_SPIR_SUPPORT); POCL_GOTO_ERROR_ON (1, build_error_code, REQUIRES_SPIR_SUPPORT " %s\n", device->short_name); #endif } #else APPEND_TO_MAIN_BUILD_LOG ( "Cannot build program from LLVM IR binaries with " "pocl that does not have online compiler support\n"); POCL_GOTO_ERROR_ON (1, CL_COMPILER_NOT_AVAILABLE, "%s", program->main_build_log); #endif } else if (program->pocl_binaries[device_i]) { POCL_MSG_PRINT_INFO("having a poclbinary for device %d\n", device_i); #ifdef OCS_AVAILABLE if (program->binaries[device_i] == NULL) { POCL_MSG_WARN ( "pocl-binary for this device doesn't contain " "program.bc - you won't be able to rebuild/link it\n"); /* do not try to read program.bc or LLVM IRs * TODO maybe read LLVM IRs ?*/ continue; } #else continue; #endif } else if (link_program && (num_input_programs > 0)) { #ifdef OCS_AVAILABLE /* just link binaries. */ unsigned char *cur_device_binaries[num_input_programs]; size_t cur_device_binary_sizes[num_input_programs]; void *cur_llvm_irs[num_input_programs]; for (j = 0; j < num_input_programs; j++) { assert (device == input_programs[j]->devices[device_i]); cur_device_binaries[j] = input_programs[j]->binaries[device_i]; assert (cur_device_binaries[j]); cur_device_binary_sizes[j] = input_programs[j]->binary_sizes[device_i]; if (input_programs[j]->llvm_irs[device_i] == NULL) pocl_update_program_llvm_irs (input_programs[j], device_i); cur_llvm_irs[j] = input_programs[j]->llvm_irs[device_i]; assert (cur_llvm_irs[j]); } error = pocl_llvm_link_program ( program, device_i, program_bc_path, num_input_programs, cur_device_binaries, cur_device_binary_sizes, cur_llvm_irs, create_library, 0); POCL_GOTO_ERROR_ON ((error != CL_SUCCESS), CL_LINK_PROGRAM_FAILURE, "pocl_llvm_link_program() failed\n"); #else POCL_GOTO_ERROR_ON ((1), CL_LINK_PROGRAM_FAILURE, "clCompileProgram/clLinkProgram/clBuildProgram" " require a pocl built with LLVM\n"); #endif } else { POCL_GOTO_ERROR_ON (1, CL_INVALID_BINARY, "No sources nor binaries for device %s - can't " "build the program\n", device->short_name); } #ifdef OCS_AVAILABLE /* Read binaries from program.bc to memory */ if (program->binaries[device_i] == NULL) { errcode = pocl_read_file(program_bc_path, &binary, &fsize); POCL_GOTO_ERROR_ON(errcode, CL_BUILD_ERROR, "Failed to read binaries from program.bc to " "memory: %s\n", program_bc_path); program->binary_sizes[device_i] = (size_t)fsize; program->binaries[device_i] = (unsigned char *)binary; } if (program->llvm_irs[device_i] == NULL) { pocl_update_program_llvm_irs(program, device_i); } /* Maintain a 'last_accessed' file in every program's * cache directory. Will be useful for cache pruning script * that flushes old directories based on LRU */ pocl_cache_update_program_last_access(program, device_i); #endif } POCL_GOTO_ERROR_ON ((actually_built < num_devices), build_error_code, "Some of the devices on the argument-supplied list are" "not available for the program, or do not exist\n"); program->build_status = CL_BUILD_SUCCESS; program->binary_type = CL_PROGRAM_BINARY_TYPE_EXECUTABLE; /* if program will be compiled using clCompileProgram its binary_type * will be set to CL_PROGRAM_BINARY_TYPE_COMPILED_OBJECT. * * if program was created by clLinkProgram which is called * with the –createlibrary link option its binary_type will be set to * CL_PROGRAM_BINARY_TYPE_LIBRARY. */ if (create_library) program->binary_type = CL_PROGRAM_BINARY_TYPE_LIBRARY; if (compile_program && !link_program) program->binary_type = CL_PROGRAM_BINARY_TYPE_COMPILED_OBJECT; assert(program->num_kernels == 0); /* get non-device-specific kernel metadata. We can stop after finding * the first method that works.*/ for (device_i = 0; device_i < program->num_devices; device_i++) { #ifdef OCS_AVAILABLE if (program->binaries[device_i]) { program->num_kernels = pocl_llvm_get_kernel_count (program, device_i); if (program->num_kernels) { program->kernel_meta = calloc (program->num_kernels, sizeof (pocl_kernel_metadata_t)); pocl_llvm_get_kernels_metadata (program, device_i); } break; } #endif if (program->pocl_binaries[device_i]) { program->num_kernels = pocl_binary_get_kernel_count (program, device_i); if (program->num_kernels) { program->kernel_meta = calloc (program->num_kernels, sizeof (pocl_kernel_metadata_t)); pocl_binary_get_kernels_metadata (program, device_i); } break; } } POCL_GOTO_ERROR_ON ((device_i >= program->num_devices), CL_INVALID_BINARY, "Could find kernel metadata in the built program\n"); /* calculate device-specific kernel hashes. */ for (j = 0; j < program->num_kernels; ++j) { program->kernel_meta[j].build_hash = calloc (program->num_devices, sizeof (pocl_kernel_hash_t)); for (device_i = 0; device_i < program->num_devices; device_i++) { pocl_calculate_kernel_hash (program, j, device_i); } } errcode = CL_SUCCESS; goto FINISH; ERROR: free_meta (program); program->kernels = NULL; for (device_i = 0; device_i < program->num_devices; device_i++) { if (program->source) { POCL_MEM_FREE (program->binaries[device_i]); program->binary_sizes[device_i] = 0; } } ERROR_CLEAN_OPTIONS: if (temp_options != options) free (temp_options); program->build_status = CL_BUILD_ERROR; FINISH: POCL_UNLOCK_OBJ (program); POCL_MEM_FREE (unique_devlist); PFN_NOTIFY: if (pfn_notify) pfn_notify (program, user_data); return errcode; }
extern cl_int pocl_check_device_supports_image(const cl_mem image, const cl_command_queue command_queue) { cl_uint num_entries; cl_int errcode; const cl_device_id device = command_queue->device; cl_image_format* supported_image_formats = NULL; unsigned i; POCL_RETURN_ERROR_ON((!device->image_support), CL_INVALID_OPERATION, "Device does not support images"); if (image->type == CL_MEM_OBJECT_IMAGE1D || image->type == CL_MEM_OBJECT_IMAGE1D_ARRAY) { POCL_RETURN_ERROR_ON((image->image_width > device->image2d_max_width), CL_INVALID_IMAGE_SIZE, "Image width > device.image2d_max_width"); } if (image->type == CL_MEM_OBJECT_IMAGE2D || image->type == CL_MEM_OBJECT_IMAGE2D_ARRAY) { POCL_RETURN_ERROR_ON((image->image_width > device->image2d_max_width), CL_INVALID_IMAGE_SIZE, "Image width > device.image2d_max_width"); POCL_RETURN_ERROR_ON((image->image_height > device->image2d_max_height), CL_INVALID_IMAGE_SIZE, "Image height > device.image2d_max_height"); } if (image->type == CL_MEM_OBJECT_IMAGE3D) { POCL_RETURN_ERROR_ON((image->image_width > device->image3d_max_width), CL_INVALID_IMAGE_SIZE, "Image width > device.image3d_max_width"); POCL_RETURN_ERROR_ON((image->image_height > device->image3d_max_height), CL_INVALID_IMAGE_SIZE, "Image height > device.image3d_max_height"); POCL_RETURN_ERROR_ON((image->image_depth > device->image3d_max_depth), CL_INVALID_IMAGE_SIZE, "Image depth > device.image3d_max_depth"); } /* check if image format is supported */ errcode = POname(clGetSupportedImageFormats) (command_queue->context, 0, image->type, 0, NULL, &num_entries); POCL_RETURN_ERROR_ON((errcode != CL_SUCCESS), errcode, "clGetSupportedImageFormats call failed"); POCL_RETURN_ERROR_ON((num_entries == 0), errcode, "This device does not support these images " "(clGetSupportedImageFormats returned 0 entries)"); supported_image_formats = (cl_image_format*) malloc (num_entries * sizeof(cl_image_format)); if (supported_image_formats == NULL) return CL_OUT_OF_HOST_MEMORY; errcode = POname(clGetSupportedImageFormats) (command_queue->context, 0, image->type, num_entries, supported_image_formats, NULL); POCL_GOTO_ERROR_ON((errcode != CL_SUCCESS), errcode, "2nd call of clGetSupportedImageFormats failed"); for (i = 0; i < num_entries; i++) { if (supported_image_formats[i].image_channel_order == image->image_channel_order && supported_image_formats[i].image_channel_data_type == image->image_channel_data_type) { errcode = CL_SUCCESS; goto ERROR; } } POCL_GOTO_ERROR_ON(1, CL_INVALID_IMAGE_FORMAT_DESCRIPTOR, "The image format is not supported by the device"); ERROR: free(supported_image_formats); return errcode; }