/** * i915_gem_object_create_internal: create an object with volatile pages * @i915: the i915 device * @size: the size in bytes of backing storage to allocate for the object * * Creates a new object that wraps some internal memory for private use. * This object is not backed by swappable storage, and as such its contents * are volatile and only valid whilst pinned. If the object is reaped by the * shrinker, its pages and data will be discarded. Equally, it is not a full * GEM object and so not valid for access from userspace. This makes it useful * for hardware interfaces like ringbuffers (which are pinned from the time * the request is written to the time the hardware stops accessing it), but * not for contexts (which need to be preserved when not active for later * reuse). Note that it is not cleared upon allocation. */ struct drm_i915_gem_object * i915_gem_object_create_internal(struct drm_i915_private *i915, phys_addr_t size) { struct drm_i915_gem_object *obj; unsigned int cache_level; GEM_BUG_ON(!size); GEM_BUG_ON(!IS_ALIGNED(size, PAGE_SIZE)); if (overflows_type(size, obj->base.size)) return ERR_PTR(-E2BIG); obj = i915_gem_object_alloc(); if (!obj) return ERR_PTR(-ENOMEM); drm_gem_private_object_init(&i915->drm, &obj->base, size); i915_gem_object_init(obj, &i915_gem_object_internal_ops); obj->read_domains = I915_GEM_DOMAIN_CPU; obj->write_domain = I915_GEM_DOMAIN_CPU; cache_level = HAS_LLC(i915) ? I915_CACHE_LLC : I915_CACHE_NONE; i915_gem_object_set_cache_coherency(obj, cache_level); return obj; }
static struct drm_i915_gem_object * _i915_gem_object_create_stolen(struct drm_device *dev, struct drm_mm_node *stolen) { struct drm_i915_gem_object *obj; obj = i915_gem_object_alloc(dev); if (obj == NULL) return NULL; drm_gem_private_object_init(dev, &obj->base, stolen->size); i915_gem_object_init(obj, &i915_gem_object_stolen_ops); obj->pages = i915_pages_create_for_stolen(dev, stolen->start, stolen->size); if (obj->pages == NULL) goto cleanup; obj->has_dma_mapping = true; i915_gem_object_pin_pages(obj); obj->stolen = stolen; obj->base.read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT; obj->cache_level = HAS_LLC(dev) ? I915_CACHE_LLC : I915_CACHE_NONE; return obj; cleanup: i915_gem_object_free(obj); return NULL; }
static struct drm_i915_gem_object * _i915_gem_object_create_stolen(struct drm_i915_private *dev_priv, struct drm_mm_node *stolen) { struct drm_i915_gem_object *obj; unsigned int cache_level; obj = i915_gem_object_alloc(dev_priv); if (obj == NULL) return NULL; drm_gem_private_object_init(&dev_priv->drm, &obj->base, stolen->size); i915_gem_object_init(obj, &i915_gem_object_stolen_ops); obj->stolen = stolen; obj->read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT; cache_level = HAS_LLC(dev_priv) ? I915_CACHE_LLC : I915_CACHE_NONE; i915_gem_object_set_cache_coherency(obj, cache_level); if (i915_gem_object_pin_pages(obj)) goto cleanup; return obj; cleanup: i915_gem_object_free(obj); return NULL; }
struct drm_i915_gem_object * huge_gem_object(struct drm_i915_private *i915, phys_addr_t phys_size, dma_addr_t dma_size) { struct drm_i915_gem_object *obj; GEM_BUG_ON(!phys_size || phys_size > dma_size); GEM_BUG_ON(!IS_ALIGNED(phys_size, PAGE_SIZE)); GEM_BUG_ON(!IS_ALIGNED(dma_size, I915_GTT_PAGE_SIZE)); if (overflows_type(dma_size, obj->base.size)) return ERR_PTR(-E2BIG); obj = i915_gem_object_alloc(i915); if (!obj) return ERR_PTR(-ENOMEM); drm_gem_private_object_init(&i915->drm, &obj->base, dma_size); i915_gem_object_init(obj, &huge_ops); obj->base.write_domain = I915_GEM_DOMAIN_CPU; obj->base.read_domains = I915_GEM_DOMAIN_CPU; obj->cache_level = HAS_LLC(i915) ? I915_CACHE_LLC : I915_CACHE_NONE; obj->scratch = phys_size; return obj; }
static void gen9_gt_workarounds_apply(struct drm_i915_private *dev_priv) { /* WaContextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */ I915_WRITE(GEN9_CSFE_CHICKEN1_RCS, _MASKED_BIT_ENABLE(GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE)); /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */ I915_WRITE(BDW_SCRATCH1, I915_READ(BDW_SCRATCH1) | GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE); /* WaDisableKillLogic:bxt,skl,kbl */ if (!IS_COFFEELAKE(dev_priv)) I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | ECOCHK_DIS_TLB); if (HAS_LLC(dev_priv)) { /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl * * Must match Display Engine. See * WaCompressedResourceDisplayNewHashMode. */ I915_WRITE(MMCD_MISC_CTRL, I915_READ(MMCD_MISC_CTRL) | MMCD_PCLA | MMCD_HOTSPOT_EN); } /* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */ I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | BDW_DISABLE_HDC_INVALIDATION); /* WaProgramL3SqcReg1DefaultForPerf:bxt,glk */ if (IS_GEN9_LP(dev_priv)) { u32 val = I915_READ(GEN8_L3SQCREG1); val &= ~L3_PRIO_CREDITS_MASK; val |= L3_GENERAL_PRIO_CREDITS(62) | L3_HIGH_PRIO_CREDITS(2); I915_WRITE(GEN8_L3SQCREG1, val); } /* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */ I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) | GEN8_LQSC_FLUSH_COHERENT_LINES); /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */ I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1, _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL)); }
/** * Creates a new object that wraps some internal memory for private use. * This object is not backed by swappable storage, and as such its contents * are volatile and only valid whilst pinned. If the object is reaped by the * shrinker, its pages and data will be discarded. Equally, it is not a full * GEM object and so not valid for access from userspace. This makes it useful * for hardware interfaces like ringbuffers (which are pinned from the time * the request is written to the time the hardware stops accessing it), but * not for contexts (which need to be preserved when not active for later * reuse). Note that it is not cleared upon allocation. */ struct drm_i915_gem_object * i915_gem_object_create_internal(struct drm_i915_private *i915, unsigned int size) { struct drm_i915_gem_object *obj; obj = i915_gem_object_alloc(&i915->drm); if (!obj) return ERR_PTR(-ENOMEM); drm_gem_private_object_init(&i915->drm, &obj->base, size); i915_gem_object_init(obj, &i915_gem_object_internal_ops); obj->base.write_domain = I915_GEM_DOMAIN_CPU; obj->base.read_domains = I915_GEM_DOMAIN_CPU; obj->cache_level = HAS_LLC(i915) ? I915_CACHE_LLC : I915_CACHE_NONE; return obj; }
static int gen9_ctx_workarounds_init(struct drm_i915_private *dev_priv) { if (HAS_LLC(dev_priv)) { /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl * * Must match Display Engine. See * WaCompressedResourceDisplayNewHashMode. */ WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2, GEN9_PBE_COMPRESSED_HASH_SELECTION); WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7, GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR); } /* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */ /* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */ WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, FLOW_CONTROL_ENABLE | PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE); /* Syncing dependencies between camera and graphics:skl,bxt,kbl */ if (!IS_COFFEELAKE(dev_priv)) WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC); /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */ /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */ WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7, GEN9_ENABLE_YV12_BUGFIX | GEN9_ENABLE_GPGPU_PREEMPTION); /* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */ /* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */ WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE | GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE); /* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */ WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5, GEN9_CCS_TLB_PREFETCH_ENABLE); /* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */ WA_SET_BIT_MASKED(HDC_CHICKEN0, HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT | HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE); /* WaForceEnableNonCoherent and WaDisableHDCInvalidation are * both tied to WaForceContextSaveRestoreNonCoherent * in some hsds for skl. We keep the tie for all gen9. The * documentation is a bit hazy and so we want to get common behaviour, * even though there is no clear evidence we would need both on kbl/bxt. * This area has been source of system hangs so we play it safe * and mimic the skl regardless of what bspec says. * * Use Force Non-Coherent whenever executing a 3D context. This * is a workaround for a possible hang in the unlikely event * a TLB invalidation occurs during a PSD flush. */ /* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */ WA_SET_BIT_MASKED(HDC_CHICKEN0, HDC_FORCE_NON_COHERENT); /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */ if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv) || IS_COFFEELAKE(dev_priv)) WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, GEN8_SAMPLER_POWER_BYPASS_DIS); /* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */ WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE); /* * Supporting preemption with fine-granularity requires changes in the * batch buffer programming. Since we can't break old userspace, we * need to set our default preemption level to safe value. Userspace is * still able to use more fine-grained preemption levels, since in * WaEnablePreemptionGranularityControlByUMD we're whitelisting the * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are * not real HW workarounds, but merely a way to start using preemption * while maintaining old contract with userspace. */ /* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */ WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL); /* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */ WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_GPGPU_LEVEL_MASK, GEN9_PREEMPT_GPGPU_COMMAND_LEVEL); /* WaClearHIZ_WM_CHICKEN3:bxt,glk */ if (IS_GEN9_LP(dev_priv)) WA_SET_BIT_MASKED(GEN9_WM_CHICKEN3, GEN9_FACTOR_IN_CLR_VAL_HIZ); return 0; }
static int i915_getparam(struct drm_device *dev, void *data, struct drm_file *file_priv) { struct drm_i915_private *dev_priv = dev->dev_private; drm_i915_getparam_t *param = data; int value; switch (param->param) { case I915_PARAM_IRQ_ACTIVE: case I915_PARAM_ALLOW_BATCHBUFFER: case I915_PARAM_LAST_DISPATCH: /* Reject all old ums/dri params. */ return -ENODEV; case I915_PARAM_CHIPSET_ID: value = dev->pdev->device; break; case I915_PARAM_HAS_GEM: value = 1; break; case I915_PARAM_NUM_FENCES_AVAIL: value = dev_priv->num_fence_regs - dev_priv->fence_reg_start; break; case I915_PARAM_HAS_OVERLAY: value = dev_priv->overlay ? 1 : 0; break; case I915_PARAM_HAS_PAGEFLIPPING: value = 1; break; case I915_PARAM_HAS_EXECBUF2: /* depends on GEM */ value = 1; break; case I915_PARAM_HAS_BSD: value = intel_ring_initialized(&dev_priv->ring[VCS]); break; case I915_PARAM_HAS_BLT: value = intel_ring_initialized(&dev_priv->ring[BCS]); break; case I915_PARAM_HAS_VEBOX: value = intel_ring_initialized(&dev_priv->ring[VECS]); break; case I915_PARAM_HAS_RELAXED_FENCING: value = 1; break; case I915_PARAM_HAS_COHERENT_RINGS: value = 1; break; case I915_PARAM_HAS_EXEC_CONSTANTS: value = INTEL_INFO(dev)->gen >= 4; break; case I915_PARAM_HAS_RELAXED_DELTA: value = 1; break; case I915_PARAM_HAS_GEN7_SOL_RESET: value = 1; break; case I915_PARAM_HAS_LLC: value = HAS_LLC(dev); break; case I915_PARAM_HAS_WT: value = HAS_WT(dev); break; case I915_PARAM_HAS_ALIASING_PPGTT: value = USES_PPGTT(dev); break; case I915_PARAM_HAS_WAIT_TIMEOUT: value = 1; break; case I915_PARAM_HAS_SEMAPHORES: value = i915_semaphore_is_enabled(dev); break; case I915_PARAM_HAS_PRIME_VMAP_FLUSH: value = 1; break; case I915_PARAM_HAS_SECURE_BATCHES: value = capable(CAP_SYS_ADMIN); break; case I915_PARAM_HAS_PINNED_BATCHES: value = 1; break; case I915_PARAM_HAS_EXEC_NO_RELOC: value = 1; break; case I915_PARAM_HAS_EXEC_HANDLE_LUT: value = 1; break; case I915_PARAM_CMD_PARSER_VERSION: value = i915_cmd_parser_get_version(); break; case I915_PARAM_HAS_COHERENT_PHYS_GTT: value = 1; break; default: DRM_DEBUG("Unknown parameter %d\n", param->param); return -EINVAL; } if (copy_to_user(param->value, &value, sizeof(int))) { DRM_ERROR("copy_to_user failed\n"); return -EFAULT; } return 0; }
/* * Creates a new mm object that wraps some normal memory from the process * context - user memory. * * We impose several restrictions upon the memory being mapped * into the GPU. * 1. It must be page aligned (both start/end addresses, i.e ptr and size). * 2. It must be normal system memory, not a pointer into another map of IO * space (e.g. it must not be a GTT mmapping of another object). * 3. We only allow a bo as large as we could in theory map into the GTT, * that is we limit the size to the total size of the GTT. * 4. The bo is marked as being snoopable. The backing pages are left * accessible directly by the CPU, but reads and writes by the GPU may * incur the cost of a snoop (unless you have an LLC architecture). * * Synchronisation between multiple users and the GPU is left to userspace * through the normal set-domain-ioctl. The kernel will enforce that the * GPU relinquishes the VMA before it is returned back to the system * i.e. upon free(), munmap() or process termination. However, the userspace * malloc() library may not immediately relinquish the VMA after free() and * instead reuse it whilst the GPU is still reading and writing to the VMA. * Caveat emptor. * * Also note, that the object created here is not currently a "first class" * object, in that several ioctls are banned. These are the CPU access * ioctls: mmap(), pwrite and pread. In practice, you are expected to use * direct access via your pointer rather than use those ioctls. Another * restriction is that we do not allow userptr surfaces to be pinned to the * hardware and so we reject any attempt to create a framebuffer out of a * userptr. * * If you think this is a good interface to use to pass GPU memory between * drivers, please use dma-buf instead. In fact, wherever possible use * dma-buf instead. */ int i915_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_userptr *args = data; struct drm_i915_gem_object *obj; int ret; u32 handle; if (!HAS_LLC(dev_priv) && !HAS_SNOOP(dev_priv)) { /* We cannot support coherent userptr objects on hw without * LLC and broken snooping. */ return -ENODEV; } if (args->flags & ~(I915_USERPTR_READ_ONLY | I915_USERPTR_UNSYNCHRONIZED)) return -EINVAL; if (!args->user_size) return -EINVAL; if (offset_in_page(args->user_ptr | args->user_size)) return -EINVAL; if (!access_ok((char __user *)(unsigned long)args->user_ptr, args->user_size)) return -EFAULT; if (args->flags & I915_USERPTR_READ_ONLY) { struct i915_hw_ppgtt *ppgtt; /* * On almost all of the older hw, we cannot tell the GPU that * a page is readonly. */ ppgtt = dev_priv->kernel_context->ppgtt; if (!ppgtt || !ppgtt->vm.has_read_only) return -ENODEV; } obj = i915_gem_object_alloc(); if (obj == NULL) return -ENOMEM; drm_gem_private_object_init(dev, &obj->base, args->user_size); i915_gem_object_init(obj, &i915_gem_userptr_ops); obj->read_domains = I915_GEM_DOMAIN_CPU; obj->write_domain = I915_GEM_DOMAIN_CPU; i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC); obj->userptr.ptr = args->user_ptr; if (args->flags & I915_USERPTR_READ_ONLY) i915_gem_object_set_readonly(obj); /* And keep a pointer to the current->mm for resolving the user pages * at binding. This means that we need to hook into the mmu_notifier * in order to detect if the mmu is destroyed. */ ret = i915_gem_userptr_init__mm_struct(obj); if (ret == 0) ret = i915_gem_userptr_init__mmu_notifier(obj, args->flags); if (ret == 0) ret = drm_gem_handle_create(file, &obj->base, &handle); /* drop reference from allocate - handle holds it now */ i915_gem_object_put(obj); if (ret) return ret; args->handle = handle; return 0; }
int i915_getparam(struct drm_device *dev, void *data, struct drm_file *file_priv) { drm_i915_private_t *dev_priv = dev->dev_private; drm_i915_getparam_t *param = data; int value; if (!dev_priv) { DRM_ERROR("called with no initialization\n"); return -EINVAL; } switch (param->param) { case I915_PARAM_CHIPSET_ID: value = dev->pci_device; break; case I915_PARAM_HAS_GEM: value = 1; break; case I915_PARAM_NUM_FENCES_AVAIL: value = dev_priv->num_fence_regs - dev_priv->fence_reg_start; break; case I915_PARAM_HAS_OVERLAY: value = dev_priv->overlay ? 1 : 0; break; case I915_PARAM_HAS_PAGEFLIPPING: value = 1; break; case I915_PARAM_HAS_EXECBUF2: /* depends on GEM */ value = 1; break; case I915_PARAM_HAS_BSD: value = intel_ring_initialized(&dev_priv->ring[VCS]); break; case I915_PARAM_HAS_BLT: value = intel_ring_initialized(&dev_priv->ring[BCS]); break; case I915_PARAM_HAS_RELAXED_FENCING: value = 1; break; case I915_PARAM_HAS_COHERENT_RINGS: value = 1; break; case I915_PARAM_HAS_EXEC_CONSTANTS: value = INTEL_INFO(dev)->gen >= 4; break; case I915_PARAM_HAS_RELAXED_DELTA: value = 1; break; case I915_PARAM_HAS_GEN7_SOL_RESET: value = 1; break; case I915_PARAM_HAS_LLC: value = HAS_LLC(dev); break; case I915_PARAM_HAS_WAIT_TIMEOUT: value = 1; break; case I915_PARAM_HAS_SEMAPHORES: value = i915_semaphore_is_enabled(dev); break; case I915_PARAM_HAS_SECURE_BATCHES: value = DRM_SUSER(curproc); break; case I915_PARAM_HAS_PINNED_BATCHES: value = 1; break; default: DRM_DEBUG_DRIVER("Unknown parameter %d\n", param->param); return -EINVAL; } return -copyout(&value, param->value, sizeof(int)); }
int i915_getparam(struct inteldrm_softc *dev_priv, void *data) { drm_i915_getparam_t *param = data; struct drm_device *dev = (struct drm_device *)dev_priv->drmdev; int value; switch (param->param) { case I915_PARAM_CHIPSET_ID: value = dev->pci_device; break; case I915_PARAM_HAS_GEM: value = 1; break; case I915_PARAM_NUM_FENCES_AVAIL: value = dev_priv->num_fence_regs - dev_priv->fence_reg_start; break; case I915_PARAM_HAS_EXECBUF2: value = 1; break; case I915_PARAM_HAS_OVERLAY: value = dev_priv->overlay ? 1 : 0; break; case I915_PARAM_HAS_PAGEFLIPPING: value = 1; break; case I915_PARAM_HAS_BSD: value = intel_ring_initialized(&dev_priv->ring[VCS]); break; case I915_PARAM_HAS_BLT: value = intel_ring_initialized(&dev_priv->ring[BCS]); break; case I915_PARAM_HAS_RELAXED_FENCING: #ifdef notyet value = 1; #else return EINVAL; #endif break; case I915_PARAM_HAS_COHERENT_RINGS: value = 1; break; case I915_PARAM_HAS_EXEC_CONSTANTS: value = INTEL_INFO(dev)->gen >= 4; break; case I915_PARAM_HAS_RELAXED_DELTA: value = 1; break; case I915_PARAM_HAS_GEN7_SOL_RESET: value = 1; break; case I915_PARAM_HAS_LLC: value = HAS_LLC(dev); break; case I915_PARAM_HAS_SEMAPHORES: value = i915_semaphore_is_enabled(dev); break; case I915_PARAM_HAS_SECURE_BATCHES: value = DRM_SUSER(curproc); break; case I915_PARAM_HAS_PINNED_BATCHES: value = 1; break; default: DRM_DEBUG("Unknown parameter %d\n", param->param); return (EINVAL); } return (copyout(&value, param->value, sizeof(int))); }