static struct drm_i915_gem_object * _i915_gem_object_create_stolen(struct drm_i915_private *dev_priv, struct drm_mm_node *stolen) { struct drm_i915_gem_object *obj; unsigned int cache_level; obj = i915_gem_object_alloc(dev_priv); if (obj == NULL) return NULL; drm_gem_private_object_init(&dev_priv->drm, &obj->base, stolen->size); i915_gem_object_init(obj, &i915_gem_object_stolen_ops); obj->stolen = stolen; obj->read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT; cache_level = HAS_LLC(dev_priv) ? I915_CACHE_LLC : I915_CACHE_NONE; i915_gem_object_set_cache_coherency(obj, cache_level); if (i915_gem_object_pin_pages(obj)) goto cleanup; return obj; cleanup: i915_gem_object_free(obj); return NULL; }
static struct drm_i915_gem_object * _i915_gem_object_create_stolen(struct drm_device *dev, struct drm_mm_node *stolen) { struct drm_i915_gem_object *obj; obj = i915_gem_object_alloc(dev); if (obj == NULL) return NULL; drm_gem_private_object_init(dev, &obj->base, stolen->size); i915_gem_object_init(obj, &i915_gem_object_stolen_ops); obj->pages = i915_pages_create_for_stolen(dev, stolen->start, stolen->size); if (obj->pages == NULL) goto cleanup; obj->has_dma_mapping = true; i915_gem_object_pin_pages(obj); obj->stolen = stolen; obj->base.read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT; obj->cache_level = HAS_LLC(dev) ? I915_CACHE_LLC : I915_CACHE_NONE; return obj; cleanup: i915_gem_object_free(obj); return NULL; }
static struct drm_i915_gem_object * _kos_fb_object_create(struct drm_device *dev, struct drm_mm_node *fb_node) { struct drm_i915_gem_object *obj; obj = i915_gem_object_alloc(dev); if (obj == NULL) return NULL; drm_gem_private_object_init(dev, &obj->base, fb_node->size); i915_gem_object_init(obj, &kos_fb_object_ops); obj->pages = i915_pages_create_for_fb(dev, fb_node->start, fb_node->size); if (obj->pages == NULL) goto cleanup; obj->has_dma_mapping = true; i915_gem_object_pin_pages(obj); obj->stolen = fb_node; obj->base.read_domains = I915_GEM_DOMAIN_GTT; obj->cache_level = I915_CACHE_NONE; obj->tiling_mode = I915_TILING_X; return obj; cleanup: i915_gem_object_free(obj); return NULL; }
static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachment, enum dma_data_direction dir) { struct drm_i915_gem_object *obj = dma_buf_to_obj(attachment->dmabuf); struct sg_table *st; struct scatterlist *src, *dst; int ret, i; ret = i915_mutex_lock_interruptible(obj->base.dev); if (ret) goto err; ret = i915_gem_object_get_pages(obj); if (ret) goto err_unlock; i915_gem_object_pin_pages(obj); /* Copy sg so that we make an independent mapping */ st = kmalloc(sizeof(struct sg_table), GFP_KERNEL); if (st == NULL) { ret = -ENOMEM; goto err_unpin; } ret = sg_alloc_table(st, obj->pages->nents, GFP_KERNEL); if (ret) goto err_free; src = obj->pages->sgl; dst = st->sgl; for (i = 0; i < obj->pages->nents; i++) { sg_set_page(dst, sg_page(src), src->length, 0); dst = sg_next(dst); src = sg_next(src); } if (!dma_map_sg(attachment->dev, st->sgl, st->nents, dir)) { ret =-ENOMEM; goto err_free_sg; } mutex_unlock(&obj->base.dev->struct_mutex); return st; err_free_sg: sg_free_table(st); err_free: kfree(st); err_unpin: i915_gem_object_unpin_pages(obj); err_unlock: mutex_unlock(&obj->base.dev->struct_mutex); err: return ERR_PTR(ret); }
/** * i915_gem_batch_pool_get() - allocate a buffer from the pool * @pool: the batch buffer pool * @size: the minimum desired size of the returned buffer * * Returns an inactive buffer from @pool with at least @size bytes, * with the pages pinned. The caller must i915_gem_object_unpin_pages() * on the returned object. * * Note: Callers must hold the struct_mutex * * Return: the buffer object or an error pointer */ struct drm_i915_gem_object * i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, size_t size) { struct drm_i915_gem_object *obj = NULL; struct drm_i915_gem_object *tmp, *next; struct list_head *list; int n; WARN_ON(!mutex_is_locked(&pool->dev->struct_mutex)); /* Compute a power-of-two bucket, but throw everything greater than * 16KiB into the same bucket: i.e. the the buckets hold objects of * (1 page, 2 pages, 4 pages, 8+ pages). */ n = fls(size >> PAGE_SHIFT) - 1; if (n >= ARRAY_SIZE(pool->cache_list)) n = ARRAY_SIZE(pool->cache_list) - 1; list = &pool->cache_list[n]; list_for_each_entry_safe(tmp, next, list, batch_pool_link) { /* The batches are strictly LRU ordered */ if (tmp->active) break; /* While we're looping, do some clean up */ if (tmp->madv == __I915_MADV_PURGED) { list_del(&tmp->batch_pool_link); drm_gem_object_unreference(&tmp->base); continue; } if (tmp->base.size >= size) { obj = tmp; break; } } if (obj == NULL) { int ret; obj = i915_gem_alloc_object(pool->dev, size); if (obj == NULL) return ERR_PTR(-ENOMEM); ret = i915_gem_object_get_pages(obj); if (ret) return ERR_PTR(ret); obj->madv = I915_MADV_DONTNEED; } list_move_tail(&obj->batch_pool_link, list); i915_gem_object_pin_pages(obj); return obj; }
static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachment, enum dma_data_direction dir) { struct drm_i915_gem_object *obj = attachment->dmabuf->priv; struct sg_table *st; struct scatterlist *src, *dst; int ret, i; ret = i915_mutex_lock_interruptible(obj->base.dev); if (ret) return ERR_PTR(ret); ret = i915_gem_object_get_pages(obj); if (ret) { st = ERR_PTR(ret); goto out; } /* Copy sg so that we make an independent mapping */ st = kmalloc(sizeof(struct sg_table), GFP_KERNEL); if (st == NULL) { st = ERR_PTR(-ENOMEM); goto out; } ret = sg_alloc_table(st, obj->pages->nents, GFP_KERNEL); if (ret) { kfree(st); st = ERR_PTR(ret); goto out; } src = obj->pages->sgl; dst = st->sgl; for (i = 0; i < obj->pages->nents; i++) { sg_set_page(dst, sg_page(src), PAGE_SIZE, 0); dst = sg_next(dst); src = sg_next(src); } if (!dma_map_sg(attachment->dev, st->sgl, st->nents, dir)) { sg_free_table(st); kfree(st); st = ERR_PTR(-ENOMEM); goto out; } i915_gem_object_pin_pages(obj); out: mutex_unlock(&obj->base.dev->struct_mutex); return st; }
/** * i915_gem_batch_pool_get() - allocate a buffer from the pool * @pool: the batch buffer pool * @size: the minimum desired size of the returned buffer * * Returns an inactive buffer from @pool with at least @size bytes, * with the pages pinned. The caller must i915_gem_object_unpin_pages() * on the returned object. * * Note: Callers must hold the struct_mutex * * Return: the buffer object or an error pointer */ struct drm_i915_gem_object * i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, size_t size) { struct drm_i915_gem_object *obj = NULL; struct drm_i915_gem_object *tmp; struct list_head *list; int n, ret; lockdep_assert_held(&pool->engine->i915->drm.struct_mutex); /* Compute a power-of-two bucket, but throw everything greater than * 16KiB into the same bucket: i.e. the the buckets hold objects of * (1 page, 2 pages, 4 pages, 8+ pages). */ n = fls(size >> PAGE_SHIFT) - 1; if (n >= ARRAY_SIZE(pool->cache_list)) n = ARRAY_SIZE(pool->cache_list) - 1; list = &pool->cache_list[n]; list_for_each_entry(tmp, list, batch_pool_link) { /* The batches are strictly LRU ordered */ if (i915_gem_object_is_active(tmp)) break; GEM_BUG_ON(!reservation_object_test_signaled_rcu(tmp->resv, true)); if (tmp->base.size >= size) { /* Clear the set of shared fences early */ ww_mutex_lock(&tmp->resv->lock, NULL); reservation_object_add_excl_fence(tmp->resv, NULL); ww_mutex_unlock(&tmp->resv->lock); obj = tmp; break; } } if (obj == NULL) { obj = i915_gem_object_create_internal(pool->engine->i915, size); if (IS_ERR(obj)) return obj; } ret = i915_gem_object_pin_pages(obj); if (ret) return ERR_PTR(ret); list_move_tail(&obj->batch_pool_link, list); return obj; }
static void *i915_gem_dmabuf_vmap(struct dma_buf *dma_buf) { struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf); struct drm_device *dev = obj->base.dev; struct sg_page_iter sg_iter; struct page **pages; int ret, i; ret = i915_mutex_lock_interruptible(dev); if (ret) return ERR_PTR(ret); if (obj->dma_buf_vmapping) { obj->vmapping_count++; goto out_unlock; } ret = i915_gem_object_get_pages(obj); if (ret) goto err; i915_gem_object_pin_pages(obj); ret = -ENOMEM; pages = drm_malloc_ab(obj->base.size >> PAGE_SHIFT, sizeof(*pages)); if (pages == NULL) goto err_unpin; i = 0; for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) pages[i++] = sg_page_iter_page(&sg_iter); obj->dma_buf_vmapping = vmap(pages, i, 0, PAGE_KERNEL); drm_free_large(pages); if (!obj->dma_buf_vmapping) goto err_unpin; obj->vmapping_count = 1; out_unlock: mutex_unlock(&dev->struct_mutex); return obj->dma_buf_vmapping; err_unpin: i915_gem_object_unpin_pages(obj); err: mutex_unlock(&dev->struct_mutex); return ERR_PTR(ret); }
static int igt_dmabuf_import_ownership(void *arg) { struct drm_i915_private *i915 = arg; struct drm_i915_gem_object *obj; struct dma_buf *dmabuf; void *ptr; int err; dmabuf = mock_dmabuf(1); if (IS_ERR(dmabuf)) return PTR_ERR(dmabuf); ptr = dma_buf_vmap(dmabuf); if (!ptr) { pr_err("dma_buf_vmap failed\n"); err = -ENOMEM; goto err_dmabuf; } memset(ptr, 0xc5, PAGE_SIZE); dma_buf_vunmap(dmabuf, ptr); obj = to_intel_bo(i915_gem_prime_import(&i915->drm, dmabuf)); if (IS_ERR(obj)) { pr_err("i915_gem_prime_import failed with err=%d\n", (int)PTR_ERR(obj)); err = PTR_ERR(obj); goto err_dmabuf; } dma_buf_put(dmabuf); err = i915_gem_object_pin_pages(obj); if (err) { pr_err("i915_gem_object_pin_pages failed with err=%d\n", err); goto out_obj; } err = 0; i915_gem_object_unpin_pages(obj); out_obj: i915_gem_object_put(obj); return err; err_dmabuf: dma_buf_put(dmabuf); return err; }
static void *i915_gem_dmabuf_vmap(struct dma_buf *dma_buf) { struct drm_i915_gem_object *obj = dma_buf->priv; struct drm_device *dev = obj->base.dev; struct scatterlist *sg; struct page **pages; int ret, i; ret = i915_mutex_lock_interruptible(dev); if (ret) return ERR_PTR(ret); if (obj->dma_buf_vmapping) { obj->vmapping_count++; goto out_unlock; } ret = i915_gem_object_get_pages(obj); if (ret) goto error; ret = -ENOMEM; pages = drm_malloc_ab(obj->pages->nents, sizeof(struct page *)); if (pages == NULL) goto error; for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) pages[i] = sg_page(sg); obj->dma_buf_vmapping = vmap(pages, obj->pages->nents, 0, PAGE_KERNEL); drm_free_large(pages); if (!obj->dma_buf_vmapping) goto error; obj->vmapping_count = 1; i915_gem_object_pin_pages(obj); out_unlock: mutex_unlock(&dev->struct_mutex); return obj->dma_buf_vmapping; error: mutex_unlock(&dev->struct_mutex); return ERR_PTR(ret); }
static void i915_clflush_work(struct work_struct *work) { struct clflush *clflush = container_of(work, typeof(*clflush), work); struct drm_i915_gem_object *obj = clflush->obj; if (i915_gem_object_pin_pages(obj)) { DRM_ERROR("Failed to acquire obj->pages for clflushing\n"); goto out; } __i915_do_clflush(obj); i915_gem_object_unpin_pages(obj); out: i915_gem_object_put(obj); dma_fence_signal(&clflush->dma); dma_fence_put(&clflush->dma); }
static int igt_gem_huge(void *arg) { const unsigned int nreal = 509; /* just to be awkward */ struct drm_i915_private *i915 = arg; struct drm_i915_gem_object *obj; unsigned int n; int err; /* Basic sanitycheck of our huge fake object allocation */ obj = huge_gem_object(i915, nreal * PAGE_SIZE, i915->ggtt.vm.total + PAGE_SIZE); if (IS_ERR(obj)) return PTR_ERR(obj); err = i915_gem_object_pin_pages(obj); if (err) { pr_err("Failed to allocate %u pages (%lu total), err=%d\n", nreal, obj->base.size / PAGE_SIZE, err); goto out; } for (n = 0; n < obj->base.size / PAGE_SIZE; n++) { if (i915_gem_object_get_page(obj, n) != i915_gem_object_get_page(obj, n % nreal)) { pr_err("Page lookup mismatch at index %u [%u]\n", n, n % nreal); err = -EINVAL; goto out_unpin; } } out_unpin: i915_gem_object_unpin_pages(obj); out: i915_gem_object_put(obj); return err; }
/** * i915_gem_set_tiling - IOCTL handler to set tiling mode * @dev: DRM device * @data: data pointer for the ioctl * @file: DRM file for the ioctl call * * Sets the tiling mode of an object, returning the required swizzling of * bit 6 of addresses in the object. * * Called by the user via ioctl. * * Returns: * Zero on success, negative errno on failure. */ int i915_gem_set_tiling(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_set_tiling *args = data; struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_object *obj; int ret = 0; obj = to_intel_bo(drm_gem_object_lookup(file, args->handle)); if (&obj->base == NULL) return -ENOENT; if (!i915_tiling_ok(dev, args->stride, obj->base.size, args->tiling_mode)) { drm_gem_object_unreference_unlocked(&obj->base); return -EINVAL; } intel_runtime_pm_get(dev_priv); mutex_lock(&dev->struct_mutex); if (obj->pin_display || obj->framebuffer_references) { ret = -EBUSY; goto err; } if (args->tiling_mode == I915_TILING_NONE) { args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE; args->stride = 0; } else { if (args->tiling_mode == I915_TILING_X) args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x; else args->swizzle_mode = dev_priv->mm.bit_6_swizzle_y; /* Hide bit 17 swizzling from the user. This prevents old Mesa * from aborting the application on sw fallbacks to bit 17, * and we use the pread/pwrite bit17 paths to swizzle for it. * If there was a user that was relying on the swizzle * information for drm_intel_bo_map()ed reads/writes this would * break it, but we don't have any of those. */ if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17) args->swizzle_mode = I915_BIT_6_SWIZZLE_9; if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17) args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10; /* If we can't handle the swizzling, make it untiled. */ if (args->swizzle_mode == I915_BIT_6_SWIZZLE_UNKNOWN) { args->tiling_mode = I915_TILING_NONE; args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE; args->stride = 0; } } if (args->tiling_mode != obj->tiling_mode || args->stride != obj->stride) { /* We need to rebind the object if its current allocation * no longer meets the alignment restrictions for its new * tiling mode. Otherwise we can just leave it alone, but * need to ensure that any fence register is updated before * the next fenced (either through the GTT or by the BLT unit * on older GPUs) access. * * After updating the tiling parameters, we then flag whether * we need to update an associated fence register. Note this * has to also include the unfenced register the GPU uses * whilst executing a fenced command for an untiled object. */ if (obj->map_and_fenceable && !i915_gem_object_fence_ok(obj, args->tiling_mode)) ret = i915_vma_unbind(i915_gem_obj_to_ggtt(obj)); if (ret == 0) { if (obj->pages && obj->madv == I915_MADV_WILLNEED && dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) { if (args->tiling_mode == I915_TILING_NONE) i915_gem_object_unpin_pages(obj); if (obj->tiling_mode == I915_TILING_NONE) i915_gem_object_pin_pages(obj); } obj->fence_dirty = obj->last_fenced_req || obj->fence_reg != I915_FENCE_REG_NONE; obj->tiling_mode = args->tiling_mode; obj->stride = args->stride; /* Force the fence to be reacquired for GTT access */ i915_gem_release_mmap(obj); } } /* we have to maintain this existing ABI... */ args->stride = obj->stride; args->tiling_mode = obj->tiling_mode; /* Try to preallocate memory required to save swizzling on put-pages */ if (i915_gem_object_needs_bit17_swizzle(obj)) { if (obj->bit_17 == NULL) { obj->bit_17 = kcalloc(BITS_TO_LONGS(obj->base.size >> PAGE_SHIFT), sizeof(long), GFP_KERNEL); } } else {
/** * i915_gem_batch_pool_get() - allocate a buffer from the pool * @pool: the batch buffer pool * @size: the minimum desired size of the returned buffer * * Returns an inactive buffer from @pool with at least @size bytes, * with the pages pinned. The caller must i915_gem_object_unpin_pages() * on the returned object. * * Note: Callers must hold the struct_mutex * * Return: the buffer object or an error pointer */ struct drm_i915_gem_object * i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, size_t size) { struct drm_i915_gem_object *obj; struct list_head *list; int n, ret; lockdep_assert_held(&pool->engine->i915->drm.struct_mutex); /* Compute a power-of-two bucket, but throw everything greater than * 16KiB into the same bucket: i.e. the the buckets hold objects of * (1 page, 2 pages, 4 pages, 8+ pages). */ n = fls(size >> PAGE_SHIFT) - 1; if (n >= ARRAY_SIZE(pool->cache_list)) n = ARRAY_SIZE(pool->cache_list) - 1; list = &pool->cache_list[n]; list_for_each_entry(obj, list, batch_pool_link) { /* The batches are strictly LRU ordered */ if (i915_gem_object_is_active(obj)) { struct reservation_object *resv = obj->resv; if (!reservation_object_test_signaled_rcu(resv, true)) break; i915_retire_requests(pool->engine->i915); GEM_BUG_ON(i915_gem_object_is_active(obj)); /* * The object is now idle, clear the array of shared * fences before we add a new request. Although, we * remain on the same engine, we may be on a different * timeline and so may continually grow the array, * trapping a reference to all the old fences, rather * than replace the existing fence. */ if (rcu_access_pointer(resv->fence)) { reservation_object_lock(resv, NULL); reservation_object_add_excl_fence(resv, NULL); reservation_object_unlock(resv); } } GEM_BUG_ON(!reservation_object_test_signaled_rcu(obj->resv, true)); if (obj->base.size >= size) goto found; } obj = i915_gem_object_create_internal(pool->engine->i915, size); if (IS_ERR(obj)) return obj; found: ret = i915_gem_object_pin_pages(obj); if (ret) return ERR_PTR(ret); list_move_tail(&obj->batch_pool_link, list); return obj; }
struct drm_i915_gem_object * i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv, resource_size_t stolen_offset, resource_size_t gtt_offset, resource_size_t size) { struct i915_ggtt *ggtt = &dev_priv->ggtt; struct drm_i915_gem_object *obj; struct drm_mm_node *stolen; struct i915_vma *vma; int ret; if (!drm_mm_initialized(&dev_priv->mm.stolen)) return NULL; lockdep_assert_held(&dev_priv->drm.struct_mutex); DRM_DEBUG_DRIVER("creating preallocated stolen object: stolen_offset=%pa, gtt_offset=%pa, size=%pa\n", &stolen_offset, >t_offset, &size); /* KISS and expect everything to be page-aligned */ if (WARN_ON(size == 0) || WARN_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)) || WARN_ON(!IS_ALIGNED(stolen_offset, I915_GTT_MIN_ALIGNMENT))) return NULL; stolen = kzalloc(sizeof(*stolen), GFP_KERNEL); if (!stolen) return NULL; stolen->start = stolen_offset; stolen->size = size; mutex_lock(&dev_priv->mm.stolen_lock); ret = drm_mm_reserve_node(&dev_priv->mm.stolen, stolen); mutex_unlock(&dev_priv->mm.stolen_lock); if (ret) { DRM_DEBUG_DRIVER("failed to allocate stolen space\n"); kfree(stolen); return NULL; } obj = _i915_gem_object_create_stolen(dev_priv, stolen); if (obj == NULL) { DRM_DEBUG_DRIVER("failed to allocate stolen object\n"); i915_gem_stolen_remove_node(dev_priv, stolen); kfree(stolen); return NULL; } /* Some objects just need physical mem from stolen space */ if (gtt_offset == I915_GTT_OFFSET_NONE) return obj; ret = i915_gem_object_pin_pages(obj); if (ret) goto err; vma = i915_vma_instance(obj, &ggtt->vm, NULL); if (IS_ERR(vma)) { ret = PTR_ERR(vma); goto err_pages; } /* To simplify the initialisation sequence between KMS and GTT, * we allow construction of the stolen object prior to * setting up the GTT space. The actual reservation will occur * later. */ ret = i915_gem_gtt_reserve(&ggtt->vm, &vma->node, size, gtt_offset, obj->cache_level, 0); if (ret) { DRM_DEBUG_DRIVER("failed to allocate stolen GTT space\n"); goto err_pages; } GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); vma->pages = obj->mm.pages; vma->flags |= I915_VMA_GLOBAL_BIND; __i915_vma_set_map_and_fenceable(vma); mutex_lock(&ggtt->vm.mutex); list_move_tail(&vma->vm_link, &ggtt->vm.bound_list); mutex_unlock(&ggtt->vm.mutex); spin_lock(&dev_priv->mm.obj_lock); list_move_tail(&obj->mm.link, &dev_priv->mm.bound_list); obj->bind_count++; spin_unlock(&dev_priv->mm.obj_lock); return obj; err_pages: i915_gem_object_unpin_pages(obj); err: i915_gem_object_put(obj); return NULL; }
/** * i915_vma_insert - finds a slot for the vma in its address space * @vma: the vma * @size: requested size in bytes (can be larger than the VMA) * @alignment: required alignment * @flags: mask of PIN_* flags to use * * First we try to allocate some free space that meets the requirements for * the VMA. Failiing that, if the flags permit, it will evict an old VMA, * preferrably the oldest idle entry to make room for the new VMA. * * Returns: * 0 on success, negative error code otherwise. */ static int i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) { struct drm_i915_private *dev_priv = to_i915(vma->vm->dev); struct drm_i915_gem_object *obj = vma->obj; u64 start, end; int ret; GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND)); GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); size = max(size, vma->size); if (flags & PIN_MAPPABLE) size = i915_gem_get_ggtt_size(dev_priv, size, i915_gem_object_get_tiling(obj)); alignment = max(max(alignment, vma->display_alignment), i915_gem_get_ggtt_alignment(dev_priv, size, i915_gem_object_get_tiling(obj), flags & PIN_MAPPABLE)); start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0; end = vma->vm->total; if (flags & PIN_MAPPABLE) end = min_t(u64, end, dev_priv->ggtt.mappable_end); if (flags & PIN_ZONE_4G) end = min_t(u64, end, (1ULL << 32) - PAGE_SIZE); /* If binding the object/GGTT view requires more space than the entire * aperture has, reject it early before evicting everything in a vain * attempt to find space. */ if (size > end) { DRM_DEBUG("Attempting to bind an object larger than the aperture: request=%llu [object=%zd] > %s aperture=%llu\n", size, obj->base.size, flags & PIN_MAPPABLE ? "mappable" : "total", end); return -E2BIG; } ret = i915_gem_object_pin_pages(obj); if (ret) return ret; if (flags & PIN_OFFSET_FIXED) { u64 offset = flags & PIN_OFFSET_MASK; if (offset & (alignment - 1) || offset > end - size) { ret = -EINVAL; goto err_unpin; } vma->node.start = offset; vma->node.size = size; vma->node.color = obj->cache_level; ret = drm_mm_reserve_node(&vma->vm->mm, &vma->node); if (ret) { ret = i915_gem_evict_for_vma(vma); if (ret == 0) ret = drm_mm_reserve_node(&vma->vm->mm, &vma->node); if (ret) goto err_unpin; } } else { u32 search_flag, alloc_flag; if (flags & PIN_HIGH) { search_flag = DRM_MM_SEARCH_BELOW; alloc_flag = DRM_MM_CREATE_TOP; } else { search_flag = DRM_MM_SEARCH_DEFAULT; alloc_flag = DRM_MM_CREATE_DEFAULT; } /* We only allocate in PAGE_SIZE/GTT_PAGE_SIZE (4096) chunks, * so we know that we always have a minimum alignment of 4096. * The drm_mm range manager is optimised to return results * with zero alignment, so where possible use the optimal * path. */ if (alignment <= 4096) alignment = 0; search_free: ret = drm_mm_insert_node_in_range_generic(&vma->vm->mm, &vma->node, size, alignment, obj->cache_level, start, end, search_flag, alloc_flag); if (ret) { ret = i915_gem_evict_something(vma->vm, size, alignment, obj->cache_level, start, end, flags); if (ret == 0) goto search_free; goto err_unpin; } GEM_BUG_ON(vma->node.start < start); GEM_BUG_ON(vma->node.start + vma->node.size > end); } GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level)); list_move_tail(&obj->global_link, &dev_priv->mm.bound_list); list_move_tail(&vma->vm_link, &vma->vm->inactive_list); obj->bind_count++; GEM_BUG_ON(atomic_read(&obj->mm.pages_pin_count) < obj->bind_count); return 0; err_unpin: i915_gem_object_unpin_pages(obj); return ret; }