int msm_gem_map_vma(struct msm_gem_address_space *aspace, struct msm_gem_vma *vma, struct sg_table *sgt, int npages) { int ret; spin_lock(&aspace->lock); if (WARN_ON(drm_mm_node_allocated(&vma->node))) { spin_unlock(&aspace->lock); return 0; } ret = drm_mm_insert_node(&aspace->mm, &vma->node, npages); spin_unlock(&aspace->lock); if (ret) return ret; vma->iova = vma->node.start << PAGE_SHIFT; if (aspace->mmu) { unsigned size = npages << PAGE_SHIFT; ret = aspace->mmu->funcs->map(aspace->mmu, vma->iova, sgt, size, IOMMU_READ | IOMMU_WRITE); } /* Get a reference to the aspace to keep it around */ kref_get(&aspace->kref); return ret; }
bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long cache_level) { struct drm_mm_node *gtt_space = &vma->node; struct drm_mm_node *other; /* * On some machines we have to be careful when putting differing types * of snoopable memory together to avoid the prefetcher crossing memory * domains and dying. During vm initialisation, we decide whether or not * these constraints apply and set the drm_mm.color_adjust * appropriately. */ if (vma->vm->mm.color_adjust == NULL) return true; if (!drm_mm_node_allocated(gtt_space)) return true; if (list_empty(>t_space->node_list)) return true; other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list); if (other->allocated && !other->hole_follows && other->color != cache_level) return false; other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list); if (other->allocated && !gtt_space->hole_follows && other->color != cache_level) return false; return true; }
bool i915_vma_misplaced(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) { if (!drm_mm_node_allocated(&vma->node)) return false; if (vma->node.size < size) return true; if (alignment && vma->node.start & (alignment - 1)) return true; if (flags & PIN_MAPPABLE && !i915_vma_is_map_and_fenceable(vma)) return true; if (flags & PIN_OFFSET_BIAS && vma->node.start < (flags & PIN_OFFSET_MASK)) return true; if (flags & PIN_OFFSET_FIXED && vma->node.start != (flags & PIN_OFFSET_MASK)) return true; return false; }
static int intel_fbc_alloc_cfb(struct intel_crtc *crtc) { struct drm_i915_private *dev_priv = crtc->base.dev->dev_private; struct intel_fbc *fbc = &dev_priv->fbc; struct drm_mm_node *uninitialized_var(compressed_llb); int size, fb_cpp, ret; WARN_ON(drm_mm_node_allocated(&fbc->compressed_fb)); size = intel_fbc_calculate_cfb_size(dev_priv, &fbc->state_cache); fb_cpp = drm_format_plane_cpp(fbc->state_cache.fb.pixel_format, 0); ret = find_compression_threshold(dev_priv, &fbc->compressed_fb, size, fb_cpp); if (!ret) goto err_llb; else if (ret > 1) { DRM_INFO("Reducing the compressed framebuffer size. This may lead to less power savings than a non-reduced-size. Try to increase stolen memory size if available in BIOS.\n"); } fbc->threshold = ret; if (INTEL_INFO(dev_priv)->gen >= 5) I915_WRITE(ILK_DPFC_CB_BASE, fbc->compressed_fb.start); else if (IS_GM45(dev_priv)) { I915_WRITE(DPFC_CB_BASE, fbc->compressed_fb.start); } else { compressed_llb = kzalloc(sizeof(*compressed_llb), GFP_KERNEL); if (!compressed_llb) goto err_fb; ret = i915_gem_stolen_insert_node(dev_priv, compressed_llb, 4096, 4096); if (ret) goto err_fb; fbc->compressed_llb = compressed_llb; I915_WRITE(FBC_CFB_BASE, dev_priv->mm.stolen_base + fbc->compressed_fb.start); I915_WRITE(FBC_LL_BASE, dev_priv->mm.stolen_base + compressed_llb->start); } DRM_DEBUG_KMS("reserved %llu bytes of contiguous stolen space for FBC, threshold: %d\n", fbc->compressed_fb.size, fbc->threshold); return 0; err_fb: kfree(compressed_llb); i915_gem_stolen_remove_node(dev_priv, &fbc->compressed_fb); err_llb: pr_info_once("drm: not enough stolen space for compressed buffer (need %d more bytes), disabling. Hint: you may be able to increase stolen memory size in the BIOS to avoid this.\n", size); return -ENOSPC; }
static void __intel_fbc_cleanup_cfb(struct drm_i915_private *dev_priv) { struct intel_fbc *fbc = &dev_priv->fbc; if (drm_mm_node_allocated(&fbc->compressed_fb)) i915_gem_stolen_remove_node(dev_priv, &fbc->compressed_fb); if (fbc->compressed_llb) { i915_gem_stolen_remove_node(dev_priv, fbc->compressed_llb); kfree(fbc->compressed_llb); } }
/** * drm_vma_offset_remove() - Remove offset node from manager * @mgr: Manager object * @node: Node to be removed * * Remove a node from the offset manager. If the node wasn't added before, this * does nothing. After this call returns, the offset and size will be 0 until a * new offset is allocated via drm_vma_offset_add() again. Helper functions like * drm_vma_node_start() and drm_vma_node_offset_addr() will return 0 if no * offset is allocated. */ void drm_vma_offset_remove(struct drm_vma_offset_manager *mgr, struct drm_vma_offset_node *node) { lockmgr(&mgr->vm_lock, LK_EXCLUSIVE); if (drm_mm_node_allocated(&node->vm_node)) { rb_erase(&node->vm_rb, &mgr->vm_addr_space_rb); drm_mm_remove_node(&node->vm_node); memset(&node->vm_node, 0, sizeof(node->vm_node)); } lockmgr(&mgr->vm_lock, LK_RELEASE); }
static void i915_vma_retire(struct i915_gem_active *active, struct i915_request *rq) { const unsigned int idx = rq->engine->id; struct i915_vma *vma = container_of(active, struct i915_vma, last_read[idx]); struct drm_i915_gem_object *obj = vma->obj; GEM_BUG_ON(!i915_vma_has_active_engine(vma, idx)); i915_vma_clear_active(vma, idx); if (i915_vma_is_active(vma)) return; GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); list_move_tail(&vma->vm_link, &vma->vm->inactive_list); if (unlikely(i915_vma_is_closed(vma) && !i915_vma_is_pinned(vma))) WARN_ON(i915_vma_unbind(vma)); GEM_BUG_ON(!i915_gem_object_is_active(obj)); if (--obj->active_count) return; /* Prune the shared fence arrays iff completely idle (inc. external) */ if (reservation_object_trylock(obj->resv)) { if (reservation_object_test_signaled_rcu(obj->resv, true)) reservation_object_add_excl_fence(obj->resv, NULL); reservation_object_unlock(obj->resv); } /* Bump our place on the bound list to keep it roughly in LRU order * so that we don't steal from recently used but inactive objects * (unless we are forced to ofc!) */ spin_lock(&rq->i915->mm.obj_lock); if (obj->bind_count) list_move_tail(&obj->mm.link, &rq->i915->mm.bound_list); spin_unlock(&rq->i915->mm.obj_lock); obj->mm.dirty = true; /* be paranoid */ if (i915_gem_object_has_active_reference(obj)) { i915_gem_object_clear_active_reference(obj); i915_gem_object_put(obj); } }
/** * drm_vma_offset_add() - Add offset node to manager * @mgr: Manager object * @node: Node to be added * @pages: Allocation size visible to user-space (in number of pages) * * Add a node to the offset-manager. If the node was already added, this does * nothing and return 0. @pages is the size of the object given in number of * pages. * After this call succeeds, you can access the offset of the node until it * is removed again. * * If this call fails, it is safe to retry the operation or call * drm_vma_offset_remove(), anyway. However, no cleanup is required in that * case. * * @pages is not required to be the same size as the underlying memory object * that you want to map. It only limits the size that user-space can map into * their address space. * * RETURNS: * 0 on success, negative error code on failure. */ int drm_vma_offset_add(struct drm_vma_offset_manager *mgr, struct drm_vma_offset_node *node, unsigned long pages) { int ret; lockmgr(&mgr->vm_lock, LK_EXCLUSIVE); if (drm_mm_node_allocated(&node->vm_node)) { ret = 0; goto out_unlock; } ret = drm_mm_insert_node(&mgr->vm_addr_space_mm, &node->vm_node, pages, 0, DRM_MM_SEARCH_DEFAULT); if (ret) goto out_unlock; _drm_vma_offset_add_rb(mgr, node); out_unlock: lockmgr(&mgr->vm_lock, LK_RELEASE); return ret; }
struct drm_i915_gem_object * i915_gem_object_create_stolen_for_preallocated(struct drm_i915_private *dev_priv, resource_size_t stolen_offset, resource_size_t gtt_offset, resource_size_t size) { struct i915_ggtt *ggtt = &dev_priv->ggtt; struct drm_i915_gem_object *obj; struct drm_mm_node *stolen; struct i915_vma *vma; int ret; if (!drm_mm_initialized(&dev_priv->mm.stolen)) return NULL; lockdep_assert_held(&dev_priv->drm.struct_mutex); DRM_DEBUG_DRIVER("creating preallocated stolen object: stolen_offset=%pa, gtt_offset=%pa, size=%pa\n", &stolen_offset, >t_offset, &size); /* KISS and expect everything to be page-aligned */ if (WARN_ON(size == 0) || WARN_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)) || WARN_ON(!IS_ALIGNED(stolen_offset, I915_GTT_MIN_ALIGNMENT))) return NULL; stolen = kzalloc(sizeof(*stolen), GFP_KERNEL); if (!stolen) return NULL; stolen->start = stolen_offset; stolen->size = size; mutex_lock(&dev_priv->mm.stolen_lock); ret = drm_mm_reserve_node(&dev_priv->mm.stolen, stolen); mutex_unlock(&dev_priv->mm.stolen_lock); if (ret) { DRM_DEBUG_DRIVER("failed to allocate stolen space\n"); kfree(stolen); return NULL; } obj = _i915_gem_object_create_stolen(dev_priv, stolen); if (obj == NULL) { DRM_DEBUG_DRIVER("failed to allocate stolen object\n"); i915_gem_stolen_remove_node(dev_priv, stolen); kfree(stolen); return NULL; } /* Some objects just need physical mem from stolen space */ if (gtt_offset == I915_GTT_OFFSET_NONE) return obj; ret = i915_gem_object_pin_pages(obj); if (ret) goto err; vma = i915_vma_instance(obj, &ggtt->vm, NULL); if (IS_ERR(vma)) { ret = PTR_ERR(vma); goto err_pages; } /* To simplify the initialisation sequence between KMS and GTT, * we allow construction of the stolen object prior to * setting up the GTT space. The actual reservation will occur * later. */ ret = i915_gem_gtt_reserve(&ggtt->vm, &vma->node, size, gtt_offset, obj->cache_level, 0); if (ret) { DRM_DEBUG_DRIVER("failed to allocate stolen GTT space\n"); goto err_pages; } GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); vma->pages = obj->mm.pages; vma->flags |= I915_VMA_GLOBAL_BIND; __i915_vma_set_map_and_fenceable(vma); mutex_lock(&ggtt->vm.mutex); list_move_tail(&vma->vm_link, &ggtt->vm.bound_list); mutex_unlock(&ggtt->vm.mutex); spin_lock(&dev_priv->mm.obj_lock); list_move_tail(&obj->mm.link, &dev_priv->mm.bound_list); obj->bind_count++; spin_unlock(&dev_priv->mm.obj_lock); return obj; err_pages: i915_gem_object_unpin_pages(obj); err: i915_gem_object_put(obj); return NULL; }
/** * i915_vma_insert - finds a slot for the vma in its address space * @vma: the vma * @size: requested size in bytes (can be larger than the VMA) * @alignment: required alignment * @flags: mask of PIN_* flags to use * * First we try to allocate some free space that meets the requirements for * the VMA. Failiing that, if the flags permit, it will evict an old VMA, * preferrably the oldest idle entry to make room for the new VMA. * * Returns: * 0 on success, negative error code otherwise. */ static int i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) { struct drm_i915_private *dev_priv = to_i915(vma->vm->dev); struct drm_i915_gem_object *obj = vma->obj; u64 start, end; int ret; GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND)); GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); size = max(size, vma->size); if (flags & PIN_MAPPABLE) size = i915_gem_get_ggtt_size(dev_priv, size, i915_gem_object_get_tiling(obj)); alignment = max(max(alignment, vma->display_alignment), i915_gem_get_ggtt_alignment(dev_priv, size, i915_gem_object_get_tiling(obj), flags & PIN_MAPPABLE)); start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0; end = vma->vm->total; if (flags & PIN_MAPPABLE) end = min_t(u64, end, dev_priv->ggtt.mappable_end); if (flags & PIN_ZONE_4G) end = min_t(u64, end, (1ULL << 32) - PAGE_SIZE); /* If binding the object/GGTT view requires more space than the entire * aperture has, reject it early before evicting everything in a vain * attempt to find space. */ if (size > end) { DRM_DEBUG("Attempting to bind an object larger than the aperture: request=%llu [object=%zd] > %s aperture=%llu\n", size, obj->base.size, flags & PIN_MAPPABLE ? "mappable" : "total", end); return -E2BIG; } ret = i915_gem_object_pin_pages(obj); if (ret) return ret; if (flags & PIN_OFFSET_FIXED) { u64 offset = flags & PIN_OFFSET_MASK; if (offset & (alignment - 1) || offset > end - size) { ret = -EINVAL; goto err_unpin; } vma->node.start = offset; vma->node.size = size; vma->node.color = obj->cache_level; ret = drm_mm_reserve_node(&vma->vm->mm, &vma->node); if (ret) { ret = i915_gem_evict_for_vma(vma); if (ret == 0) ret = drm_mm_reserve_node(&vma->vm->mm, &vma->node); if (ret) goto err_unpin; } } else { u32 search_flag, alloc_flag; if (flags & PIN_HIGH) { search_flag = DRM_MM_SEARCH_BELOW; alloc_flag = DRM_MM_CREATE_TOP; } else { search_flag = DRM_MM_SEARCH_DEFAULT; alloc_flag = DRM_MM_CREATE_DEFAULT; } /* We only allocate in PAGE_SIZE/GTT_PAGE_SIZE (4096) chunks, * so we know that we always have a minimum alignment of 4096. * The drm_mm range manager is optimised to return results * with zero alignment, so where possible use the optimal * path. */ if (alignment <= 4096) alignment = 0; search_free: ret = drm_mm_insert_node_in_range_generic(&vma->vm->mm, &vma->node, size, alignment, obj->cache_level, start, end, search_flag, alloc_flag); if (ret) { ret = i915_gem_evict_something(vma->vm, size, alignment, obj->cache_level, start, end, flags); if (ret == 0) goto search_free; goto err_unpin; } GEM_BUG_ON(vma->node.start < start); GEM_BUG_ON(vma->node.start + vma->node.size > end); } GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level)); list_move_tail(&obj->global_link, &dev_priv->mm.bound_list); list_move_tail(&vma->vm_link, &vma->vm->inactive_list); obj->bind_count++; GEM_BUG_ON(atomic_read(&obj->mm.pages_pin_count) < obj->bind_count); return 0; err_unpin: i915_gem_object_unpin_pages(obj); return ret; }