static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, bool interruptible, bool no_wait_reserve, bool no_wait_gpu) { struct ttm_bo_device *bdev = bo->bdev; struct ttm_bo_global *glob = bo->glob; int put_count; int ret = 0; retry: spin_lock(&bdev->fence_lock); ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); spin_unlock(&bdev->fence_lock); if (unlikely(ret != 0)) return ret; spin_lock(&glob->lru_lock); if (unlikely(list_empty(&bo->ddestroy))) { spin_unlock(&glob->lru_lock); return 0; } ret = ttm_bo_reserve_locked(bo, interruptible, no_wait_reserve, false, 0); if (unlikely(ret != 0)) { spin_unlock(&glob->lru_lock); return ret; } /** * We can re-check for sync object without taking * the bo::lock since setting the sync object requires * also bo::reserved. A busy object at this point may * be caused by another thread recently starting an accelerated * eviction. */ if (unlikely(bo->sync_obj)) { atomic_set(&bo->reserved, 0); wake_up_all(&bo->event_queue); spin_unlock(&glob->lru_lock); goto retry; } put_count = ttm_bo_del_from_lru(bo); list_del_init(&bo->ddestroy); ++put_count; spin_unlock(&glob->lru_lock); ttm_bo_cleanup_memtype_use(bo); ttm_bo_list_ref_sub(bo, put_count, true); return 0; }
static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) { struct ttm_bo_device *bdev = bo->bdev; struct ttm_bo_global *glob = bo->glob; struct ttm_bo_driver *driver; void *sync_obj = NULL; void *sync_obj_arg; int put_count; int ret; spin_lock(&bdev->fence_lock); (void) ttm_bo_wait(bo, false, false, true); if (!bo->sync_obj) { spin_lock(&glob->lru_lock); /** * Lock inversion between bo:reserve and bdev::fence_lock here, * but that's OK, since we're only trylocking. */ ret = ttm_bo_reserve_locked(bo, false, true, false, 0); if (unlikely(ret == -EBUSY)) goto queue; spin_unlock(&bdev->fence_lock); put_count = ttm_bo_del_from_lru(bo); spin_unlock(&glob->lru_lock); ttm_bo_cleanup_memtype_use(bo); ttm_bo_list_ref_sub(bo, put_count, true); return; } else { spin_lock(&glob->lru_lock); } queue: driver = bdev->driver; if (bo->sync_obj) sync_obj = driver->sync_obj_ref(bo->sync_obj); sync_obj_arg = bo->sync_obj_arg; kref_get(&bo->list_kref); list_add_tail(&bo->ddestroy, &bdev->ddestroy); spin_unlock(&glob->lru_lock); spin_unlock(&bdev->fence_lock); if (sync_obj) { driver->sync_obj_flush(sync_obj, sync_obj_arg); driver->sync_obj_unref(&sync_obj); } schedule_delayed_work(&bdev->wq, ((HZ / 100) < 1) ? 1 : HZ / 100); }
static int ttm_mem_evict_first(struct ttm_bo_device *bdev, uint32_t mem_type, bool interruptible, bool no_wait_reserve, bool no_wait_gpu) { struct ttm_bo_global *glob = bdev->glob; struct ttm_mem_type_manager *man = &bdev->man[mem_type]; struct ttm_buffer_object *bo; int ret, put_count = 0; retry: spin_lock(&glob->lru_lock); if (list_empty(&man->lru)) { spin_unlock(&glob->lru_lock); return -EBUSY; } bo = list_first_entry(&man->lru, struct ttm_buffer_object, lru); kref_get(&bo->list_kref); ret = ttm_bo_reserve_locked(bo, false, no_wait_reserve, false, 0); if (unlikely(ret == -EBUSY)) { spin_unlock(&glob->lru_lock); if (likely(!no_wait_gpu)) ret = ttm_bo_wait_unreserved(bo, interruptible); kref_put(&bo->list_kref, ttm_bo_release_list); /** * We *need* to retry after releasing the lru lock. */ if (unlikely(ret != 0)) return ret; goto retry; } put_count = ttm_bo_del_from_lru(bo); spin_unlock(&glob->lru_lock); BUG_ON(ret != 0); while (put_count--) kref_put(&bo->list_kref, ttm_bo_ref_bug); ret = ttm_bo_evict(bo, interruptible, no_wait_reserve, no_wait_gpu); ttm_bo_unreserve(bo); kref_put(&bo->list_kref, ttm_bo_release_list); return ret; }
static void ttm_eu_del_from_lru_locked(struct list_head *list) { struct ttm_validate_buffer *entry; list_for_each_entry(entry, list, head) { struct ttm_buffer_object *bo = entry->bo; if (!entry->reserved) continue; if (!entry->removed) { entry->put_count = ttm_bo_del_from_lru(bo); entry->removed = true; } } }
int ttm_bo_reserve(struct ttm_buffer_object *bo, bool interruptible, bool no_wait, bool use_sequence, uint32_t sequence) { struct ttm_bo_global *glob = bo->glob; int put_count = 0; int ret; spin_lock(&glob->lru_lock); ret = ttm_bo_reserve_locked(bo, interruptible, no_wait, use_sequence, sequence); if (likely(ret == 0)) put_count = ttm_bo_del_from_lru(bo); spin_unlock(&glob->lru_lock); ttm_bo_list_ref_sub(bo, put_count, true); return ret; }
static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) { struct ttm_bo_global *glob = container_of(shrink, struct ttm_bo_global, shrink); struct ttm_buffer_object *bo; int ret = -EBUSY; int put_count; uint32_t swap_placement = (TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM); spin_lock(&glob->lru_lock); while (ret == -EBUSY) { if (unlikely(list_empty(&glob->swap_lru))) { spin_unlock(&glob->lru_lock); return -EBUSY; } bo = list_first_entry(&glob->swap_lru, struct ttm_buffer_object, swap); kref_get(&bo->list_kref); if (!list_empty(&bo->ddestroy)) { spin_unlock(&glob->lru_lock); (void) ttm_bo_cleanup_refs(bo, false, false, false); kref_put(&bo->list_kref, ttm_bo_release_list); continue; } /** * Reserve buffer. Since we unlock while sleeping, we need * to re-check that nobody removed us from the swap-list while * we slept. */ ret = ttm_bo_reserve_locked(bo, false, true, false, 0); if (unlikely(ret == -EBUSY)) { spin_unlock(&glob->lru_lock); ttm_bo_wait_unreserved(bo, false); kref_put(&bo->list_kref, ttm_bo_release_list); spin_lock(&glob->lru_lock); } } BUG_ON(ret != 0); put_count = ttm_bo_del_from_lru(bo); spin_unlock(&glob->lru_lock); ttm_bo_list_ref_sub(bo, put_count, true); /** * Wait for GPU, then move to system cached. */ spin_lock(&bo->bdev->fence_lock); ret = ttm_bo_wait(bo, false, false, false); spin_unlock(&bo->bdev->fence_lock); if (unlikely(ret != 0)) goto out; if ((bo->mem.placement & swap_placement) != swap_placement) { struct ttm_mem_reg evict_mem; evict_mem = bo->mem; evict_mem.mm_node = NULL; evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED; evict_mem.mem_type = TTM_PL_SYSTEM; ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, false, false, false); if (unlikely(ret != 0)) goto out; } ttm_bo_unmap_virtual(bo); /** * Swap out. Buffer will be swapped in again as soon as * anyone tries to access a ttm page. */ if (bo->bdev->driver->swap_notify) bo->bdev->driver->swap_notify(bo); ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage); out: /** * * Unreserve without putting on LRU to avoid swapping out an * already swapped buffer. */ atomic_set(&bo->reserved, 0); wake_up_all(&bo->event_queue); kref_put(&bo->list_kref, ttm_bo_release_list); return ret; }
static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, bool remove_all) { struct ttm_bo_device *bdev = bo->bdev; struct ttm_bo_global *glob = bo->glob; struct ttm_bo_driver *driver = bdev->driver; int ret; spin_lock(&bo->lock); (void) ttm_bo_wait(bo, false, false, !remove_all); if (!bo->sync_obj) { int put_count; spin_unlock(&bo->lock); spin_lock(&glob->lru_lock); put_count = ttm_bo_del_from_lru(bo); ret = ttm_bo_reserve_locked(bo, false, false, false, 0); BUG_ON(ret); if (bo->ttm) ttm_tt_unbind(bo->ttm); if (!list_empty(&bo->ddestroy)) { list_del_init(&bo->ddestroy); ++put_count; } if (bo->mem.mm_node) { drm_mm_put_block(bo->mem.mm_node); bo->mem.mm_node = NULL; } spin_unlock(&glob->lru_lock); atomic_set(&bo->reserved, 0); while (put_count--) kref_put(&bo->list_kref, ttm_bo_ref_bug); return 0; } spin_lock(&glob->lru_lock); if (list_empty(&bo->ddestroy)) { void *sync_obj = bo->sync_obj; void *sync_obj_arg = bo->sync_obj_arg; kref_get(&bo->list_kref); list_add_tail(&bo->ddestroy, &bdev->ddestroy); spin_unlock(&glob->lru_lock); spin_unlock(&bo->lock); if (sync_obj) driver->sync_obj_flush(sync_obj, sync_obj_arg); schedule_delayed_work(&bdev->wq, ((HZ / 100) < 1) ? 1 : HZ / 100); ret = 0; } else { spin_unlock(&glob->lru_lock); spin_unlock(&bo->lock); ret = -EBUSY; } return ret; }