int ttm_bo_move_ttm(struct ttm_buffer_object *bo, bool evict, bool no_wait, struct ttm_mem_reg *new_mem) { struct ttm_tt *ttm = bo->ttm; struct ttm_mem_reg *old_mem = &bo->mem; uint32_t save_flags = old_mem->placement; int ret; if (old_mem->mem_type != TTM_PL_SYSTEM) { ttm_tt_unbind(ttm); ttm_bo_free_old_node(bo); ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM, TTM_PL_MASK_MEM); old_mem->mem_type = TTM_PL_SYSTEM; save_flags = old_mem->placement; } ret = ttm_tt_set_placement_caching(ttm, new_mem->placement); if (unlikely(ret != 0)) return ret; if (new_mem->mem_type != TTM_PL_SYSTEM) { ret = ttm_tt_bind(ttm, new_mem); if (unlikely(ret != 0)) return ret; } *old_mem = *new_mem; new_mem->mm_node = NULL; ttm_flag_masked(&save_flags, new_mem->placement, TTM_PL_MASK_MEMTYPE); return 0; }
int ttm_bo_validate(struct ttm_buffer_object *bo, struct ttm_placement *placement, bool interruptible, bool no_wait_reserve, bool no_wait_gpu) { int ret; BUG_ON(!atomic_read(&bo->reserved)); /* Check that range is valid */ if (placement->lpfn || placement->fpfn) if (placement->fpfn > placement->lpfn || (placement->lpfn - placement->fpfn) < bo->num_pages) return -EINVAL; /* * Check whether we need to move buffer. */ ret = ttm_bo_mem_compat(placement, &bo->mem); if (ret < 0) { ret = ttm_bo_move_buffer(bo, placement, interruptible, no_wait_reserve, no_wait_gpu); if (ret) return ret; } else { /* * Use the access and other non-mapping-related flag bits from * the compatible memory placement flags to the active flags */ ttm_flag_masked(&bo->mem.placement, placement->placement[ret], ~TTM_PL_MASK_MEMTYPE); } /* * We might need to add a TTM. */ if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { ret = ttm_bo_add_ttm(bo, true); if (ret) return ret; } return 0; }
int ttm_bo_move_ttm(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx, struct ttm_mem_reg *new_mem) { struct ttm_tt *ttm = bo->ttm; struct ttm_mem_reg *old_mem = &bo->mem; int ret; if (old_mem->mem_type != TTM_PL_SYSTEM) { ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu); if (unlikely(ret != 0)) { if (ret != -ERESTARTSYS) pr_err("Failed to expire sync object before unbinding TTM\n"); return ret; } ttm_tt_unbind(ttm); ttm_bo_free_old_node(bo); ttm_flag_masked(&old_mem->placement, TTM_PL_FLAG_SYSTEM, TTM_PL_MASK_MEM); old_mem->mem_type = TTM_PL_SYSTEM; } ret = ttm_tt_set_placement_caching(ttm, new_mem->placement); if (unlikely(ret != 0)) return ret; if (new_mem->mem_type != TTM_PL_SYSTEM) { ret = ttm_tt_bind(ttm, new_mem, ctx); if (unlikely(ret != 0)) return ret; } *old_mem = *new_mem; new_mem->mm_node = NULL; return 0; }
/** * Creates space for memory region @mem according to its type. * * This function first searches for free space in compatible memory types in * the priority order defined by the driver. If free space isn't found, then * ttm_bo_mem_force_space is attempted in priority order to evict and find * space. */ int ttm_bo_mem_space(struct ttm_buffer_object *bo, struct ttm_placement *placement, struct ttm_mem_reg *mem, bool interruptible, bool no_wait_reserve, bool no_wait_gpu) { struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_type_manager *man; uint32_t mem_type = TTM_PL_SYSTEM; uint32_t cur_flags = 0; bool type_found = false; bool type_ok = false; bool has_erestartsys = false; int i, ret; mem->mm_node = NULL; for (i = 0; i < placement->num_placement; ++i) { ret = ttm_mem_type_from_flags(placement->placement[i], &mem_type); if (ret) return ret; man = &bdev->man[mem_type]; type_ok = ttm_bo_mt_compatible(man, bo->type == ttm_bo_type_user, mem_type, placement->placement[i], &cur_flags); if (!type_ok) continue; cur_flags = ttm_bo_select_caching(man, bo->mem.placement, cur_flags); /* * Use the access and other non-mapping-related flag bits from * the memory placement flags to the current flags */ ttm_flag_masked(&cur_flags, placement->placement[i], ~TTM_PL_MASK_MEMTYPE); if (mem_type == TTM_PL_SYSTEM) break; if (man->has_type && man->use_type) { type_found = true; ret = (*man->func->get_node)(man, bo, placement, mem); if (unlikely(ret)) return ret; } if (mem->mm_node) break; } if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) { mem->mem_type = mem_type; mem->placement = cur_flags; return 0; } if (!type_found) return -EINVAL; for (i = 0; i < placement->num_busy_placement; ++i) { ret = ttm_mem_type_from_flags(placement->busy_placement[i], &mem_type); if (ret) return ret; man = &bdev->man[mem_type]; if (!man->has_type) continue; if (!ttm_bo_mt_compatible(man, bo->type == ttm_bo_type_user, mem_type, placement->busy_placement[i], &cur_flags)) continue; cur_flags = ttm_bo_select_caching(man, bo->mem.placement, cur_flags); /* * Use the access and other non-mapping-related flag bits from * the memory placement flags to the current flags */ ttm_flag_masked(&cur_flags, placement->busy_placement[i], ~TTM_PL_MASK_MEMTYPE); if (mem_type == TTM_PL_SYSTEM) { mem->mem_type = mem_type; mem->placement = cur_flags; mem->mm_node = NULL; return 0; } ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem, interruptible, no_wait_reserve, no_wait_gpu); if (ret == 0 && mem->mm_node) { mem->placement = cur_flags; return 0; } if (ret == -ERESTARTSYS) has_erestartsys = true; } ret = (has_erestartsys) ? -ERESTARTSYS : -ENOMEM; return ret; }