static int gpu_set(struct drm_i915_gem_object *obj, unsigned long offset, u32 v) { struct drm_i915_private *i915 = to_i915(obj->base.dev); struct drm_i915_gem_request *rq; struct i915_vma *vma; u32 *cs; int err; err = i915_gem_object_set_to_gtt_domain(obj, true); if (err) return err; vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0); if (IS_ERR(vma)) return PTR_ERR(vma); rq = i915_gem_request_alloc(i915->engine[RCS], i915->kernel_context); if (IS_ERR(rq)) { i915_vma_unpin(vma); return PTR_ERR(rq); } cs = intel_ring_begin(rq, 4); if (IS_ERR(cs)) { __i915_add_request(rq, false); i915_vma_unpin(vma); return PTR_ERR(cs); } if (INTEL_GEN(i915) >= 8) { *cs++ = MI_STORE_DWORD_IMM_GEN4 | 1 << 22; *cs++ = lower_32_bits(i915_ggtt_offset(vma) + offset); *cs++ = upper_32_bits(i915_ggtt_offset(vma) + offset); *cs++ = v; } else if (INTEL_GEN(i915) >= 4) { *cs++ = MI_STORE_DWORD_IMM_GEN4 | 1 << 22; *cs++ = 0; *cs++ = i915_ggtt_offset(vma) + offset; *cs++ = v; } else { *cs++ = MI_STORE_DWORD_IMM | 1 << 22; *cs++ = i915_ggtt_offset(vma) + offset; *cs++ = v; *cs++ = MI_NOOP; } intel_ring_advance(rq, cs); i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); i915_vma_unpin(vma); reservation_object_lock(obj->resv, NULL); reservation_object_add_excl_fence(obj->resv, &rq->fence); reservation_object_unlock(obj->resv); __i915_add_request(rq, true); return 0; }
static int gtt_set(struct drm_i915_gem_object *obj, unsigned long offset, u32 v) { struct i915_vma *vma; u32 __iomem *map; int err; err = i915_gem_object_set_to_gtt_domain(obj, true); if (err) return err; vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE); if (IS_ERR(vma)) return PTR_ERR(vma); map = i915_vma_pin_iomap(vma); i915_vma_unpin(vma); if (IS_ERR(map)) return PTR_ERR(map); iowrite32(v, &map[offset / sizeof(*map)]); i915_vma_unpin_iomap(vma); return 0; }
int i915_gem_render_state_init(struct drm_i915_gem_request *req) { struct render_state so; struct drm_i915_gem_object *obj; int ret; if (WARN_ON(req->engine->id != RCS)) return -ENOENT; so.rodata = render_state_get_rodata(req); if (!so.rodata) return 0; if (so.rodata->batch_items * 4 > 4096) return -EINVAL; obj = i915_gem_object_create(&req->i915->drm, 4096); if (IS_ERR(obj)) return PTR_ERR(obj); so.vma = i915_vma_create(obj, &req->i915->ggtt.base, NULL); if (IS_ERR(so.vma)) { ret = PTR_ERR(so.vma); goto err_obj; } ret = i915_vma_pin(so.vma, 0, 0, PIN_GLOBAL); if (ret) goto err_obj; ret = render_state_setup(&so); if (ret) goto err_unpin; ret = req->engine->emit_bb_start(req, so.vma->node.start, so.rodata->batch_items * 4, I915_DISPATCH_SECURE); if (ret) goto err_unpin; if (so.aux_batch_size > 8) { ret = req->engine->emit_bb_start(req, (so.vma->node.start + so.aux_batch_offset), so.aux_batch_size, I915_DISPATCH_SECURE); if (ret) goto err_unpin; } i915_vma_move_to_active(so.vma, req, 0); err_unpin: i915_vma_unpin(so.vma); err_obj: i915_gem_object_put(obj); return ret; }
int i915_gem_render_state_emit(struct i915_request *rq) { struct intel_engine_cs *engine = rq->engine; struct intel_render_state so = {}; /* keep the compiler happy */ int err; so.rodata = render_state_get_rodata(engine); if (!so.rodata) return 0; if (so.rodata->batch_items * 4 > PAGE_SIZE) return -EINVAL; so.obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE); if (IS_ERR(so.obj)) return PTR_ERR(so.obj); so.vma = i915_vma_instance(so.obj, &engine->i915->ggtt.vm, NULL); if (IS_ERR(so.vma)) { err = PTR_ERR(so.vma); goto err_obj; } err = i915_vma_pin(so.vma, 0, 0, PIN_GLOBAL | PIN_HIGH); if (err) goto err_vma; err = render_state_setup(&so, rq->i915); if (err) goto err_unpin; err = engine->emit_bb_start(rq, so.batch_offset, so.batch_size, I915_DISPATCH_SECURE); if (err) goto err_unpin; if (so.aux_size > 8) { err = engine->emit_bb_start(rq, so.aux_offset, so.aux_size, I915_DISPATCH_SECURE); if (err) goto err_unpin; } i915_vma_lock(so.vma); err = i915_vma_move_to_active(so.vma, rq, 0); i915_vma_unlock(so.vma); err_unpin: i915_vma_unpin(so.vma); err_vma: i915_vma_close(so.vma); err_obj: i915_gem_object_put(so.obj); return err; }
static void unpin_ggtt(struct drm_i915_private *i915) { struct i915_ggtt *ggtt = &i915->ggtt; struct i915_vma *vma; mutex_lock(&ggtt->vm.mutex); list_for_each_entry(vma, &i915->ggtt.vm.bound_list, vm_link) if (vma->obj->mm.quirked) i915_vma_unpin(vma); mutex_unlock(&ggtt->vm.mutex); }
int i915_gem_render_state_emit(struct drm_i915_gem_request *req) { struct intel_render_state *so; int ret; lockdep_assert_held(&req->i915->drm.struct_mutex); so = req->engine->render_state; if (!so) return 0; /* Recreate the page after shrinking */ if (!so->vma->obj->mm.pages) so->batch_offset = -1; ret = i915_vma_pin(so->vma, 0, 0, PIN_GLOBAL | PIN_HIGH); if (ret) return ret; if (so->vma->node.start != so->batch_offset) { ret = render_state_setup(so, req->i915); if (ret) goto err_unpin; } ret = req->engine->emit_flush(req, EMIT_INVALIDATE); if (ret) goto err_unpin; ret = req->engine->emit_bb_start(req, so->batch_offset, so->batch_size, I915_DISPATCH_SECURE); if (ret) goto err_unpin; if (so->aux_size > 8) { ret = req->engine->emit_bb_start(req, so->aux_offset, so->aux_size, I915_DISPATCH_SECURE); if (ret) goto err_unpin; } i915_vma_move_to_active(so->vma, req, 0); err_unpin: i915_vma_unpin(so->vma); return ret; }
void i915_vma_unpin_and_release(struct i915_vma **p_vma) { struct i915_vma *vma; struct drm_i915_gem_object *obj; vma = fetch_and_zero(p_vma); if (!vma) return; obj = vma->obj; i915_vma_unpin(vma); i915_vma_close(vma); __i915_gem_object_release_unless_active(obj); }
static int emit_recurse_batch(struct hang *h, struct i915_request *rq) { struct drm_i915_private *i915 = h->i915; struct i915_address_space *vm = rq->gem_context->ppgtt ? &rq->gem_context->ppgtt->vm : &i915->ggtt.vm; struct i915_vma *hws, *vma; unsigned int flags; u32 *batch; int err; vma = i915_vma_instance(h->obj, vm, NULL); if (IS_ERR(vma)) return PTR_ERR(vma); hws = i915_vma_instance(h->hws, vm, NULL); if (IS_ERR(hws)) return PTR_ERR(hws); err = i915_vma_pin(vma, 0, 0, PIN_USER); if (err) return err; err = i915_vma_pin(hws, 0, 0, PIN_USER); if (err) goto unpin_vma; err = i915_vma_move_to_active(vma, rq, 0); if (err) goto unpin_hws; if (!i915_gem_object_has_active_reference(vma->obj)) { i915_gem_object_get(vma->obj); i915_gem_object_set_active_reference(vma->obj); } err = i915_vma_move_to_active(hws, rq, 0); if (err) goto unpin_hws; if (!i915_gem_object_has_active_reference(hws->obj)) { i915_gem_object_get(hws->obj); i915_gem_object_set_active_reference(hws->obj); } batch = h->batch; if (INTEL_GEN(i915) >= 8) { *batch++ = MI_STORE_DWORD_IMM_GEN4; *batch++ = lower_32_bits(hws_address(hws, rq)); *batch++ = upper_32_bits(hws_address(hws, rq)); *batch++ = rq->fence.seqno; *batch++ = MI_ARB_CHECK; memset(batch, 0, 1024); batch += 1024 / sizeof(*batch); *batch++ = MI_ARB_CHECK; *batch++ = MI_BATCH_BUFFER_START | 1 << 8 | 1; *batch++ = lower_32_bits(vma->node.start); *batch++ = upper_32_bits(vma->node.start); } else if (INTEL_GEN(i915) >= 6) { *batch++ = MI_STORE_DWORD_IMM_GEN4; *batch++ = 0; *batch++ = lower_32_bits(hws_address(hws, rq)); *batch++ = rq->fence.seqno; *batch++ = MI_ARB_CHECK; memset(batch, 0, 1024); batch += 1024 / sizeof(*batch); *batch++ = MI_ARB_CHECK; *batch++ = MI_BATCH_BUFFER_START | 1 << 8; *batch++ = lower_32_bits(vma->node.start); } else if (INTEL_GEN(i915) >= 4) { *batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; *batch++ = 0; *batch++ = lower_32_bits(hws_address(hws, rq)); *batch++ = rq->fence.seqno; *batch++ = MI_ARB_CHECK; memset(batch, 0, 1024); batch += 1024 / sizeof(*batch); *batch++ = MI_ARB_CHECK; *batch++ = MI_BATCH_BUFFER_START | 2 << 6; *batch++ = lower_32_bits(vma->node.start); } else { *batch++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL; *batch++ = lower_32_bits(hws_address(hws, rq)); *batch++ = rq->fence.seqno; *batch++ = MI_ARB_CHECK; memset(batch, 0, 1024); batch += 1024 / sizeof(*batch); *batch++ = MI_ARB_CHECK; *batch++ = MI_BATCH_BUFFER_START | 2 << 6; *batch++ = lower_32_bits(vma->node.start); } *batch++ = MI_BATCH_BUFFER_END; /* not reached */ i915_gem_chipset_flush(h->i915); flags = 0; if (INTEL_GEN(vm->i915) <= 5) flags |= I915_DISPATCH_SECURE; err = rq->engine->emit_bb_start(rq, vma->node.start, PAGE_SIZE, flags); unpin_hws: i915_vma_unpin(hws); unpin_vma: i915_vma_unpin(vma); return err; }
struct i915_request * igt_spinner_create_request(struct igt_spinner *spin, struct i915_gem_context *ctx, struct intel_engine_cs *engine, u32 arbitration_command) { struct i915_address_space *vm = &ctx->ppgtt->vm; struct i915_request *rq = NULL; struct i915_vma *hws, *vma; u32 *batch; int err; vma = i915_vma_instance(spin->obj, vm, NULL); if (IS_ERR(vma)) return ERR_CAST(vma); hws = i915_vma_instance(spin->hws, vm, NULL); if (IS_ERR(hws)) return ERR_CAST(hws); err = i915_vma_pin(vma, 0, 0, PIN_USER); if (err) return ERR_PTR(err); err = i915_vma_pin(hws, 0, 0, PIN_USER); if (err) goto unpin_vma; rq = i915_request_alloc(engine, ctx); if (IS_ERR(rq)) { err = PTR_ERR(rq); goto unpin_hws; } err = move_to_active(vma, rq, 0); if (err) goto cancel_rq; err = move_to_active(hws, rq, 0); if (err) goto cancel_rq; batch = spin->batch; *batch++ = MI_STORE_DWORD_IMM_GEN4; *batch++ = lower_32_bits(hws_address(hws, rq)); *batch++ = upper_32_bits(hws_address(hws, rq)); *batch++ = rq->fence.seqno; *batch++ = arbitration_command; *batch++ = MI_BATCH_BUFFER_START | 1 << 8 | 1; *batch++ = lower_32_bits(vma->node.start); *batch++ = upper_32_bits(vma->node.start); *batch++ = MI_BATCH_BUFFER_END; /* not reached */ i915_gem_chipset_flush(spin->i915); err = engine->emit_bb_start(rq, vma->node.start, PAGE_SIZE, 0); cancel_rq: if (err) { i915_request_skip(rq, err); i915_request_add(rq); } unpin_hws: i915_vma_unpin(hws); unpin_vma: i915_vma_unpin(vma); return err ? ERR_PTR(err) : rq; }