static void __init save_end_v1(struct host1x_hwctx_handler *p, u32 *ptr)
{
/* write end of restore buffer */
ptr[0] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID,
AR3D_DW_MEMORY_OUTPUT_DATA, 1);
nvhost_3dctx_restore_end(p, ptr + 1);
ptr += RESTORE_END_SIZE;
/* reset to dual reg if necessary */
ptr[1] = nvhost_opcode_imm(AR3D_GSHIM_WRITE_MASK,
(1 << register_sets) - 1);
/* op_done syncpt incr to flush FDC */
ptr[2] = nvhost_opcode_imm_incr_syncpt(NV_SYNCPT_OP_DONE, p->syncpt);
/* host wait for that syncpt incr, and advance the wait base */
ptr[3] = nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
NV_CLASS_HOST_WAIT_SYNCPT_BASE,
nvhost_mask2(
NV_CLASS_HOST_WAIT_SYNCPT_BASE,
NV_CLASS_HOST_INCR_SYNCPT_BASE));
ptr[4] = nvhost_class_host_wait_syncpt_base(p->syncpt,
p->waitbase, p->save_incrs - 1);
ptr[5] = nvhost_class_host_incr_syncpt_base(p->waitbase,
p->save_incrs);
/* set class back to 3d */
ptr[6] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0);
/* send reg reads back to host */
ptr[7] = nvhost_opcode_imm(AR3D_GLOBAL_MEMORY_OUTPUT_READS, 0);
/* final syncpt increment to release waiters */
ptr[8] = nvhost_opcode_imm(0, p->syncpt);
}
static void save_push_v1(struct nvhost_hwctx *nctx, struct nvhost_cdma *cdma)
{
struct host1x_hwctx *ctx = to_host1x_hwctx(nctx);
struct host1x_hwctx_handler *p = host1x_hwctx_handler(ctx);
/* wait for 3d idle */
nvhost_cdma_push(cdma,
nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0),
nvhost_opcode_imm_incr_syncpt(
host1x_uclass_incr_syncpt_cond_op_done_v(),
p->syncpt));
nvhost_cdma_push(cdma,
nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
host1x_uclass_wait_syncpt_base_r(), 1),
nvhost_class_host_wait_syncpt_base(p->syncpt,
p->waitbase, 1));
/* back to 3d */
nvhost_cdma_push(cdma,
nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0),
NVHOST_OPCODE_NOOP);
/* invalidate the FDC to prevent cache-coherency issues across GPUs
note that we assume FDC_CONTROL_0 is left in the reset state by all
contexts. the invalidate bit will clear itself, so the register
should be unchanged after this */
nvhost_cdma_push(cdma,
nvhost_opcode_imm(AR3D_FDC_CONTROL_0,
AR3D_FDC_CONTROL_0_RESET_VAL
| AR3D_FDC_CONTROL_0_INVALIDATE),
NVHOST_OPCODE_NOOP);
/* set register set 0 and 1 register read memory output addresses,
and send their reads to memory */
nvhost_cdma_push(cdma,
nvhost_opcode_imm(AR3D_GSHIM_WRITE_MASK, 2),
nvhost_opcode_imm(AR3D_GLOBAL_MEMORY_OUTPUT_READS, 1));
nvhost_cdma_push(cdma,
nvhost_opcode_nonincr(AR3D_DW_MEMORY_OUTPUT_ADDRESS, 1),
ctx->restore_phys + restore_set1_offset * 4);
nvhost_cdma_push(cdma,
nvhost_opcode_imm(AR3D_GSHIM_WRITE_MASK, 1),
nvhost_opcode_imm(AR3D_GLOBAL_MEMORY_OUTPUT_READS, 1));
nvhost_cdma_push(cdma,
nvhost_opcode_nonincr(AR3D_DW_MEMORY_OUTPUT_ADDRESS, 1),
ctx->restore_phys);
/* gather the save buffer */
nvhost_cdma_push_gather(cdma,
nvhost_get_host(nctx->channel->dev)->memmgr,
p->save_buf,
0,
nvhost_opcode_gather(p->save_size),
p->save_phys);
}
static void save_end(u32 *ptr, u32 syncpt_id, u32 waitbase)
{
/* Wait for context read service */
ptr[0] = nvhost_opcode_nonincr(NV_CLASS_HOST_WAIT_SYNCPT_BASE, 1);
ptr[1] = nvhost_class_host_wait_syncpt_base(syncpt_id, waitbase, 3);
/* Increment syncpoint base */
ptr[2] = nvhost_opcode_nonincr(NV_CLASS_HOST_INCR_SYNCPT_BASE, 1);
ptr[3] = nvhost_class_host_incr_syncpt_base(waitbase, 3);
/* set class back to the unit */
ptr[4] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0);
}
static void __init save_end_v0(struct host1x_hwctx_handler *h, u32 *ptr)
{
/* Wait for context read service to finish (cpu incr 3) */
ptr[0] = nvhost_opcode_nonincr(host1x_uclass_wait_syncpt_base_r(), 1);
ptr[1] = nvhost_class_host_wait_syncpt_base(h->syncpt,
h->waitbase, h->save_incrs);
/* Advance syncpoint base */
ptr[2] = nvhost_opcode_nonincr(host1x_uclass_incr_syncpt_base_r(), 1);
ptr[3] = nvhost_class_host_incr_syncpt_base(h->waitbase,
h->save_incrs);
/* set class back to the unit */
ptr[4] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0);
}
static void save_push_v1(struct nvhost_hwctx *nctx, struct nvhost_cdma *cdma)
{
struct host1x_hwctx *ctx = to_host1x_hwctx(nctx);
struct host1x_hwctx_handler *p = host1x_hwctx_handler(ctx);
/* wait for 3d idle */
nvhost_cdma_push(cdma,
nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0),
nvhost_opcode_imm_incr_syncpt(
host1x_uclass_incr_syncpt_cond_op_done_v(),
p->h.syncpt));
nvhost_cdma_push(cdma,
nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
host1x_uclass_wait_syncpt_base_r(), 1),
nvhost_class_host_wait_syncpt_base(p->h.syncpt,
p->h.waitbase, 1));
/* back to 3d */
nvhost_cdma_push(cdma,
nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0),
NVHOST_OPCODE_NOOP);
/* invalidate the FDC to prevent cache-coherency issues across GPUs
note that we assume FDC_CONTROL_0 is left in the reset state by all
contexts. the invalidate bit will clear itself, so the register
should be unchanged after this */
/* bug 990395 T114 HW no longer can automatically clear the invalidate
bit. Luckily that the ctx switching always happens on the push
buffer boundary, and 3d driver inserts a FDC flush & invalidate &
clear the invalidate bit in the beginning of the each push buffer.
So we do not need to explicitly clear the invalidate bit here. */
nvhost_cdma_push(cdma,
nvhost_opcode_imm(AR3D_FDC_CONTROL_0,
AR3D_FDC_CONTROL_0_RESET_VAL
| AR3D_FDC_CONTROL_0_INVALIDATE),
nvhost_opcode_imm(AR3D_GLOBAL_MEMORY_OUTPUT_READS, 1));
/* bug 972588 requires SW to clear the reg 0x403 and 0xe45 */
nvhost_cdma_push(cdma,
nvhost_opcode_imm(0xe45, 0),
nvhost_opcode_imm(0x403, 0));
nvhost_cdma_push(cdma,
nvhost_opcode_nonincr(AR3D_DW_MEMORY_OUTPUT_ADDRESS, 1),
ctx->restore_phys);
/* gather the save buffer */
nvhost_cdma_push_gather(cdma,
nvhost_get_host(nctx->channel->dev)->memmgr,
p->save_buf,
0,
nvhost_opcode_gather(p->save_size),
p->save_phys);
}
static void __init save_begin_v0(struct host1x_hwctx_handler *h, u32 *ptr)
{
/* 3d: when done, increment syncpt to base+1 */
ptr[0] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0);
ptr[1] = nvhost_opcode_imm_incr_syncpt(
host1x_uclass_incr_syncpt_cond_op_done_v(),
h->syncpt); /* incr 1 */
/* host: wait for syncpt base+1 */
ptr[2] = nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
host1x_uclass_wait_syncpt_base_r(), 1);
ptr[3] = nvhost_class_host_wait_syncpt_base(h->syncpt,
h->waitbase, 1);
/* host: signal context read thread to start reading */
ptr[4] = nvhost_opcode_imm_incr_syncpt(
host1x_uclass_incr_syncpt_cond_immediate_v(),
h->syncpt); /* incr 2 */
}
static void save_begin(u32 *ptr, u32 syncpt_id, u32 waitbase)
{
/* set class to the unit to flush */
ptr[0] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0);
/*
* Flush pipe and signal context read thread to start reading
* sync point increment
*/
ptr[1] = nvhost_opcode_imm(0, 0x100 | syncpt_id);
ptr[2] = nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
NV_CLASS_HOST_WAIT_SYNCPT_BASE, 1);
/* wait for base+1 */
ptr[3] = nvhost_class_host_wait_syncpt_base(syncpt_id, waitbase, 1);
ptr[4] = nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0);
ptr[5] = nvhost_opcode_imm(0, syncpt_id);
ptr[6] = nvhost_opcode_setclass(NV_HOST1X_CLASS_ID, 0, 0);
}
static void save_push_v1(struct nvhost_hwctx *nctx, struct nvhost_cdma *cdma)
{
struct host1x_hwctx *ctx = to_host1x_hwctx(nctx);
struct host1x_hwctx_handler *p = host1x_hwctx_handler(ctx);
/* wait for 3d idle */
nvhost_cdma_push(cdma,
nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0),
nvhost_opcode_imm_incr_syncpt(NV_SYNCPT_OP_DONE,
p->syncpt));
nvhost_cdma_push(cdma,
nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
NV_CLASS_HOST_WAIT_SYNCPT_BASE, 1),
nvhost_class_host_wait_syncpt_base(p->syncpt,
p->waitbase, 1));
/* back to 3d */
nvhost_cdma_push(cdma,
nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0),
NVHOST_OPCODE_NOOP);
/* set register set 0 and 1 register read memory output addresses,
and send their reads to memory */
if (register_sets == 2) {
nvhost_cdma_push(cdma,
nvhost_opcode_imm(AR3D_GSHIM_WRITE_MASK, 2),
nvhost_opcode_imm(AR3D_GLOBAL_MEMORY_OUTPUT_READS,
1));
nvhost_cdma_push(cdma,
nvhost_opcode_nonincr(0x904, 1),
ctx->restore_phys + restore_set1_offset * 4);
}
nvhost_cdma_push(cdma,
nvhost_opcode_imm(AR3D_GSHIM_WRITE_MASK, 1),
nvhost_opcode_imm(AR3D_GLOBAL_MEMORY_OUTPUT_READS, 1));
nvhost_cdma_push(cdma,
nvhost_opcode_nonincr(AR3D_DW_MEMORY_OUTPUT_ADDRESS, 1),
ctx->restore_phys);
/* gather the save buffer */
nvhost_cdma_push_gather(cdma,
nvhost_get_host(nctx->channel->dev)->nvmap,
p->save_buf->handle,
0,
nvhost_opcode_gather(p->save_size),
p->save_phys);
}
static void save_push_v1(struct nvhost_cdma *cdma,
struct nvhost_hwctx *ctx)
{
/* wait for 3d idle */
nvhost_cdma_push(cdma,
nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0),
nvhost_opcode_imm_incr_syncpt(NV_SYNCPT_OP_DONE,
NVSYNCPT_3D));
nvhost_cdma_push(cdma,
nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
NV_CLASS_HOST_WAIT_SYNCPT_BASE, 1),
nvhost_class_host_wait_syncpt_base(NVSYNCPT_3D,
NVWAITBASE_3D, 1));
/* back to 3d */
nvhost_cdma_push(cdma,
nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0),
NVHOST_OPCODE_NOOP);
/* set register set 0 and 1 register read memory output addresses,
and send their reads to memory */
if (register_sets == 2) {
nvhost_cdma_push(cdma,
nvhost_opcode_imm(AR3D_GSHIM_WRITE_MASK, 2),
nvhost_opcode_imm(AR3D_GLOBAL_MEMORY_OUTPUT_READS,
1));
nvhost_cdma_push(cdma,
nvhost_opcode_nonincr(0x904, 1),
ctx->restore_phys + restore_set1_offset * 4);
}
nvhost_cdma_push(cdma,
nvhost_opcode_imm(AR3D_GSHIM_WRITE_MASK, 1),
nvhost_opcode_imm(AR3D_GLOBAL_MEMORY_OUTPUT_READS, 1));
nvhost_cdma_push(cdma,
nvhost_opcode_nonincr(AR3D_DW_MEMORY_OUTPUT_ADDRESS, 1),
ctx->restore_phys);
/* gather the save buffer */
nvhost_cdma_push_gather(cdma,
(void *)NVHOST_CDMA_PUSH_GATHER_CTXSAVE,
(void *)NVHOST_CDMA_PUSH_GATHER_CTXSAVE,
nvhost_opcode_gather(save_size),
save_phys);
}
int nvhost_gr3d_t20_read_reg(
struct nvhost_device *dev,
struct nvhost_channel *channel,
struct nvhost_hwctx *hwctx,
u32 offset,
u32 *value)
{
struct host1x_hwctx *hwctx_to_save = NULL;
struct nvhost_hwctx_handler *h = hwctx->h;
struct host1x_hwctx_handler *p = to_host1x_hwctx_handler(h);
bool need_restore = false;
u32 syncpt_incrs = 4;
unsigned int pending = 0;
DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
void *ref;
void *ctx_waiter, *read_waiter, *completed_waiter;
struct nvhost_job *job;
u32 syncval;
int err;
if (hwctx->has_timedout)
return -ETIMEDOUT;
ctx_waiter = nvhost_intr_alloc_waiter();
read_waiter = nvhost_intr_alloc_waiter();
completed_waiter = nvhost_intr_alloc_waiter();
if (!ctx_waiter || !read_waiter || !completed_waiter) {
err = -ENOMEM;
goto done;
}
job = nvhost_job_alloc(channel, hwctx,
NULL,
nvhost_get_host(dev)->memmgr, 0, 0);
if (!job) {
err = -ENOMEM;
goto done;
}
/* keep module powered */
nvhost_module_busy(dev);
/* get submit lock */
err = mutex_lock_interruptible(&channel->submitlock);
if (err) {
nvhost_module_idle(dev);
return err;
}
/* context switch */
if (channel->cur_ctx != hwctx) {
hwctx_to_save = channel->cur_ctx ?
to_host1x_hwctx(channel->cur_ctx) : NULL;
if (hwctx_to_save) {
syncpt_incrs += hwctx_to_save->save_incrs;
hwctx_to_save->hwctx.valid = true;
nvhost_job_get_hwctx(job, &hwctx_to_save->hwctx);
}
channel->cur_ctx = hwctx;
if (channel->cur_ctx && channel->cur_ctx->valid) {
need_restore = true;
syncpt_incrs += to_host1x_hwctx(channel->cur_ctx)
->restore_incrs;
}
}
syncval = nvhost_syncpt_incr_max(&nvhost_get_host(dev)->syncpt,
p->syncpt, syncpt_incrs);
job->syncpt_id = p->syncpt;
job->syncpt_incrs = syncpt_incrs;
job->syncpt_end = syncval;
/* begin a CDMA submit */
nvhost_cdma_begin(&channel->cdma, job);
/* push save buffer (pre-gather setup depends on unit) */
if (hwctx_to_save)
h->save_push(&hwctx_to_save->hwctx, &channel->cdma);
/* gather restore buffer */
if (need_restore)
nvhost_cdma_push(&channel->cdma,
nvhost_opcode_gather(to_host1x_hwctx(channel->cur_ctx)
->restore_size),
to_host1x_hwctx(channel->cur_ctx)->restore_phys);
/* Switch to 3D - wait for it to complete what it was doing */
nvhost_cdma_push(&channel->cdma,
nvhost_opcode_setclass(NV_GRAPHICS_3D_CLASS_ID, 0, 0),
nvhost_opcode_imm_incr_syncpt(
host1x_uclass_incr_syncpt_cond_op_done_v(),
p->syncpt));
nvhost_cdma_push(&channel->cdma,
nvhost_opcode_setclass(NV_HOST1X_CLASS_ID,
host1x_uclass_wait_syncpt_base_r(), 1),
nvhost_class_host_wait_syncpt_base(p->syncpt,
p->waitbase, 1));
/* Tell 3D to send register value to FIFO */
nvhost_cdma_push(&channel->cdma,
nvhost_opcode_nonincr(host1x_uclass_indoff_r(), 1),
nvhost_class_host_indoff_reg_read(
host1x_uclass_indoff_indmodid_gr3d_v(),
offset, false));
nvhost_cdma_push(&channel->cdma,
nvhost_opcode_imm(host1x_uclass_inddata_r(), 0),
NVHOST_OPCODE_NOOP);
/* Increment syncpt to indicate that FIFO can be read */
nvhost_cdma_push(&channel->cdma,
nvhost_opcode_imm_incr_syncpt(
host1x_uclass_incr_syncpt_cond_immediate_v(),
p->syncpt),
NVHOST_OPCODE_NOOP);
/* Wait for value to be read from FIFO */
nvhost_cdma_push(&channel->cdma,
nvhost_opcode_nonincr(host1x_uclass_wait_syncpt_base_r(), 1),
nvhost_class_host_wait_syncpt_base(p->syncpt,
p->waitbase, 3));
/* Indicate submit complete */
nvhost_cdma_push(&channel->cdma,
nvhost_opcode_nonincr(host1x_uclass_incr_syncpt_base_r(), 1),
nvhost_class_host_incr_syncpt_base(p->waitbase, 4));
nvhost_cdma_push(&channel->cdma,
NVHOST_OPCODE_NOOP,
nvhost_opcode_imm_incr_syncpt(
host1x_uclass_incr_syncpt_cond_immediate_v(),
p->syncpt));
/* end CDMA submit */
nvhost_cdma_end(&channel->cdma, job);
nvhost_job_put(job);
job = NULL;
/*
* schedule a context save interrupt (to drain the host FIFO
* if necessary, and to release the restore buffer)
*/
if (hwctx_to_save) {
err = nvhost_intr_add_action(
&nvhost_get_host(dev)->intr,
p->syncpt,
syncval - syncpt_incrs
+ hwctx_to_save->save_incrs
- 1,
NVHOST_INTR_ACTION_CTXSAVE, hwctx_to_save,
ctx_waiter,
NULL);
ctx_waiter = NULL;
WARN(err, "Failed to set context save interrupt");
}
/* Wait for FIFO to be ready */
err = nvhost_intr_add_action(&nvhost_get_host(dev)->intr,
p->syncpt, syncval - 2,
NVHOST_INTR_ACTION_WAKEUP, &wq,
read_waiter,
&ref);
read_waiter = NULL;
WARN(err, "Failed to set wakeup interrupt");
wait_event(wq,
nvhost_syncpt_is_expired(&nvhost_get_host(dev)->syncpt,
p->syncpt, syncval - 2));
nvhost_intr_put_ref(&nvhost_get_host(dev)->intr, p->syncpt,
ref);
/* Read the register value from FIFO */
err = nvhost_channel_drain_read_fifo(channel, value, 1, &pending);
/* Indicate we've read the value */
nvhost_syncpt_cpu_incr(&nvhost_get_host(dev)->syncpt,
p->syncpt);
/* Schedule a submit complete interrupt */
err = nvhost_intr_add_action(&nvhost_get_host(dev)->intr,
p->syncpt, syncval,
NVHOST_INTR_ACTION_SUBMIT_COMPLETE, channel,
completed_waiter, NULL);
completed_waiter = NULL;
WARN(err, "Failed to set submit complete interrupt");
mutex_unlock(&channel->submitlock);
done:
kfree(ctx_waiter);
kfree(read_waiter);
kfree(completed_waiter);
return err;
}
