static void show_syncpts(struct nvhost_master *m, struct output *o)
{
int i;
BUG_ON(!nvhost_get_chip_ops()->syncpt.name);
nvhost_debug_output(o, "---- syncpts ----\n");
for (i = 0; i < nvhost_syncpt_nb_pts(&m->syncpt); i++) {
u32 max = nvhost_syncpt_read_max(&m->syncpt, i);
u32 min = nvhost_syncpt_update_min(&m->syncpt, i);
if (!min && !max)
continue;
nvhost_debug_output(o, "id %d (%s) min %d max %d\n",
i, nvhost_get_chip_ops()->syncpt.name(&m->syncpt, i),
min, max);
}
for (i = 0; i < nvhost_syncpt_nb_pts(&m->syncpt); i++) {
u32 base_val;
base_val = nvhost_syncpt_read_wait_base(&m->syncpt, i);
if (base_val)
nvhost_debug_output(o, "waitbase id %d val %d\n",
i, base_val);
}
nvhost_debug_output(o, "\n");
}
/**
* Get the current syncpoint value
*/
u32 nvhost_syncpt_read(struct nvhost_syncpt *sp, u32 id)
{
u32 val;
nvhost_module_busy(&syncpt_to_dev(sp)->mod);
val = nvhost_syncpt_update_min(sp, id);
nvhost_module_idle(&syncpt_to_dev(sp)->mod);
return val;
}
int nvhost_job_pin(struct nvhost_job *job, struct nvhost_syncpt *sp)
{
int err = 0, i = 0;
phys_addr_t gather_phys = 0;
void *gather_addr = NULL;
unsigned long waitchk_mask = job->waitchk_mask;
/* get current syncpt values for waitchk */
for_each_set_bit(i, &waitchk_mask, sizeof(job->waitchk_mask))
nvhost_syncpt_update_min(sp, i);
/* pin gathers */
for (i = 0; i < job->num_gathers; i++) {
struct nvhost_job_gather *g = &job->gathers[i];
/* process each gather mem only once */
if (!g->ref) {
g->ref = mem_op().get(job->memmgr,
job->gathers[i].mem_id);
if (IS_ERR(g->ref)) {
err = PTR_ERR(g->ref);
g->ref = NULL;
break;
}
gather_phys = mem_op().pin(job->memmgr, g->ref);
if (IS_ERR((void *)gather_phys)) {
mem_op().put(job->memmgr, g->ref);
err = gather_phys;
break;
}
/* store the gather ref into unpin array */
job->unpins[job->num_unpins++] = g->ref;
gather_addr = mem_op().mmap(g->ref);
if (!gather_addr) {
err = -ENOMEM;
break;
}
err = do_relocs(job, g->mem_id, gather_addr);
if (!err)
err = do_waitchks(job, sp,
g->mem_id, gather_addr);
mem_op().munmap(g->ref, gather_addr);
if (err)
break;
}
g->mem = gather_phys + g->offset;
}
wmb();
return err;
}
void nvhost_syncpt_debug(struct nvhost_syncpt *sp)
{
u32 i;
for (i = 0; i < NV_HOST1X_SYNCPT_NB_PTS; i++) {
u32 max = nvhost_syncpt_read_max(sp, i);
if (!max)
continue;
dev_info(&syncpt_to_dev(sp)->pdev->dev,
"id %d (%s) min %d max %d\n",
i, nvhost_syncpt_name(i),
nvhost_syncpt_update_min(sp, i), max);
}
}
/**
* Updates sw shadow state for client managed registers
*/
void nvhost_syncpt_save(struct nvhost_syncpt *sp)
{
u32 i;
for (i = 0; i < NV_HOST1X_SYNCPT_NB_PTS; i++) {
if (client_managed(i))
nvhost_syncpt_update_min(sp, i);
else
BUG_ON(!nvhost_syncpt_min_eq_max(sp, i));
}
for (i = 0; i < NV_HOST1X_SYNCPT_NB_BASES; i++)
read_syncpt_wait_base(sp, i);
}
/* check for old WAITs to be removed (avoiding a wrap) */
static int t20_syncpt_wait_check(struct nvhost_syncpt *sp,
struct nvmap_client *nvmap,
u32 waitchk_mask,
struct nvhost_waitchk *wait,
int num_waitchk)
{
u32 idx;
int err = 0;
/* get current syncpt values */
for (idx = 0; idx < NV_HOST1X_SYNCPT_NB_PTS; idx++) {
if (BIT(idx) & waitchk_mask)
nvhost_syncpt_update_min(sp, idx);
}
BUG_ON(!wait && !num_waitchk);
/* compare syncpt vs wait threshold */
while (num_waitchk) {
u32 override;
BUG_ON(wait->syncpt_id >= NV_HOST1X_SYNCPT_NB_PTS);
trace_nvhost_syncpt_wait_check(wait->mem, wait->offset,
wait->syncpt_id, wait->thresh);
if (nvhost_syncpt_is_expired(sp,
wait->syncpt_id, wait->thresh)) {
/*
* NULL an already satisfied WAIT_SYNCPT host method,
* by patching its args in the command stream. The
* method data is changed to reference a reserved
* (never given out or incr) NVSYNCPT_GRAPHICS_HOST
* syncpt with a matching threshold value of 0, so
* is guaranteed to be popped by the host HW.
*/
dev_dbg(&syncpt_to_dev(sp)->dev->dev,
"drop WAIT id %d (%s) thresh 0x%x, min 0x%x\n",
wait->syncpt_id,
syncpt_op(sp).name(sp, wait->syncpt_id),
wait->thresh,
nvhost_syncpt_read_min(sp, wait->syncpt_id));
/* patch the wait */
override = nvhost_class_host_wait_syncpt(
NVSYNCPT_GRAPHICS_HOST, 0);
err = nvmap_patch_word(nvmap,
(struct nvmap_handle *)wait->mem,
wait->offset, override);
if (err)
break;
}
/**
* Updates sw shadow state for client managed registers
*/
void nvhost_syncpt_save(struct nvhost_syncpt *sp)
{
u32 i;
for (i = 0; i < NV_HOST1X_SYNCPT_NB_PTS; i++) {
if (client_managed(i))
nvhost_syncpt_update_min(sp, i);
else
//BUG_ON(!nvhost_syncpt_min_eq_max(sp, i));
if (!nvhost_syncpt_min_eq_max(sp, i)) //A temporary workaround....
printk("BUG! sp=%p, i=%d",sp,i);
}
for (i = 0; i < NV_HOST1X_SYNCPT_NB_BASES; i++)
read_syncpt_wait_base(sp, i);
}
/**
* Updates sw shadow state for client managed registers
*/
void nvhost_syncpt_save(struct nvhost_syncpt *sp)
{
u32 i;
for (i = 0; i < NV_HOST1X_SYNCPT_NB_PTS; i++) {
if (client_managed(i))
nvhost_syncpt_update_min(sp, i);
else
BUG_ON(!nvhost_syncpt_min_eq_max(sp, i));
}
for (i = 0; i < NV_HOST1X_SYNCPT_NB_BASES; i++)
read_syncpt_wait_base(sp, i);
#ifdef CONFIG_MACH_N1
sp->restore_needed = true;
#endif
}
/**
* Begin a cdma submit
*/
int nvhost_cdma_begin(struct nvhost_cdma *cdma,
struct nvhost_userctx_timeout *timeout)
{
mutex_lock(&cdma->lock);
if (timeout && timeout->has_timedout) {
struct nvhost_master *dev = cdma_to_dev(cdma);
u32 min, max;
min = nvhost_syncpt_update_min(&dev->syncpt,
cdma->timeout.syncpt_id);
max = nvhost_syncpt_read_min(&dev->syncpt,
cdma->timeout.syncpt_id);
dev_dbg(&dev->pdev->dev,
"%s: skip timed out ctx submit (min = %d, max = %d)\n",
__func__, min, max);
mutex_unlock(&cdma->lock);
return -ETIMEDOUT;
}
if (timeout->timeout) {
/* init state on first submit with timeout value */
if (!cdma->timeout.initialized) {
int err;
BUG_ON(!cdma_op(cdma).timeout_init);
err = cdma_op(cdma).timeout_init(cdma,
timeout->syncpt_id);
if (err) {
mutex_unlock(&cdma->lock);
return err;
}
}
}
if (!cdma->running) {
BUG_ON(!cdma_op(cdma).start);
cdma_op(cdma).start(cdma);
}
cdma->slots_free = 0;
cdma->slots_used = 0;
cdma->first_get = cdma_pb_op(cdma).putptr(&cdma->push_buffer);
return 0;
}
/* check for old WAITs to be removed (avoiding a wrap) */
int nvhost_syncpt_wait_check(struct nvmap_client *nvmap,
struct nvhost_syncpt *sp, u32 waitchk_mask,
struct nvhost_waitchk *waitp, u32 waitchks)
{
u32 idx;
int err = 0;
/* get current syncpt values */
for (idx = 0; idx < NV_HOST1X_SYNCPT_NB_PTS; idx++) {
if (BIT(idx) & waitchk_mask) {
nvhost_syncpt_update_min(sp, idx);
}
}
BUG_ON(!waitp);
/* compare syncpt vs wait threshold */
while (waitchks) {
u32 syncpt, override;
BUG_ON(waitp->syncpt_id >= NV_HOST1X_SYNCPT_NB_PTS);
syncpt = atomic_read(&sp->min_val[waitp->syncpt_id]);
if (nvhost_syncpt_wrapping_comparison(syncpt, waitp->thresh)) {
/* wait has completed already, so can be removed */
dev_dbg(&syncpt_to_dev(sp)->pdev->dev,
"drop WAIT id %d (%s) thresh 0x%x, syncpt 0x%x\n",
waitp->syncpt_id, nvhost_syncpt_name(waitp->syncpt_id),
waitp->thresh, syncpt);
/* move wait to a kernel reserved syncpt (that's always 0) */
override = nvhost_class_host_wait_syncpt(NVSYNCPT_GRAPHICS_HOST, 0);
/* patch the wait */
err = nvmap_patch_wait(nvmap,
(struct nvmap_handle *)waitp->mem,
waitp->offset, override);
if (err)
break;
}
/**
* Main entrypoint for syncpoint value waits.
*/
int nvhost_syncpt_wait_timeout(struct nvhost_syncpt *sp, u32 id,
u32 thresh, u32 timeout, u32 *value)
{
DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
void *ref;
int err = 0;
if (value)
*value = 0;
BUG_ON(!check_max(sp, id, thresh));
/* first check cache */
if (nvhost_syncpt_min_cmp(sp, id, thresh)) {
if (value)
*value = nvhost_syncpt_read_min(sp, id);
return 0;
}
/* keep host alive */
nvhost_module_busy(&syncpt_to_dev(sp)->mod);
if (client_managed(id) || !nvhost_syncpt_min_eq_max(sp, id)) {
/* try to read from register */
u32 val = nvhost_syncpt_update_min(sp, id);
if ((s32)(val - thresh) >= 0) {
if (value)
*value = val;
goto done;
}
}
if (!timeout) {
err = -EAGAIN;
goto done;
}
/* schedule a wakeup when the syncpoint value is reached */
err = nvhost_intr_add_action(&(syncpt_to_dev(sp)->intr), id, thresh,
NVHOST_INTR_ACTION_WAKEUP_INTERRUPTIBLE, &wq, &ref);
if (err)
goto done;
err = -EAGAIN;
/* wait for the syncpoint, or timeout, or signal */
while (timeout) {
u32 check = min_t(u32, SYNCPT_CHECK_PERIOD, timeout);
int remain = wait_event_interruptible_timeout(wq,
nvhost_syncpt_min_cmp(sp, id, thresh),
check);
if (remain > 0 || nvhost_syncpt_min_cmp(sp, id, thresh)) {
if (value)
*value = nvhost_syncpt_read_min(sp, id);
err = 0;
break;
}
if (remain < 0) {
err = remain;
break;
}
if (timeout != NVHOST_NO_TIMEOUT)
timeout -= check;
if (timeout) {
dev_warn(&syncpt_to_dev(sp)->pdev->dev,
"syncpoint id %d (%s) stuck waiting %d\n",
id, nvhost_syncpt_name(id), thresh);
nvhost_syncpt_debug(sp);
}
};
nvhost_intr_put_ref(&(syncpt_to_dev(sp)->intr), ref);
done:
nvhost_module_idle(&syncpt_to_dev(sp)->mod);
return err;
}
/**
* Main entrypoint for syncpoint value waits.
*/
int nvhost_syncpt_wait_timeout(struct nvhost_syncpt *sp, u32 id,
u32 thresh, u32 timeout)
{
DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
void *ref;
int err = 0;
//struct nvhost_dev *dev = syncpt_to_dev(sp);
BUG_ON(!check_max(sp, id, thresh));
/* first check cache */
if (nvhost_syncpt_min_cmp(sp, id, thresh))
return 0;
/* keep host alive */
nvhost_module_busy(&syncpt_to_dev(sp)->mod);
if (client_managed(id) || !nvhost_syncpt_min_eq_max(sp, id)) {
/* try to read from register */
u32 val = nvhost_syncpt_update_min(sp, id);
if ((s32)(val - thresh) >= 0)
goto done;
}
if (!timeout) {
err = -EAGAIN;
goto done;
}
/* schedule a wakeup when the syncpoint value is reached */
err = nvhost_intr_add_action(&(syncpt_to_dev(sp)->intr), id, thresh,
NVHOST_INTR_ACTION_WAKEUP_INTERRUPTIBLE, &wq, &ref);
if (err)
goto done;
/* wait for the syncpoint, or timeout, or signal */
while (timeout) {
u32 check = min_t(u32, SYNCPT_CHECK_PERIOD, timeout);
err = wait_event_interruptible_timeout(wq,
nvhost_syncpt_min_cmp(sp, id, thresh),
check);
if (err != 0)
break;
if (timeout != NVHOST_NO_TIMEOUT)
timeout -= SYNCPT_CHECK_PERIOD;
if (timeout) {
dev_warn(&syncpt_to_dev(sp)->pdev->dev,
"syncpoint id %d (%s) stuck waiting %d timeout=%d\n",
id, nvhost_syncpt_name(id), thresh, timeout);
/* A wait queue in nvhost driver maybe run frequently
when early suspend/late resume. These log will be
printed,then the early suspend/late resume
maybe blocked,then it will triger early suspend/late
resume watchdog. Now we cancel these log. */
/*
nvhost_syncpt_debug(sp);
nvhost_channel_fifo_debug(dev);
nvhost_sync_reg_dump(dev);
*/
}
};
if (err > 0)
err = 0;
else if (err == 0)
err = -EAGAIN;
nvhost_intr_put_ref(&(syncpt_to_dev(sp)->intr), ref);
done:
nvhost_module_idle(&syncpt_to_dev(sp)->mod);
return err;
}
void nvhost_cdma_update_sync_queue(struct nvhost_cdma *cdma,
struct nvhost_syncpt *syncpt, struct device *dev)
{
u32 first_get, get_restart;
u32 syncpt_incrs, nr_slots;
bool clear_ctxsave, exec_ctxsave;
struct sync_queue *queue = &cdma->sync_queue;
u32 *sync = sync_queue_head(queue);
u32 syncpt_val = nvhost_syncpt_update_min(syncpt,
cdma->timeout.syncpt_id);
dev_dbg(dev,
"%s: starting cleanup (thresh %d, queue rd 0x%x wr 0x%x)\n",
__func__,
syncpt_val, queue->read, queue->write);
/*
* Move the sync_queue read pointer to the first entry that hasn't
* completed based on the current HW syncpt value. It's likely there
* won't be any (i.e. we're still at the head), but covers the case
* where a syncpt incr happens just prior/during the teardown.
*/
dev_dbg(dev,
"%s: skip completed buffers still in sync_queue\n",
__func__);
while (sync != (queue->buffer + queue->write)) {
/* move read ptr to first blocked entry */
if (syncpt_val < sync[SQ_IDX_SYNCPT_VAL])
break; /* not completed */
dump_sync_queue_entry(cdma, sync);
sync = advance_next_entry(cdma, sync);
}
/*
* Walk the sync_queue, first incrementing with the CPU syncpts that
* are partially executed (the first buffer) or fully skipped while
* still in the current context (slots are also NOP-ed).
*
* At the point contexts are interleaved, syncpt increments must be
* done inline with the pushbuffer from a GATHER buffer to maintain
* the order (slots are modified to be a GATHER of syncpt incrs).
*
* Note: save in get_restart the location where the timed out buffer
* started in the PB, so we can start the refetch from there (with the
* modified NOP-ed PB slots). This lets things appear to have completed
* properly for this buffer and resources are freed.
*/
dev_dbg(dev,
"%s: perform CPU incr on pending same ctx buffers\n",
__func__);
get_restart = cdma->last_put;
if (sync != (queue->buffer + queue->write))
get_restart = sync[SQ_IDX_FIRST_GET];
/* do CPU increments */
while (sync != (queue->buffer + queue->write)) {
/* different context, gets us out of this loop */
if ((void *)sync[SQ_IDX_TIMEOUT_CTX] !=
cdma->timeout.ctx_timeout)
break;
syncpt_incrs = (sync[SQ_IDX_SYNCPT_VAL] - syncpt_val);
first_get = sync[SQ_IDX_FIRST_GET];
nr_slots = sync[SQ_IDX_NUM_SLOTS];
/* won't need a timeout when replayed */
sync[SQ_IDX_TIMEOUT] = 0;
dev_dbg(dev,
"%s: CPU incr (%d)\n", __func__, syncpt_incrs);
dump_sync_queue_entry(cdma, sync);
/* safe to use CPU to incr syncpts */
cdma_op(cdma).timeout_cpu_incr(cdma, first_get,
syncpt_incrs, nr_slots);
syncpt_val += syncpt_incrs;
sync = advance_next_entry(cdma, sync);
}
dev_dbg(dev,
"%s: GPU incr blocked interleaved ctx buffers\n",
__func__);
clear_ctxsave = true;
exec_ctxsave = false;
/* setup GPU increments */
while (sync != (queue->buffer + queue->write)) {
syncpt_incrs = (sync[SQ_IDX_SYNCPT_VAL] - syncpt_val);
first_get = sync[SQ_IDX_FIRST_GET];
nr_slots = sync[SQ_IDX_NUM_SLOTS];
/* same context, increment in the pushbuffer */
if ((void *)sync[SQ_IDX_TIMEOUT_CTX] ==
cdma->timeout.ctx_timeout) {
/* won't need a timeout when replayed */
sync[SQ_IDX_TIMEOUT] = 0;
/* update buffer's syncpts in the pushbuffer */
cdma_op(cdma).timeout_pb_incr(cdma, first_get,
syncpt_incrs, nr_slots, exec_ctxsave);
clear_ctxsave = true;
exec_ctxsave = false;
} else {
dev_dbg(dev,
"%s: switch to a different userctx\n",
__func__);
/*
* If previous context was the timed out context
* then clear its CTXSAVE in this slot.
*/
if (clear_ctxsave) {
cdma_op(cdma).timeout_clear_ctxsave(cdma,
first_get, nr_slots);
clear_ctxsave = false;
}
exec_ctxsave = true;
}
dump_sync_queue_entry(cdma, sync);
syncpt_val = sync[SQ_IDX_SYNCPT_VAL];
sync = advance_next_entry(cdma, sync);
}
dev_dbg(dev,
"%s: finished sync_queue modification\n", __func__);
/* roll back DMAGET and start up channel again */
cdma_op(cdma).timeout_teardown_end(cdma, get_restart);
cdma->timeout.ctx_timeout->has_timedout = true;
mutex_unlock(&cdma->lock);
}