/** * Updates the last value read from hardware. * (was nvhost_syncpt_update_min) */ static u32 t20_syncpt_update_min(struct nvhost_syncpt *sp, u32 id) { struct nvhost_master *dev = syncpt_to_dev(sp); void __iomem *sync_regs = dev->sync_aperture; u32 old, live; do { old = nvhost_syncpt_read_min(sp, id); live = readl(sync_regs + (HOST1X_SYNC_SYNCPT_0 + id * 4)); } while ((u32)atomic_cmpxchg(&sp->min_val[id], old, live) != old); if (!nvhost_syncpt_check_max(sp, id, live)) dev_err(&syncpt_to_dev(sp)->dev->dev, "%s failed: id=%u, min=%d, max=%d\n", __func__, nvhost_syncpt_read_min(sp, id), nvhost_syncpt_read_max(sp, id), id); return live; }
/** * Updates the last value read from hardware. * (was nvhost_syncpt_update_min) */ static u32 t20_syncpt_update_min(struct nvhost_syncpt *sp, u32 id) { struct nvhost_master *dev = syncpt_to_dev(sp); void __iomem *sync_regs = dev->sync_aperture; u32 old, live; do { old = nvhost_syncpt_read_min(sp, id); live = readl(sync_regs + (host1x_sync_syncpt_0_r() + id * 4)); } while ((u32)atomic_cmpxchg(&sp->min_val[id], old, live) != old); return live; }
/* check for old WAITs to be removed (avoiding a wrap) */ static int t20_syncpt_wait_check(struct nvhost_syncpt *sp, struct nvmap_client *nvmap, u32 waitchk_mask, struct nvhost_waitchk *wait, int num_waitchk) { u32 idx; int err = 0; /* get current syncpt values */ for (idx = 0; idx < NV_HOST1X_SYNCPT_NB_PTS; idx++) { if (BIT(idx) & waitchk_mask) nvhost_syncpt_update_min(sp, idx); } BUG_ON(!wait && !num_waitchk); /* compare syncpt vs wait threshold */ while (num_waitchk) { u32 override; BUG_ON(wait->syncpt_id >= NV_HOST1X_SYNCPT_NB_PTS); trace_nvhost_syncpt_wait_check(wait->mem, wait->offset, wait->syncpt_id, wait->thresh); if (nvhost_syncpt_is_expired(sp, wait->syncpt_id, wait->thresh)) { /* * NULL an already satisfied WAIT_SYNCPT host method, * by patching its args in the command stream. The * method data is changed to reference a reserved * (never given out or incr) NVSYNCPT_GRAPHICS_HOST * syncpt with a matching threshold value of 0, so * is guaranteed to be popped by the host HW. */ dev_dbg(&syncpt_to_dev(sp)->dev->dev, "drop WAIT id %d (%s) thresh 0x%x, min 0x%x\n", wait->syncpt_id, syncpt_op(sp).name(sp, wait->syncpt_id), wait->thresh, nvhost_syncpt_read_min(sp, wait->syncpt_id)); /* patch the wait */ override = nvhost_class_host_wait_syncpt( NVSYNCPT_GRAPHICS_HOST, 0); err = nvmap_patch_word(nvmap, (struct nvmap_handle *)wait->mem, wait->offset, override); if (err) break; }
/* * Check driver supplied waitchk structs for syncpt thresholds * that have already been satisfied and NULL the comparison (to * avoid a wrap condition in the HW). */ static int do_waitchks(struct nvhost_job *job, struct nvhost_syncpt *sp, u32 patch_mem, void *patch_addr) { int i; /* compare syncpt vs wait threshold */ for (i = 0; i < job->num_waitchk; i++) { struct nvhost_waitchk *wait = &job->waitchk[i]; /* skip all other gathers */ if (patch_mem != wait->mem) continue; trace_nvhost_syncpt_wait_check(wait->mem, wait->offset, wait->syncpt_id, wait->thresh, nvhost_syncpt_read(sp, wait->syncpt_id)); if (nvhost_syncpt_is_expired(sp, wait->syncpt_id, wait->thresh)) { /* * NULL an already satisfied WAIT_SYNCPT host method, * by patching its args in the command stream. The * method data is changed to reference a reserved * (never given out or incr) NVSYNCPT_GRAPHICS_HOST * syncpt with a matching threshold value of 0, so * is guaranteed to be popped by the host HW. */ dev_dbg(&syncpt_to_dev(sp)->dev->dev, "drop WAIT id %d (%s) thresh 0x%x, min 0x%x\n", wait->syncpt_id, syncpt_op().name(sp, wait->syncpt_id), wait->thresh, nvhost_syncpt_read_min(sp, wait->syncpt_id)); /* patch the wait */ nvhost_syncpt_patch_wait(sp, (patch_addr + wait->offset)); } wait->mem = 0; } return 0; }
/** * Begin a cdma submit */ int nvhost_cdma_begin(struct nvhost_cdma *cdma, struct nvhost_userctx_timeout *timeout) { mutex_lock(&cdma->lock); if (timeout && timeout->has_timedout) { struct nvhost_master *dev = cdma_to_dev(cdma); u32 min, max; min = nvhost_syncpt_update_min(&dev->syncpt, cdma->timeout.syncpt_id); max = nvhost_syncpt_read_min(&dev->syncpt, cdma->timeout.syncpt_id); dev_dbg(&dev->pdev->dev, "%s: skip timed out ctx submit (min = %d, max = %d)\n", __func__, min, max); mutex_unlock(&cdma->lock); return -ETIMEDOUT; } if (timeout->timeout) { /* init state on first submit with timeout value */ if (!cdma->timeout.initialized) { int err; BUG_ON(!cdma_op(cdma).timeout_init); err = cdma_op(cdma).timeout_init(cdma, timeout->syncpt_id); if (err) { mutex_unlock(&cdma->lock); return err; } } } if (!cdma->running) { BUG_ON(!cdma_op(cdma).start); cdma_op(cdma).start(cdma); } cdma->slots_free = 0; cdma->slots_used = 0; cdma->first_get = cdma_pb_op(cdma).putptr(&cdma->push_buffer); return 0; }
/** * Main entrypoint for syncpoint value waits. */ int nvhost_syncpt_wait_timeout(struct nvhost_syncpt *sp, u32 id, u32 thresh, u32 timeout, u32 *value) { DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq); void *ref; int err = 0; if (value) *value = 0; BUG_ON(!check_max(sp, id, thresh)); /* first check cache */ if (nvhost_syncpt_min_cmp(sp, id, thresh)) { if (value) *value = nvhost_syncpt_read_min(sp, id); return 0; } /* keep host alive */ nvhost_module_busy(&syncpt_to_dev(sp)->mod); if (client_managed(id) || !nvhost_syncpt_min_eq_max(sp, id)) { /* try to read from register */ u32 val = nvhost_syncpt_update_min(sp, id); if ((s32)(val - thresh) >= 0) { if (value) *value = val; goto done; } } if (!timeout) { err = -EAGAIN; goto done; } /* schedule a wakeup when the syncpoint value is reached */ err = nvhost_intr_add_action(&(syncpt_to_dev(sp)->intr), id, thresh, NVHOST_INTR_ACTION_WAKEUP_INTERRUPTIBLE, &wq, &ref); if (err) goto done; err = -EAGAIN; /* wait for the syncpoint, or timeout, or signal */ while (timeout) { u32 check = min_t(u32, SYNCPT_CHECK_PERIOD, timeout); int remain = wait_event_interruptible_timeout(wq, nvhost_syncpt_min_cmp(sp, id, thresh), check); if (remain > 0 || nvhost_syncpt_min_cmp(sp, id, thresh)) { if (value) *value = nvhost_syncpt_read_min(sp, id); err = 0; break; } if (remain < 0) { err = remain; break; } if (timeout != NVHOST_NO_TIMEOUT) timeout -= check; if (timeout) { dev_warn(&syncpt_to_dev(sp)->pdev->dev, "syncpoint id %d (%s) stuck waiting %d\n", id, nvhost_syncpt_name(id), thresh); nvhost_syncpt_debug(sp); } }; nvhost_intr_put_ref(&(syncpt_to_dev(sp)->intr), ref); done: nvhost_module_idle(&syncpt_to_dev(sp)->mod); return err; }
/** * Write the current syncpoint value back to hw. */ static void t20_syncpt_reset(struct nvhost_syncpt *sp, u32 id) { struct nvhost_master *dev = syncpt_to_dev(sp); int min = nvhost_syncpt_read_min(sp, id); writel(min, dev->sync_aperture + (HOST1X_SYNC_SYNCPT_0 + id * 4)); }
/** * Main entrypoint for syncpoint value waits. */ int nvhost_syncpt_wait_timeout(struct nvhost_syncpt *sp, u32 id, u32 thresh, u32 timeout, u32 *value) { DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq); void *ref; void *waiter; int err = 0, check_count = 0, low_timeout = 0; u32 val; if (value) *value = 0; /* first check cache */ if (nvhost_syncpt_is_expired(sp, id, thresh)) { if (value) *value = nvhost_syncpt_read_min(sp, id); return 0; } /* keep host alive */ nvhost_module_busy(syncpt_to_dev(sp)->dev); /* try to read from register */ val = syncpt_op().update_min(sp, id); if (nvhost_syncpt_is_expired(sp, id, thresh)) { if (value) *value = val; goto done; } if (!timeout) { err = -EAGAIN; goto done; } /* schedule a wakeup when the syncpoint value is reached */ waiter = nvhost_intr_alloc_waiter(); if (!waiter) { err = -ENOMEM; goto done; } err = nvhost_intr_add_action(&(syncpt_to_dev(sp)->intr), id, thresh, NVHOST_INTR_ACTION_WAKEUP_INTERRUPTIBLE, &wq, waiter, &ref); if (err) goto done; err = -EAGAIN; /* Caller-specified timeout may be impractically low */ if (timeout < SYNCPT_CHECK_PERIOD) low_timeout = timeout; /* wait for the syncpoint, or timeout, or signal */ while (timeout) { u32 check = min_t(u32, SYNCPT_CHECK_PERIOD, timeout); int remain = wait_event_interruptible_timeout(wq, syncpt_update_min_is_expired(sp, id, thresh), check); if (remain > 0 || nvhost_syncpt_is_expired(sp, id, thresh)) { if (value) *value = nvhost_syncpt_read_min(sp, id); err = 0; break; } if (remain < 0) { err = remain; break; } if (timeout != NVHOST_NO_TIMEOUT) timeout -= check; if (timeout && check_count <= MAX_STUCK_CHECK_COUNT) { dev_warn(&syncpt_to_dev(sp)->dev->dev, "%s: syncpoint id %d (%s) stuck waiting %d, timeout=%d\n", current->comm, id, syncpt_op().name(sp, id), thresh, timeout); syncpt_op().debug(sp); if (check_count == MAX_STUCK_CHECK_COUNT) { if (low_timeout) { dev_warn(&syncpt_to_dev(sp)->dev->dev, "is timeout %d too low?\n", low_timeout); } nvhost_debug_dump(syncpt_to_dev(sp)); } check_count++; } } nvhost_intr_put_ref(&(syncpt_to_dev(sp)->intr), id, ref); done: nvhost_module_idle(syncpt_to_dev(sp)->dev); return err; }
/** * Main entrypoint for syncpoint value waits. */ int nvhost_syncpt_wait_timeout(struct nvhost_syncpt *sp, u32 id, u32 thresh, u32 timeout, u32 *value) { DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq); void *ref; void *waiter; int err = 0, check_count = 0, low_timeout = 0; static int print_once = 0; if (value) *value = 0; BUG_ON(!syncpt_op(sp).update_min); if (!nvhost_syncpt_check_max(sp, id, thresh)) { dev_warn(&syncpt_to_dev(sp)->pdev->dev, "wait %d (%s) for (%d) wouldn't be met (max %d)\n", id, syncpt_op(sp).name(sp, id), thresh, nvhost_syncpt_read_max(sp, id)); nvhost_debug_dump(syncpt_to_dev(sp)); return -EINVAL; } /* first check cache */ if (nvhost_syncpt_min_cmp(sp, id, thresh)) { if (value) *value = nvhost_syncpt_read_min(sp, id); return 0; } /* keep host alive */ nvhost_module_busy(syncpt_to_dev(sp)->dev); if (client_managed(id) || !nvhost_syncpt_min_eq_max(sp, id)) { /* try to read from register */ u32 val = syncpt_op(sp).update_min(sp, id); if ((s32)(val - thresh) >= 0) { if (value) *value = val; goto done; } } if (!timeout) { err = -EAGAIN; goto done; } /* schedule a wakeup when the syncpoint value is reached */ waiter = nvhost_intr_alloc_waiter(); if (!waiter) { err = -ENOMEM; goto done; } err = nvhost_intr_add_action(&(syncpt_to_dev(sp)->intr), id, thresh, NVHOST_INTR_ACTION_WAKEUP_INTERRUPTIBLE, &wq, waiter, &ref); if (err) goto done; err = -EAGAIN; /* wait for the syncpoint, or timeout, or signal */ while (timeout) { u32 check = min_t(u32, SYNCPT_CHECK_PERIOD, timeout); int remain = wait_event_interruptible_timeout(wq, nvhost_syncpt_min_cmp(sp, id, thresh), check); if (remain > 0 || nvhost_syncpt_min_cmp(sp, id, thresh)) { if (value) *value = nvhost_syncpt_read_min(sp, id); err = 0; break; } if (remain < 0) { err = remain; break; } if (timeout != NVHOST_NO_TIMEOUT) { if (timeout < SYNCPT_CHECK_PERIOD) { /* Caller-specified timeout may be impractically low */ low_timeout = timeout; } timeout -= check; } if (timeout) { dev_warn(&syncpt_to_dev(sp)->pdev->dev, "%s: syncpoint id %d (%s) stuck waiting %d, timeout=%d\n", current->comm, id, syncpt_op(sp).name(sp, id), thresh, timeout); syncpt_op(sp).debug(sp); print_once++; if (print_once == 1) { nvhost_debug_dump(syncpt_to_dev(sp)); debug_stuck_syncpoint(); } if (check_count > MAX_STUCK_CHECK_COUNT) { if (low_timeout) { dev_warn(&syncpt_to_dev(sp)->pdev->dev, "is timeout %d too low?\n", low_timeout); } nvhost_debug_dump(syncpt_to_dev(sp)); BUG(); } check_count++; } } nvhost_intr_put_ref(&(syncpt_to_dev(sp)->intr), ref); done: nvhost_module_idle(syncpt_to_dev(sp)->dev); return err; }
/** * Write the current syncpoint value back to hw. */ static void t20_syncpt_reset(struct nvhost_syncpt *sp, u32 id) { struct nvhost_master *dev = syncpt_to_dev(sp); int min = nvhost_syncpt_read_min(sp, id); writel(min, dev->sync_aperture + (host1x_sync_syncpt_0_r() + id * 4)); }