/** * radeon_fence_check_signaled - callback from fence_queue * * this function is called with fence_queue lock held, which is also used * for the fence locking itself, so unlocked variants are used for * fence_signal, and remove_wait_queue. */ static int radeon_fence_check_signaled(wait_queue_t *wait, unsigned mode, int flags, void *key) { struct radeon_fence *fence; u64 seq; fence = container_of(wait, struct radeon_fence, fence_wake); /* * We cannot use radeon_fence_process here because we're already * in the waitqueue, in a call from wake_up_all. */ seq = atomic64_read(&fence->rdev->fence_drv[fence->ring].last_seq); if (seq >= fence->seq) { int ret = fence_signal_locked(&fence->base); if (!ret) FENCE_TRACE(&fence->base, "signaled from irq context\n"); else FENCE_TRACE(&fence->base, "was already signaled\n"); radeon_irq_kms_sw_irq_put(fence->rdev, fence->ring); // __remove_wait_queue(&fence->rdev->fence_queue, &fence->fence_wake); fence_put(&fence->base); } else FENCE_TRACE(&fence->base, "pending\n"); return 0; }
/** * amdgpu_fence_process - check for fence activity * * @ring: pointer to struct amdgpu_ring * * Checks the current fence value and calculates the last * signalled fence value. Wakes the fence queue if the * sequence number has increased. */ void amdgpu_fence_process(struct amdgpu_ring *ring) { struct amdgpu_fence_driver *drv = &ring->fence_drv; uint32_t seq, last_seq; int r; do { last_seq = atomic_read(&ring->fence_drv.last_seq); seq = amdgpu_fence_read(ring); } while (atomic_cmpxchg(&drv->last_seq, last_seq, seq) != last_seq); if (seq != ring->fence_drv.sync_seq) amdgpu_fence_schedule_fallback(ring); while (last_seq != seq) { struct fence *fence, **ptr; ptr = &drv->fences[++last_seq & drv->num_fences_mask]; /* There is always exactly one thread signaling this fence slot */ fence = rcu_dereference_protected(*ptr, 1); RCU_INIT_POINTER(*ptr, NULL); BUG_ON(!fence); r = fence_signal(fence); if (!r) FENCE_TRACE(fence, "signaled from irq context\n"); else BUG(); fence_put(fence); } }
/** * radeon_fence_wait - wait for a fence to signal * * @fence: radeon fence object * @intr: use interruptible sleep * * Wait for the requested fence to signal (all asics). * @intr selects whether to use interruptable (true) or non-interruptable * (false) sleep when waiting for the fence. * Returns 0 if the fence has passed, error for all other cases. */ int radeon_fence_wait(struct radeon_fence *fence, bool intr) { uint64_t seq[RADEON_NUM_RINGS] = {}; long r; /* * This function should not be called on !radeon fences. * If this is the case, it would mean this function can * also be called on radeon fences belonging to another card. * exclusive_lock is not held in that case. */ if (WARN_ON_ONCE(!to_radeon_fence(&fence->base))) return fence_wait(&fence->base, intr); seq[fence->ring] = fence->seq; r = radeon_fence_wait_seq_timeout(fence->rdev, seq, intr, MAX_SCHEDULE_TIMEOUT); if (r < 0) { return r; } r = fence_signal(&fence->base); if (!r) FENCE_TRACE(&fence->base, "signaled from fence_wait\n"); return 0; }
/** * amdgpu_fence_enable_signaling - enable signalling on fence * @fence: fence * * This function is called with fence_queue lock held, and adds a callback * to fence_queue that checks if this fence is signaled, and if so it * signals the fence and removes itself. */ static bool amdgpu_fence_enable_signaling(struct fence *f) { struct amdgpu_fence *fence = to_amdgpu_fence(f); struct amdgpu_ring *ring = fence->ring; if (!timer_pending(&ring->fence_drv.fallback_timer)) amdgpu_fence_schedule_fallback(ring); FENCE_TRACE(&fence->base, "armed on ring %i!\n", ring->idx); return true; }
/** * radeon_fence_signaled - check if a fence has signaled * * @fence: radeon fence object * * Check if the requested fence has signaled (all asics). * Returns true if the fence has signaled or false if it has not. */ bool radeon_fence_signaled(struct radeon_fence *fence) { if (!fence) return true; if (radeon_fence_seq_signaled(fence->rdev, fence->seq, fence->ring)) { int ret; ret = fence_signal(&fence->base); if (!ret) FENCE_TRACE(&fence->base, "signaled from radeon_fence_signaled\n"); return true; } return false; }
/** * radeon_fence_enable_signaling - enable signalling on fence * @fence: fence * * This function is called with fence_queue lock held, and adds a callback * to fence_queue that checks if this fence is signaled, and if so it * signals the fence and removes itself. */ static bool radeon_fence_enable_signaling(struct fence *f) { struct radeon_fence *fence = to_radeon_fence(f); struct radeon_device *rdev = fence->rdev; if (atomic64_read(&rdev->fence_drv[fence->ring].last_seq) >= fence->seq) return false; // if (down_read_trylock(&rdev->exclusive_lock)) { radeon_irq_kms_sw_irq_get(rdev, fence->ring); // if (radeon_fence_activity(rdev, fence->ring)) // wake_up_all_locked(&rdev->fence_queue); /* did fence get signaled after we enabled the sw irq? */ if (atomic64_read(&rdev->fence_drv[fence->ring].last_seq) >= fence->seq) { radeon_irq_kms_sw_irq_put(rdev, fence->ring); // up_read(&rdev->exclusive_lock); return false; } // up_read(&rdev->exclusive_lock); // } else { /* we're probably in a lockup, lets not fiddle too much */ // if (radeon_irq_kms_sw_irq_get_delayed(rdev, fence->ring)) // rdev->fence_drv[fence->ring].delayed_irq = true; // radeon_fence_schedule_check(rdev, fence->ring); } // fence->fence_wake.flags = 0; // fence->fence_wake.private = NULL; fence->fence_wake.func = radeon_fence_check_signaled; __add_wait_queue(&rdev->fence_queue, &fence->fence_wake); fence_get(f); FENCE_TRACE(&fence->base, "armed on ring %i!\n", fence->ring); return true; }