static struct dma_fence *
etnaviv_sched_dependency(struct drm_sched_job *sched_job,
struct drm_sched_entity *entity)
{
struct etnaviv_gem_submit *submit = to_etnaviv_submit(sched_job);
struct dma_fence *fence;
int i;
if (unlikely(submit->in_fence)) {
fence = submit->in_fence;
submit->in_fence = NULL;
if (!dma_fence_is_signaled(fence))
return fence;
dma_fence_put(fence);
}
for (i = 0; i < submit->nr_bos; i++) {
struct etnaviv_gem_submit_bo *bo = &submit->bos[i];
int j;
if (bo->excl) {
fence = bo->excl;
bo->excl = NULL;
if (!dma_fence_is_signaled(fence))
return fence;
dma_fence_put(fence);
}
for (j = 0; j < bo->nr_shared; j++) {
if (!bo->shared[j])
continue;
fence = bo->shared[j];
bo->shared[j] = NULL;
if (!dma_fence_is_signaled(fence))
return fence;
dma_fence_put(fence);
}
kfree(bo->shared);
bo->nr_shared = 0;
bo->shared = NULL;
}
return NULL;
}
int amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, struct amdgpu_ring *ring,
struct dma_fence *fence, uint64_t* handler)
{
struct amdgpu_ctx_ring *cring = & ctx->rings[ring->idx];
uint64_t seq = cring->sequence;
unsigned idx = 0;
struct dma_fence *other = NULL;
idx = seq & (amdgpu_sched_jobs - 1);
other = cring->fences[idx];
if (other)
BUG_ON(!dma_fence_is_signaled(other));
dma_fence_get(fence);
spin_lock(&ctx->ring_lock);
cring->fences[idx] = fence;
cring->sequence++;
spin_unlock(&ctx->ring_lock);
dma_fence_put(other);
if (handler)
*handler = seq;
return 0;
}
/*
* vgem_fence_signal_ioctl (DRM_IOCTL_VGEM_FENCE_SIGNAL):
*
* Signal and consume a fence ealier attached to a vGEM handle using
* vgem_fence_attach_ioctl (DRM_IOCTL_VGEM_FENCE_ATTACH).
*
* All fences must be signaled within 10s of attachment or otherwise they
* will automatically expire (and a vgem_fence_signal_ioctl returns -ETIMEDOUT).
*
* Signaling a fence indicates to all consumers of the dma-buf that the
* client has completed the operation associated with the fence, and that the
* buffer is then ready for consumption.
*
* If the fence does not exist (or has already been signaled by the client),
* vgem_fence_signal_ioctl returns -ENOENT.
*/
int vgem_fence_signal_ioctl(struct drm_device *dev,
void *data,
struct drm_file *file)
{
struct vgem_file *vfile = file->driver_priv;
struct drm_vgem_fence_signal *arg = data;
struct dma_fence *fence;
int ret = 0;
if (arg->flags)
return -EINVAL;
mutex_lock(&vfile->fence_mutex);
fence = idr_replace(&vfile->fence_idr, NULL, arg->fence);
mutex_unlock(&vfile->fence_mutex);
if (!fence)
return -ENOENT;
if (IS_ERR(fence))
return PTR_ERR(fence);
if (dma_fence_is_signaled(fence))
ret = -ETIMEDOUT;
dma_fence_signal(fence);
dma_fence_put(fence);
return ret;
}
void amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx,
struct drm_sched_entity *entity,
struct dma_fence *fence, uint64_t* handle)
{
struct amdgpu_ctx_entity *centity = to_amdgpu_ctx_entity(entity);
uint64_t seq = centity->sequence;
struct dma_fence *other = NULL;
unsigned idx = 0;
idx = seq & (amdgpu_sched_jobs - 1);
other = centity->fences[idx];
if (other)
BUG_ON(!dma_fence_is_signaled(other));
dma_fence_get(fence);
spin_lock(&ctx->ring_lock);
centity->fences[idx] = fence;
centity->sequence++;
spin_unlock(&ctx->ring_lock);
dma_fence_put(other);
if (handle)
*handle = seq;
}
static long sync_file_ioctl_fence_info(struct sync_file *sync_file,
unsigned long arg)
{
struct sync_file_info info;
struct sync_fence_info *fence_info = NULL;
struct dma_fence **fences;
__u32 size;
int num_fences, ret, i;
if (copy_from_user(&info, (void __user *)arg, sizeof(info)))
return -EFAULT;
if (info.flags || info.pad)
return -EINVAL;
fences = get_fences(sync_file, &num_fences);
/*
* Passing num_fences = 0 means that userspace doesn't want to
* retrieve any sync_fence_info. If num_fences = 0 we skip filling
* sync_fence_info and return the actual number of fences on
* info->num_fences.
*/
if (!info.num_fences)
goto no_fences;
if (info.num_fences < num_fences)
return -EINVAL;
size = num_fences * sizeof(*fence_info);
fence_info = kzalloc(size, GFP_KERNEL);
if (!fence_info)
return -ENOMEM;
for (i = 0; i < num_fences; i++)
sync_fill_fence_info(fences[i], &fence_info[i]);
if (copy_to_user(u64_to_user_ptr(info.sync_fence_info), fence_info,
size)) {
ret = -EFAULT;
goto out;
}
no_fences:
strlcpy(info.name, sync_file->name, sizeof(info.name));
info.status = dma_fence_is_signaled(sync_file->fence);
info.num_fences = num_fences;
if (copy_to_user((void __user *)arg, &info, sizeof(info)))
ret = -EFAULT;
else
ret = 0;
out:
kfree(fence_info);
return ret;
}
static void add_fence(struct dma_fence **fences,
int *i, struct dma_fence *fence)
{
fences[*i] = fence;
if (!dma_fence_is_signaled(fence)) {
dma_fence_get(fence);
(*i)++;
}
}
static void sync_fill_fence_info(struct dma_fence *fence,
struct sync_fence_info *info)
{
strlcpy(info->obj_name, fence->ops->get_timeline_name(fence),
sizeof(info->obj_name));
strlcpy(info->driver_name, fence->ops->get_driver_name(fence),
sizeof(info->driver_name));
if (dma_fence_is_signaled(fence))
info->status = fence->status >= 0 ? 1 : fence->status;
else
info->status = 0;
info->timestamp_ns = ktime_to_ns(fence->timestamp);
}
static unsigned int sync_file_poll(struct file *file, poll_table *wait)
{
struct sync_file *sync_file = file->private_data;
poll_wait(file, &sync_file->wq, wait);
if (!test_and_set_bit(POLL_ENABLED, &sync_file->fence->flags)) {
if (dma_fence_add_callback(sync_file->fence, &sync_file->cb,
fence_check_cb_func) < 0)
wake_up_all(&sync_file->wq);
}
return dma_fence_is_signaled(sync_file->fence) ? POLLIN : 0;
}
static void etnaviv_sched_timedout_job(struct drm_sched_job *sched_job)
{
struct etnaviv_gem_submit *submit = to_etnaviv_submit(sched_job);
struct etnaviv_gpu *gpu = submit->gpu;
u32 dma_addr;
int change;
/*
* If the GPU managed to complete this jobs fence, the timout is
* spurious. Bail out.
*/
if (dma_fence_is_signaled(submit->out_fence))
return;
/*
* If the GPU is still making forward progress on the front-end (which
* should never loop) we shift out the timeout to give it a chance to
* finish the job.
*/
dma_addr = gpu_read(gpu, VIVS_FE_DMA_ADDRESS);
change = dma_addr - gpu->hangcheck_dma_addr;
if (change < 0 || change > 16) {
gpu->hangcheck_dma_addr = dma_addr;
return;
}
/* block scheduler */
drm_sched_stop(&gpu->sched);
if(sched_job)
drm_sched_increase_karma(sched_job);
/* get the GPU back into the init state */
etnaviv_core_dump(gpu);
etnaviv_gpu_recover_hang(gpu);
drm_sched_resubmit_jobs(&gpu->sched);
/* restart scheduler after GPU is usable again */
drm_sched_start(&gpu->sched, true);
}
/** kgd2kfd_schedule_evict_and_restore_process - Schedules work queue that will
* prepare for safe eviction of KFD BOs that belong to the specified
* process.
*
* @mm: mm_struct that identifies the specified KFD process
* @fence: eviction fence attached to KFD process BOs
*
*/
int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm,
struct dma_fence *fence)
{
struct kfd_process *p;
unsigned long active_time;
unsigned long delay_jiffies = msecs_to_jiffies(PROCESS_ACTIVE_TIME_MS);
if (!fence)
return -EINVAL;
if (dma_fence_is_signaled(fence))
return 0;
p = kfd_lookup_process_by_mm(mm);
if (!p)
return -ENODEV;
if (fence->seqno == p->last_eviction_seqno)
goto out;
p->last_eviction_seqno = fence->seqno;
/* Avoid KFD process starvation. Wait for at least
* PROCESS_ACTIVE_TIME_MS before evicting the process again
*/
active_time = get_jiffies_64() - p->last_restore_timestamp;
if (delay_jiffies > active_time)
delay_jiffies -= active_time;
else
delay_jiffies = 0;
/* During process initialization eviction_work.dwork is initialized
* to kfd_evict_bo_worker
*/
schedule_delayed_work(&p->eviction_work, delay_jiffies);
out:
kfd_unref_process(p);
return 0;
}
/**
* amdgpu_fence_emit - emit a fence on the requested ring
*
* @ring: ring the fence is associated with
* @f: resulting fence object
*
* Emits a fence command on the requested ring (all asics).
* Returns 0 on success, -ENOMEM on failure.
*/
int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f)
{
struct amdgpu_device *adev = ring->adev;
struct amdgpu_fence *fence;
struct dma_fence *old, **ptr;
uint32_t seq;
fence = kmem_cache_alloc(amdgpu_fence_slab, GFP_KERNEL);
if (fence == NULL)
return -ENOMEM;
seq = ++ring->fence_drv.sync_seq;
fence->ring = ring;
dma_fence_init(&fence->base, &amdgpu_fence_ops,
&ring->fence_drv.lock,
adev->fence_context + ring->idx,
seq);
amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
seq, AMDGPU_FENCE_FLAG_INT);
ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];
/* This function can't be called concurrently anyway, otherwise
* emitting the fence would mess up the hardware ring buffer.
*/
old = rcu_dereference_protected(*ptr, 1);
if (old && !dma_fence_is_signaled(old)) {
DRM_INFO("rcu slot is busy\n");
dma_fence_wait(old, false);
}
rcu_assign_pointer(*ptr, dma_fence_get(&fence->base));
*f = &fence->base;
return 0;
}
static void vgem_fence_timeline_value_str(struct dma_fence *fence, char *str,
int size)
{
snprintf(str, size, "%u",
dma_fence_is_signaled(fence) ? fence->seqno : 0);
}
