int nfp_flower_metadata_init(struct nfp_app *app, u64 host_ctx_count,
unsigned int host_num_mems)
{
struct nfp_flower_priv *priv = app->priv;
int err, stats_size;
hash_init(priv->mask_table);
err = rhashtable_init(&priv->flow_table, &nfp_flower_table_params);
if (err)
return err;
get_random_bytes(&priv->mask_id_seed, sizeof(priv->mask_id_seed));
/* Init ring buffer and unallocated mask_ids. */
priv->mask_ids.mask_id_free_list.buf =
kmalloc_array(NFP_FLOWER_MASK_ENTRY_RS,
NFP_FLOWER_MASK_ELEMENT_RS, GFP_KERNEL);
if (!priv->mask_ids.mask_id_free_list.buf)
goto err_free_flow_table;
priv->mask_ids.init_unallocated = NFP_FLOWER_MASK_ENTRY_RS - 1;
/* Init timestamps for mask id*/
priv->mask_ids.last_used =
kmalloc_array(NFP_FLOWER_MASK_ENTRY_RS,
sizeof(*priv->mask_ids.last_used), GFP_KERNEL);
if (!priv->mask_ids.last_used)
goto err_free_mask_id;
/* Init ring buffer and unallocated stats_ids. */
priv->stats_ids.free_list.buf =
vmalloc(array_size(NFP_FL_STATS_ELEM_RS,
priv->stats_ring_size));
if (!priv->stats_ids.free_list.buf)
goto err_free_last_used;
priv->stats_ids.init_unalloc = div_u64(host_ctx_count, host_num_mems);
stats_size = FIELD_PREP(NFP_FL_STAT_ID_STAT, host_ctx_count) |
FIELD_PREP(NFP_FL_STAT_ID_MU_NUM, host_num_mems - 1);
priv->stats = kvmalloc_array(stats_size, sizeof(struct nfp_fl_stats),
GFP_KERNEL);
if (!priv->stats)
goto err_free_ring_buf;
spin_lock_init(&priv->stats_lock);
return 0;
err_free_ring_buf:
vfree(priv->stats_ids.free_list.buf);
err_free_last_used:
kfree(priv->mask_ids.last_used);
err_free_mask_id:
kfree(priv->mask_ids.mask_id_free_list.buf);
err_free_flow_table:
rhashtable_destroy(&priv->flow_table);
return -ENOMEM;
}
static int udl_prime_create(struct drm_device *dev,
size_t size,
struct sg_table *sg,
struct udl_gem_object **obj_p)
{
struct udl_gem_object *obj;
int npages;
npages = size / PAGE_SIZE;
*obj_p = NULL;
obj = udl_gem_alloc_object(dev, npages * PAGE_SIZE);
if (!obj)
return -ENOMEM;
obj->sg = sg;
obj->pages = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
if (obj->pages == NULL) {
DRM_ERROR("obj pages is NULL %d\n", npages);
return -ENOMEM;
}
drm_prime_sg_to_page_addr_arrays(sg, obj->pages, NULL, npages);
*obj_p = obj;
return 0;
}
/**
* Allocates storage for pointers to the pages that back the ttm.
*/
static int ttm_tt_alloc_page_directory(struct ttm_tt *ttm)
{
ttm->pages = kvmalloc_array(ttm->num_pages, sizeof(void*),
GFP_KERNEL | __GFP_ZERO);
if (!ttm->pages)
return -ENOMEM;
return 0;
}
/*
* BLKREPORTZONE ioctl processing.
* Called from blkdev_ioctl.
*/
int blkdev_report_zones_ioctl(struct block_device *bdev, fmode_t mode,
unsigned int cmd, unsigned long arg)
{
void __user *argp = (void __user *)arg;
struct request_queue *q;
struct blk_zone_report rep;
struct blk_zone *zones;
int ret;
if (!argp)
return -EINVAL;
q = bdev_get_queue(bdev);
if (!q)
return -ENXIO;
if (!blk_queue_is_zoned(q))
return -ENOTTY;
if (!capable(CAP_SYS_ADMIN))
return -EACCES;
if (copy_from_user(&rep, argp, sizeof(struct blk_zone_report)))
return -EFAULT;
if (!rep.nr_zones)
return -EINVAL;
rep.nr_zones = min(blkdev_nr_zones(bdev), rep.nr_zones);
zones = kvmalloc_array(rep.nr_zones, sizeof(struct blk_zone),
GFP_KERNEL | __GFP_ZERO);
if (!zones)
return -ENOMEM;
ret = blkdev_report_zones(bdev, rep.sector,
zones, &rep.nr_zones,
GFP_KERNEL);
if (ret)
goto out;
if (copy_to_user(argp, &rep, sizeof(struct blk_zone_report))) {
ret = -EFAULT;
goto out;
}
if (rep.nr_zones) {
if (copy_to_user(argp + sizeof(struct blk_zone_report), zones,
sizeof(struct blk_zone) * rep.nr_zones))
ret = -EFAULT;
}
out:
kvfree(zones);
return ret;
}
static int ttm_sg_tt_alloc_page_directory(struct ttm_dma_tt *ttm)
{
ttm->dma_address = kvmalloc_array(ttm->ttm.num_pages,
sizeof(*ttm->dma_address),
GFP_KERNEL | __GFP_ZERO);
if (!ttm->dma_address)
return -ENOMEM;
return 0;
}
static struct fdtable * alloc_fdtable(unsigned int nr)
{
struct fdtable *fdt;
void *data;
/*
* Figure out how many fds we actually want to support in this fdtable.
* Allocation steps are keyed to the size of the fdarray, since it
* grows far faster than any of the other dynamic data. We try to fit
* the fdarray into comfortable page-tuned chunks: starting at 1024B
* and growing in powers of two from there on.
*/
nr /= (1024 / sizeof(struct file *));
nr = roundup_pow_of_two(nr + 1);
nr *= (1024 / sizeof(struct file *));
/*
* Note that this can drive nr *below* what we had passed if sysctl_nr_open
* had been set lower between the check in expand_files() and here. Deal
* with that in caller, it's cheaper that way.
*
* We make sure that nr remains a multiple of BITS_PER_LONG - otherwise
* bitmaps handling below becomes unpleasant, to put it mildly...
*/
if (unlikely(nr > sysctl_nr_open))
nr = ((sysctl_nr_open - 1) | (BITS_PER_LONG - 1)) + 1;
fdt = kmalloc(sizeof(struct fdtable), GFP_KERNEL_ACCOUNT);
if (!fdt)
goto out;
fdt->max_fds = nr;
data = kvmalloc_array(nr, sizeof(struct file *), GFP_KERNEL_ACCOUNT);
if (!data)
goto out_fdt;
fdt->fd = data;
data = kvmalloc(max_t(size_t,
2 * nr / BITS_PER_BYTE + BITBIT_SIZE(nr), L1_CACHE_BYTES),
GFP_KERNEL_ACCOUNT);
if (!data)
goto out_arr;
fdt->open_fds = data;
data += nr / BITS_PER_BYTE;
fdt->close_on_exec = data;
data += nr / BITS_PER_BYTE;
fdt->full_fds_bits = data;
return fdt;
out_arr:
kvfree(fdt->fd);
out_fdt:
kfree(fdt);
out:
return NULL;
}
static struct drm_gem_object *vgem_prime_import_sg_table(struct drm_device *dev,
struct dma_buf_attachment *attach, struct sg_table *sg)
{
struct drm_vgem_gem_object *obj;
int npages;
obj = __vgem_gem_create(dev, attach->dmabuf->size);
if (IS_ERR(obj))
return ERR_CAST(obj);
npages = PAGE_ALIGN(attach->dmabuf->size) / PAGE_SIZE;
obj->table = sg;
obj->pages = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
if (!obj->pages) {
__vgem_gem_destroy(obj);
return ERR_PTR(-ENOMEM);
}
obj->pages_pin_count++; /* perma-pinned */
drm_prime_sg_to_page_addr_arrays(obj->table, obj->pages, NULL,
npages);
return &obj->base;
}
static int choke_change(struct Qdisc *sch, struct nlattr *opt,
struct netlink_ext_ack *extack)
{
struct choke_sched_data *q = qdisc_priv(sch);
struct nlattr *tb[TCA_CHOKE_MAX + 1];
const struct tc_red_qopt *ctl;
int err;
struct sk_buff **old = NULL;
unsigned int mask;
u32 max_P;
if (opt == NULL)
return -EINVAL;
err = nla_parse_nested(tb, TCA_CHOKE_MAX, opt, choke_policy, NULL);
if (err < 0)
return err;
if (tb[TCA_CHOKE_PARMS] == NULL ||
tb[TCA_CHOKE_STAB] == NULL)
return -EINVAL;
max_P = tb[TCA_CHOKE_MAX_P] ? nla_get_u32(tb[TCA_CHOKE_MAX_P]) : 0;
ctl = nla_data(tb[TCA_CHOKE_PARMS]);
if (!red_check_params(ctl->qth_min, ctl->qth_max, ctl->Wlog))
return -EINVAL;
if (ctl->limit > CHOKE_MAX_QUEUE)
return -EINVAL;
mask = roundup_pow_of_two(ctl->limit + 1) - 1;
if (mask != q->tab_mask) {
struct sk_buff **ntab;
ntab = kvmalloc_array((mask + 1), sizeof(struct sk_buff *), GFP_KERNEL | __GFP_ZERO);
if (!ntab)
return -ENOMEM;
sch_tree_lock(sch);
old = q->tab;
if (old) {
unsigned int oqlen = sch->q.qlen, tail = 0;
unsigned dropped = 0;
while (q->head != q->tail) {
struct sk_buff *skb = q->tab[q->head];
q->head = (q->head + 1) & q->tab_mask;
if (!skb)
continue;
if (tail < mask) {
ntab[tail++] = skb;
continue;
}
dropped += qdisc_pkt_len(skb);
qdisc_qstats_backlog_dec(sch, skb);
--sch->q.qlen;
rtnl_qdisc_drop(skb, sch);
}
qdisc_tree_reduce_backlog(sch, oqlen - sch->q.qlen, dropped);
q->head = 0;
q->tail = tail;
}
q->tab_mask = mask;
q->tab = ntab;
} else
sch_tree_lock(sch);
q->flags = ctl->flags;
q->limit = ctl->limit;
red_set_parms(&q->parms, ctl->qth_min, ctl->qth_max, ctl->Wlog,
ctl->Plog, ctl->Scell_log,
nla_data(tb[TCA_CHOKE_STAB]),
max_P);
red_set_vars(&q->vars);
if (q->head == q->tail)
red_end_of_idle_period(&q->vars);
sch_tree_unlock(sch);
choke_free(old);
return 0;
}
int amdgpu_bo_list_ioctl(struct drm_device *dev, void *data,
struct drm_file *filp)
{
const uint32_t info_size = sizeof(struct drm_amdgpu_bo_list_entry);
struct amdgpu_device *adev = dev->dev_private;
struct amdgpu_fpriv *fpriv = filp->driver_priv;
union drm_amdgpu_bo_list *args = data;
uint32_t handle = args->in.list_handle;
const void __user *uptr = u64_to_user_ptr(args->in.bo_info_ptr);
struct drm_amdgpu_bo_list_entry *info;
struct amdgpu_bo_list *list;
int r;
info = kvmalloc_array(args->in.bo_number,
sizeof(struct drm_amdgpu_bo_list_entry), GFP_KERNEL);
if (!info)
return -ENOMEM;
/* copy the handle array from userspace to a kernel buffer */
r = -EFAULT;
if (likely(info_size == args->in.bo_info_size)) {
unsigned long bytes = args->in.bo_number *
args->in.bo_info_size;
if (copy_from_user(info, uptr, bytes))
goto error_free;
} else {
unsigned long bytes = min(args->in.bo_info_size, info_size);
unsigned i;
memset(info, 0, args->in.bo_number * info_size);
for (i = 0; i < args->in.bo_number; ++i) {
if (copy_from_user(&info[i], uptr, bytes))
goto error_free;
uptr += args->in.bo_info_size;
}
}
switch (args->in.operation) {
case AMDGPU_BO_LIST_OP_CREATE:
r = amdgpu_bo_list_create(adev, filp, info, args->in.bo_number,
&handle);
if (r)
goto error_free;
break;
case AMDGPU_BO_LIST_OP_DESTROY:
amdgpu_bo_list_destroy(fpriv, handle);
handle = 0;
break;
case AMDGPU_BO_LIST_OP_UPDATE:
r = -ENOENT;
list = amdgpu_bo_list_get(fpriv, handle);
if (!list)
goto error_free;
r = amdgpu_bo_list_set(adev, filp, list, info,
args->in.bo_number);
amdgpu_bo_list_put(list);
if (r)
goto error_free;
break;
default:
r = -EINVAL;
goto error_free;
}
memset(args, 0, sizeof(*args));
args->out.list_handle = handle;
kvfree(info);
return 0;
error_free:
kvfree(info);
return r;
}
static int amdgpu_bo_list_set(struct amdgpu_device *adev,
struct drm_file *filp,
struct amdgpu_bo_list *list,
struct drm_amdgpu_bo_list_entry *info,
unsigned num_entries)
{
struct amdgpu_bo_list_entry *array;
struct amdgpu_bo *gds_obj = adev->gds.gds_gfx_bo;
struct amdgpu_bo *gws_obj = adev->gds.gws_gfx_bo;
struct amdgpu_bo *oa_obj = adev->gds.oa_gfx_bo;
unsigned last_entry = 0, first_userptr = num_entries;
unsigned i;
int r;
unsigned long total_size = 0;
array = kvmalloc_array(num_entries, sizeof(struct amdgpu_bo_list_entry), GFP_KERNEL);
if (!array)
return -ENOMEM;
memset(array, 0, num_entries * sizeof(struct amdgpu_bo_list_entry));
for (i = 0; i < num_entries; ++i) {
struct amdgpu_bo_list_entry *entry;
struct drm_gem_object *gobj;
struct amdgpu_bo *bo;
struct mm_struct *usermm;
gobj = drm_gem_object_lookup(filp, info[i].bo_handle);
if (!gobj) {
r = -ENOENT;
goto error_free;
}
bo = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj));
drm_gem_object_put_unlocked(gobj);
usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm);
if (usermm) {
if (usermm != current->mm) {
amdgpu_bo_unref(&bo);
r = -EPERM;
goto error_free;
}
entry = &array[--first_userptr];
} else {
entry = &array[last_entry++];
}
entry->robj = bo;
entry->priority = min(info[i].bo_priority,
AMDGPU_BO_LIST_MAX_PRIORITY);
entry->tv.bo = &entry->robj->tbo;
entry->tv.shared = !entry->robj->prime_shared_count;
if (entry->robj->preferred_domains == AMDGPU_GEM_DOMAIN_GDS)
gds_obj = entry->robj;
if (entry->robj->preferred_domains == AMDGPU_GEM_DOMAIN_GWS)
gws_obj = entry->robj;
if (entry->robj->preferred_domains == AMDGPU_GEM_DOMAIN_OA)
oa_obj = entry->robj;
total_size += amdgpu_bo_size(entry->robj);
trace_amdgpu_bo_list_set(list, entry->robj);
}
for (i = 0; i < list->num_entries; ++i)
amdgpu_bo_unref(&list->array[i].robj);
kvfree(list->array);
list->gds_obj = gds_obj;
list->gws_obj = gws_obj;
list->oa_obj = oa_obj;
list->first_userptr = first_userptr;
list->array = array;
list->num_entries = num_entries;
trace_amdgpu_cs_bo_status(list->num_entries, total_size);
return 0;
error_free:
while (i--)
amdgpu_bo_unref(&array[i].robj);
kvfree(array);
return r;
}
static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
{
const int num_pages = obj->base.size >> PAGE_SHIFT;
struct mm_struct *mm = obj->userptr.mm->mm;
struct page **pvec;
struct sg_table *pages;
bool active;
int pinned;
/* If userspace should engineer that these pages are replaced in
* the vma between us binding this page into the GTT and completion
* of rendering... Their loss. If they change the mapping of their
* pages they need to create a new bo to point to the new vma.
*
* However, that still leaves open the possibility of the vma
* being copied upon fork. Which falls under the same userspace
* synchronisation issue as a regular bo, except that this time
* the process may not be expecting that a particular piece of
* memory is tied to the GPU.
*
* Fortunately, we can hook into the mmu_notifier in order to
* discard the page references prior to anything nasty happening
* to the vma (discard or cloning) which should prevent the more
* egregious cases from causing harm.
*/
if (obj->userptr.work) {
/* active flag should still be held for the pending work */
if (IS_ERR(obj->userptr.work))
return PTR_ERR(obj->userptr.work);
else
return -EAGAIN;
}
pvec = NULL;
pinned = 0;
if (mm == current->mm) {
pvec = kvmalloc_array(num_pages, sizeof(struct page *),
GFP_KERNEL |
__GFP_NORETRY |
__GFP_NOWARN);
if (pvec) /* defer to worker if malloc fails */
pinned = __get_user_pages_fast(obj->userptr.ptr,
num_pages,
!i915_gem_object_is_readonly(obj),
pvec);
}
active = false;
if (pinned < 0) {
pages = ERR_PTR(pinned);
pinned = 0;
} else if (pinned < num_pages) {
pages = __i915_gem_userptr_get_pages_schedule(obj);
active = pages == ERR_PTR(-EAGAIN);
} else {
pages = __i915_gem_userptr_alloc_pages(obj, pvec, num_pages);
active = !IS_ERR(pages);
}
if (active)
__i915_gem_userptr_set_active(obj, true);
if (IS_ERR(pages))
release_pages(pvec, pinned);
kvfree(pvec);
return PTR_ERR_OR_ZERO(pages);
}
static void
__i915_gem_userptr_get_pages_worker(struct work_struct *_work)
{
struct get_pages_work *work = container_of(_work, typeof(*work), work);
struct drm_i915_gem_object *obj = work->obj;
const int npages = obj->base.size >> PAGE_SHIFT;
struct page **pvec;
int pinned, ret;
ret = -ENOMEM;
pinned = 0;
pvec = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
if (pvec != NULL) {
struct mm_struct *mm = obj->userptr.mm->mm;
unsigned int flags = 0;
if (!i915_gem_object_is_readonly(obj))
flags |= FOLL_WRITE;
ret = -EFAULT;
if (mmget_not_zero(mm)) {
down_read(&mm->mmap_sem);
while (pinned < npages) {
ret = get_user_pages_remote
(work->task, mm,
obj->userptr.ptr + pinned * PAGE_SIZE,
npages - pinned,
flags,
pvec + pinned, NULL, NULL);
if (ret < 0)
break;
pinned += ret;
}
up_read(&mm->mmap_sem);
mmput(mm);
}
}
mutex_lock(&obj->mm.lock);
if (obj->userptr.work == &work->work) {
struct sg_table *pages = ERR_PTR(ret);
if (pinned == npages) {
pages = __i915_gem_userptr_alloc_pages(obj, pvec,
npages);
if (!IS_ERR(pages)) {
pinned = 0;
pages = NULL;
}
}
obj->userptr.work = ERR_CAST(pages);
if (IS_ERR(pages))
__i915_gem_userptr_set_active(obj, false);
}
mutex_unlock(&obj->mm.lock);
release_pages(pvec, pinned);
kvfree(pvec);
i915_gem_object_put(obj);
put_task_struct(work->task);
kfree(work);
}
static int igt_wakeup(void *arg)
{
I915_RND_STATE(prng);
struct intel_engine_cs *engine = arg;
struct igt_wakeup *waiters;
DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
const int count = 4096;
const u32 max_seqno = count / 4;
atomic_t ready, set, done;
int err = -ENOMEM;
int n, step;
mock_engine_reset(engine);
waiters = kvmalloc_array(count, sizeof(*waiters), GFP_KERNEL);
if (!waiters)
goto out_engines;
/* Create a large number of threads, each waiting on a random seqno.
* Multiple waiters will be waiting for the same seqno.
*/
atomic_set(&ready, count);
for (n = 0; n < count; n++) {
waiters[n].wq = &wq;
waiters[n].ready = &ready;
waiters[n].set = &set;
waiters[n].done = &done;
waiters[n].engine = engine;
waiters[n].flags = BIT(IDLE);
waiters[n].tsk = kthread_run(igt_wakeup_thread, &waiters[n],
"i915/igt:%d", n);
if (IS_ERR(waiters[n].tsk))
goto out_waiters;
get_task_struct(waiters[n].tsk);
}
for (step = 1; step <= max_seqno; step <<= 1) {
u32 seqno;
/* The waiter threads start paused as we assign them a random
* seqno and reset the engine. Once the engine is reset,
* we signal that the threads may begin their wait upon their
* seqno.
*/
for (n = 0; n < count; n++) {
GEM_BUG_ON(!test_bit(IDLE, &waiters[n].flags));
waiters[n].seqno =
1 + prandom_u32_state(&prng) % max_seqno;
}
mock_seqno_advance(engine, 0);
igt_wake_all_sync(&ready, &set, &done, &wq, count);
/* Simulate the GPU doing chunks of work, with one or more
* seqno appearing to finish at the same time. A random number
* of threads will be waiting upon the update and hopefully be
* woken.
*/
for (seqno = 1; seqno <= max_seqno + step; seqno += step) {
usleep_range(50, 500);
mock_seqno_advance(engine, seqno);
}
GEM_BUG_ON(intel_engine_get_seqno(engine) < 1 + max_seqno);
/* With the seqno now beyond any of the waiting threads, they
* should all be woken, see that they are complete and signal
* that they are ready for the next test. We wait until all
* threads are complete and waiting for us (i.e. not a seqno).
*/
if (!wait_var_event_timeout(&done,
!atomic_read(&done), 10 * HZ)) {
pr_err("Timed out waiting for %d remaining waiters\n",
atomic_read(&done));
err = -ETIMEDOUT;
break;
}
err = check_rbtree_empty(engine);
if (err)
break;
}
out_waiters:
for (n = 0; n < count; n++) {
if (IS_ERR(waiters[n].tsk))
break;
set_bit(STOP, &waiters[n].flags);
}
mock_seqno_advance(engine, INT_MAX); /* wakeup any broken waiters */
igt_wake_all_sync(&ready, &set, &done, &wq, n);
for (n = 0; n < count; n++) {
if (IS_ERR(waiters[n].tsk))
break;
kthread_stop(waiters[n].tsk);
put_task_struct(waiters[n].tsk);
}
kvfree(waiters);
out_engines:
mock_engine_flush(engine);
return err;
}
static int igt_insert_complete(void *arg)
{
const u32 seqno_bias = 0x1000;
struct intel_engine_cs *engine = arg;
struct intel_wait *waiters;
const int count = 4096;
unsigned long *bitmap;
int err = -ENOMEM;
int n, m;
mock_engine_reset(engine);
waiters = kvmalloc_array(count, sizeof(*waiters), GFP_KERNEL);
if (!waiters)
goto out_engines;
bitmap = kcalloc(DIV_ROUND_UP(count, BITS_PER_LONG), sizeof(*bitmap),
GFP_KERNEL);
if (!bitmap)
goto out_waiters;
for (n = 0; n < count; n++) {
intel_wait_init_for_seqno(&waiters[n], n + seqno_bias);
intel_engine_add_wait(engine, &waiters[n]);
__set_bit(n, bitmap);
}
err = check_rbtree(engine, bitmap, waiters, count);
if (err)
goto out_bitmap;
/* On each step, we advance the seqno so that several waiters are then
* complete (we increase the seqno by increasingly larger values to
* retire more and more waiters at once). All retired waiters should
* be woken and removed from the rbtree, and so that we check.
*/
for (n = 0; n < count; n = m) {
int seqno = 2 * n;
GEM_BUG_ON(find_first_bit(bitmap, count) != n);
if (intel_wait_complete(&waiters[n])) {
pr_err("waiter[%d, seqno=%d] completed too early\n",
n, waiters[n].seqno);
err = -EINVAL;
goto out_bitmap;
}
/* complete the following waiters */
mock_seqno_advance(engine, seqno + seqno_bias);
for (m = n; m <= seqno; m++) {
if (m == count)
break;
GEM_BUG_ON(!test_bit(m, bitmap));
__clear_bit(m, bitmap);
}
intel_engine_remove_wait(engine, &waiters[n]);
RB_CLEAR_NODE(&waiters[n].node);
err = check_rbtree(engine, bitmap, waiters, count);
if (err) {
pr_err("rbtree corrupt after seqno advance to %d\n",
seqno + seqno_bias);
goto out_bitmap;
}
err = check_completion(engine, bitmap, waiters, count);
if (err) {
pr_err("completions after seqno advance to %d failed\n",
seqno + seqno_bias);
goto out_bitmap;
}
}
err = check_rbtree_empty(engine);
out_bitmap:
kfree(bitmap);
out_waiters:
kvfree(waiters);
out_engines:
mock_engine_flush(engine);
return err;
}
static int igt_random_insert_remove(void *arg)
{
const u32 seqno_bias = 0x1000;
I915_RND_STATE(prng);
struct intel_engine_cs *engine = arg;
struct intel_wait *waiters;
const int count = 4096;
unsigned int *order;
unsigned long *bitmap;
int err = -ENOMEM;
int n;
mock_engine_reset(engine);
waiters = kvmalloc_array(count, sizeof(*waiters), GFP_KERNEL);
if (!waiters)
goto out_engines;
bitmap = kcalloc(DIV_ROUND_UP(count, BITS_PER_LONG), sizeof(*bitmap),
GFP_KERNEL);
if (!bitmap)
goto out_waiters;
order = i915_random_order(count, &prng);
if (!order)
goto out_bitmap;
for (n = 0; n < count; n++)
intel_wait_init_for_seqno(&waiters[n], seqno_bias + n);
err = check_rbtree(engine, bitmap, waiters, count);
if (err)
goto out_order;
/* Add and remove waiters into the rbtree in random order. At each
* step, we verify that the rbtree is correctly ordered.
*/
for (n = 0; n < count; n++) {
int i = order[n];
intel_engine_add_wait(engine, &waiters[i]);
__set_bit(i, bitmap);
err = check_rbtree(engine, bitmap, waiters, count);
if (err)
goto out_order;
}
i915_random_reorder(order, count, &prng);
for (n = 0; n < count; n++) {
int i = order[n];
intel_engine_remove_wait(engine, &waiters[i]);
__clear_bit(i, bitmap);
err = check_rbtree(engine, bitmap, waiters, count);
if (err)
goto out_order;
}
err = check_rbtree_empty(engine);
out_order:
kfree(order);
out_bitmap:
kfree(bitmap);
out_waiters:
kvfree(waiters);
out_engines:
mock_engine_flush(engine);
return err;
}
int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data,
struct drm_file *file)
{
struct etnaviv_drm_private *priv = dev->dev_private;
struct drm_etnaviv_gem_submit *args = data;
struct drm_etnaviv_gem_submit_reloc *relocs;
struct drm_etnaviv_gem_submit_pmr *pmrs;
struct drm_etnaviv_gem_submit_bo *bos;
struct etnaviv_gem_submit *submit;
struct etnaviv_gpu *gpu;
struct sync_file *sync_file = NULL;
struct ww_acquire_ctx ticket;
int out_fence_fd = -1;
void *stream;
int ret;
if (args->pipe >= ETNA_MAX_PIPES)
return -EINVAL;
gpu = priv->gpu[args->pipe];
if (!gpu)
return -ENXIO;
if (args->stream_size % 4) {
DRM_ERROR("non-aligned cmdstream buffer size: %u\n",
args->stream_size);
return -EINVAL;
}
if (args->exec_state != ETNA_PIPE_3D &&
args->exec_state != ETNA_PIPE_2D &&
args->exec_state != ETNA_PIPE_VG) {
DRM_ERROR("invalid exec_state: 0x%x\n", args->exec_state);
return -EINVAL;
}
if (args->flags & ~ETNA_SUBMIT_FLAGS) {
DRM_ERROR("invalid flags: 0x%x\n", args->flags);
return -EINVAL;
}
/*
* Copy the command submission and bo array to kernel space in
* one go, and do this outside of any locks.
*/
bos = kvmalloc_array(args->nr_bos, sizeof(*bos), GFP_KERNEL);
relocs = kvmalloc_array(args->nr_relocs, sizeof(*relocs), GFP_KERNEL);
pmrs = kvmalloc_array(args->nr_pmrs, sizeof(*pmrs), GFP_KERNEL);
stream = kvmalloc_array(1, args->stream_size, GFP_KERNEL);
if (!bos || !relocs || !pmrs || !stream) {
ret = -ENOMEM;
goto err_submit_cmds;
}
ret = copy_from_user(bos, u64_to_user_ptr(args->bos),
args->nr_bos * sizeof(*bos));
if (ret) {
ret = -EFAULT;
goto err_submit_cmds;
}
ret = copy_from_user(relocs, u64_to_user_ptr(args->relocs),
args->nr_relocs * sizeof(*relocs));
if (ret) {
ret = -EFAULT;
goto err_submit_cmds;
}
ret = copy_from_user(pmrs, u64_to_user_ptr(args->pmrs),
args->nr_pmrs * sizeof(*pmrs));
if (ret) {
ret = -EFAULT;
goto err_submit_cmds;
}
ret = copy_from_user(stream, u64_to_user_ptr(args->stream),
args->stream_size);
if (ret) {
ret = -EFAULT;
goto err_submit_cmds;
}
if (args->flags & ETNA_SUBMIT_FENCE_FD_OUT) {
out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
if (out_fence_fd < 0) {
ret = out_fence_fd;
goto err_submit_cmds;
}
}
ww_acquire_init(&ticket, &reservation_ww_class);
submit = submit_create(dev, gpu, args->nr_bos, args->nr_pmrs);
if (!submit) {
ret = -ENOMEM;
goto err_submit_ww_acquire;
}
ret = etnaviv_cmdbuf_init(gpu->cmdbuf_suballoc, &submit->cmdbuf,
ALIGN(args->stream_size, 8) + 8);
if (ret)
goto err_submit_objects;
submit->cmdbuf.ctx = file->driver_priv;
submit->exec_state = args->exec_state;
submit->flags = args->flags;
ret = submit_lookup_objects(submit, file, bos, args->nr_bos);
if (ret)
goto err_submit_objects;
ret = submit_lock_objects(submit, &ticket);
if (ret)
goto err_submit_objects;
if (!etnaviv_cmd_validate_one(gpu, stream, args->stream_size / 4,
relocs, args->nr_relocs)) {
ret = -EINVAL;
goto err_submit_objects;
}
if (args->flags & ETNA_SUBMIT_FENCE_FD_IN) {
submit->in_fence = sync_file_get_fence(args->fence_fd);
if (!submit->in_fence) {
ret = -EINVAL;
goto err_submit_objects;
}
}
ret = submit_fence_sync(submit);
if (ret)
goto err_submit_objects;
ret = submit_pin_objects(submit);
if (ret)
goto err_submit_objects;
ret = submit_reloc(submit, stream, args->stream_size / 4,
relocs, args->nr_relocs);
if (ret)
goto err_submit_objects;
ret = submit_perfmon_validate(submit, args->exec_state, pmrs);
if (ret)
goto err_submit_objects;
memcpy(submit->cmdbuf.vaddr, stream, args->stream_size);
submit->cmdbuf.user_size = ALIGN(args->stream_size, 8);
ret = etnaviv_gpu_submit(gpu, submit);
if (ret)
goto err_submit_objects;
submit_attach_object_fences(submit);
if (args->flags & ETNA_SUBMIT_FENCE_FD_OUT) {
/*
* This can be improved: ideally we want to allocate the sync
* file before kicking off the GPU job and just attach the
* fence to the sync file here, eliminating the ENOMEM
* possibility at this stage.
*/
sync_file = sync_file_create(submit->out_fence);
if (!sync_file) {
ret = -ENOMEM;
goto err_submit_objects;
}
fd_install(out_fence_fd, sync_file->file);
}
args->fence_fd = out_fence_fd;
args->fence = submit->out_fence->seqno;
err_submit_objects:
etnaviv_submit_put(submit);
err_submit_ww_acquire:
ww_acquire_fini(&ticket);
err_submit_cmds:
if (ret && (out_fence_fd >= 0))
put_unused_fd(out_fence_fd);
if (stream)
kvfree(stream);
if (bos)
kvfree(bos);
if (relocs)
kvfree(relocs);
if (pmrs)
kvfree(pmrs);
return ret;
}