struct nvmap_handle_ref *nvmap_duplicate_handle(struct nvmap_client *client,
struct nvmap_handle *h, bool skip_val)
{
struct nvmap_handle_ref *ref = NULL;
BUG_ON(!client);
/* on success, the reference count for the handle should be
* incremented, so the success paths will not call nvmap_handle_put */
h = nvmap_validate_get(h);
if (!h) {
pr_debug("%s duplicate handle failed\n",
current->group_leader->comm);
return ERR_PTR(-EPERM);
}
if (!h->alloc) {
pr_err("%s duplicating unallocated handle\n",
current->group_leader->comm);
nvmap_handle_put(h);
return ERR_PTR(-EINVAL);
}
nvmap_ref_lock(client);
ref = __nvmap_validate_locked(client, h);
if (ref) {
/* handle already duplicated in client; just increment
* the reference count rather than re-duplicating it */
atomic_inc(&ref->dupes);
nvmap_ref_unlock(client);
return ref;
}
nvmap_ref_unlock(client);
ref = kzalloc(sizeof(*ref), GFP_KERNEL);
if (!ref) {
nvmap_handle_put(h);
return ERR_PTR(-ENOMEM);
}
atomic_set(&ref->dupes, 1);
ref->handle = h;
atomic_set(&ref->pin, 0);
add_handle_ref(client, ref);
/*
* Ref counting on the dma_bufs follows the creation and destruction of
* nvmap_handle_refs. That is every time a handle_ref is made the
* dma_buf ref count goes up and everytime a handle_ref is destroyed
* the dma_buf ref count goes down.
*/
get_dma_buf(h->dmabuf);
trace_nvmap_duplicate_handle(client, h, ref);
return ref;
}
phys_addr_t nvmap_pin(struct nvmap_client *client,
struct nvmap_handle_ref *ref)
{
struct nvmap_handle *h;
if (!ref)
return -EINVAL;
if (WARN_ON(!ref->handle))
return -EINVAL;
nvmap_ref_lock(client);
ref = _nvmap_validate_id_locked(client, (unsigned long)ref->handle);
if (ref)
h = ref->handle;
nvmap_ref_unlock(client);
return _nvmap_pin(client, ref);
}
void nvmap_unpin_ids(struct nvmap_client *client,
unsigned int nr, const unsigned long *ids)
{
unsigned int i;
int do_wake = 0;
for (i = 0; i < nr; i++) {
struct nvmap_handle_ref *ref;
if (!ids[i])
continue;
nvmap_ref_lock(client);
ref = _nvmap_validate_id_locked(client, ids[i]);
if (ref) {
struct nvmap_handle *h = ref->handle;
int e = atomic_add_unless(&ref->pin, -1, 0);
nvmap_ref_unlock(client);
if (!e) {
nvmap_err(client, "%s unpinning unpinned "
"handle %08lx\n",
current->group_leader->comm, ids[i]);
} else {
do_wake |= handle_unpin(client, h, false);
}
} else {
nvmap_ref_unlock(client);
if (client->super)
do_wake |= handle_unpin_noref(client, ids[i]);
else
nvmap_err(client, "%s unpinning invalid "
"handle %08lx\n",
current->group_leader->comm, ids[i]);
}
}
if (do_wake)
wake_up(&client->share->pin_wait);
}
static void add_handle_ref(struct nvmap_client *client,
struct nvmap_handle_ref *ref)
{
struct rb_node **p, *parent = NULL;
nvmap_ref_lock(client);
p = &client->handle_refs.rb_node;
while (*p) {
struct nvmap_handle_ref *node;
parent = *p;
node = rb_entry(parent, struct nvmap_handle_ref, node);
if (ref->handle > node->handle)
p = &parent->rb_right;
else
p = &parent->rb_left;
}
rb_link_node(&ref->node, parent, p);
rb_insert_color(&ref->node, &client->handle_refs);
client->handle_count++;
if (client->handle_count > nvmap_max_handle_count)
nvmap_max_handle_count = client->handle_count;
atomic_inc(&ref->handle->share_count);
nvmap_ref_unlock(client);
}
static int nvmap_validate_get_pin_array(struct nvmap_client *client,
unsigned long *h,
long unsigned id_type_mask,
long unsigned id_type,
int nr,
struct nvmap_handle **unique_handles,
struct nvmap_handle_ref **unique_handle_refs)
{
int i;
int err = 0;
int count = 0;
unsigned long last_h = 0;
struct nvmap_handle_ref *last_ref = 0;
nvmap_ref_lock(client);
for (i = 0; i < nr; i++) {
struct nvmap_handle_ref *ref;
if ((h[i] & id_type_mask) != id_type)
continue;
if (last_h == h[i])
continue;
ref = _nvmap_validate_id_locked(client, h[i]);
if (!ref)
nvmap_err(client, "failed to validate id\n");
else if (!ref->handle)
nvmap_err(client, "id had no associated handle\n");
else if (!ref->handle->alloc)
nvmap_err(client, "handle had no allocation\n");
if (!ref || !ref->handle || !ref->handle->alloc) {
err = -EPERM;
break;
}
last_h = h[i];
last_ref = ref;
/* a handle may be referenced multiple times in arr, but
* it will only be pinned once; this ensures that the
* minimum number of sync-queue slots in the host driver
* are dedicated to storing unpin lists, which allows
* for greater parallelism between the CPU and graphics
* processor */
if (ref->handle->flags & NVMAP_HANDLE_VISITED)
continue;
ref->handle->flags |= NVMAP_HANDLE_VISITED;
unique_handles[count] = nvmap_handle_get(ref->handle);
/* Duplicate handle */
atomic_inc(&ref->dupes);
nvmap_handle_get(ref->handle);
unique_handle_refs[count] = ref;
BUG_ON(!unique_handles[count]);
count++;
}
nvmap_ref_unlock(client);
if (err) {
for (i = 0; i < count; i++) {
unique_handles[i]->flags &= ~NVMAP_HANDLE_VISITED;
/* pin ref */
nvmap_handle_put(unique_handles[i]);
/* ref count */
atomic_dec(&unique_handle_refs[i]->dupes);
nvmap_handle_put(unique_handles[i]);
}
}
return err ? err : count;
}
/* pins a list of handle_ref objects; same conditions apply as to
* _nvmap_handle_pin, but also bumps the pin count of each handle_ref. */
int nvmap_pin_ids(struct nvmap_client *client,
unsigned int nr, const unsigned long *ids)
{
int ret = 0;
unsigned int i;
struct nvmap_handle **h = (struct nvmap_handle **)ids;
struct nvmap_handle_ref *ref;
/* to optimize for the common case (client provided valid handle
* references and the pin succeeds), increment the handle_ref pin
* count during validation. in error cases, the tree will need to
* be re-walked, since the handle_ref is discarded so that an
* allocation isn't required. if a handle_ref is not found,
* locally validate that the caller has permission to pin the handle;
* handle_refs are not created in this case, so it is possible that
* if the caller crashes after pinning a global handle, the handle
* will be permanently leaked. */
nvmap_ref_lock(client);
for (i = 0; i < nr; i++) {
ref = _nvmap_validate_id_locked(client, ids[i]);
if (ref) {
atomic_inc(&ref->pin);
nvmap_handle_get(h[i]);
} else {
struct nvmap_handle *verify;
nvmap_ref_unlock(client);
verify = nvmap_validate_get(client, ids[i]);
if (verify) {
nvmap_warn(client, "%s pinning unreferenced "
"handle %p\n",
current->group_leader->comm, h[i]);
} else {
ret = -EPERM;
nr = i;
break;
}
nvmap_ref_lock(client);
}
if (!h[i]->alloc) {
ret = -EFAULT;
nr = i + 1;
break;
}
}
nvmap_ref_unlock(client);
if (ret)
goto out;
ret = mutex_lock_interruptible(&client->share->pin_lock);
if (WARN_ON(ret))
goto out;
ret = wait_pin_array_locked(client, h, nr);
mutex_unlock(&client->share->pin_lock);
if (ret) {
ret = -EINTR;
} else {
for (i = 0; i < nr; i++) {
if (h[i]->heap_pgalloc && h[i]->pgalloc.dirty)
map_iovmm_area(h[i]);
}
}
out:
if (ret) {
nvmap_ref_lock(client);
for (i = 0; i < nr; i++) {
if (!ids[i])
continue;
ref = _nvmap_validate_id_locked(client, ids[i]);
if (!ref) {
nvmap_warn(client, "%s freed handle %p "
"during pinning\n",
current->group_leader->comm,
(void *)ids[i]);
continue;
}
atomic_dec(&ref->pin);
}
nvmap_ref_unlock(client);
for (i = 0; i < nr; i++)
if(h[i])
nvmap_handle_put(h[i]);
}
return ret;
}
