void nvmap_unpin(struct nvmap_client *client, struct nvmap_handle_ref *ref)
{
if (!ref)
return;
atomic_dec(&ref->pin);
if (handle_unpin(client, ref->handle, false))
wake_up(&client->share->pin_wait);
}
static int pin_array_locked(struct nvmap_client *client,
struct nvmap_handle **h, int count)
{
int pinned;
int i;
int err = 0;
/* Flush deferred cache maintenance if needed */
for (pinned = 0; pinned < count; pinned++)
if (nvmap_find_cache_maint_op(client->dev, h[pinned]))
nvmap_cache_maint_ops_flush(client->dev, h[pinned]);
nvmap_mru_lock(client->share);
for (pinned = 0; pinned < count; pinned++) {
err = pin_locked(client, h[pinned]);
if (err)
break;
}
nvmap_mru_unlock(client->share);
if (err) {
/* unpin pinned handles */
for (i = 0; i < pinned; i++) {
/* inc ref counter, because
* handle_unpin decrements it */
nvmap_handle_get(h[i]);
/* unpin handles and free vm */
handle_unpin(client, h[i], true);
}
}
if (err && tegra_iovmm_get_max_free(client->share->iovmm) >=
client->iovm_limit) {
/* First attempt to pin in empty iovmm
* may still fail because of fragmentation caused by
* placing handles in MRU areas. After such failure
* all MRU gets cleaned and iovm space is freed.
*
* We have to do pinning again here since there might be is
* no more incoming pin_wait wakeup calls from unpin
* operations */
nvmap_mru_lock(client->share);
for (pinned = 0; pinned < count; pinned++) {
err = pin_locked(client, h[pinned]);
if (err)
break;
}
nvmap_mru_unlock(client->share);
if (err) {
pr_err("Pinning in empty iovmm failed!!!\n");
BUG_ON(1);
}
}
return err;
}
void nvmap_unpin_handles(struct nvmap_client *client,
struct nvmap_handle **h, int nr)
{
int i;
int do_wake = 0;
for (i = 0; i < nr; i++) {
if (WARN_ON(!h[i]))
continue;
do_wake |= handle_unpin(client, h[i], false);
}
if (do_wake)
wake_up(&client->share->pin_wait);
}
/*
* Pin handle without slow validation step
*/
phys_addr_t _nvmap_pin(struct nvmap_client *client,
struct nvmap_handle_ref *ref)
{
int ret = 0;
struct nvmap_handle *h;
phys_addr_t phys;
if (!ref)
return -EINVAL;
h = ref->handle;
if (WARN_ON(!h))
return -EINVAL;
h = nvmap_handle_get(h);
atomic_inc(&ref->pin);
if (WARN_ON(mutex_lock_interruptible(&client->share->pin_lock))) {
ret = -EINTR;
} else {
ret = wait_pin_array_locked(client, &h, 1);
mutex_unlock(&client->share->pin_lock);
}
if (ret) {
goto err_out;
} else {
if (h->heap_pgalloc && h->pgalloc.dirty)
ret = map_iovmm_area(h);
if (ret)
goto err_out_unpin;
phys = handle_phys(h);
}
return phys;
err_out_unpin:
nvmap_handle_get(h);
handle_unpin(client, h, true);
err_out:
atomic_dec(&ref->pin);
nvmap_handle_put(h);
return ret;
}
static int handle_unpin_noref(struct nvmap_client *client, unsigned long id)
{
struct nvmap_handle *h;
int w;
h = nvmap_validate_get(client, id);
if (unlikely(!h)) {
nvmap_err(client, "%s attempting to unpin invalid handle %p\n",
current->group_leader->comm, (void *)id);
return 0;
}
nvmap_err(client, "%s unpinning unreferenced handle %p\n",
current->group_leader->comm, h);
WARN_ON(1);
w = handle_unpin(client, h, false);
nvmap_handle_put(h);
return w;
}
void nvmap_unpin_ids(struct nvmap_client *client,
unsigned int nr, const unsigned long *ids)
{
unsigned int i;
int do_wake = 0;
for (i = 0; i < nr; i++) {
struct nvmap_handle_ref *ref;
if (!ids[i])
continue;
nvmap_ref_lock(client);
ref = _nvmap_validate_id_locked(client, ids[i]);
if (ref) {
struct nvmap_handle *h = ref->handle;
int e = atomic_add_unless(&ref->pin, -1, 0);
nvmap_ref_unlock(client);
if (!e) {
nvmap_err(client, "%s unpinning unpinned "
"handle %08lx\n",
current->group_leader->comm, ids[i]);
} else {
do_wake |= handle_unpin(client, h, false);
}
} else {
nvmap_ref_unlock(client);
if (client->super)
do_wake |= handle_unpin_noref(client, ids[i]);
else
nvmap_err(client, "%s unpinning invalid "
"handle %08lx\n",
current->group_leader->comm, ids[i]);
}
}
if (do_wake)
wake_up(&client->share->pin_wait);
}
/*
* @client: nvmap_client which should be used for validation;
* should be owned by the process which is submitting
* command buffers
* @ids: array of nvmap_handles to pin
* @id_type_mask: bitmask which defines handle type field in handle id.
* @id_type: only handles with of this type will be pinned. Handles with
* other type are ignored.
* @nr: number of entries in arr
* @unique_arr: list of nvmap_handle objects which were pinned by
* nvmap_pin_array. Must be unpinned after use
* @unique_arr_ref: list of duplicated nvmap_handle_refs corresponding
* to unique_arr. Must be freed after use.
*/
int nvmap_pin_array(struct nvmap_client *client,
unsigned long *ids,
long unsigned id_type_mask,
long unsigned id_type,
int nr,
struct nvmap_handle **unique_arr,
struct nvmap_handle_ref **unique_arr_refs)
{
int count = 0;
int ret = 0;
int i;
if (mutex_lock_interruptible(&client->share->pin_lock)) {
nvmap_err(client, "%s interrupted when acquiring pin lock\n",
current->group_leader->comm);
return -EINTR;
}
count = nvmap_validate_get_pin_array(client, ids,
id_type_mask, id_type, nr,
unique_arr, unique_arr_refs);
if (count < 0) {
mutex_unlock(&client->share->pin_lock);
nvmap_warn(client, "failed to validate pin array\n");
return count;
}
for (i = 0; i < count; i++)
unique_arr[i]->flags &= ~NVMAP_HANDLE_VISITED;
ret = wait_pin_array_locked(client, unique_arr, count);
mutex_unlock(&client->share->pin_lock);
if (WARN_ON(ret)) {
goto err_out;
} else {
for (i = 0; i < count; i++) {
if (unique_arr[i]->heap_pgalloc &&
unique_arr[i]->pgalloc.dirty) {
ret = map_iovmm_area(unique_arr[i]);
while (ret && --i >= 0) {
tegra_iovmm_zap_vm(
unique_arr[i]->pgalloc.area);
atomic_dec(&unique_arr_refs[i]->pin);
}
if (ret)
goto err_out_unpin;
}
atomic_inc(&unique_arr_refs[i]->pin);
}
}
return count;
err_out_unpin:
for (i = 0; i < count; i++) {
/* inc ref counter, because handle_unpin decrements it */
nvmap_handle_get(unique_arr[i]);
/* unpin handles and free vm */
handle_unpin(client, unique_arr[i], true);
}
err_out:
for (i = 0; i < count; i++) {
/* pin ref */
nvmap_handle_put(unique_arr[i]);
/* remove duplicate */
atomic_dec(&unique_arr_refs[i]->dupes);
nvmap_handle_put(unique_arr[i]);
}
return ret;
}
/* pins a list of handle_ref objects; same conditions apply as to
* _nvmap_handle_pin, but also bumps the pin count of each handle_ref. */
int nvmap_pin_ids(struct nvmap_client *client,
unsigned int nr, const unsigned long *ids)
{
int ret = 0;
int cnt = 0;
unsigned int i;
struct nvmap_handle **h = (struct nvmap_handle **)ids;
struct nvmap_handle_ref *ref;
/* to optimize for the common case (client provided valid handle
* references and the pin succeeds), increment the handle_ref pin
* count during validation. in error cases, the tree will need to
* be re-walked, since the handle_ref is discarded so that an
* allocation isn't required. if a handle_ref is not found,
* locally validate that the caller has permission to pin the handle;
* handle_refs are not created in this case, so it is possible that
* if the caller crashes after pinning a global handle, the handle
* will be permanently leaked. */
nvmap_ref_lock(client);
for (i = 0; i < nr && !ret; i++) {
ref = _nvmap_validate_id_locked(client, ids[i]);
if (ref) {
atomic_inc(&ref->pin);
nvmap_handle_get(h[i]);
} else {
struct nvmap_handle *verify;
nvmap_ref_unlock(client);
verify = nvmap_validate_get(client, ids[i]);
if (verify)
nvmap_warn(client, "%s pinning unreferenced "
"handle %p\n",
current->group_leader->comm, h[i]);
else
ret = -EPERM;
nvmap_ref_lock(client);
}
}
nvmap_ref_unlock(client);
nr = i;
if (ret)
goto out;
ret = mutex_lock_interruptible(&client->share->pin_lock);
if (WARN_ON(ret))
goto out;
for (cnt = 0; cnt < nr && !ret; cnt++) {
ret = wait_pin_locked(client, h[cnt]);
}
mutex_unlock(&client->share->pin_lock);
if (ret) {
int do_wake = 0;
for (i = 0; i < cnt; i++)
do_wake |= handle_unpin(client, h[i]);
if (do_wake)
wake_up(&client->share->pin_wait);
ret = -EINTR;
} else {
for (i = 0; i < nr; i++) {
if (h[i]->heap_pgalloc && h[i]->pgalloc.dirty)
map_iovmm_area(h[i]);
}
}
out:
if (ret) {
nvmap_ref_lock(client);
for (i = 0; i < nr; i++) {
ref = _nvmap_validate_id_locked(client, ids[i]);
if (!ref) {
nvmap_warn(client, "%s freed handle %p "
"during pinning\n",
current->group_leader->comm,
(void *)ids[i]);
continue;
}
atomic_dec(&ref->pin);
}
nvmap_ref_unlock(client);
for (i = cnt; i < nr; i++)
nvmap_handle_put(h[i]);
}
return ret;
}