static int fimg2d_check_address_range(unsigned long addr, size_t size)
{
struct vm_area_struct *vma;
struct vm_area_struct *nvma;
int ret = 0;
if (addr + size <= addr) {
fimg2d_err("address overflow. addr:0x%lx, size:%d\n",
addr, size);
return -EINVAL;
}
down_read(¤t->mm->mmap_sem);
vma = find_vma(current->mm, addr);
if (vma == NULL) {
fimg2d_err("vma is NULL\n");
ret = -EINVAL;
} else {
nvma = vma->vm_next;
while ((vma->vm_end < (addr + size)) &&
(vma != NULL) && (nvma != NULL) &&
(vma->vm_end == nvma->vm_start)) {
vma = vma->vm_next;
nvma = nvma->vm_next;
}
if (vma->vm_end < (addr + size)) {
fimg2d_err("addr : %#lx, size : %#x - out of vma[%#lx, %#lx] range\n",
addr, size, vma->vm_start, vma->vm_end);
ret = -EFAULT;
}
}
up_read(¤t->mm->mmap_sem);
if (!ret) {
/*
* Invoking COW gainst the first and last page if they can be
* accessed by CPU and G2D concurrently.
* checking the return value of get_user_pages_fast() is not
* required because the calls to get_user_pages_fast() are to
* invoke COW so that COW is not invoked against the first and
* the last pages while G2D is working.
*/
if (!IS_ALIGNED(addr, PAGE_SIZE))
get_user_pages_fast(addr, 1, 1, NULL);
if (!IS_ALIGNED(addr + size, PAGE_SIZE))
get_user_pages_fast(addr + size, 1, 1, NULL);
}
return ret;
}
/*
* Sets up a given context for notify to work. Maps the notify
* boolean in user VA into kernel space.
*/
static int vmci_host_setup_notify(struct vmci_ctx *context,
unsigned long uva)
{
int retval;
if (context->notify_page) {
pr_devel("%s: Notify mechanism is already set up\n", __func__);
return VMCI_ERROR_DUPLICATE_ENTRY;
}
/*
* We are using 'bool' internally, but let's make sure we explicit
* about the size.
*/
BUILD_BUG_ON(sizeof(bool) != sizeof(u8));
if (!access_ok((void __user *)uva, sizeof(u8)))
return VMCI_ERROR_GENERIC;
/*
* Lock physical page backing a given user VA.
*/
retval = get_user_pages_fast(uva, 1, FOLL_WRITE, &context->notify_page);
if (retval != 1) {
context->notify_page = NULL;
return VMCI_ERROR_GENERIC;
}
/*
* Map the locked page and set up notify pointer.
*/
context->notify = kmap(context->notify_page) + (uva & (PAGE_SIZE - 1));
vmci_ctx_check_signal_notify(context);
return VMCI_SUCCESS;
}
/*
* best effort, GUP based copy_from_user() that assumes IRQ or NMI context
*/
static unsigned long
copy_from_user_nmi(void *to, const void __user *from, unsigned long n)
{
unsigned long offset, addr = (unsigned long)from;
unsigned long size, len = 0;
struct page *page;
void *map;
int ret;
do {
#ifndef CONFIG_L4
ret = __get_user_pages_fast(addr, 1, 0, &page);
#else
ret = get_user_pages_fast(addr, 1, 0, &page);
#endif
if (!ret)
break;
offset = addr & (PAGE_SIZE - 1);
size = min(PAGE_SIZE - offset, n - len);
map = kmap_atomic(page);
memcpy(to, map+offset, size);
kunmap_atomic(map);
put_page(page);
len += size;
to += size;
addr += size;
} while (len < n);
return len;
}
static ssize_t get_pages_iovec(struct iov_iter *i,
struct page **pages, size_t maxsize,
size_t *start)
{
size_t offset = i->iov_offset;
const struct iovec *iov = i->iov;
size_t len;
unsigned long addr;
int n;
int res;
len = iov->iov_len - offset;
if (len > i->count)
len = i->count;
if (len > maxsize)
len = maxsize;
addr = (unsigned long)iov->iov_base + offset;
len += *start = addr & (PAGE_SIZE - 1);
addr &= ~(PAGE_SIZE - 1);
n = (len + PAGE_SIZE - 1) / PAGE_SIZE;
res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
if (unlikely(res < 0))
return res;
return (res == n ? len : res * PAGE_SIZE) - *start;
}
static int pin_user_pages(unsigned long first_page,
unsigned long last_page,
unsigned int last_page_size,
int is_write,
struct page *pages[MAX_BUFFERS_PER_COMMAND],
unsigned int *iter_last_page_size)
{
int ret;
int requested_pages = ((last_page - first_page) >> PAGE_SHIFT) + 1;
if (requested_pages > MAX_BUFFERS_PER_COMMAND) {
requested_pages = MAX_BUFFERS_PER_COMMAND;
*iter_last_page_size = PAGE_SIZE;
} else {
*iter_last_page_size = last_page_size;
}
ret = get_user_pages_fast(first_page, requested_pages, !is_write,
pages);
if (ret <= 0)
return -EFAULT;
if (ret < requested_pages)
*iter_last_page_size = PAGE_SIZE;
return ret;
}
ssize_t iov_iter_get_pages(struct iov_iter *i,
struct page **pages, size_t maxsize, unsigned maxpages,
size_t *start)
{
if (maxsize > i->count)
maxsize = i->count;
if (!maxsize)
return 0;
iterate_all_kinds(i, maxsize, v, ({
unsigned long addr = (unsigned long)v.iov_base;
size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1));
int n;
int res;
if (len > maxpages * PAGE_SIZE)
len = maxpages * PAGE_SIZE;
addr &= ~(PAGE_SIZE - 1);
n = DIV_ROUND_UP(len, PAGE_SIZE);
res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages);
if (unlikely(res < 0))
return res;
return (res == n ? len : res * PAGE_SIZE) - *start;
0;}),({
/*H:350
* This routine takes a page number given by the Guest and converts it to
* an actual, physical page number. It can fail for several reasons: the
* virtual address might not be mapped by the Launcher, the write flag is set
* and the page is read-only, or the write flag was set and the page was
* shared so had to be copied, but we ran out of memory.
*
* This holds a reference to the page, so release_pte() is careful to put that
* back.
*/
static unsigned long get_pfn(unsigned long virtpfn, int write)
{
struct page *page;
/* gup me one page at this address please! */
if (get_user_pages_fast(virtpfn << PAGE_SHIFT, 1, write, &page) == 1)
return page_to_pfn(page);
/* This value indicates failure. */
return -1UL;
}
int p9_payload_gup(char *data, int *nr_pages, struct page **pages, int write)
{
int nr_mapped_pages;
nr_mapped_pages = get_user_pages_fast((unsigned long)data,
*nr_pages, write, pages);
if (nr_mapped_pages <= 0)
return nr_mapped_pages;
*nr_pages = nr_mapped_pages;
return 0;
}
static int ept_set_epte(struct vmx_vcpu *vcpu, int make_write,
unsigned long gpa, unsigned long hva)
{
int ret;
epte_t *epte, flags;
struct page *page;
ret = get_user_pages_fast(hva, 1, make_write, &page);
if (ret != 1) {
printk(KERN_ERR "ept: failed to get user page %lx\n", hva);
return ret;
}
spin_lock(&vcpu->ept_lock);
ret = ept_lookup_gpa(vcpu, (void *) gpa,
PageHuge(page) ? 1 : 0, 1, &epte);
if (ret) {
spin_unlock(&vcpu->ept_lock);
printk(KERN_ERR "ept: failed to lookup EPT entry\n");
return ret;
}
if (epte_present(*epte)) {
if (!epte_big(*epte) && PageHuge(page))
ept_clear_l1_epte(epte);
else
ept_clear_epte(epte);
}
flags = __EPTE_READ | __EPTE_EXEC |
__EPTE_TYPE(EPTE_TYPE_WB) | __EPTE_IPAT;
if (make_write)
flags |= __EPTE_WRITE;
if (vcpu->ept_ad_enabled) {
/* premark A/D to avoid extra memory references */
flags |= __EPTE_A;
if (make_write)
flags |= __EPTE_D;
}
if (PageHuge(page)) {
flags |= __EPTE_SZ;
*epte = epte_addr(page_to_phys(page) & ~((1 << 21) - 1)) |
flags;
} else
*epte = epte_addr(page_to_phys(page)) | flags;
spin_unlock(&vcpu->ept_lock);
return 0;
}
/* return number of pages pinned... */
static int ipath_user_sdma_pin_pages(const struct ipath_devdata *dd,
struct ipath_user_sdma_pkt *pkt,
unsigned long addr, int tlen, int npages)
{
struct page *pages[2];
int j;
int ret;
ret = get_user_pages_fast(addr, npages, 0, pages);
if (ret != npages) {
int i;
for (i = 0; i < ret; i++)
put_page(pages[i]);
ret = -ENOMEM;
goto done;
}
for (j = 0; j < npages; j++) {
/* map the pages... */
const int flen =
ipath_user_sdma_page_length(addr, tlen);
dma_addr_t dma_addr =
dma_map_page(&dd->pcidev->dev,
pages[j], 0, flen, DMA_TO_DEVICE);
unsigned long fofs = addr & ~PAGE_MASK;
if (dma_mapping_error(&dd->pcidev->dev, dma_addr)) {
ret = -ENOMEM;
goto done;
}
ipath_user_sdma_init_frag(pkt, pkt->naddr, fofs, flen, 1, 1,
pages[j], kmap(pages[j]),
dma_addr);
pkt->naddr++;
addr += flen;
tlen -= flen;
}
done:
return ret;
}
/* TODO: This is really inefficient. We need something like get_user()
* (instruction directly accesses the data, with an exception table entry
* returning -EFAULT). See Documentation/x86/exception-tables.txt.
*/
static int set_bit_to_user(int nr, void __user *addr)
{
unsigned long log = (unsigned long)addr;
struct page *page;
void *base;
int bit = nr + (log % PAGE_SIZE) * 8;
int r;
r = get_user_pages_fast(log, 1, 1, &page);
if (r < 0)
return r;
BUG_ON(r != 1);
base = kmap_atomic(page, KM_USER0);
set_bit(bit, base);
kunmap_atomic(base, KM_USER0);
set_page_dirty_lock(page);
put_page(page);
return 0;
}
/**
* lttng_logger_write - write a userspace string into the trace system
* @file: file pointer
* @user_buf: user string
* @count: length to copy
* @ppos: file position
*
* Copy a userspace string into a trace event named "lttng:logger".
* Copies at most @count bytes into the event "msg" dynamic array.
* Truncates the count at LTTNG_LOGGER_COUNT_MAX. Returns the number of
* bytes copied from the source.
* Return -1 on error, with EFAULT errno.
*/
static
ssize_t lttng_logger_write(struct file *file, const char __user *user_buf,
size_t count, loff_t *ppos)
{
unsigned int nr_pages = 1, i;
unsigned long uaddr = (unsigned long) user_buf;
struct page *pages[2];
ssize_t written;
int ret;
/* Truncate count */
if (unlikely(count > LTTNG_LOGGER_COUNT_MAX))
count = LTTNG_LOGGER_COUNT_MAX;
/* How many pages are we dealing with ? */
if (unlikely((uaddr & PAGE_MASK) != ((uaddr + count) & PAGE_MASK)))
nr_pages = 2;
/* Pin userspace pages */
ret = get_user_pages_fast(uaddr, nr_pages, 0, pages);
if (unlikely(ret < nr_pages)) {
if (ret > 0) {
BUG_ON(ret != 1);
put_page(pages[0]);
}
written = -EFAULT;
goto end;
}
/* Trace the event */
trace_lttng_logger(user_buf, count);
written = count;
*ppos += written;
for (i = 0; i < nr_pages; i++)
put_page(pages[i]);
end:
return written;
}
static ssize_t get_pages_alloc_iovec(struct iov_iter *i,
struct page ***pages, size_t maxsize,
size_t *start)
{
size_t offset = i->iov_offset;
const struct iovec *iov = i->iov;
size_t len;
unsigned long addr;
void *p;
int n;
int res;
len = iov->iov_len - offset;
if (len > i->count)
len = i->count;
if (len > maxsize)
len = maxsize;
addr = (unsigned long)iov->iov_base + offset;
len += *start = addr & (PAGE_SIZE - 1);
addr &= ~(PAGE_SIZE - 1);
n = (len + PAGE_SIZE - 1) / PAGE_SIZE;
p = kmalloc(n * sizeof(struct page *), GFP_KERNEL);
if (!p)
p = vmalloc(n * sizeof(struct page *));
if (!p)
return -ENOMEM;
res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p);
if (unlikely(res < 0)) {
kvfree(p);
return res;
}
*pages = p;
return (res == n ? len : res * PAGE_SIZE) - *start;
}
struct tee_shm *tee_shm_register(struct tee_context *ctx, unsigned long addr,
size_t length, u32 flags)
{
struct tee_device *teedev = ctx->teedev;
const u32 req_flags = TEE_SHM_DMA_BUF | TEE_SHM_USER_MAPPED;
struct tee_shm *shm;
void *ret;
int rc;
int num_pages;
unsigned long start;
if (flags != req_flags)
return ERR_PTR(-ENOTSUPP);
if (!tee_device_get(teedev))
return ERR_PTR(-EINVAL);
if (!teedev->desc->ops->shm_register ||
!teedev->desc->ops->shm_unregister) {
tee_device_put(teedev);
return ERR_PTR(-ENOTSUPP);
}
teedev_ctx_get(ctx);
shm = kzalloc(sizeof(*shm), GFP_KERNEL);
if (!shm) {
ret = ERR_PTR(-ENOMEM);
goto err;
}
shm->flags = flags | TEE_SHM_REGISTER;
shm->teedev = teedev;
shm->ctx = ctx;
shm->id = -1;
start = rounddown(addr, PAGE_SIZE);
shm->offset = addr - start;
shm->size = length;
num_pages = (roundup(addr + length, PAGE_SIZE) - start) / PAGE_SIZE;
shm->pages = kcalloc(num_pages, sizeof(*shm->pages), GFP_KERNEL);
if (!shm->pages) {
ret = ERR_PTR(-ENOMEM);
goto err;
}
rc = get_user_pages_fast(start, num_pages, 1, shm->pages);
if (rc > 0)
shm->num_pages = rc;
if (rc != num_pages) {
if (rc >= 0)
rc = -ENOMEM;
ret = ERR_PTR(rc);
goto err;
}
mutex_lock(&teedev->mutex);
shm->id = idr_alloc(&teedev->idr, shm, 1, 0, GFP_KERNEL);
mutex_unlock(&teedev->mutex);
if (shm->id < 0) {
ret = ERR_PTR(shm->id);
goto err;
}
rc = teedev->desc->ops->shm_register(ctx, shm, shm->pages,
shm->num_pages, start);
if (rc) {
ret = ERR_PTR(rc);
goto err;
}
if (flags & TEE_SHM_DMA_BUF) {
DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
exp_info.ops = &tee_shm_dma_buf_ops;
exp_info.size = shm->size;
exp_info.flags = O_RDWR;
exp_info.priv = shm;
shm->dmabuf = dma_buf_export(&exp_info);
if (IS_ERR(shm->dmabuf)) {
ret = ERR_CAST(shm->dmabuf);
teedev->desc->ops->shm_unregister(ctx, shm);
goto err;
}
}
mutex_lock(&teedev->mutex);
list_add_tail(&shm->link, &ctx->list_shm);
mutex_unlock(&teedev->mutex);
return shm;
err:
if (shm) {
size_t n;
if (shm->id >= 0) {
mutex_lock(&teedev->mutex);
idr_remove(&teedev->idr, shm->id);
mutex_unlock(&teedev->mutex);
}
if (shm->pages) {
for (n = 0; n < shm->num_pages; n++)
put_page(shm->pages[n]);
kfree(shm->pages);
}
}
kfree(shm);
teedev_ctx_put(ctx);
tee_device_put(teedev);
return ret;
}
/*
* @optval points to the userspace buffer that the information snapshot
* will be copied into.
*
* @optlen on input is the size of the buffer in userspace. @optlen
* on output is the size of the requested snapshot in bytes.
*
* This function returns -errno if there is a failure, particularly -ENOSPC
* if the given userspace buffer was not large enough to fit the snapshot.
* On success it returns the positive number of bytes of each array element
* in the snapshot.
*/
int rds_info_getsockopt(struct socket *sock, int optname, char __user *optval,
int __user *optlen)
{
struct rds_info_iterator iter;
struct rds_info_lengths lens;
unsigned long nr_pages = 0;
unsigned long start;
unsigned long i;
rds_info_func func;
struct page **pages = NULL;
int ret;
int len;
int total;
if (get_user(len, optlen)) {
ret = -EFAULT;
goto out;
}
/* check for all kinds of wrapping and the like */
start = (unsigned long)optval;
if (len < 0 || len + PAGE_SIZE - 1 < len || start + len < start) {
ret = -EINVAL;
goto out;
}
/* a 0 len call is just trying to probe its length */
if (len == 0)
goto call_func;
nr_pages = (PAGE_ALIGN(start + len) - (start & PAGE_MASK))
>> PAGE_SHIFT;
pages = kmalloc(nr_pages * sizeof(struct page *), GFP_KERNEL);
if (pages == NULL) {
ret = -ENOMEM;
goto out;
}
ret = get_user_pages_fast(start, nr_pages, 1, pages);
if (ret != nr_pages) {
if (ret > 0)
nr_pages = ret;
else
nr_pages = 0;
ret = -EAGAIN; /* XXX ? */
goto out;
}
rdsdebug("len %d nr_pages %lu\n", len, nr_pages);
call_func:
func = rds_info_funcs[optname - RDS_INFO_FIRST];
if (func == NULL) {
ret = -ENOPROTOOPT;
goto out;
}
iter.pages = pages;
iter.addr = NULL;
iter.offset = start & (PAGE_SIZE - 1);
func(sock, len, &iter, &lens);
BUG_ON(lens.each == 0);
total = lens.nr * lens.each;
rds_info_iter_unmap(&iter);
if (total > len) {
len = total;
ret = -ENOSPC;
} else {
len = total;
ret = lens.each;
}
if (put_user(len, optlen))
ret = -EFAULT;
out:
for (i = 0; pages != NULL && i < nr_pages; i++)
put_page(pages[i]);
kfree(pages);
return ret;
}
static int ept_set_epte(struct vmx_vcpu *vcpu, int make_write,
unsigned long gpa, unsigned long hva)
{
int ret;
epte_t *epte, flags;
struct page *page;
unsigned huge_shift;
int level;
ret = get_user_pages_fast(hva, 1, make_write, &page);
if (ret != 1) {
ret = ept_set_pfnmap_epte(vcpu, make_write, gpa, hva);
if (ret)
printk(KERN_ERR "ept: failed to get user page %lx\n", hva);
return ret;
}
spin_lock(&vcpu->ept_lock);
huge_shift = compound_order(compound_head(page)) + PAGE_SHIFT;
level = 0;
if (huge_shift == 30)
level = 2;
else if (huge_shift == 21)
level = 1;
ret = ept_lookup_gpa(vcpu, (void *) gpa,
level, 1, &epte);
if (ret) {
spin_unlock(&vcpu->ept_lock);
put_page(page);
printk(KERN_ERR "ept: failed to lookup EPT entry\n");
return ret;
}
if (epte_present(*epte)) {
if (!epte_big(*epte) && level == 2)
ept_clear_l2_epte(epte);
else if (!epte_big(*epte) && level == 1)
ept_clear_l1_epte(epte);
else
ept_clear_epte(epte);
}
flags = __EPTE_READ | __EPTE_EXEC |
__EPTE_TYPE(EPTE_TYPE_WB) | __EPTE_IPAT;
if (make_write)
flags |= __EPTE_WRITE;
if (vcpu->ept_ad_enabled) {
/* premark A/D to avoid extra memory references */
flags |= __EPTE_A;
if (make_write)
flags |= __EPTE_D;
}
if (level) {
struct page *tmp = page;
page = compound_head(page);
get_page(page);
put_page(tmp);
flags |= __EPTE_SZ;
}
*epte = epte_addr(page_to_phys(page)) | flags;
spin_unlock(&vcpu->ept_lock);
return 0;
}
/*
* Map an iov into an array of pages and offset/length tupples. With the
* partial_page structure, we can map several non-contiguous ranges into
* our ones pages[] map instead of splitting that operation into pieces.
* Could easily be exported as a generic helper for other users, in which
* case one would probably want to add a 'max_nr_pages' parameter as well.
*/
static int get_iovec_page_array(const struct iovec __user *iov,
unsigned int nr_vecs, struct page **pages,
struct partial_page *partial, int aligned)
{
int buffers = 0, error = 0;
while (nr_vecs) {
unsigned long off, npages;
void __user *base;
size_t len;
int i;
/*
* Get user address base and length for this iovec.
*/
error = get_user(base, &iov->iov_base);
if (unlikely(error))
break;
error = get_user(len, &iov->iov_len);
if (unlikely(error))
break;
/*
* Sanity check this iovec. 0 read succeeds.
*/
if (unlikely(!len))
break;
error = -EFAULT;
if (unlikely(!base))
break;
if (unlikely(!access_ok(VERIFY_READ, base, len)))
break;
/*
* Get this base offset and number of pages, then map
* in the user pages.
*/
off = (unsigned long) base & ~PAGE_MASK;
/*
* If asked for alignment, the offset must be zero and the
* length a multiple of the PAGE_SIZE.
*/
error = -EINVAL;
if (aligned && (off || len & ~PAGE_MASK))
break;
npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
if (npages > PIPE_BUFFERS - buffers)
npages = PIPE_BUFFERS - buffers;
error = get_user_pages_fast((unsigned long)base, npages,
0, &pages[buffers]);
if (unlikely(error <= 0))
break;
/*
* Fill this contiguous range into the partial page map.
*/
for (i = 0; i < error; i++) {
const int plen = min_t(size_t, len, PAGE_SIZE - off);
partial[buffers].offset = off;
partial[buffers].len = plen;
off = 0;
len -= plen;
buffers++;
}
/*
* We didn't complete this iov, stop here since it probably
* means we have to move some of this into a pipe to
* be able to continue.
*/
if (len)
break;
/*
* Don't continue if we mapped fewer pages than we asked for,
* or if we mapped the max number of pages that we have
* room for.
*/
if (error < npages || buffers == PIPE_BUFFERS)
break;
nr_vecs--;
iov++;
}
if (buffers)
return buffers;
return error;
}
static int __gup_benchmark_ioctl(unsigned int cmd,
struct gup_benchmark *gup)
{
ktime_t start_time, end_time;
unsigned long i, nr_pages, addr, next;
int nr;
struct page **pages;
if (gup->size > ULONG_MAX)
return -EINVAL;
nr_pages = gup->size / PAGE_SIZE;
pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL);
if (!pages)
return -ENOMEM;
i = 0;
nr = gup->nr_pages_per_call;
start_time = ktime_get();
for (addr = gup->addr; addr < gup->addr + gup->size; addr = next) {
if (nr != gup->nr_pages_per_call)
break;
next = addr + nr * PAGE_SIZE;
if (next > gup->addr + gup->size) {
next = gup->addr + gup->size;
nr = (next - addr) / PAGE_SIZE;
}
switch (cmd) {
case GUP_FAST_BENCHMARK:
nr = get_user_pages_fast(addr, nr, gup->flags & 1,
pages + i);
break;
case GUP_LONGTERM_BENCHMARK:
nr = get_user_pages_longterm(addr, nr, gup->flags & 1,
pages + i, NULL);
break;
case GUP_BENCHMARK:
nr = get_user_pages(addr, nr, gup->flags & 1, pages + i,
NULL);
break;
default:
return -1;
}
if (nr <= 0)
break;
i += nr;
}
end_time = ktime_get();
gup->get_delta_usec = ktime_us_delta(end_time, start_time);
gup->size = addr - gup->addr;
start_time = ktime_get();
for (i = 0; i < nr_pages; i++) {
if (!pages[i])
break;
put_page(pages[i]);
}
end_time = ktime_get();
gup->put_delta_usec = ktime_us_delta(end_time, start_time);
kvfree(pages);
return 0;
}
