/* * best effort, GUP based copy_from_user() that assumes IRQ or NMI context */ static unsigned long copy_from_user_nmi(void *to, const void __user *from, unsigned long n) { unsigned long offset, addr = (unsigned long)from; int type = in_nmi() ? KM_NMI : KM_IRQ0; unsigned long size, len = 0; struct page *page; void *map; int ret; do { ret = __get_user_pages_fast(addr, 1, 0, &page); if (!ret) break; offset = addr & (PAGE_SIZE - 1); size = min(PAGE_SIZE - offset, n - len); map = kmap_atomic(page, type); memcpy(to, map+offset, size); kunmap_atomic(map, type); put_page(page); len += size; to += size; addr += size; } while (len < n); return len; }
int get_user_pages_fast(unsigned long start, int nr_pages, int write, struct page **pages) { struct mm_struct *mm = current->mm; int nr, ret; start &= PAGE_MASK; nr = __get_user_pages_fast(start, nr_pages, write, pages); ret = nr; if (nr < nr_pages) { pr_devel(" slow path ! nr = %d\n", nr); /* Try to get the remaining pages with get_user_pages */ start += nr << PAGE_SHIFT; pages += nr; down_read(&mm->mmap_sem); ret = get_user_pages(current, mm, start, nr_pages - nr, write, 0, pages, NULL); up_read(&mm->mmap_sem); /* Have to be a bit careful with return values */ if (nr > 0) { if (ret < 0) ret = nr; else ret += nr; } } return ret; }
/* * best effort, GUP based copy_from_user() that is NMI-safe */ unsigned long copy_from_user_nmi(void *to, const void __user *from, unsigned long n) { unsigned long offset, addr = (unsigned long)from; unsigned long size, len = 0; struct page *page; void *map; int ret; if (__range_not_ok(from, n, TASK_SIZE)) return len; do { ret = __get_user_pages_fast(addr, 1, 0, &page); if (!ret) break; offset = addr & (PAGE_SIZE - 1); size = min(PAGE_SIZE - offset, n - len); map = kmap_atomic(page); memcpy(to, map+offset, size); kunmap_atomic(map); put_page(page); len += size; to += size; addr += size; } while (len < n); return len; }
/* * best effort, GUP based copy_from_user() that assumes IRQ or NMI context */ static unsigned long copy_from_user_nmi(void *to, const void __user *from, unsigned long n) { unsigned long offset, addr = (unsigned long)from; unsigned long size, len = 0; struct page *page; void *map; int ret; do { #ifndef CONFIG_L4 ret = __get_user_pages_fast(addr, 1, 0, &page); #else ret = get_user_pages_fast(addr, 1, 0, &page); #endif if (!ret) break; offset = addr & (PAGE_SIZE - 1); size = min(PAGE_SIZE - offset, n - len); map = kmap_atomic(page); memcpy(to, map+offset, size); kunmap_atomic(map); put_page(page); len += size; to += size; addr += size; } while (len < n); return len; }
static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) { const int num_pages = obj->base.size >> PAGE_SHIFT; struct mm_struct *mm = obj->userptr.mm->mm; struct page **pvec; struct sg_table *pages; bool active; int pinned; /* If userspace should engineer that these pages are replaced in * the vma between us binding this page into the GTT and completion * of rendering... Their loss. If they change the mapping of their * pages they need to create a new bo to point to the new vma. * * However, that still leaves open the possibility of the vma * being copied upon fork. Which falls under the same userspace * synchronisation issue as a regular bo, except that this time * the process may not be expecting that a particular piece of * memory is tied to the GPU. * * Fortunately, we can hook into the mmu_notifier in order to * discard the page references prior to anything nasty happening * to the vma (discard or cloning) which should prevent the more * egregious cases from causing harm. */ if (obj->userptr.work) { /* active flag should still be held for the pending work */ if (IS_ERR(obj->userptr.work)) return PTR_ERR(obj->userptr.work); else return -EAGAIN; } pvec = NULL; pinned = 0; if (mm == current->mm) { pvec = kvmalloc_array(num_pages, sizeof(struct page *), GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN); if (pvec) /* defer to worker if malloc fails */ pinned = __get_user_pages_fast(obj->userptr.ptr, num_pages, !i915_gem_object_is_readonly(obj), pvec); } active = false; if (pinned < 0) { pages = ERR_PTR(pinned); pinned = 0; } else if (pinned < num_pages) { pages = __i915_gem_userptr_get_pages_schedule(obj); active = pages == ERR_PTR(-EAGAIN); } else { pages = __i915_gem_userptr_alloc_pages(obj, pvec, num_pages); active = !IS_ERR(pages); } if (active) __i915_gem_userptr_set_active(obj, true); if (IS_ERR(pages)) release_pages(pvec, pinned); kvfree(pvec); return PTR_ERR_OR_ZERO(pages); }