unsigned long noinline __copy_to_user_memcpy(void __user *to, const void *from, unsigned long n) { unsigned long ua_flags; int atomic; if (unlikely(segment_eq(get_fs(), KERNEL_DS))) { memcpy((void *)to, from, n); return 0; } /* the mmap semaphore is taken only if not in an atomic context */ atomic = faulthandler_disabled(); if (!atomic) down_read(¤t->mm->mmap_sem); while (n) { pte_t *pte; spinlock_t *ptl; int tocopy; while (!pin_page_for_write(to, &pte, &ptl)) { if (!atomic) up_read(¤t->mm->mmap_sem); if (__put_user(0, (char __user *)to)) goto out; if (!atomic) down_read(¤t->mm->mmap_sem); } tocopy = (~(unsigned long)to & ~PAGE_MASK) + 1; if (tocopy > n) tocopy = n; ua_flags = uaccess_save_and_enable(); memcpy((void *)to, from, tocopy); uaccess_restore(ua_flags); to += tocopy; from += tocopy; n -= tocopy; if (pte) pte_unmap_unlock(pte, ptl); else spin_unlock(ptl); } if (!atomic) up_read(¤t->mm->mmap_sem); out: return n; }
static unsigned long noinline __clear_user_memset(void __user *addr, unsigned long n) { unsigned long ua_flags; if (unlikely(segment_eq(get_fs(), KERNEL_DS))) { memset((void *)addr, 0, n); return 0; } down_read(¤t->mm->mmap_sem); while (n) { pte_t *pte; spinlock_t *ptl; int tocopy; while (!pin_page_for_write(addr, &pte, &ptl)) { up_read(¤t->mm->mmap_sem); if (__put_user(0, (char __user *)addr)) goto out; down_read(¤t->mm->mmap_sem); } tocopy = (~(unsigned long)addr & ~PAGE_MASK) + 1; if (tocopy > n) tocopy = n; ua_flags = uaccess_save_and_enable(); memset((void *)addr, 0, tocopy); uaccess_restore(ua_flags); addr += tocopy; n -= tocopy; if (pte) pte_unmap_unlock(pte, ptl); else spin_unlock(ptl); } up_read(¤t->mm->mmap_sem); out: return n; }
unsigned long ___copy_to_user(void *to, const void *from, unsigned long n) { if (get_fs() == KERNEL_DS) { memcpy(to, from, n); return 0; } if (n < 256) return __arch_copy_to_user(to, from, n); down_read(¤t->mm->mmap_sem); while (n) { pte_t *pte; spinlock_t *ptl; int tocopy; while (unlikely(!pin_page_for_write(to, &pte, &ptl))) { up_read(¤t->mm->mmap_sem); if (put_user(*((u8 *)from), (u8 *)to)) goto out; down_read(¤t->mm->mmap_sem); } tocopy = ((~((unsigned long)to)) & (PAGE_SIZE - 1)) + 1; if (tocopy > n) tocopy = n; memcpy(to, from, tocopy); to += tocopy; from += tocopy; n -= tocopy; pte_unmap_unlock(pte, ptl); } out: up_read(¤t->mm->mmap_sem); return n; }