unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n)
{
if (uaccess_kernel()) {
memcpy((__force void *) to, from, n);
return 0;
}
return buffer_op((unsigned long) to, n, 1, copy_chunk_to_user, &from);
}
unsigned long __clear_user(void __user *mem, unsigned long len)
{
if (uaccess_kernel()) {
memset((__force void*)mem, 0, len);
return 0;
}
return buffer_op((unsigned long) mem, len, 1, clear_chunk, NULL);
}
unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n)
{
if (uaccess_kernel()) {
memcpy(to, (__force void*)from, n);
return 0;
}
return buffer_op((unsigned long) from, n, 0, copy_chunk_from_user, &to);
}
long __strnlen_user(const void __user *str, long len)
{
int count = 0, n;
if (uaccess_kernel())
return strnlen((__force char*)str, len) + 1;
n = buffer_op((unsigned long) str, len, 0, strnlen_chunk, &count);
if (n == 0)
return count + 1;
return 0;
}
unsigned long noinline
__copy_to_user_memcpy(void __user *to, const void *from, unsigned long n)
{
unsigned long ua_flags;
int atomic;
if (uaccess_kernel()) {
memcpy((void *)to, from, n);
return 0;
}
/* the mmap semaphore is taken only if not in an atomic context */
atomic = faulthandler_disabled();
if (!atomic)
down_read(¤t->mm->mmap_sem);
while (n) {
pte_t *pte;
spinlock_t *ptl;
int tocopy;
while (!pin_page_for_write(to, &pte, &ptl)) {
if (!atomic)
up_read(¤t->mm->mmap_sem);
if (__put_user(0, (char __user *)to))
goto out;
if (!atomic)
down_read(¤t->mm->mmap_sem);
}
tocopy = (~(unsigned long)to & ~PAGE_MASK) + 1;
if (tocopy > n)
tocopy = n;
ua_flags = uaccess_save_and_enable();
memcpy((void *)to, from, tocopy);
uaccess_restore(ua_flags);
to += tocopy;
from += tocopy;
n -= tocopy;
if (pte)
pte_unmap_unlock(pte, ptl);
else
spin_unlock(ptl);
}
if (!atomic)
up_read(¤t->mm->mmap_sem);
out:
return n;
}
long __strncpy_from_user(char *dst, const char __user *src, long count)
{
long n;
char *ptr = dst;
if (uaccess_kernel()) {
strncpy(dst, (__force void *) src, count);
return strnlen(dst, count);
}
n = buffer_op((unsigned long) src, count, 0, strncpy_chunk_from_user,
&ptr);
if (n != 0)
return -EFAULT;
return strnlen(dst, count);
}
static unsigned long noinline
__clear_user_memset(void __user *addr, unsigned long n)
{
unsigned long ua_flags;
if (uaccess_kernel()) {
memset((void *)addr, 0, n);
return 0;
}
down_read(¤t->mm->mmap_sem);
while (n) {
pte_t *pte;
spinlock_t *ptl;
int tocopy;
while (!pin_page_for_write(addr, &pte, &ptl)) {
up_read(¤t->mm->mmap_sem);
if (__put_user(0, (char __user *)addr))
goto out;
down_read(¤t->mm->mmap_sem);
}
tocopy = (~(unsigned long)addr & ~PAGE_MASK) + 1;
if (tocopy > n)
tocopy = n;
ua_flags = uaccess_save_and_enable();
memset((void *)addr, 0, tocopy);
uaccess_restore(ua_flags);
addr += tocopy;
n -= tocopy;
if (pte)
pte_unmap_unlock(pte, ptl);
else
spin_unlock(ptl);
}
up_read(¤t->mm->mmap_sem);
out:
return n;
}
BPF_CALL_3(bpf_probe_write_user, void *, unsafe_ptr, const void *, src,
u32, size)
{
/*
* Ensure we're in user context which is safe for the helper to
* run. This helper has no business in a kthread.
*
* access_ok() should prevent writing to non-user memory, but in
* some situations (nommu, temporary switch, etc) access_ok() does
* not provide enough validation, hence the check on KERNEL_DS.
*/
if (unlikely(in_interrupt() ||
current->flags & (PF_KTHREAD | PF_EXITING)))
return -EPERM;
if (unlikely(uaccess_kernel()))
return -EPERM;
if (!access_ok(VERIFY_WRITE, unsafe_ptr, size))
return -EPERM;
return probe_kernel_write(unsafe_ptr, src, size);
}