static size_t copy_page_from_iter_bvec(struct page *page, size_t offset, size_t bytes, struct iov_iter *i) { size_t skip, copy, wanted; const struct bio_vec *bvec; void *kaddr, *to; if (unlikely(bytes > i->count)) bytes = i->count; if (unlikely(!bytes)) return 0; wanted = bytes; bvec = i->bvec; skip = i->iov_offset; kaddr = kmap_atomic(page); to = kaddr + offset; copy = min(bytes, bvec->bv_len - skip); memcpy_from_page(to, bvec->bv_page, bvec->bv_offset + skip, copy); to += copy; skip += copy; bytes -= copy; while (bytes) { bvec++; copy = min(bytes, (size_t)bvec->bv_len); memcpy_from_page(to, bvec->bv_page, bvec->bv_offset, copy); skip = copy; to += copy; bytes -= copy; } kunmap_atomic(kaddr); if (skip == bvec->bv_len) { bvec++; skip = 0; } i->count -= wanted; i->nr_segs -= bvec - i->bvec; i->bvec = bvec; i->iov_offset = skip; return wanted; }
size_t iov_iter_copy_from_user_atomic(struct page *page, struct iov_iter *i, unsigned long offset, size_t bytes) { char *kaddr = kmap_atomic(page), *p = kaddr + offset; iterate_all_kinds(i, bytes, v, __copy_from_user_inatomic((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page, v.bv_offset, v.bv_len), memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) ) kunmap_atomic(kaddr); return bytes; }
size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) { char *to = addr; if (unlikely(bytes > i->count)) bytes = i->count; if (unlikely(!bytes)) return 0; iterate_and_advance(i, bytes, v, __copy_from_user_nocache((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, v.bv_offset, v.bv_len), memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) ) return bytes; }
static size_t copy_from_user_bvec(struct page *page, struct iov_iter *i, unsigned long offset, size_t bytes) { char *kaddr; size_t left; const struct bio_vec *bvec; size_t base = i->iov_offset; kaddr = kmap_atomic(page); for (left = bytes, bvec = i->bvec; left; bvec++, base = 0) { size_t copy = min(left, bvec->bv_len - base); if (!bvec->bv_len) continue; memcpy_from_page(kaddr + offset, bvec->bv_page, bvec->bv_offset + base, copy); offset += copy; left -= copy; } kunmap_atomic(kaddr); return bytes; }