/* * The performance critical leaf functions are made noinline otherwise gcc * inlines everything into a single function which results in too much * register pressure. */ static noinline int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, int write, struct page **pages, int *nr) { unsigned long mask, result; pte_t *ptep; result = _PAGE_PRESENT|_PAGE_USER; if (write) result |= _PAGE_RW; mask = result | _PAGE_SPECIAL; ptep = pte_offset_kernel(&pmd, addr); do { pte_t pte = *ptep; struct page *page; if ((pte_val(pte) & mask) != result) return 0; VM_BUG_ON(!pfn_valid(pte_pfn(pte))); page = pte_page(pte); if (!page_cache_get_speculative(page)) return 0; if (unlikely(pte_val(pte) != pte_val(*ptep))) { put_page(page); return 0; } pages[*nr] = page; (*nr)++; } while (ptep++, addr += PAGE_SIZE, addr != end); return 1; }
/* * The performance critical leaf functions are made noinline otherwise gcc * inlines everything into a single function which results in too much * register pressure. */ static noinline int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, int write, struct page **pages, int *nr) { unsigned long mask, result; pte_t *ptep; if (tlb_type == hypervisor) { result = _PAGE_PRESENT_4V|_PAGE_P_4V; if (write) result |= _PAGE_WRITE_4V; } else { result = _PAGE_PRESENT_4U|_PAGE_P_4U; if (write) result |= _PAGE_WRITE_4U; } mask = result | _PAGE_SPECIAL; ptep = pte_offset_kernel(&pmd, addr); do { struct page *page, *head; pte_t pte = *ptep; if ((pte_val(pte) & mask) != result) return 0; VM_BUG_ON(!pfn_valid(pte_pfn(pte))); /* The hugepage case is simplified on sparc64 because * we encode the sub-page pfn offsets into the * hugepage PTEs. We could optimize this in the future * use page_cache_add_speculative() for the hugepage case. */ page = pte_page(pte); head = compound_head(page); if (!page_cache_get_speculative(head)) return 0; if (unlikely(pte_val(pte) != pte_val(*ptep))) { put_page(head); return 0; } pages[*nr] = page; (*nr)++; } while (ptep++, addr += PAGE_SIZE, addr != end); return 1; }
static noinline int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end, int write, struct page **pages, int *nr) { unsigned long mask, result; pte_t *ptep; if (tlb_type == hypervisor) { result = _PAGE_PRESENT_4V|_PAGE_P_4V; if (write) result |= _PAGE_WRITE_4V; } else { result = _PAGE_PRESENT_4U|_PAGE_P_4U; if (write) result |= _PAGE_WRITE_4U; } mask = result | _PAGE_SPECIAL; ptep = pte_offset_kernel(&pmd, addr); do { struct page *page, *head; pte_t pte = *ptep; if ((pte_val(pte) & mask) != result) return 0; VM_BUG_ON(!pfn_valid(pte_pfn(pte))); page = pte_page(pte); head = compound_head(page); if (!page_cache_get_speculative(head)) return 0; if (unlikely(pte_val(pte) != pte_val(*ptep))) { put_page(head); return 0; } if (head != page) get_huge_page_tail(page); pages[*nr] = page; (*nr)++; } while (ptep++, addr += PAGE_SIZE, addr != end); return 1; }