/*
* This function is exported but must not be called by anything other
* than get_page(). It implements the slow path of get_page().
*/
bool __get_page_tail(struct page *page)
{
/*
* This takes care of get_page() if run on a tail page
* returned by one of the get_user_pages/follow_page variants.
* get_user_pages/follow_page itself doesn't need the compound
* lock because it runs __get_page_tail_foll() under the
* proper PT lock that already serializes against
* split_huge_page().
*/
unsigned long flags;
bool got;
struct page *page_head = compound_trans_head(page);
/* Ref to put_compound_page() comment. */
if (!__compound_tail_refcounted(page_head)) {
smp_rmb();
if (likely(PageTail(page))) {
/*
* This is a hugetlbfs page or a slab
* page. __split_huge_page_refcount
* cannot race here.
*/
VM_BUG_ON(!PageHead(page_head));
__get_page_tail_foll(page, true);
return true;
} else {
/*
* __split_huge_page_refcount run
* before us, "page" was a THP
* tail. The split page_head has been
* freed and reallocated as slab or
* hugetlbfs page of smaller order
* (only possible if reallocated as
* slab on x86).
*/
return false;
}
}
got = false;
if (likely(page != page_head && get_page_unless_zero(page_head))) {
/*
* page_head wasn't a dangling pointer but it
* may not be a head page anymore by the time
* we obtain the lock. That is ok as long as it
* can't be freed from under us.
*/
flags = compound_lock_irqsave(page_head);
/* here __split_huge_page_refcount won't run anymore */
if (likely(PageTail(page))) {
__get_page_tail_foll(page, false);
got = true;
}
compound_unlock_irqrestore(page_head, flags);
if (unlikely(!got))
put_page(page_head);
}
return got;
}
/*
* This function is exported but must not be called by anything other
* than get_page(). It implements the slow path of get_page().
*/
bool __get_page_tail(struct page *page)
{
/*
* This takes care of get_page() if run on a tail page
* returned by one of the get_user_pages/follow_page variants.
* get_user_pages/follow_page itself doesn't need the compound
* lock because it runs __get_page_tail_foll() under the
* proper PT lock that already serializes against
* split_huge_page().
*/
bool got = false;
struct page *page_head;
/*
* If this is a hugetlbfs page it cannot be split under us. Simply
* increment refcount for the head page.
*/
if (PageHuge(page)) {
page_head = compound_head(page);
atomic_inc(&page_head->_count);
got = true;
} else {
unsigned long flags;
page_head = compound_trans_head(page);
if (likely(page != page_head &&
get_page_unless_zero(page_head))) {
/* Ref to put_compound_page() comment. */
if (PageSlab(page_head)) {
if (likely(PageTail(page))) {
__get_page_tail_foll(page, false);
return true;
} else {
put_page(page_head);
return false;
}
}
/*
* page_head wasn't a dangling pointer but it
* may not be a head page anymore by the time
* we obtain the lock. That is ok as long as it
* can't be freed from under us.
*/
flags = compound_lock_irqsave(page_head);
/* here __split_huge_page_refcount won't run anymore */
if (likely(PageTail(page))) {
__get_page_tail_foll(page, false);
got = true;
}
compound_unlock_irqrestore(page_head, flags);
if (unlikely(!got))
put_page(page_head);
}
}
return got;
}
/*
* This function is exported but must not be called by anything other
* than get_page(). It implements the slow path of get_page().
*/
bool __get_page_tail(struct page *page)
{
/*
* This takes care of get_page() if run on a tail page
* returned by one of the get_user_pages/follow_page variants.
* get_user_pages/follow_page itself doesn't need the compound
* lock because it runs __get_page_tail_foll() under the
* proper PT lock that already serializes against
* split_huge_page().
*/
unsigned long flags;
bool got = false;
struct page *page_head = compound_trans_head(page);
if (likely(page != page_head && get_page_unless_zero(page_head))) {
/*
* page_head wasn't a dangling pointer but it
* may not be a head page anymore by the time
* we obtain the lock. That is ok as long as it
* can't be freed from under us.
*/
flags = compound_lock_irqsave(page_head);
/* here __split_huge_page_refcount won't run anymore */
if (likely(PageTail(page))) {
__get_page_tail_foll(page, false);
got = true;
}
compound_unlock_irqrestore(page_head, flags);
if (unlikely(!got))
put_page(page_head);
}
return got;
}