static void put_compound_page(struct page *page) { if (unlikely(PageTail(page))) { /* __split_huge_page_refcount can run under us */ struct page *page_head = compound_trans_head(page); if (likely(page != page_head && get_page_unless_zero(page_head))) { unsigned long flags; /* * page_head wasn't a dangling pointer but it * may not be a head page anymore by the time * we obtain the lock. That is ok as long as it * can't be freed from under us. */ flags = compound_lock_irqsave(page_head); if (unlikely(!PageTail(page))) { /* __split_huge_page_refcount run before us */ compound_unlock_irqrestore(page_head, flags); VM_BUG_ON(PageHead(page_head)); if (put_page_testzero(page_head)) __put_single_page(page_head); out_put_single: if (put_page_testzero(page)) __put_single_page(page); return; } VM_BUG_ON(page_head != page->first_page); /* * We can release the refcount taken by * get_page_unless_zero() now that * __split_huge_page_refcount() is blocked on * the compound_lock. */ if (put_page_testzero(page_head)) VM_BUG_ON(1); /* __split_huge_page_refcount will wait now */ VM_BUG_ON(page_mapcount(page) <= 0); atomic_dec(&page->_mapcount); VM_BUG_ON(atomic_read(&page_head->_count) <= 0); VM_BUG_ON(atomic_read(&page->_count) != 0); compound_unlock_irqrestore(page_head, flags); if (put_page_testzero(page_head)) { if (PageHead(page_head)) __put_compound_page(page_head); else __put_single_page(page_head); } } else { /* page_head is a dangling pointer */ VM_BUG_ON(PageTail(page)); goto out_put_single; } } else if (put_page_testzero(page)) { if (PageHead(page)) __put_compound_page(page); else __put_single_page(page); } }
void put_page(struct page *page) { if (unlikely(PageCompound(page))) { page = (struct page *)page_private(page); if (put_page_testzero(page)) { void (*dtor)(struct page *page); dtor = (void (*)(struct page *))page[1].mapping; (*dtor)(page); } return; } if (put_page_testzero(page)) __page_cache_release(page); }
/* Cleanup a mapping */ static void fb_vma_free(struct pme_fb_vma *node) { int order, num_pages; struct page *p; if (atomic_dec_and_test(&node->ref_count)) { /* This buffer can be recycled * (Buffers can be NULL in the case where * the mapped area is an iovec structure) */ if (node->buffers) pme_fbchain_recycle(node->buffers); if (node->iovec_pages) { order = get_order(node->kmem_size); if (order) { p = node->iovec_pages; num_pages = 1 << order; while (num_pages--) { put_page_testzero(p); p++; } } __free_pages(node->iovec_pages, order); } vfree(node); } }
void put_page(struct page *page) { if (unlikely(PageCompound(page))) put_compound_page(page); else if (put_page_testzero(page)) __put_single_page(page); }
void put_page(struct page *page) { if (unlikely(PageCompound(page))) put_compound_page(page); else if (put_page_testzero(page)) __page_cache_release(page); }
static void vmlfb_free_vram_area(struct vram_area *va) { unsigned long j; if (va->logical) { /* * Reset the linear kernel map caching policy. */ set_pages_wb(virt_to_page(va->logical), va->size >> PAGE_SHIFT); /* * Decrease the usage count on the pages we've used * to compensate for upping when allocating. */ for (j = va->logical; j < va->logical + va->size; j += PAGE_SIZE) { (void)put_page_testzero(virt_to_page(j)); } printk(KERN_DEBUG MODULE_NAME ": Freeing %ld bytes vram area at 0x%08lx\n", va->size, va->phys); free_pages(va->logical, va->order); va->logical = 0; }
/** * Two special cases here: we could avoid taking compound_lock_irqsave * and could skip the tail refcounting(in _mapcount). * * 1. Hugetlbfs page: * * PageHeadHuge will remain true until the compound page * is released and enters the buddy allocator, and it could * not be split by __split_huge_page_refcount(). * * So if we see PageHeadHuge set, and we have the tail page pin, * then we could safely put head page. * * 2. Slab THP page: * * PG_slab is cleared before the slab frees the head page, and * tail pin cannot be the last reference left on the head page, * because the slab code is free to reuse the compound page * after a kfree/kmem_cache_free without having to check if * there's any tail pin left. In turn all tail pinsmust be always * released while the head is still pinned by the slab code * and so we know PG_slab will be still set too. * * So if we see PageSlab set, and we have the tail page pin, * then we could safely put head page. */ static __always_inline void put_unrefcounted_compound_page(struct page *page_head, struct page *page) { /* * If @page is a THP tail, we must read the tail page * flags after the head page flags. The * __split_huge_page_refcount side enforces write memory barriers * between clearing PageTail and before the head page * can be freed and reallocated. */ smp_rmb(); if (likely(PageTail(page))) { /* * __split_huge_page_refcount cannot race * here, see the comment above this function. */ VM_BUG_ON_PAGE(!PageHead(page_head), page_head); if (put_page_testzero(page_head)) { /* * If this is the tail of a slab THP page, * the tail pin must not be the last reference * held on the page, because the PG_slab cannot * be cleared before all tail pins (which skips * the _mapcount tail refcounting) have been * released. * * If this is the tail of a hugetlbfs page, * the tail pin may be the last reference on * the page instead, because PageHeadHuge will * not go away until the compound page enters * the buddy allocator. */ VM_BUG_ON_PAGE(PageSlab(page_head), page_head); __put_compound_page(page_head); } } else /* * __split_huge_page_refcount run before us, * @page was a THP tail. The split @page_head * has been freed and reallocated as slab or * hugetlbfs page of smaller order (only * possible if reallocated as slab on x86). */ if (put_page_testzero(page)) __put_single_page(page); }
void pte_fragment_free(unsigned long *table, int kernel) { struct page *page = virt_to_page(table); if (put_page_testzero(page)) { if (!kernel) pgtable_page_dtor(page); free_hot_cold_page(page, 0); } }
/** * release_pages:释放pages页框数组(只有count = 0时,才正真的被回收) */ void release_pages(struct page **pages, int nr, int cold) { int i; struct pagevec pages_to_free; struct zone *zone = NULL; unsigned long uninitialized_var(flags); pagevec_init(&pages_to_free, cold); for (i = 0; i < nr; i++) { struct page *page = pages[i]; if (unlikely(PageCompound(page))) { if (zone) { spin_unlock_irqrestore(&zone->lru_lock, flags); zone = NULL; } put_compound_page(page); continue; } /*减少page的引用次数*/ if (!put_page_testzero(page)) continue; /*page的引用次数为0,则page从zone LRU链表中删除*/ if (PageLRU(page)) { struct zone *pagezone = page_zone(page); if (pagezone != zone) { if (zone) spin_unlock_irqrestore(&zone->lru_lock, flags); zone = pagezone; spin_lock_irqsave(&zone->lru_lock, flags); } VM_BUG_ON(!PageLRU(page)); /*清空页的LRU属性*/ __ClearPageLRU(page); del_page_from_lru(zone, page); } /*page加入到pages_to_free页缓存中*/ if (!pagevec_add(&pages_to_free, page)) { if (zone) { spin_unlock_irqrestore(&zone->lru_lock, flags); zone = NULL; } __pagevec_free(&pages_to_free); pagevec_reinit(&pages_to_free); } } if (zone) spin_unlock_irqrestore(&zone->lru_lock, flags); pagevec_free(&pages_to_free); }
static void put_compound_page(struct page *page) { page = compound_head(page); if (put_page_testzero(page)) { compound_page_dtor *dtor; dtor = get_compound_page_dtor(page); (*dtor)(page); } }
static void put_compound_page(struct page *page) { page = (struct page *)page_private(page); if (put_page_testzero(page)) { void (*dtor)(struct page *page); dtor = (void (*)(struct page *))page[1].lru.next; (*dtor)(page); } }
/* * Batched page_cache_release(). Decrement the reference count on all the * passed pages. If it fell to zero then remove the page from the LRU and * free it. * * Avoid taking zone->lru_lock if possible, but if it is taken, retain it * for the remainder of the operation. * * The locking in this function is against shrink_inactive_list(): we recheck * the page count inside the lock to see whether shrink_inactive_list() * grabbed the page via the LRU. If it did, give up: shrink_inactive_list() * will free it. */ void release_pages(struct page **pages, int nr, int cold) { int i; struct pagevec pages_to_free; struct zone *zone = NULL; unsigned long uninitialized_var(flags); pagevec_init(&pages_to_free, cold); for (i = 0; i < nr; i++) { struct page *page = pages[i]; #ifdef CONFIG_OXNAS_FAST_READS_AND_WRITES WARN_ON(PageIncoherentSendfile(page)); #endif // CONFIG_OXNAS_FAST_READS_AND_WRITES if (unlikely(PageCompound(page))) { if (zone) { spin_unlock_irqrestore(&zone->lru_lock, flags); zone = NULL; } put_compound_page(page); continue; } if (!put_page_testzero(page)) continue; if (PageLRU(page)) { struct zone *pagezone = page_zone(page); if (pagezone != zone) { if (zone) spin_unlock_irqrestore(&zone->lru_lock, flags); zone = pagezone; spin_lock_irqsave(&zone->lru_lock, flags); } VM_BUG_ON(!PageLRU(page)); __ClearPageLRU(page); del_page_from_lru(zone, page); } if (!pagevec_add(&pages_to_free, page)) { if (zone) { spin_unlock_irqrestore(&zone->lru_lock, flags); zone = NULL; } __pagevec_free(&pages_to_free); pagevec_reinit(&pages_to_free); } } if (zone) spin_unlock_irqrestore(&zone->lru_lock, flags); pagevec_free(&pages_to_free); }
/* * Batched page_cache_release(). Decrement the reference count on all the * passed pages. If it fell to zero then remove the page from the LRU and * free it. * * Avoid taking zone->lru_lock if possible, but if it is taken, retain it * for the remainder of the operation. * * The locking in this function is against shrink_inactive_list(): we recheck * the page count inside the lock to see whether shrink_inactive_list() * grabbed the page via the LRU. If it did, give up: shrink_inactive_list() * will free it. */ void release_pages(struct page **pages, int nr, int cold) { int i; struct pagevec pages_to_free; struct zone *zone = NULL; unsigned long uninitialized_var(flags); pagevec_init(&pages_to_free, cold); for (i = 0; i < nr; i++) { struct page *page = pages[i]; if (unlikely(PageCompound(page))) { if (zone) { spin_unlock_irqrestore(&zone->lru_lock, flags); zone = NULL; } put_compound_page(page); continue; } if (!put_page_testzero(page)) continue; if (PageLRU(page)) { struct zone *pagezone = page_zone(page); if (pagezone != zone) { if (zone) spin_unlock_irqrestore(&zone->lru_lock, flags); zone = pagezone; spin_lock_irqsave(&zone->lru_lock, flags); } VM_BUG_ON(!PageLRU(page)); __ClearPageLRU(page); del_page_from_lru(zone, page); } else if (PageIONBacked(page)) { ClearPageActive(page); ClearPageUnevictable(page); } if (!pagevec_add(&pages_to_free, page)) { if (zone) { spin_unlock_irqrestore(&zone->lru_lock, flags); zone = NULL; } __pagevec_free(&pages_to_free); pagevec_reinit(&pages_to_free); } } if (zone) spin_unlock_irqrestore(&zone->lru_lock, flags); pagevec_free(&pages_to_free); }
void __homecache_free_pages(struct page *page, unsigned int order) { if (put_page_testzero(page)) { homecache_change_page_home(page, order, PAGE_HOME_HASH); if (order == 0) { free_hot_cold_page(page, false); } else { init_page_count(page); __free_pages(page, order); } } }
/* * Batched page_cache_release(). Decrement the reference count on all the * passed pages. If it fell to zero then remove the page from the LRU and * free it. * * Avoid taking zone->lru_lock if possible, but if it is taken, retain it * for the remainder of the operation. * * The locking in this function is against shrink_cache(): we recheck the * page count inside the lock to see whether shrink_cache grabbed the page * via the LRU. If it did, give up: shrink_cache will free it. */ void release_pages(struct page **pages, int nr, int cold) { int i; struct pagevec pages_to_free; struct zone *zone = NULL; unsigned long uninitialized_var(flags); pagevec_init(&pages_to_free, cold); for (i = 0; i < nr; i++) { struct page *page = pages[i]; if (unlikely(PageCompound(page))) { if (zone) { spin_unlock_irqrestore(&zone->lru_lock, flags); zone = NULL; } put_compound_page(page); continue; } // dyc: if page->ref not zero, continue if (!put_page_testzero(page)) continue; // dyc: if in url, remove from it if (PageLRU(page)) { struct zone *pagezone = page_zone(page); if (pagezone != zone) { if (zone) spin_unlock_irqrestore(&zone->lru_lock, flags); zone = pagezone; spin_lock_irqsave(&zone->lru_lock, flags); } VM_BUG_ON(!PageLRU(page)); __ClearPageLRU(page); del_page_from_lru(zone, page); } // dyc: if no space available after adding if (!pagevec_add(&pages_to_free, page)) { if (zone) { spin_unlock_irqrestore(&zone->lru_lock, flags); zone = NULL; } // dyc: return page to buddy system __pagevec_free(&pages_to_free); pagevec_reinit(&pages_to_free); } } // for (i = 0; i < nr; i++) if (zone) spin_unlock_irqrestore(&zone->lru_lock, flags); pagevec_free(&pages_to_free); }
/* * Batched page_cache_release(). Decrement the reference count on all the * passed pages. If it fell to zero then remove the page from the LRU and * free it. * * Avoid taking zone->lru_lock if possible, but if it is taken, retain it * for the remainder of the operation. * * The locking in this function is against shrink_inactive_list(): we recheck * the page count inside the lock to see whether shrink_inactive_list() * grabbed the page via the LRU. If it did, give up: shrink_inactive_list() * will free it. */ void release_pages(struct page **pages, int nr, int cold) { int i; LIST_HEAD(pages_to_free); struct zone *zone = NULL; struct lruvec *lruvec; unsigned long uninitialized_var(flags); for (i = 0; i < nr; i++) { struct page *page = pages[i]; if (unlikely(PageCompound(page))) { if (zone) { spin_unlock_irqrestore(&zone->lru_lock, flags); zone = NULL; } put_compound_page(page); continue; } if (!put_page_testzero(page)) continue; if (PageLRU(page)) { struct zone *pagezone = page_zone(page); if (pagezone != zone) { if (zone) spin_unlock_irqrestore(&zone->lru_lock, flags); zone = pagezone; spin_lock_irqsave(&zone->lru_lock, flags); } lruvec = mem_cgroup_page_lruvec(page, zone); VM_BUG_ON(!PageLRU(page)); __ClearPageLRU(page); del_page_from_lru_list(page, lruvec, page_off_lru(page)); } /* Clear Active bit in case of parallel mark_page_accessed */ ClearPageActive(page); list_add(&page->lru, &pages_to_free); } if (zone) spin_unlock_irqrestore(&zone->lru_lock, flags); free_hot_cold_page_list(&pages_to_free, cold); }
void homecache_free_pages(unsigned long addr, unsigned int order) { struct page *page; if (addr == 0) return; VM_BUG_ON(!virt_addr_valid((void *)addr)); page = virt_to_page((void *)addr); if (put_page_testzero(page)) { int pages = (1 << order); homecache_change_page_home(page, order, initial_page_home()); while (pages--) __free_page(page++); } }
/* * pagevec_release() for pages which are known to not be on the LRU * * This function reinitialises the caller's pagevec. */ void __pagevec_release_nonlru(struct pagevec *pvec) { int i; struct pagevec pages_to_free; pagevec_init(&pages_to_free, pvec->cold); for (i = 0; i < pagevec_count(pvec); i++) { struct page *page = pvec->pages[i]; VM_BUG_ON(PageLRU(page)); if (put_page_testzero(page)) pagevec_add(&pages_to_free, page); } pagevec_free(&pages_to_free); pagevec_reinit(pvec); }
void put_page(struct page *page) { #ifdef CONFIG_OXNAS_FAST_READS_AND_WRITES if (PageIncoherentSendfile(page)) { fast_put(page); } else { #endif // CONFIG_OXNAS_FAST_READS_AND_WRITES if (unlikely(PageCompound(page))) { put_compound_page(page); } else if (put_page_testzero(page)) { __page_cache_release(page); } #ifdef CONFIG_OXNAS_FAST_READS_AND_WRITES } #endif // CONFIG_OXNAS_FAST_READS_AND_WRITES }
/* * Batched page_cache_release(). Decrement the reference count on all the * passed pages. If it fell to zero then remove the page from the LRU and * free it. * * Avoid taking zone->lru_lock if possible, but if it is taken, retain it * for the remainder of the operation. * * The locking in this function is against shrink_cache(): we recheck the * page count inside the lock to see whether shrink_cache grabbed the page * via the LRU. If it did, give up: shrink_cache will free it. */ void release_pages(struct page **pages, int nr, int cold) { int i; struct pagevec pages_to_free; struct zone *zone = NULL; pagevec_init(&pages_to_free, cold); for (i = 0; i < nr; i++) { struct page *page = pages[i]; struct zone *pagezone; if (unlikely(PageCompound(page))) { if (zone) { spin_unlock_irq(&zone->lru_lock); zone = NULL; } put_compound_page(page); continue; } if (!put_page_testzero(page)) continue; pagezone = page_zone(page); if (pagezone != zone) { if (zone) spin_unlock_irq(&zone->lru_lock); zone = pagezone; spin_lock_irq(&zone->lru_lock); } if (TestClearPageLRU(page)) del_page_from_lru(zone, page); if (page_count(page) == 0) { if (!pagevec_add(&pages_to_free, page)) { spin_unlock_irq(&zone->lru_lock); __pagevec_free(&pages_to_free); pagevec_reinit(&pages_to_free); zone = NULL; /* No lock is held */ } } } if (zone) spin_unlock_irq(&zone->lru_lock); pagevec_free(&pages_to_free); }
/* * Batched page_cache_release(). Decrement the reference count on all the * passed pages. If it fell to zero then remove the page from the LRU and * free it. * * Avoid taking zone->lru_lock if possible, but if it is taken, retain it * for the remainder of the operation. * * The locking in this function is against shrink_inactive_list(): we recheck * the page count inside the lock to see whether shrink_inactive_list() * grabbed the page via the LRU. If it did, give up: shrink_inactive_list() * will free it. */ void release_pages(struct page **pages, int nr, int cold) { int i; LIST_HEAD(pages_to_free); struct zone *zone = NULL; unsigned long uninitialized_var(flags); for (i = 0; i < nr; i++) { struct page *page = pages[i]; if (unlikely(PageCompound(page))) { if (zone) { spin_unlock_irqrestore(&zone->lru_lock, flags); zone = NULL; } put_compound_page(page); continue; } if (!put_page_testzero(page)) continue; if (PageLRU(page)) { struct zone *pagezone = page_zone(page); if (pagezone != zone) { if (zone) spin_unlock_irqrestore(&zone->lru_lock, flags); zone = pagezone; spin_lock_irqsave(&zone->lru_lock, flags); } VM_BUG_ON(!PageLRU(page)); __ClearPageLRU(page); del_page_from_lru_list(zone, page, page_off_lru(page)); } list_add(&page->lru, &pages_to_free); } if (zone) spin_unlock_irqrestore(&zone->lru_lock, flags); free_hot_cold_page_list(&pages_to_free, cold); }
static void put_compound_page(struct page *page) { struct page *page_head; /* * We see the PageCompound set and PageTail not set, so @page maybe: * 1. hugetlbfs head page, or * 2. THP head page. */ if (likely(!PageTail(page))) { if (put_page_testzero(page)) { /* * By the time all refcounts have been released * split_huge_page cannot run anymore from under us. */ if (PageHead(page)) __put_compound_page(page); else __put_single_page(page); } return; } /* * We see the PageCompound set and PageTail set, so @page maybe: * 1. a tail hugetlbfs page, or * 2. a tail THP page, or * 3. a split THP page. * * Case 3 is possible, as we may race with * __split_huge_page_refcount tearing down a THP page. */ page_head = compound_head(page); if (!__compound_tail_refcounted(page_head)) put_unrefcounted_compound_page(page_head, page); else put_refcounted_compound_page(page_head, page); }
static void put_compound_page(struct page *page) { if (unlikely(PageTail(page))) { /* __split_huge_page_refcount can run under us */ struct page *page_head = compound_trans_head(page); if (likely(page != page_head && get_page_unless_zero(page_head))) { unsigned long flags; if (PageHeadHuge(page_head)) { if (likely(PageTail(page))) { /* * __split_huge_page_refcount * cannot race here. */ VM_BUG_ON(!PageHead(page_head)); atomic_dec(&page->_mapcount); if (put_page_testzero(page_head)) VM_BUG_ON(1); if (put_page_testzero(page_head)) __put_compound_page(page_head); return; } else { /* * __split_huge_page_refcount * run before us, "page" was a * THP tail. The split * page_head has been freed * and reallocated as slab or * hugetlbfs page of smaller * order (only possible if * reallocated as slab on * x86). */ goto skip_lock; } } /* * page_head wasn't a dangling pointer but it * may not be a head page anymore by the time * we obtain the lock. That is ok as long as it * can't be freed from under us. */ flags = compound_lock_irqsave(page_head); if (unlikely(!PageTail(page))) { /* __split_huge_page_refcount run before us */ compound_unlock_irqrestore(page_head, flags); VM_BUG_ON(PageHead(page_head)); skip_lock: if (put_page_testzero(page_head)) { /* * The head page may have been * freed and reallocated as a * compound page of smaller * order and then freed again. * All we know is that it * cannot have become: a THP * page, a compound page of * higher order, a tail page. * That is because we still * hold the refcount of the * split THP tail and * page_head was the THP head * before the split. */ if (PageHead(page_head)) __put_compound_page(page_head); else __put_single_page(page_head); } out_put_single: if (put_page_testzero(page)) __put_single_page(page); return; } VM_BUG_ON(page_head != page->first_page); /* * We can release the refcount taken by * get_page_unless_zero() now that * __split_huge_page_refcount() is blocked on * the compound_lock. */ if (put_page_testzero(page_head)) VM_BUG_ON(1); /* __split_huge_page_refcount will wait now */ VM_BUG_ON(page_mapcount(page) <= 0); atomic_dec(&page->_mapcount); VM_BUG_ON(atomic_read(&page_head->_count) <= 0); VM_BUG_ON(atomic_read(&page->_count) != 0); compound_unlock_irqrestore(page_head, flags); if (put_page_testzero(page_head)) { if (PageHead(page_head)) __put_compound_page(page_head); else __put_single_page(page_head); } } else { /* page_head is a dangling pointer */ VM_BUG_ON(PageTail(page)); goto out_put_single; } } else if (put_page_testzero(page)) { if (PageHead(page)) __put_compound_page(page); else __put_single_page(page); } }
static void put_compound_page(struct page *page) { struct page *page_head; if (likely(!PageTail(page))) { if (put_page_testzero(page)) { /* * By the time all refcounts have been released * split_huge_page cannot run anymore from under us. */ if (PageHead(page)) __put_compound_page(page); else __put_single_page(page); } return; } /* __split_huge_page_refcount can run under us */ page_head = compound_trans_head(page); /* * THP can not break up slab pages so avoid taking * compound_lock() and skip the tail page refcounting (in * _mapcount) too. Slab performs non-atomic bit ops on * page->flags for better performance. In particular * slab_unlock() in slub used to be a hot path. It is still * hot on arches that do not support * this_cpu_cmpxchg_double(). * * If "page" is part of a slab or hugetlbfs page it cannot be * splitted and the head page cannot change from under us. And * if "page" is part of a THP page under splitting, if the * head page pointed by the THP tail isn't a THP head anymore, * we'll find PageTail clear after smp_rmb() and we'll treat * it as a single page. */ if (!__compound_tail_refcounted(page_head)) { /* * If "page" is a THP tail, we must read the tail page * flags after the head page flags. The * split_huge_page side enforces write memory barriers * between clearing PageTail and before the head page * can be freed and reallocated. */ smp_rmb(); if (likely(PageTail(page))) { /* * __split_huge_page_refcount cannot race * here. */ VM_BUG_ON(!PageHead(page_head)); VM_BUG_ON(page_mapcount(page) != 0); if (put_page_testzero(page_head)) { /* * If this is the tail of a slab * compound page, the tail pin must * not be the last reference held on * the page, because the PG_slab * cannot be cleared before all tail * pins (which skips the _mapcount * tail refcounting) have been * released. For hugetlbfs the tail * pin may be the last reference on * the page instead, because * PageHeadHuge will not go away until * the compound page enters the buddy * allocator. */ VM_BUG_ON(PageSlab(page_head)); __put_compound_page(page_head); } return; } else /* * __split_huge_page_refcount run before us, * "page" was a THP tail. The split page_head * has been freed and reallocated as slab or * hugetlbfs page of smaller order (only * possible if reallocated as slab on x86). */ goto out_put_single; } if (likely(page != page_head && get_page_unless_zero(page_head))) { unsigned long flags; /* * page_head wasn't a dangling pointer but it may not * be a head page anymore by the time we obtain the * lock. That is ok as long as it can't be freed from * under us. */ flags = compound_lock_irqsave(page_head); if (unlikely(!PageTail(page))) { /* __split_huge_page_refcount run before us */ compound_unlock_irqrestore(page_head, flags); if (put_page_testzero(page_head)) { /* * The head page may have been freed * and reallocated as a compound page * of smaller order and then freed * again. All we know is that it * cannot have become: a THP page, a * compound page of higher order, a * tail page. That is because we * still hold the refcount of the * split THP tail and page_head was * the THP head before the split. */ if (PageHead(page_head)) __put_compound_page(page_head); else __put_single_page(page_head); } out_put_single: if (put_page_testzero(page)) __put_single_page(page); return; } VM_BUG_ON(page_head != page->first_page); /* * We can release the refcount taken by * get_page_unless_zero() now that * __split_huge_page_refcount() is blocked on the * compound_lock. */ if (put_page_testzero(page_head)) VM_BUG_ON(1); /* __split_huge_page_refcount will wait now */ VM_BUG_ON(page_mapcount(page) <= 0); atomic_dec(&page->_mapcount); VM_BUG_ON(atomic_read(&page_head->_count) <= 0); VM_BUG_ON(atomic_read(&page->_count) != 0); compound_unlock_irqrestore(page_head, flags); if (put_page_testzero(page_head)) { if (PageHead(page_head)) __put_compound_page(page_head); else __put_single_page(page_head); } } else { /* page_head is a dangling pointer */ VM_BUG_ON(PageTail(page)); goto out_put_single; } }
static __always_inline void put_refcounted_compound_page(struct page *page_head, struct page *page) { if (likely(page != page_head && get_page_unless_zero(page_head))) { unsigned long flags; /* * @page_head wasn't a dangling pointer but it may not * be a head page anymore by the time we obtain the * lock. That is ok as long as it can't be freed from * under us. */ flags = compound_lock_irqsave(page_head); if (unlikely(!PageTail(page))) { /* __split_huge_page_refcount run before us */ compound_unlock_irqrestore(page_head, flags); if (put_page_testzero(page_head)) { /* * The @page_head may have been freed * and reallocated as a compound page * of smaller order and then freed * again. All we know is that it * cannot have become: a THP page, a * compound page of higher order, a * tail page. That is because we * still hold the refcount of the * split THP tail and page_head was * the THP head before the split. */ if (PageHead(page_head)) __put_compound_page(page_head); else __put_single_page(page_head); } out_put_single: if (put_page_testzero(page)) __put_single_page(page); return; } VM_BUG_ON_PAGE(page_head != compound_head(page), page); /* * We can release the refcount taken by * get_page_unless_zero() now that * __split_huge_page_refcount() is blocked on the * compound_lock. */ if (put_page_testzero(page_head)) VM_BUG_ON_PAGE(1, page_head); /* __split_huge_page_refcount will wait now */ VM_BUG_ON_PAGE(page_mapcount(page) <= 0, page); atomic_dec(&page->_mapcount); VM_BUG_ON_PAGE(atomic_read(&page_head->_count) <= 0, page_head); VM_BUG_ON_PAGE(atomic_read(&page->_count) != 0, page); compound_unlock_irqrestore(page_head, flags); if (put_page_testzero(page_head)) { if (PageHead(page_head)) __put_compound_page(page_head); else __put_single_page(page_head); } } else { /* @page_head is a dangling pointer */ VM_BUG_ON_PAGE(PageTail(page), page); goto out_put_single; } }
/** * release_pages - batched page_cache_release() * @pages: array of pages to release * @nr: number of pages * @cold: whether the pages are cache cold * * Decrement the reference count on all the pages in @pages. If it * fell to zero, remove the page from the LRU and free it. */ void release_pages(struct page **pages, int nr, bool cold) { int i; LIST_HEAD(pages_to_free); struct zone *zone = NULL; struct lruvec *lruvec; unsigned long uninitialized_var(flags); unsigned int uninitialized_var(lock_batch); for (i = 0; i < nr; i++) { struct page *page = pages[i]; if (unlikely(PageCompound(page))) { if (zone) { spin_unlock_irqrestore(&zone->lru_lock, flags); zone = NULL; } put_compound_page(page); continue; } /* * Make sure the IRQ-safe lock-holding time does not get * excessive with a continuous string of pages from the * same zone. The lock is held only if zone != NULL. */ if (zone && ++lock_batch == SWAP_CLUSTER_MAX) { spin_unlock_irqrestore(&zone->lru_lock, flags); zone = NULL; } if (!put_page_testzero(page)) continue; if (PageLRU(page)) { struct zone *pagezone = page_zone(page); if (pagezone != zone) { if (zone) spin_unlock_irqrestore(&zone->lru_lock, flags); lock_batch = 0; zone = pagezone; spin_lock_irqsave(&zone->lru_lock, flags); } lruvec = mem_cgroup_page_lruvec(page, zone); VM_BUG_ON_PAGE(!PageLRU(page), page); __ClearPageLRU(page); del_page_from_lru_list(page, lruvec, page_off_lru(page)); } /* Clear Active bit in case of parallel mark_page_accessed */ __ClearPageActive(page); list_add(&page->lru, &pages_to_free); } if (zone) spin_unlock_irqrestore(&zone->lru_lock, flags); mem_cgroup_uncharge_list(&pages_to_free); free_hot_cold_page_list(&pages_to_free, cold); }
static void put_compound_page(struct page *page) { if (unlikely(PageTail(page))) { /* __split_huge_page_refcount can run under us */ struct page *page_head = page->first_page; smp_rmb(); /* * If PageTail is still set after smp_rmb() we can be sure * that the page->first_page we read wasn't a dangling pointer. * See __split_huge_page_refcount() smp_wmb(). */ if (likely(PageTail(page) && get_page_unless_zero(page_head))) { unsigned long flags; /* * Verify that our page_head wasn't converted * to a a regular page before we got a * reference on it. */ if (unlikely(!PageHead(page_head))) { /* PageHead is cleared after PageTail */ smp_rmb(); VM_BUG_ON(PageTail(page)); goto out_put_head; } /* * Only run compound_lock on a valid PageHead, * after having it pinned with * get_page_unless_zero() above. */ smp_mb(); /* page_head wasn't a dangling pointer */ flags = compound_lock_irqsave(page_head); if (unlikely(!PageTail(page))) { /* __split_huge_page_refcount run before us */ compound_unlock_irqrestore(page_head, flags); VM_BUG_ON(PageHead(page_head)); out_put_head: if (put_page_testzero(page_head)) __put_single_page(page_head); out_put_single: if (put_page_testzero(page)) __put_single_page(page); return; } VM_BUG_ON(page_head != page->first_page); /* * We can release the refcount taken by * get_page_unless_zero now that * split_huge_page_refcount is blocked on the * compound_lock. */ if (put_page_testzero(page_head)) VM_BUG_ON(1); /* __split_huge_page_refcount will wait now */ VM_BUG_ON(atomic_read(&page->_count) <= 0); atomic_dec(&page->_count); VM_BUG_ON(atomic_read(&page_head->_count) <= 0); compound_unlock_irqrestore(page_head, flags); if (put_page_testzero(page_head)) { if (PageHead(page_head)) __put_compound_page(page_head); else __put_single_page(page_head); } } else { /* page_head is a dangling pointer */ VM_BUG_ON(PageTail(page)); goto out_put_single; } } else if (put_page_testzero(page)) { if (PageHead(page)) __put_compound_page(page); else __put_single_page(page); } }
/** * release_pages - batched put_page() * @pages: array of pages to release * @nr: number of pages * * Decrement the reference count on all the pages in @pages. If it * fell to zero, remove the page from the LRU and free it. */ void release_pages(struct page **pages, int nr) { int i; LIST_HEAD(pages_to_free); struct pglist_data *locked_pgdat = NULL; struct lruvec *lruvec; unsigned long uninitialized_var(flags); unsigned int uninitialized_var(lock_batch); for (i = 0; i < nr; i++) { struct page *page = pages[i]; /* * Make sure the IRQ-safe lock-holding time does not get * excessive with a continuous string of pages from the * same pgdat. The lock is held only if pgdat != NULL. */ if (locked_pgdat && ++lock_batch == SWAP_CLUSTER_MAX) { spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags); locked_pgdat = NULL; } if (is_huge_zero_page(page)) continue; /* Device public page can not be huge page */ if (is_device_public_page(page)) { if (locked_pgdat) { spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags); locked_pgdat = NULL; } put_devmap_managed_page(page); continue; } page = compound_head(page); if (!put_page_testzero(page)) continue; if (PageCompound(page)) { if (locked_pgdat) { spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags); locked_pgdat = NULL; } __put_compound_page(page); continue; } if (PageLRU(page)) { struct pglist_data *pgdat = page_pgdat(page); if (pgdat != locked_pgdat) { if (locked_pgdat) spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags); lock_batch = 0; locked_pgdat = pgdat; spin_lock_irqsave(&locked_pgdat->lru_lock, flags); } lruvec = mem_cgroup_page_lruvec(page, locked_pgdat); VM_BUG_ON_PAGE(!PageLRU(page), page); __ClearPageLRU(page); del_page_from_lru_list(page, lruvec, page_off_lru(page)); } /* Clear Active bit in case of parallel mark_page_accessed */ __ClearPageActive(page); __ClearPageWaiters(page); list_add(&page->lru, &pages_to_free); } if (locked_pgdat) spin_unlock_irqrestore(&locked_pgdat->lru_lock, flags); mem_cgroup_uncharge_list(&pages_to_free); free_unref_page_list(&pages_to_free); }
static void put_compound_page(struct page *page) { if (unlikely(PageTail(page))) { /* __split_huge_page_refcount can run under us */ struct page *page_head = compound_trans_head(page); if (likely(page != page_head && get_page_unless_zero(page_head))) { unsigned long flags; /* * THP can not break up slab pages so avoid taking * compound_lock(). Slab performs non-atomic bit ops * on page->flags for better performance. In particular * slab_unlock() in slub used to be a hot path. It is * still hot on arches that do not support * this_cpu_cmpxchg_double(). */ if (PageSlab(page_head)) { if (PageTail(page)) { if (put_page_testzero(page_head)) VM_BUG_ON(1); atomic_dec(&page->_mapcount); goto skip_lock_tail; } else goto skip_lock; } /* * page_head wasn't a dangling pointer but it * may not be a head page anymore by the time * we obtain the lock. That is ok as long as it * can't be freed from under us. */ flags = compound_lock_irqsave(page_head); if (unlikely(!PageTail(page))) { /* __split_huge_page_refcount run before us */ compound_unlock_irqrestore(page_head, flags); skip_lock: if (put_page_testzero(page_head)) __put_single_page(page_head); out_put_single: if (put_page_testzero(page)) __put_single_page(page); return; } VM_BUG_ON(page_head != page->first_page); /* * We can release the refcount taken by * get_page_unless_zero() now that * __split_huge_page_refcount() is blocked on * the compound_lock. */ if (put_page_testzero(page_head)) VM_BUG_ON(1); /* __split_huge_page_refcount will wait now */ VM_BUG_ON(page_mapcount(page) <= 0); atomic_dec(&page->_mapcount); VM_BUG_ON(atomic_read(&page_head->_count) <= 0); VM_BUG_ON(atomic_read(&page->_count) != 0); compound_unlock_irqrestore(page_head, flags); skip_lock_tail: if (put_page_testzero(page_head)) { if (PageHead(page_head)) __put_compound_page(page_head); else __put_single_page(page_head); } } else { /* page_head is a dangling pointer */ VM_BUG_ON(PageTail(page)); goto out_put_single; } } else if (put_page_testzero(page)) { if (PageHead(page)) __put_compound_page(page); else __put_single_page(page); } }