static int mlx4_en_alloc_frags(struct mlx4_en_priv *priv, struct mlx4_en_rx_desc *rx_desc, struct mlx4_en_rx_alloc *frags, struct mlx4_en_rx_alloc *ring_alloc, gfp_t gfp) { struct mlx4_en_rx_alloc page_alloc[MLX4_EN_MAX_RX_FRAGS]; const struct mlx4_en_frag_info *frag_info; struct page *page; int i; for (i = 0; i < priv->num_frags; i++) { frag_info = &priv->frag_info[i]; page_alloc[i] = ring_alloc[i]; page_alloc[i].page_offset += frag_info->frag_stride; if (page_alloc[i].page_offset + frag_info->frag_stride <= ring_alloc[i].page_size) continue; if (unlikely(mlx4_alloc_pages(priv, &page_alloc[i], frag_info, gfp))) goto out; } for (i = 0; i < priv->num_frags; i++) { frags[i] = ring_alloc[i]; frags[i].page_offset += priv->frag_info[i].rx_headroom; rx_desc->data[i].addr = cpu_to_be64(frags[i].dma + frags[i].page_offset); ring_alloc[i] = page_alloc[i]; } return 0; out: while (i--) { if (page_alloc[i].page != ring_alloc[i].page) { dma_unmap_page(priv->ddev, page_alloc[i].dma, page_alloc[i].page_size, priv->frag_info[i].dma_dir); page = page_alloc[i].page; /* Revert changes done by mlx4_alloc_pages */ page_ref_sub(page, page_alloc[i].page_size / priv->frag_info[i].frag_stride - 1); put_page(page); } } return -ENOMEM; }
/* * This must be called only on pages that have * been verified to be in the swap cache and locked. * It will never put the page into the free list, * the caller has a reference on the page. */ void delete_from_swap_cache(struct page *page) { swp_entry_t entry; struct address_space *address_space; entry.val = page_private(page); address_space = swap_address_space(entry); spin_lock_irq(&address_space->tree_lock); __delete_from_swap_cache(page); spin_unlock_irq(&address_space->tree_lock); put_swap_page(page, entry); page_ref_sub(page, hpage_nr_pages(page)); }
/* * __add_to_swap_cache resembles add_to_page_cache_locked on swapper_space, * but sets SwapCache flag and private instead of mapping and index. */ int __add_to_swap_cache(struct page *page, swp_entry_t entry) { int error, i, nr = hpage_nr_pages(page); struct address_space *address_space; pgoff_t idx = swp_offset(entry); VM_BUG_ON_PAGE(!PageLocked(page), page); VM_BUG_ON_PAGE(PageSwapCache(page), page); VM_BUG_ON_PAGE(!PageSwapBacked(page), page); page_ref_add(page, nr); SetPageSwapCache(page); address_space = swap_address_space(entry); spin_lock_irq(&address_space->tree_lock); for (i = 0; i < nr; i++) { set_page_private(page + i, entry.val + i); error = radix_tree_insert(&address_space->page_tree, idx + i, page + i); if (unlikely(error)) break; } if (likely(!error)) { address_space->nrpages += nr; __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr); ADD_CACHE_INFO(add_total, nr); } else { /* * Only the context which have set SWAP_HAS_CACHE flag * would call add_to_swap_cache(). * So add_to_swap_cache() doesn't returns -EEXIST. */ VM_BUG_ON(error == -EEXIST); set_page_private(page + i, 0UL); while (i--) { radix_tree_delete(&address_space->page_tree, idx + i); set_page_private(page + i, 0UL); } ClearPageSwapCache(page); page_ref_sub(page, nr); } spin_unlock_irq(&address_space->tree_lock); return error; }
static int mlx4_en_init_allocator(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring) { int i; struct mlx4_en_rx_alloc *page_alloc; for (i = 0; i < priv->num_frags; i++) { const struct mlx4_en_frag_info *frag_info = &priv->frag_info[i]; if (mlx4_alloc_pages(priv, &ring->page_alloc[i], frag_info, GFP_KERNEL | __GFP_COLD)) goto out; en_dbg(DRV, priv, " frag %d allocator: - size:%d frags:%d\n", i, ring->page_alloc[i].page_size, page_ref_count(ring->page_alloc[i].page)); } return 0; out: while (i--) { struct page *page; page_alloc = &ring->page_alloc[i]; dma_unmap_page(priv->ddev, page_alloc->dma, page_alloc->page_size, priv->frag_info[i].dma_dir); page = page_alloc->page; /* Revert changes done by mlx4_alloc_pages */ page_ref_sub(page, page_alloc->page_size / priv->frag_info[i].frag_stride - 1); put_page(page); page_alloc->page = NULL; } return -ENOMEM; }