/* caller must hold the tree lock
* remove from the tree and free it, if nobody reference the entry
*/
static void zswap_entry_put(struct zswap_tree *tree,
struct zswap_entry *entry)
{
int refcount = --entry->refcount;
BUG_ON(refcount < 0);
if (refcount == 0) {
zswap_rb_erase(&tree->rbroot, entry);
zswap_free_entry(entry);
}
}
/* frees an entry in zswap */
static void zswap_frontswap_invalidate_page(unsigned type, pgoff_t offset)
{
struct zswap_tree *tree = zswap_trees[type];
struct zswap_entry *entry;
/* find */
spin_lock(&tree->lock);
entry = zswap_rb_search(&tree->rbroot, offset);
if (!entry) {
/* entry was written back */
spin_unlock(&tree->lock);
return;
}
/* remove from rbtree */
zswap_rb_erase(&tree->rbroot, entry);
/* drop the initial reference from entry creation */
zswap_entry_put(tree, entry);
spin_unlock(&tree->lock);
}
/* attempts to compress and store an single page */
static int zswap_frontswap_store(unsigned type, pgoff_t offset,
struct page *page)
{
struct zswap_tree *tree = zswap_trees[type];
struct zswap_entry *entry, *dupentry;
int ret;
unsigned int dlen = PAGE_SIZE, len;
unsigned long handle;
char *buf;
u8 *src, *dst;
struct zswap_header *zhdr;
if (!tree) {
ret = -ENODEV;
goto reject;
}
/* if this page got EIO on pageout before, give up immediately */
if (PageError(page)) {
ret = -ENOMEM;
goto reject;
}
/* reclaim space if needed */
if (zswap_is_full()) {
zswap_pool_limit_hit++;
if (zpool_shrink(zswap_pool, 1, NULL)) {
zswap_reject_reclaim_fail++;
ret = -ENOMEM;
goto reject;
}
}
/* allocate entry */
entry = zswap_entry_cache_alloc(GFP_KERNEL);
if (!entry) {
zswap_reject_kmemcache_fail++;
ret = -ENOMEM;
goto reject;
}
/* compress */
src = kmap_atomic(page);
if (page_zero_filled(src)) {
atomic_inc(&zswap_zero_pages);
entry->zero_flag = 1;
kunmap_atomic(src);
handle = 0;
dlen = PAGE_SIZE;
goto zeropage_out;
}
dst = get_cpu_var(zswap_dstmem);
ret = zswap_comp_op(ZSWAP_COMPOP_COMPRESS, src, PAGE_SIZE, dst, &dlen);
kunmap_atomic(src);
if (ret) {
ret = -EINVAL;
goto freepage;
}
/* store */
len = dlen + sizeof(struct zswap_header);
ret = zpool_malloc(zswap_pool, len, __GFP_NORETRY | __GFP_NOWARN,
&handle);
if (ret == -ENOSPC) {
zswap_reject_compress_poor++;
goto freepage;
}
if (ret) {
zswap_reject_alloc_fail++;
goto freepage;
}
zhdr = zpool_map_handle(zswap_pool, handle, ZPOOL_MM_RW);
zhdr->swpentry = swp_entry(type, offset);
buf = (u8 *)(zhdr + 1);
memcpy(buf, dst, dlen);
zpool_unmap_handle(zswap_pool, handle);
put_cpu_var(zswap_dstmem);
zeropage_out:
/* populate entry */
entry->offset = offset;
entry->handle = handle;
entry->length = dlen;
/* map */
spin_lock(&tree->lock);
do {
ret = zswap_rb_insert(&tree->rbroot, entry, &dupentry);
if (ret == -EEXIST) {
zswap_duplicate_entry++;
/* remove from rbtree */
zswap_rb_erase(&tree->rbroot, dupentry);
zswap_entry_put(tree, dupentry);
}
} while (ret == -EEXIST);
spin_unlock(&tree->lock);
/* update stats */
atomic_inc(&zswap_stored_pages);
zswap_pool_total_size = zpool_get_total_size(zswap_pool);
zswap_pool_pages = zswap_pool_total_size >> PAGE_SHIFT;
return 0;
freepage:
put_cpu_var(zswap_dstmem);
zswap_entry_cache_free(entry);
reject:
return ret;
}
/* attempts to compress and store an single page */
static int zswap_frontswap_store(unsigned type, pgoff_t offset,
struct page *page)
{
struct zswap_tree *tree = zswap_trees[type];
struct zswap_entry *entry, *dupentry;
int ret;
unsigned int dlen = PAGE_SIZE, len;
unsigned long handle;
char *buf;
u8 *src, *dst;
struct zswap_header *zhdr;
if (!tree) {
ret = -ENODEV;
goto reject;
}
/* reclaim space if needed */
if (zswap_is_full()) {
zswap_pool_limit_hit++;
if (zbud_reclaim_page(tree->pool, 8)) {
zswap_reject_reclaim_fail++;
ret = -ENOMEM;
goto reject;
}
}
/* allocate entry */
entry = zswap_entry_cache_alloc(GFP_KERNEL);
if (!entry) {
zswap_reject_kmemcache_fail++;
ret = -ENOMEM;
goto reject;
}
/* compress */
dst = get_cpu_var(zswap_dstmem);
src = kmap_atomic(page);
ret = zswap_comp_op(ZSWAP_COMPOP_COMPRESS, src, PAGE_SIZE, dst, &dlen);
kunmap_atomic(src);
if (ret) {
ret = -EINVAL;
goto freepage;
}
/* store */
len = dlen + sizeof(struct zswap_header);
ret = zbud_alloc(tree->pool, len, __GFP_NORETRY | __GFP_NOWARN,
&handle);
if (ret == -ENOSPC) {
zswap_reject_compress_poor++;
goto freepage;
}
if (ret) {
zswap_reject_alloc_fail++;
goto freepage;
}
zhdr = zbud_map(tree->pool, handle);
zhdr->swpentry = swp_entry(type, offset);
buf = (u8 *)(zhdr + 1);
memcpy(buf, dst, dlen);
zbud_unmap(tree->pool, handle);
put_cpu_var(zswap_dstmem);
/* populate entry */
entry->offset = offset;
entry->handle = handle;
entry->length = dlen;
/* map */
spin_lock(&tree->lock);
do {
ret = zswap_rb_insert(&tree->rbroot, entry, &dupentry);
if (ret == -EEXIST) {
zswap_duplicate_entry++;
/* remove from rbtree */
zswap_rb_erase(&tree->rbroot, dupentry);
zswap_entry_put(tree, dupentry);
}
} while (ret == -EEXIST);
spin_unlock(&tree->lock);
/* update stats */
atomic_inc(&zswap_stored_pages);
zswap_pool_pages = zbud_get_pool_size(tree->pool);
return 0;
freepage:
put_cpu_var(zswap_dstmem);
zswap_entry_cache_free(entry);
reject:
return ret;
}