static void zram_free_page(struct zram *zram, size_t index)
{
struct zram_meta *meta = zram->meta;
unsigned long handle = meta->table[index].handle;
u16 size = meta->table[index].size;
if (unlikely(!handle)) {
/*
* No memory is allocated for zero filled pages.
* Simply clear zero page flag.
*/
if (zram_test_flag(meta, index, ZRAM_ZERO)) {
zram_clear_flag(meta, index, ZRAM_ZERO);
zram->stats.pages_zero--;
}
return;
}
if (unlikely(size > max_zpage_size))
zram->stats.bad_compress--;
zs_free(meta->mem_pool, handle);
if (size <= PAGE_SIZE / 2)
zram->stats.good_compress--;
zram_stat64_sub(zram, &zram->stats.compr_size,
meta->table[index].size);
zram->stats.pages_stored--;
meta->table[index].handle = 0;
meta->table[index].size = 0;
}
static int zram_decompress_page(struct zram *zram, char *mem, u32 index)
{
int ret = LZO_E_OK;
size_t clen = PAGE_SIZE;
unsigned char *cmem;
struct zram_meta *meta = zram->meta;
unsigned long handle = meta->table[index].handle;
if (!handle || zram_test_flag(meta, index, ZRAM_ZERO)) {
memset(mem, 0, PAGE_SIZE);
return 0;
}
cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_RO);
if (meta->table[index].size == PAGE_SIZE)
memcpy(mem, cmem, PAGE_SIZE);
else
ret = lzo1x_decompress_safe(cmem, meta->table[index].size,
mem, &clen);
zs_unmap_object(meta->mem_pool, handle);
/* Should NEVER happen. Return bio error if it does. */
if (unlikely(ret != LZO_E_OK)) {
pr_err("Decompression failed! err=%d, page=%u\n", ret, index);
zram_stat64_inc(zram, &zram->stats.failed_reads);
return ret;
}
return 0;
}
static int zram_decompress_page(struct zram *zram, char *mem, u32 index)
{
int ret = LZO_E_OK;
size_t clen = PAGE_SIZE;
unsigned char *cmem;
struct zram_meta *meta = zram->meta;
unsigned long handle = meta->table[index].handle;
if (!handle || zram_test_flag(meta, index, ZRAM_ZERO)) {
memset(mem, 0, PAGE_SIZE);
return 0;
}
cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_RO);
if (meta->table[index].size == PAGE_SIZE)
memcpy(mem, cmem, PAGE_SIZE);
else
#ifdef CONFIG_MT_ENG_BUILD
{
/* Move to the start of bitstream */
cmem += GUIDE_BYTES_HALFLEN;
#endif
ret = lzo1x_decompress_safe(cmem, meta->table[index].size,
mem, &clen);
#ifdef CONFIG_MT_ENG_BUILD
}
#endif
zs_unmap_object(meta->mem_pool, handle);
/* Should NEVER happen. Return bio error if it does. */
if (unlikely(ret != LZO_E_OK)) {
pr_err("Decompression failed! err=%d, page=%u\n", ret, index);
zram_stat64_inc(zram, &zram->stats.failed_reads);
#ifdef CONFIG_MT_ENG_BUILD
{
int idx;
size_t tlen;
printk(KERN_ALERT "\n@@@@@@@@@@\n");
tlen = meta->table[index].size + GUIDE_BYTES_LENGTH;
cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_RO);
/* Head guide bytes */
for (idx = 0; idx < GUIDE_BYTES_HALFLEN; idx++) {
printk(KERN_ALERT "%x ",(int)*cmem++);
}
printk(KERN_ALERT "\n=========\n");
for (;idx < tlen; idx++) {
printk(KERN_ALERT "%x ",(int)*cmem++);
}
zs_unmap_object(meta->mem_pool, handle);
printk(KERN_ALERT "\n!!!!!!!!!\n");
}
#endif
return ret;
}
return 0;
}
static void zram_free_page(struct zram *zram, size_t index)
{
u32 clen;
void *obj;
struct page *page = zram->table[index].page;
u32 offset = zram->table[index].offset;
if (unlikely(!page)) {
/*
* No memory is allocated for zero filled pages.
* Simply clear zero page flag.
*/
if (zram_test_flag(zram, index, ZRAM_ZERO)) {
zram_clear_flag(zram, index, ZRAM_ZERO);
zram_stat_dec(&zram->stats.pages_zero);
}
return;
}
if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED))) {
clen = PAGE_SIZE;
__free_page(page);
zram_clear_flag(zram, index, ZRAM_UNCOMPRESSED);
zram_stat_dec(&zram->stats.pages_expand);
goto out;
}
obj = kmap_atomic(page, KM_USER0) + offset;
clen = xv_get_object_size(obj) - sizeof(struct zobj_header);
kunmap_atomic(obj, KM_USER0);
xv_free(zram->mem_pool, page, offset);
if (clen <= PAGE_SIZE / 2)
zram_stat_dec(&zram->stats.good_compress);
out:
zram_stat64_sub(zram, &zram->stats.compr_size, clen);
zram_stat_dec(&zram->stats.pages_stored);
zram->table[index].page = NULL;
zram->table[index].offset = 0;
}
static int zram_read_before_write(struct zram *zram, char *mem, u32 index)
{
int ret;
size_t clen = PAGE_SIZE;
struct zobj_header *zheader;
unsigned char *cmem;
if (zram_test_flag(zram, index, ZRAM_ZERO) ||
!zram->table[index].page) {
memset(mem, 0, PAGE_SIZE);
return 0;
}
cmem = kmap_atomic(zram->table[index].page, KM_USER0) +
zram->table[index].offset;
/* Page is stored uncompressed since it's incompressible */
if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED))) {
memcpy(mem, cmem, PAGE_SIZE);
kunmap_atomic(cmem, KM_USER0);
return 0;
}
ret = DECOMPRESS(cmem + sizeof(*zheader),
xv_get_object_size(cmem) - sizeof(*zheader),
mem, &clen);
kunmap_atomic(cmem, KM_USER0);
/* Should NEVER happen. Return bio error if it does. */
if (unlikely(ret)) {
pr_err("Decompression failed! err=%d, page=%u\n", ret, index);
zram_stat64_inc(zram, &zram->stats.failed_reads);
return ret;
}
return 0;
}
static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
u32 index, int offset, struct bio *bio)
{
int ret;
struct page *page;
unsigned char *user_mem, *uncmem = NULL;
struct zram_meta *meta = zram->meta;
page = bvec->bv_page;
if (unlikely(!meta->table[index].handle) ||
zram_test_flag(meta, index, ZRAM_ZERO)) {
handle_zero_page(bvec);
return 0;
}
if (is_partial_io(bvec))
/* Use a temporary buffer to decompress the page */
uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);
user_mem = kmap_atomic(page);
if (!is_partial_io(bvec))
uncmem = user_mem;
if (!uncmem) {
pr_info("Unable to allocate temp memory\n");
ret = -ENOMEM;
goto out_cleanup;
}
ret = zram_decompress_page(zram, uncmem, index);
/* Should NEVER happen. Return bio error if it does. */
if (unlikely(ret != LZO_E_OK)) {
pr_err("Decompression failed! err=%d, page=%u\n", ret, index);
zram_stat64_inc(zram, &zram->stats.failed_reads);
goto out_cleanup;
}
if (is_partial_io(bvec))
memcpy(user_mem + bvec->bv_offset, uncmem + offset,
bvec->bv_len);
flush_dcache_page(page);
ret = 0;
out_cleanup:
kunmap_atomic(user_mem);
if (is_partial_io(bvec))
kfree(uncmem);
return ret;
}
static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
int offset)
{
int ret;
u32 store_offset;
size_t clen;
struct zobj_header *zheader;
struct page *page, *page_store;
unsigned char *user_mem, *cmem, *src, *uncmem = NULL;
page = bvec->bv_page;
src = zram->compress_buffer;
if (is_partial_io(bvec)) {
/*
* This is a partial IO. We need to read the full page
* before to write the changes.
*/
uncmem = kmalloc(PAGE_SIZE, GFP_KERNEL);
if (!uncmem) {
pr_info("Error allocating temp memory!\n");
ret = -ENOMEM;
goto out;
}
ret = zram_read_before_write(zram, uncmem, index);
if (ret) {
kfree(uncmem);
goto out;
}
}
/*
* System overwrites unused sectors. Free memory associated
* with this sector now.
*/
if (zram->table[index].page ||
zram_test_flag(zram, index, ZRAM_ZERO))
zram_free_page(zram, index);
user_mem = kmap_atomic(page, KM_USER0);
if (is_partial_io(bvec))
memcpy(uncmem + offset, user_mem + bvec->bv_offset,
bvec->bv_len);
else
uncmem = user_mem;
if (page_zero_filled(uncmem)) {
kunmap_atomic(user_mem, KM_USER0);
if (is_partial_io(bvec))
kfree(uncmem);
zram_stat_inc(&zram->stats.pages_zero);
zram_set_flag(zram, index, ZRAM_ZERO);
ret = 0;
goto out;
}
COMPRESS(uncmem, PAGE_SIZE, src, &clen,
zram->compress_workmem);
ret = 0;
kunmap_atomic(user_mem, KM_USER0);
if (is_partial_io(bvec))
kfree(uncmem);
if (unlikely(ret != 0)) {
pr_err("Compression failed! err=%d\n", ret);
goto out;
}
/*
* Page is incompressible. Store it as-is (uncompressed)
* since we do not want to return too many disk write
* errors which has side effect of hanging the system.
*/
if (unlikely(clen > max_zpage_size)) {
clen = PAGE_SIZE;
page_store = alloc_page(GFP_NOIO | __GFP_HIGHMEM);
if (unlikely(!page_store)) {
pr_info("Error allocating memory for "
"incompressible page: %u\n", index);
ret = -ENOMEM;
goto out;
}
store_offset = 0;
zram_set_flag(zram, index, ZRAM_UNCOMPRESSED);
zram_stat_inc(&zram->stats.pages_expand);
zram->table[index].page = page_store;
src = kmap_atomic(page, KM_USER0);
goto memstore;
}
if (xv_malloc(zram->mem_pool, clen + sizeof(*zheader),
&zram->table[index].page, &store_offset,
GFP_NOIO | __GFP_HIGHMEM)) {
pr_info("Error allocating memory for compressed "
"page: %u, size=%zu\n", index, clen);
ret = -ENOMEM;
goto out;
}
memstore:
zram->table[index].offset = store_offset;
cmem = kmap_atomic(zram->table[index].page, KM_USER1) +
zram->table[index].offset;
#if 0
/* Back-reference needed for memory defragmentation */
if (!zram_test_flag(zram, index, ZRAM_UNCOMPRESSED)) {
zheader = (struct zobj_header *)cmem;
zheader->table_idx = index;
cmem += sizeof(*zheader);
}
#endif
memcpy(cmem, src, clen);
kunmap_atomic(cmem, KM_USER1);
if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED)))
kunmap_atomic(src, KM_USER0);
/* Update stats */
zram_stat64_add(zram, &zram->stats.compr_size, clen);
zram_stat_inc(&zram->stats.pages_stored);
if (clen <= PAGE_SIZE / 2)
zram_stat_inc(&zram->stats.good_compress);
return 0;
out:
if (ret)
zram_stat64_inc(zram, &zram->stats.failed_writes);
return ret;
}
static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
u32 index, int offset, struct bio *bio)
{
int ret;
size_t clen;
struct page *page;
struct zobj_header *zheader;
unsigned char *user_mem, *cmem, *uncmem = NULL;
page = bvec->bv_page;
if (zram_test_flag(zram, index, ZRAM_ZERO)) {
handle_zero_page(bvec);
return 0;
}
/* Requested page is not present in compressed area */
if (unlikely(!zram->table[index].page)) {
pr_debug("Read before write: sector=%lu, size=%u",
(ulong)(bio->bi_sector), bio->bi_size);
handle_zero_page(bvec);
return 0;
}
/* Page is stored uncompressed since it's incompressible */
if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED))) {
handle_uncompressed_page(zram, bvec, index, offset);
return 0;
}
if (is_partial_io(bvec)) {
/* Use a temporary buffer to decompress the page */
uncmem = kmalloc(PAGE_SIZE, GFP_KERNEL);
if (!uncmem) {
pr_info("Error allocating temp memory!\n");
return -ENOMEM;
}
}
user_mem = kmap_atomic(page, KM_USER0);
if (!is_partial_io(bvec))
uncmem = user_mem;
clen = PAGE_SIZE;
cmem = kmap_atomic(zram->table[index].page, KM_USER1) +
zram->table[index].offset;
ret = DECOMPRESS(
cmem + sizeof(*zheader),
xv_get_object_size(cmem) - sizeof(*zheader),
uncmem, &clen);
if (is_partial_io(bvec)) {
memcpy(user_mem + bvec->bv_offset, uncmem + offset,
bvec->bv_len);
kfree(uncmem);
}
kunmap_atomic(cmem, KM_USER1);
kunmap_atomic(user_mem, KM_USER0);
/* Should NEVER happen. Return bio error if it does. */
if (unlikely(ret)) {
pr_err("Decompression failed! err=%d, page=%u\n", ret, index);
zram_stat64_inc(zram, &zram->stats.failed_reads);
return ret;
}
flush_dcache_page(page);
return 0;
}
static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
int offset)
{
int ret = 0;
size_t clen;
unsigned long handle;
struct page *page;
unsigned char *user_mem, *cmem, *src, *uncmem = NULL;
struct zram_meta *meta = zram->meta;
page = bvec->bv_page;
src = meta->compress_buffer;
if (is_partial_io(bvec)) {
/*
* This is a partial IO. We need to read the full page
* before to write the changes.
*/
uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);
if (!uncmem) {
ret = -ENOMEM;
goto out;
}
ret = zram_decompress_page(zram, uncmem, index);
if (ret)
goto out;
}
/*
* System overwrites unused sectors. Free memory associated
* with this sector now.
*/
if (meta->table[index].handle ||
zram_test_flag(meta, index, ZRAM_ZERO))
zram_free_page(zram, index);
user_mem = kmap_atomic(page);
if (is_partial_io(bvec)) {
memcpy(uncmem + offset, user_mem + bvec->bv_offset,
bvec->bv_len);
kunmap_atomic(user_mem);
user_mem = NULL;
} else {
uncmem = user_mem;
}
if (page_zero_filled(uncmem)) {
kunmap_atomic(user_mem);
zram->stats.pages_zero++;
zram_set_flag(meta, index, ZRAM_ZERO);
ret = 0;
goto out;
}
ret = lzo1x_1_compress(uncmem, PAGE_SIZE, src, &clen,
meta->compress_workmem);
if (!is_partial_io(bvec)) {
kunmap_atomic(user_mem);
user_mem = NULL;
uncmem = NULL;
}
if (unlikely(ret != LZO_E_OK)) {
pr_err("Compression failed! err=%d\n", ret);
goto out;
}
if (unlikely(clen > max_zpage_size)) {
zram->stats.bad_compress++;
clen = PAGE_SIZE;
src = NULL;
if (is_partial_io(bvec))
src = uncmem;
}
handle = zs_malloc(meta->mem_pool, clen);
if (!handle) {
pr_info("Error allocating memory for compressed "
"page: %u, size=%zu\n", index, clen);
ret = -ENOMEM;
goto out;
}
cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_WO);
if ((clen == PAGE_SIZE) && !is_partial_io(bvec))
src = kmap_atomic(page);
memcpy(cmem, src, clen);
if ((clen == PAGE_SIZE) && !is_partial_io(bvec))
kunmap_atomic(src);
zs_unmap_object(meta->mem_pool, handle);
meta->table[index].handle = handle;
meta->table[index].size = clen;
/* Update stats */
zram_stat64_add(zram, &zram->stats.compr_size, clen);
zram->stats.pages_stored++;
if (clen <= PAGE_SIZE / 2)
zram->stats.good_compress++;
out:
if (is_partial_io(bvec))
kfree(uncmem);
if (ret)
zram_stat64_inc(zram, &zram->stats.failed_writes);
return ret;
}
static int zram_read(struct zram *zram, struct bio *bio)
{
int i;
u32 index;
struct bio_vec *bvec;
if (unlikely(!zram->init_done)) {
set_bit(BIO_UPTODATE, &bio->bi_flags);
bio_endio(bio, 0);
return 0;
}
zram_stat64_inc(zram, &zram->stats.num_reads);
index = bio->bi_sector >> SECTORS_PER_PAGE_SHIFT;
bio_for_each_segment(bvec, bio, i) {
int ret;
size_t clen;
struct page *page;
struct zobj_header *zheader;
unsigned char *user_mem, *cmem;
page = bvec->bv_page;
if (zram_test_flag(zram, index, ZRAM_ZERO)) {
handle_zero_page(page);
index++;
continue;
}
/* Requested page is not present in compressed area */
if (unlikely(!zram->table[index].page)) {
pr_debug("Read before write: sector=%lu, size=%u",
(ulong)(bio->bi_sector), bio->bi_size);
/* Do nothing */
index++;
continue;
}
/* Page is stored uncompressed since it's incompressible */
if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED))) {
handle_uncompressed_page(zram, page, index);
index++;
continue;
}
user_mem = kmap_atomic(page, KM_USER0);
clen = PAGE_SIZE;
cmem = kmap_atomic(zram->table[index].page, KM_USER1) +
zram->table[index].offset;
ret = lzo1x_decompress_safe(
cmem + sizeof(*zheader),
xv_get_object_size(cmem) - sizeof(*zheader),
user_mem, &clen);
kunmap_atomic(user_mem, KM_USER0);
kunmap_atomic(cmem, KM_USER1);
/* Should NEVER happen. Return bio error if it does. */
if (unlikely(ret != LZO_E_OK)) {
pr_err("Decompression failed! err=%d, page=%u\n",
ret, index);
zram_stat64_inc(zram, &zram->stats.failed_reads);
goto out;
}
flush_dcache_page(page);
index++;
}
