static void direct_finish_page(struct squashfs_page_actor *actor) { if (actor->pageaddr) kunmap_atomic(actor->pageaddr); }
static int nilfs_cpfile_set_snapshot(struct inode *cpfile, __u64 cno) { struct buffer_head *header_bh, *curr_bh, *prev_bh, *cp_bh; struct nilfs_cpfile_header *header; struct nilfs_checkpoint *cp; struct nilfs_snapshot_list *list; __u64 curr, prev; unsigned long curr_blkoff, prev_blkoff; void *kaddr; int ret; if (cno == 0) return -ENOENT; /* checkpoint number 0 is invalid */ down_write(&NILFS_MDT(cpfile)->mi_sem); ret = nilfs_cpfile_get_checkpoint_block(cpfile, cno, 0, &cp_bh); if (ret < 0) goto out_sem; kaddr = kmap_atomic(cp_bh->b_page); cp = nilfs_cpfile_block_get_checkpoint(cpfile, cno, cp_bh, kaddr); if (nilfs_checkpoint_invalid(cp)) { ret = -ENOENT; kunmap_atomic(kaddr); goto out_cp; } if (nilfs_checkpoint_snapshot(cp)) { ret = 0; kunmap_atomic(kaddr); goto out_cp; } kunmap_atomic(kaddr); ret = nilfs_cpfile_get_header_block(cpfile, &header_bh); if (ret < 0) goto out_cp; kaddr = kmap_atomic(header_bh->b_page); header = nilfs_cpfile_block_get_header(cpfile, header_bh, kaddr); list = &header->ch_snapshot_list; curr_bh = header_bh; get_bh(curr_bh); curr = 0; curr_blkoff = 0; prev = le64_to_cpu(list->ssl_prev); while (prev > cno) { prev_blkoff = nilfs_cpfile_get_blkoff(cpfile, prev); curr = prev; if (curr_blkoff != prev_blkoff) { kunmap_atomic(kaddr); brelse(curr_bh); ret = nilfs_cpfile_get_checkpoint_block(cpfile, curr, 0, &curr_bh); if (ret < 0) goto out_header; kaddr = kmap_atomic(curr_bh->b_page); } curr_blkoff = prev_blkoff; cp = nilfs_cpfile_block_get_checkpoint( cpfile, curr, curr_bh, kaddr); list = &cp->cp_snapshot_list; prev = le64_to_cpu(list->ssl_prev); } kunmap_atomic(kaddr); if (prev != 0) { ret = nilfs_cpfile_get_checkpoint_block(cpfile, prev, 0, &prev_bh); if (ret < 0) goto out_curr; } else { prev_bh = header_bh; get_bh(prev_bh); } kaddr = kmap_atomic(curr_bh->b_page); list = nilfs_cpfile_block_get_snapshot_list( cpfile, curr, curr_bh, kaddr); list->ssl_prev = cpu_to_le64(cno); kunmap_atomic(kaddr); kaddr = kmap_atomic(cp_bh->b_page); cp = nilfs_cpfile_block_get_checkpoint(cpfile, cno, cp_bh, kaddr); cp->cp_snapshot_list.ssl_next = cpu_to_le64(curr); cp->cp_snapshot_list.ssl_prev = cpu_to_le64(prev); nilfs_checkpoint_set_snapshot(cp); kunmap_atomic(kaddr); kaddr = kmap_atomic(prev_bh->b_page); list = nilfs_cpfile_block_get_snapshot_list( cpfile, prev, prev_bh, kaddr); list->ssl_next = cpu_to_le64(cno); kunmap_atomic(kaddr); kaddr = kmap_atomic(header_bh->b_page); header = nilfs_cpfile_block_get_header(cpfile, header_bh, kaddr); le64_add_cpu(&header->ch_nsnapshots, 1); kunmap_atomic(kaddr); mark_buffer_dirty(prev_bh); mark_buffer_dirty(curr_bh); mark_buffer_dirty(cp_bh); mark_buffer_dirty(header_bh); nilfs_mdt_mark_dirty(cpfile); brelse(prev_bh); out_curr: brelse(curr_bh); out_header: brelse(header_bh); out_cp: brelse(cp_bh); out_sem: up_write(&NILFS_MDT(cpfile)->mi_sem); return ret; }
int ttm_tt_swapout(struct ttm_tt *ttm, struct file *persistant_swap_storage) { struct address_space *swap_space; struct file *swap_storage; struct page *from_page; struct page *to_page; void *from_virtual; void *to_virtual; int i; int ret = -ENOMEM; BUG_ON(ttm->state != tt_unbound && ttm->state != tt_unpopulated); BUG_ON(ttm->caching_state != tt_cached); /* * For user buffers, just unpin the pages, as there should be * vma references. */ if (ttm->page_flags & TTM_PAGE_FLAG_USER) { ttm_tt_free_user_pages(ttm); ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED; ttm->swap_storage = NULL; return 0; } if (!persistant_swap_storage) { swap_storage = shmem_file_setup("ttm swap", ttm->num_pages << PAGE_SHIFT, 0); if (unlikely(IS_ERR(swap_storage))) { printk(KERN_ERR "Failed allocating swap storage.\n"); return PTR_ERR(swap_storage); } } else swap_storage = persistant_swap_storage; swap_space = swap_storage->f_path.dentry->d_inode->i_mapping; for (i = 0; i < ttm->num_pages; ++i) { from_page = ttm->pages[i]; if (unlikely(from_page == NULL)) continue; to_page = read_mapping_page(swap_space, i, NULL); if (unlikely(IS_ERR(to_page))) { ret = PTR_ERR(to_page); goto out_err; } preempt_disable(); from_virtual = kmap_atomic(from_page, KM_USER0); to_virtual = kmap_atomic(to_page, KM_USER1); memcpy(to_virtual, from_virtual, PAGE_SIZE); kunmap_atomic(to_virtual, KM_USER1); kunmap_atomic(from_virtual, KM_USER0); preempt_enable(); set_page_dirty(to_page); mark_page_accessed(to_page); page_cache_release(to_page); } ttm_tt_free_alloced_pages(ttm); ttm->swap_storage = swap_storage; ttm->page_flags |= TTM_PAGE_FLAG_SWAPPED; if (persistant_swap_storage) ttm->page_flags |= TTM_PAGE_FLAG_PERSISTANT_SWAP; return 0; out_err: if (!persistant_swap_storage) fput(swap_storage); return ret; }
/* read the superblock from the bitmap file and initialize some bitmap fields */ static int bitmap_read_sb(struct bitmap *bitmap) { char *reason = NULL; bitmap_super_t *sb; unsigned long chunksize, daemon_sleep, write_behind; unsigned long long events; int err = -EINVAL; /* page 0 is the superblock, read it... */ if (bitmap->file) { loff_t isize = i_size_read(bitmap->file->f_mapping->host); int bytes = isize > PAGE_SIZE ? PAGE_SIZE : isize; bitmap->sb_page = read_page(bitmap->file, 0, bitmap, bytes); } else { bitmap->sb_page = read_sb_page(bitmap->mddev, bitmap->offset, 0); } if (IS_ERR(bitmap->sb_page)) { err = PTR_ERR(bitmap->sb_page); bitmap->sb_page = NULL; return err; } sb = (bitmap_super_t *)kmap_atomic(bitmap->sb_page, KM_USER0); chunksize = le32_to_cpu(sb->chunksize); daemon_sleep = le32_to_cpu(sb->daemon_sleep); write_behind = le32_to_cpu(sb->write_behind); /* verify that the bitmap-specific fields are valid */ if (sb->magic != cpu_to_le32(BITMAP_MAGIC)) reason = "bad magic"; else if (le32_to_cpu(sb->version) < BITMAP_MAJOR_LO || le32_to_cpu(sb->version) > BITMAP_MAJOR_HI) reason = "unrecognized superblock version"; else if (chunksize < PAGE_SIZE) reason = "bitmap chunksize too small"; else if ((1 << ffz(~chunksize)) != chunksize) reason = "bitmap chunksize not a power of 2"; else if (daemon_sleep < 1 || daemon_sleep > MAX_SCHEDULE_TIMEOUT / HZ) reason = "daemon sleep period out of range"; else if (write_behind > COUNTER_MAX) reason = "write-behind limit out of range (0 - 16383)"; if (reason) { printk(KERN_INFO "%s: invalid bitmap file superblock: %s\n", bmname(bitmap), reason); goto out; } /* keep the array size field of the bitmap superblock up to date */ sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors); if (!bitmap->mddev->persistent) goto success; /* * if we have a persistent array superblock, compare the * bitmap's UUID and event counter to the mddev's */ if (memcmp(sb->uuid, bitmap->mddev->uuid, 16)) { printk(KERN_INFO "%s: bitmap superblock UUID mismatch\n", bmname(bitmap)); goto out; } events = le64_to_cpu(sb->events); if (events < bitmap->mddev->events) { printk(KERN_INFO "%s: bitmap file is out of date (%llu < %llu) " "-- forcing full recovery\n", bmname(bitmap), events, (unsigned long long) bitmap->mddev->events); sb->state |= cpu_to_le32(BITMAP_STALE); } success: /* assign fields using values from superblock */ bitmap->chunksize = chunksize; bitmap->daemon_sleep = daemon_sleep; bitmap->daemon_lastrun = jiffies; bitmap->max_write_behind = write_behind; bitmap->flags |= le32_to_cpu(sb->state); if (le32_to_cpu(sb->version) == BITMAP_MAJOR_HOSTENDIAN) bitmap->flags |= BITMAP_HOSTENDIAN; bitmap->events_cleared = le64_to_cpu(sb->events_cleared); if (sb->state & cpu_to_le32(BITMAP_STALE)) bitmap->events_cleared = bitmap->mddev->events; err = 0; out: kunmap_atomic(sb, KM_USER0); if (err) bitmap_print_sb(bitmap); return err; }
/** * nilfs_cpfile_delete_checkpoints - delete checkpoints * @cpfile: inode of checkpoint file * @start: start checkpoint number * @end: end checkpoint numer * * Description: nilfs_cpfile_delete_checkpoints() deletes the checkpoints in * the period from @start to @end, excluding @end itself. The checkpoints * which have been already deleted are ignored. * * Return Value: On success, 0 is returned. On error, one of the following * negative error codes is returned. * * %-EIO - I/O error. * * %-ENOMEM - Insufficient amount of memory available. * * %-EINVAL - invalid checkpoints. */ int nilfs_cpfile_delete_checkpoints(struct inode *cpfile, __u64 start, __u64 end) { struct buffer_head *header_bh, *cp_bh; struct nilfs_cpfile_header *header; struct nilfs_checkpoint *cp; size_t cpsz = NILFS_MDT(cpfile)->mi_entry_size; __u64 cno; void *kaddr; unsigned long tnicps; int ret, ncps, nicps, nss, count, i; if (unlikely(start == 0 || start > end)) { printk(KERN_ERR "%s: invalid range of checkpoint numbers: " "[%llu, %llu)\n", __func__, (unsigned long long)start, (unsigned long long)end); return -EINVAL; } down_write(&NILFS_MDT(cpfile)->mi_sem); ret = nilfs_cpfile_get_header_block(cpfile, &header_bh); if (ret < 0) goto out_sem; tnicps = 0; nss = 0; for (cno = start; cno < end; cno += ncps) { ncps = nilfs_cpfile_checkpoints_in_block(cpfile, cno, end); ret = nilfs_cpfile_get_checkpoint_block(cpfile, cno, 0, &cp_bh); if (ret < 0) { if (ret != -ENOENT) break; /* skip hole */ ret = 0; continue; } kaddr = kmap_atomic(cp_bh->b_page); cp = nilfs_cpfile_block_get_checkpoint( cpfile, cno, cp_bh, kaddr); nicps = 0; for (i = 0; i < ncps; i++, cp = (void *)cp + cpsz) { if (nilfs_checkpoint_snapshot(cp)) { nss++; } else if (!nilfs_checkpoint_invalid(cp)) { nilfs_checkpoint_set_invalid(cp); nicps++; } } if (nicps > 0) { tnicps += nicps; mark_buffer_dirty(cp_bh); nilfs_mdt_mark_dirty(cpfile); if (!nilfs_cpfile_is_in_first(cpfile, cno)) { count = nilfs_cpfile_block_sub_valid_checkpoints( cpfile, cp_bh, kaddr, nicps); if (count == 0) { /* make hole */ kunmap_atomic(kaddr); brelse(cp_bh); ret = nilfs_cpfile_delete_checkpoint_block( cpfile, cno); if (ret == 0) continue; printk(KERN_ERR "%s: cannot delete block\n", __func__); break; } } } kunmap_atomic(kaddr); brelse(cp_bh); } if (tnicps > 0) { kaddr = kmap_atomic(header_bh->b_page); header = nilfs_cpfile_block_get_header(cpfile, header_bh, kaddr); le64_add_cpu(&header->ch_ncheckpoints, -(u64)tnicps); mark_buffer_dirty(header_bh); nilfs_mdt_mark_dirty(cpfile); kunmap_atomic(kaddr); } brelse(header_bh); if (nss > 0) ret = -EBUSY; out_sem: up_write(&NILFS_MDT(cpfile)->mi_sem); return ret; }
/* * Clear the user page. No aliasing to deal with so we can just * attack the kernel's existing mapping of this page. */ static void v6_clear_user_highpage_nonaliasing(struct page *page, unsigned long vaddr) { void *kaddr = kmap_atomic(page, KM_USER0); clear_page(kaddr); kunmap_atomic(kaddr, KM_USER0); }
/* * Deliver read data back to initiator. * XXX TBD handle resource problems later. */ int ft_queue_data_in(struct se_cmd *se_cmd) { struct ft_cmd *cmd = container_of(se_cmd, struct ft_cmd, se_cmd); struct fc_frame *fp = NULL; struct fc_exch *ep; struct fc_lport *lport; struct scatterlist *sg = NULL; size_t remaining; u32 f_ctl = FC_FC_EX_CTX | FC_FC_REL_OFF; u32 mem_off = 0; u32 fh_off = 0; u32 frame_off = 0; size_t frame_len = 0; size_t mem_len = 0; size_t tlen; size_t off_in_page; struct page *page = NULL; int use_sg; int error; void *page_addr; void *from; void *to = NULL; ep = fc_seq_exch(cmd->seq); lport = ep->lp; cmd->seq = lport->tt.seq_start_next(cmd->seq); remaining = se_cmd->data_length; /* * Setup to use first mem list entry, unless no data. */ BUG_ON(remaining && !se_cmd->t_data_sg); if (remaining) { sg = se_cmd->t_data_sg; mem_len = sg->length; mem_off = sg->offset; page = sg_page(sg); } /* no scatter/gather in skb for odd word length due to fc_seq_send() */ use_sg = !(remaining % 4); while (remaining) { if (!mem_len) { sg = sg_next(sg); mem_len = min((size_t)sg->length, remaining); mem_off = sg->offset; page = sg_page(sg); } if (!frame_len) { /* * If lport's has capability of Large Send Offload LSO) * , then allow 'frame_len' to be as big as 'lso_max' * if indicated transfer length is >= lport->lso_max */ frame_len = (lport->seq_offload) ? lport->lso_max : cmd->sess->max_frame; frame_len = min(frame_len, remaining); fp = fc_frame_alloc(lport, use_sg ? 0 : frame_len); if (!fp) return -ENOMEM; to = fc_frame_payload_get(fp, 0); fh_off = frame_off; frame_off += frame_len; /* * Setup the frame's max payload which is used by base * driver to indicate HW about max frame size, so that * HW can do fragmentation appropriately based on * "gso_max_size" of underline netdev. */ fr_max_payload(fp) = cmd->sess->max_frame; } tlen = min(mem_len, frame_len); if (use_sg) { off_in_page = mem_off; BUG_ON(!page); get_page(page); skb_fill_page_desc(fp_skb(fp), skb_shinfo(fp_skb(fp))->nr_frags, page, off_in_page, tlen); fr_len(fp) += tlen; fp_skb(fp)->data_len += tlen; fp_skb(fp)->truesize += PAGE_SIZE << compound_order(page); } else { BUG_ON(!page); from = kmap_atomic(page + (mem_off >> PAGE_SHIFT)); page_addr = from; from += mem_off & ~PAGE_MASK; tlen = min(tlen, (size_t)(PAGE_SIZE - (mem_off & ~PAGE_MASK))); memcpy(to, from, tlen); kunmap_atomic(page_addr); to += tlen; } mem_off += tlen; mem_len -= tlen; frame_len -= tlen; remaining -= tlen; if (frame_len && (skb_shinfo(fp_skb(fp))->nr_frags < FC_FRAME_SG_LEN)) continue; if (!remaining) f_ctl |= FC_FC_END_SEQ; fc_fill_fc_hdr(fp, FC_RCTL_DD_SOL_DATA, ep->did, ep->sid, FC_TYPE_FCP, f_ctl, fh_off); error = lport->tt.seq_send(lport, cmd->seq, fp); if (error) { /* XXX For now, initiator will retry */ pr_err_ratelimited("%s: Failed to send frame %p, " "xid <0x%x>, remaining %zu, " "lso_max <0x%x>\n", __func__, fp, ep->xid, remaining, lport->lso_max); } } return ft_queue_status(se_cmd); }
/** * toi_copy_pageset1 - do the atomic copy of pageset1 * * Make the atomic copy of pageset1. We can't use copy_page (as we once did) * because we can't be sure what side effects it has. On my old Duron, with * 3DNOW, kernel_fpu_begin increments preempt count, making our preempt * count at resume time 4 instead of 3. * * We don't want to call kmap_atomic unconditionally because it has the side * effect of incrementing the preempt count, which will leave it one too high * post resume (the page containing the preempt count will be copied after * its incremented. This is essentially the same problem. **/ void toi_copy_pageset1(void) { int i; unsigned long source_index, dest_index; memory_bm_position_reset(pageset1_map); memory_bm_position_reset(pageset1_copy_map); source_index = memory_bm_next_pfn(pageset1_map); dest_index = memory_bm_next_pfn(pageset1_copy_map); for (i = 0; i < pagedir1.size; i++) { unsigned long *origvirt, *copyvirt; struct page *origpage, *copypage; int loop = (PAGE_SIZE / sizeof(unsigned long)) - 1, was_present1, was_present2; #ifdef CONFIG_MTK_HIBERNATION if (!pfn_valid(source_index) || !pfn_valid(dest_index)) { pr_emerg("[%s] (%d) dest_index:%lu, source_index:%lu\n", __func__, i, dest_index, source_index); set_abort_result(TOI_ARCH_PREPARE_FAILED); return; } #endif origpage = pfn_to_page(source_index); copypage = pfn_to_page(dest_index); origvirt = PageHighMem(origpage) ? kmap_atomic(origpage) : page_address(origpage); copyvirt = PageHighMem(copypage) ? kmap_atomic(copypage) : page_address(copypage); was_present1 = kernel_page_present(origpage); if (!was_present1) kernel_map_pages(origpage, 1, 1); was_present2 = kernel_page_present(copypage); if (!was_present2) kernel_map_pages(copypage, 1, 1); while (loop >= 0) { *(copyvirt + loop) = *(origvirt + loop); loop--; } if (!was_present1) kernel_map_pages(origpage, 1, 0); if (!was_present2) kernel_map_pages(copypage, 1, 0); if (PageHighMem(origpage)) kunmap_atomic(origvirt); if (PageHighMem(copypage)) kunmap_atomic(copyvirt); source_index = memory_bm_next_pfn(pageset1_map); dest_index = memory_bm_next_pfn(pageset1_copy_map); } }
static int zram_read(struct zram *zram, struct bio *bio) { int i; u32 index; struct bio_vec *bvec; if (unlikely(!zram->init_done)) { set_bit(BIO_UPTODATE, &bio->bi_flags); bio_endio(bio, 0); return 0; } zram_stat64_inc(zram, &zram->stats.num_reads); index = bio->bi_sector >> SECTORS_PER_PAGE_SHIFT; bio_for_each_segment(bvec, bio, i) { int ret; size_t clen; struct page *page; struct zobj_header *zheader; unsigned char *user_mem, *cmem; page = bvec->bv_page; if (zram_test_flag(zram, index, ZRAM_ZERO)) { handle_zero_page(page); index++; continue; } /* Requested page is not present in compressed area */ if (unlikely(!zram->table[index].page)) { pr_debug("Read before write: sector=%lu, size=%u", (ulong)(bio->bi_sector), bio->bi_size); /* Do nothing */ index++; continue; } /* Page is stored uncompressed since it's incompressible */ if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED))) { handle_uncompressed_page(zram, page, index); index++; continue; } user_mem = kmap_atomic(page, KM_USER0); clen = PAGE_SIZE; cmem = kmap_atomic(zram->table[index].page, KM_USER1) + zram->table[index].offset; ret = lzo1x_decompress_safe( cmem + sizeof(*zheader), xv_get_object_size(cmem) - sizeof(*zheader), user_mem, &clen); kunmap_atomic(user_mem, KM_USER0); kunmap_atomic(cmem, KM_USER1); /* Should NEVER happen. Return bio error if it does. */ if (unlikely(ret != LZO_E_OK)) { pr_err("Decompression failed! err=%d, page=%u\n", ret, index); zram_stat64_inc(zram, &zram->stats.failed_reads); goto out; } flush_dcache_page(page); index++; }
int f2fs_convert_inline_page(struct dnode_of_data *dn, struct page *page) { struct f2fs_io_info fio = { .sbi = F2FS_I_SB(dn->inode), .type = DATA, .op = REQ_OP_WRITE, .op_flags = WRITE_SYNC | REQ_PRIO, .page = page, .encrypted_page = NULL, }; int dirty, err; if (!f2fs_exist_data(dn->inode)) goto clear_out; err = f2fs_reserve_block(dn, 0); if (err) return err; f2fs_bug_on(F2FS_P_SB(page), PageWriteback(page)); read_inline_data(page, dn->inode_page); set_page_dirty(page); /* clear dirty state */ dirty = clear_page_dirty_for_io(page); /* write data page to try to make data consistent */ set_page_writeback(page); fio.old_blkaddr = dn->data_blkaddr; write_data_page(dn, &fio); f2fs_wait_on_page_writeback(page, DATA, true); if (dirty) inode_dec_dirty_pages(dn->inode); /* this converted inline_data should be recovered. */ set_inode_flag(dn->inode, FI_APPEND_WRITE); /* clear inline data and flag after data writeback */ truncate_inline_inode(dn->inode_page, 0); clear_inline_node(dn->inode_page); clear_out: stat_dec_inline_inode(dn->inode); f2fs_clear_inline_inode(dn->inode); f2fs_put_dnode(dn); return 0; } int f2fs_convert_inline_inode(struct inode *inode) { struct f2fs_sb_info *sbi = F2FS_I_SB(inode); struct dnode_of_data dn; struct page *ipage, *page; int err = 0; if (!f2fs_has_inline_data(inode)) return 0; page = f2fs_grab_cache_page(inode->i_mapping, 0, false); if (!page) return -ENOMEM; f2fs_lock_op(sbi); ipage = get_node_page(sbi, inode->i_ino); if (IS_ERR(ipage)) { err = PTR_ERR(ipage); goto out; } set_new_dnode(&dn, inode, ipage, ipage, 0); if (f2fs_has_inline_data(inode)) err = f2fs_convert_inline_page(&dn, page); f2fs_put_dnode(&dn); out: f2fs_unlock_op(sbi); f2fs_put_page(page, 1); f2fs_balance_fs(sbi, dn.node_changed); return err; } int f2fs_write_inline_data(struct inode *inode, struct page *page) { void *src_addr, *dst_addr; struct dnode_of_data dn; int err; set_new_dnode(&dn, inode, NULL, NULL, 0); err = get_dnode_of_data(&dn, 0, LOOKUP_NODE); if (err) return err; if (!f2fs_has_inline_data(inode)) { f2fs_put_dnode(&dn); return -EAGAIN; } f2fs_bug_on(F2FS_I_SB(inode), page->index); f2fs_wait_on_page_writeback(dn.inode_page, NODE, true); src_addr = kmap_atomic(page); dst_addr = inline_data_addr(dn.inode_page); memcpy(dst_addr, src_addr, MAX_INLINE_DATA); kunmap_atomic(src_addr); set_page_dirty(dn.inode_page); set_inode_flag(inode, FI_APPEND_WRITE); set_inode_flag(inode, FI_DATA_EXIST); clear_inline_node(dn.inode_page); f2fs_put_dnode(&dn); return 0; }
static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes, struct iov_iter *i) { size_t skip, copy, left, wanted; const struct iovec *iov; char __user *buf; void *kaddr, *to; if (unlikely(bytes > i->count)) bytes = i->count; if (unlikely(!bytes)) return 0; wanted = bytes; iov = i->iov; skip = i->iov_offset; buf = iov->iov_base + skip; copy = min(bytes, iov->iov_len - skip); if (!fault_in_pages_readable(buf, copy)) { kaddr = kmap_atomic(page); to = kaddr + offset; /* first chunk, usually the only one */ left = __copy_from_user_inatomic(to, buf, copy); copy -= left; skip += copy; to += copy; bytes -= copy; while (unlikely(!left && bytes)) { iov++; buf = iov->iov_base; copy = min(bytes, iov->iov_len); left = __copy_from_user_inatomic(to, buf, copy); copy -= left; skip = copy; to += copy; bytes -= copy; } if (likely(!bytes)) { kunmap_atomic(kaddr); goto done; } offset = to - kaddr; buf += copy; kunmap_atomic(kaddr); copy = min(bytes, iov->iov_len - skip); } /* Too bad - revert to non-atomic kmap */ kaddr = kmap(page); to = kaddr + offset; left = __copy_from_user(to, buf, copy); copy -= left; skip += copy; to += copy; bytes -= copy; while (unlikely(!left && bytes)) { iov++; buf = iov->iov_base; copy = min(bytes, iov->iov_len); left = __copy_from_user(to, buf, copy); copy -= left; skip = copy; to += copy; bytes -= copy; } kunmap(page); done: if (skip == iov->iov_len) { iov++; skip = 0; } i->count -= wanted - bytes; i->nr_segs -= iov - i->iov; i->iov = iov; i->iov_offset = skip; return wanted - bytes; }
/** * ecryptfs_copy_up_encrypted_with_header * @page: Sort of a ``virtual'' representation of the encrypted lower * file. The actual lower file does not have the metadata in * the header. This is locked. * @crypt_stat: The eCryptfs inode's cryptographic context * * The ``view'' is the version of the file that userspace winds up * seeing, with the header information inserted. */ static int ecryptfs_copy_up_encrypted_with_header(struct page *page, struct ecryptfs_crypt_stat *crypt_stat) { loff_t extent_num_in_page = 0; loff_t num_extents_per_page = (PAGE_CACHE_SIZE / crypt_stat->extent_size); int rc = 0; while (extent_num_in_page < num_extents_per_page) { loff_t view_extent_num = ((((loff_t)page->index) * num_extents_per_page) + extent_num_in_page); size_t num_header_extents_at_front = (crypt_stat->metadata_size / crypt_stat->extent_size); if (view_extent_num < num_header_extents_at_front) { /* This is a header extent */ char *page_virt; page_virt = kmap_atomic(page); memset(page_virt, 0, PAGE_CACHE_SIZE); /* TODO: Support more than one header extent */ if (view_extent_num == 0) { size_t written; rc = ecryptfs_read_xattr_region( page_virt, page->mapping->host); strip_xattr_flag(page_virt + 16, crypt_stat); ecryptfs_write_header_metadata(page_virt + 20, crypt_stat, &written); } kunmap_atomic(page_virt); flush_dcache_page(page); if (rc) { printk(KERN_ERR "%s: Error reading xattr " "region; rc = [%d]\n", __func__, rc); goto out; } } else { /* This is an encrypted data extent */ loff_t lower_offset = ((view_extent_num * crypt_stat->extent_size) - crypt_stat->metadata_size); rc = ecryptfs_read_lower_page_segment( page, (lower_offset >> PAGE_CACHE_SHIFT), (lower_offset & ~PAGE_CACHE_MASK), crypt_stat->extent_size, page->mapping->host); if (rc) { printk(KERN_ERR "%s: Error attempting to read " "extent at offset [%lld] in the lower " "file; rc = [%d]\n", __func__, lower_offset, rc); goto out; } } extent_num_in_page++; } out: return rc; }
/** * ecryptfs_writepage * @page: Page that is locked before this call is made * * Returns zero on success; non-zero otherwise * * This is where we encrypt the data and pass the encrypted data to * the lower filesystem. In OpenPGP-compatible mode, we operate on * entire underlying packets. */ static int ecryptfs_writepage(struct page *page, struct writeback_control *wbc) { #ifndef CONFIG_CRYPTO_DEV_KFIPS int rc; #else struct ecryptfs_page_crypt_req *page_crypt_req; int rc = 0; #endif #if 1 // FEATURE_SDCARD_ENCRYPTION struct inode *ecryptfs_inode; struct ecryptfs_crypt_stat *crypt_stat = &ecryptfs_inode_to_private(page->mapping->host)->crypt_stat; ecryptfs_inode = page->mapping->host; #endif #if 1 // FEATURE_SDCARD_ENCRYPTION if (!crypt_stat || !(crypt_stat->flags & ECRYPTFS_ENCRYPTED)) { ecryptfs_printk(KERN_DEBUG, "Passing through unencrypted page\n"); rc = ecryptfs_write_lower_page_segment(ecryptfs_inode, page, 0, PAGE_CACHE_SIZE); if (rc) { ClearPageUptodate(page); goto out; } SetPageUptodate(page); } else { #ifndef CONFIG_CRYPTO_DEV_KFIPS rc = ecryptfs_encrypt_page(page); if (rc) { ecryptfs_printk(KERN_WARNING, "Error encrypting " "page (upper index [0x%.16lx])\n", page->index); ClearPageUptodate(page); #else // rc = ecryptfs_encrypt_page(page); // if (rc) { // ecryptfs_printk(KERN_WARNING, "Error encrypting " // "page (upper index [0x%.16lx])\n", page->index); // ClearPageUptodate(page); page_crypt_req = ecryptfs_alloc_page_crypt_req( page, ecryptfs_writepage_complete); if (unlikely(!page_crypt_req)) { rc = -ENOMEM; ecryptfs_printk(KERN_ERR, "Failed to allocate page crypt request " "for encryption\n"); #endif goto out; } #ifndef CONFIG_CRYPTO_DEV_KFIPS SetPageUptodate(page); #else // SetPageUptodate(page); set_page_writeback(page); ecryptfs_encrypt_page_async(page_crypt_req); #endif } #else rc = ecryptfs_encrypt_page(page); if (rc) { ecryptfs_printk(KERN_WARNING, "Error encrypting " "page (upper index [0x%.16lx])\n", page->index); ClearPageUptodate(page); goto out; } SetPageUptodate(page); #endif out: unlock_page(page); return rc; } static void strip_xattr_flag(char *page_virt, struct ecryptfs_crypt_stat *crypt_stat) { if (crypt_stat->flags & ECRYPTFS_METADATA_IN_XATTR) { size_t written; crypt_stat->flags &= ~ECRYPTFS_METADATA_IN_XATTR; ecryptfs_write_crypt_stat_flags(page_virt, crypt_stat, &written); crypt_stat->flags |= ECRYPTFS_METADATA_IN_XATTR; } } /** * Header Extent: * Octets 0-7: Unencrypted file size (big-endian) * Octets 8-15: eCryptfs special marker * Octets 16-19: Flags * Octet 16: File format version number (between 0 and 255) * Octets 17-18: Reserved * Octet 19: Bit 1 (lsb): Reserved * Bit 2: Encrypted? * Bits 3-8: Reserved * Octets 20-23: Header extent size (big-endian) * Octets 24-25: Number of header extents at front of file * (big-endian) * Octet 26: Begin RFC 2440 authentication token packet set */ /** * ecryptfs_copy_up_encrypted_with_header * @page: Sort of a ``virtual'' representation of the encrypted lower * file. The actual lower file does not have the metadata in * the header. This is locked. * @crypt_stat: The eCryptfs inode's cryptographic context * * The ``view'' is the version of the file that userspace winds up * seeing, with the header information inserted. */ static int ecryptfs_copy_up_encrypted_with_header(struct page *page, struct ecryptfs_crypt_stat *crypt_stat) { loff_t extent_num_in_page = 0; loff_t num_extents_per_page = (PAGE_CACHE_SIZE / crypt_stat->extent_size); int rc = 0; while (extent_num_in_page < num_extents_per_page) { loff_t view_extent_num = ((((loff_t)page->index) * num_extents_per_page) + extent_num_in_page); size_t num_header_extents_at_front = (crypt_stat->metadata_size / crypt_stat->extent_size); if (view_extent_num < num_header_extents_at_front) { /* This is a header extent */ char *page_virt; page_virt = kmap_atomic(page); memset(page_virt, 0, PAGE_CACHE_SIZE); /* TODO: Support more than one header extent */ if (view_extent_num == 0) { size_t written; rc = ecryptfs_read_xattr_region( page_virt, page->mapping->host); strip_xattr_flag(page_virt + 16, crypt_stat); ecryptfs_write_header_metadata(page_virt + 20, crypt_stat, &written); } kunmap_atomic(page_virt); flush_dcache_page(page); if (rc) { printk(KERN_ERR "%s: Error reading xattr " "region; rc = [%d]\n", __func__, rc); goto out; } } else { /* This is an encrypted data extent */ loff_t lower_offset = ((view_extent_num * crypt_stat->extent_size) - crypt_stat->metadata_size); rc = ecryptfs_read_lower_page_segment( page, (lower_offset >> PAGE_CACHE_SHIFT), (lower_offset & ~PAGE_CACHE_MASK), crypt_stat->extent_size, page->mapping->host); if (rc) { printk(KERN_ERR "%s: Error attempting to read " "extent at offset [%lld] in the lower " "file; rc = [%d]\n", __func__, lower_offset, rc); goto out; } } extent_num_in_page++; } out: return rc; }
static int ext4_destroy_inline_data_nolock(handle_t *handle, struct inode *inode) { struct ext4_inode_info *ei = EXT4_I(inode); struct ext4_xattr_ibody_find is = { .s = { .not_found = 0, }, }; struct ext4_xattr_info i = { .name_index = EXT4_XATTR_INDEX_SYSTEM, .name = EXT4_XATTR_SYSTEM_DATA, .value = NULL, .value_len = 0, }; int error; if (!ei->i_inline_off) return 0; error = ext4_get_inode_loc(inode, &is.iloc); if (error) return error; error = ext4_xattr_ibody_find(inode, &i, &is); if (error) goto out; BUFFER_TRACE(is.iloc.bh, "get_write_access"); error = ext4_journal_get_write_access(handle, is.iloc.bh); if (error) goto out; error = ext4_xattr_ibody_inline_set(handle, inode, &i, &is); if (error) goto out; memset((void *)ext4_raw_inode(&is.iloc)->i_block, 0, EXT4_MIN_INLINE_DATA_SIZE); if (EXT4_HAS_INCOMPAT_FEATURE(inode->i_sb, EXT4_FEATURE_INCOMPAT_EXTENTS)) { if (S_ISDIR(inode->i_mode) || S_ISREG(inode->i_mode) || S_ISLNK(inode->i_mode)) { ext4_set_inode_flag(inode, EXT4_INODE_EXTENTS); ext4_ext_tree_init(handle, inode); } } ext4_clear_inode_flag(inode, EXT4_INODE_INLINE_DATA); get_bh(is.iloc.bh); error = ext4_mark_iloc_dirty(handle, inode, &is.iloc); EXT4_I(inode)->i_inline_off = 0; EXT4_I(inode)->i_inline_size = 0; ext4_clear_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA); out: brelse(is.iloc.bh); if (error == -ENODATA) error = 0; return error; } static int ext4_read_inline_page(struct inode *inode, struct page *page) { void *kaddr; int ret = 0; size_t len; struct ext4_iloc iloc; BUG_ON(!PageLocked(page)); BUG_ON(!ext4_has_inline_data(inode)); BUG_ON(page->index); if (!EXT4_I(inode)->i_inline_off) { ext4_warning(inode->i_sb, "inode %lu doesn't have inline data.", inode->i_ino); goto out; } ret = ext4_get_inode_loc(inode, &iloc); if (ret) goto out; len = min_t(size_t, ext4_get_inline_size(inode), i_size_read(inode)); kaddr = kmap_atomic(page); ret = ext4_read_inline_data(inode, kaddr, len, &iloc); flush_dcache_page(page); kunmap_atomic(kaddr); zero_user_segment(page, len, PAGE_CACHE_SIZE); SetPageUptodate(page); brelse(iloc.bh); out: return ret; }
static int vmw_gmr_build_descriptors(struct list_head *desc_pages, struct page *pages[], unsigned long num_pages) { struct page *page, *next; struct svga_guest_mem_descriptor *page_virtual = NULL; struct svga_guest_mem_descriptor *desc_virtual = NULL; unsigned int desc_per_page; unsigned long prev_pfn; unsigned long pfn; int ret; desc_per_page = PAGE_SIZE / sizeof(struct svga_guest_mem_descriptor) - 1; while (likely(num_pages != 0)) { page = alloc_page(__GFP_HIGHMEM); if (unlikely(page == NULL)) { ret = -ENOMEM; goto out_err; } list_add_tail(&page->lru, desc_pages); /* * Point previous page terminating descriptor to this * page before unmapping it. */ if (likely(page_virtual != NULL)) { desc_virtual->ppn = page_to_pfn(page); kunmap_atomic(page_virtual, KM_USER0); } page_virtual = kmap_atomic(page, KM_USER0); desc_virtual = page_virtual - 1; prev_pfn = ~(0UL); while (likely(num_pages != 0)) { pfn = page_to_pfn(*pages); if (pfn != prev_pfn + 1) { if (desc_virtual - page_virtual == desc_per_page - 1) break; (++desc_virtual)->ppn = cpu_to_le32(pfn); desc_virtual->num_pages = cpu_to_le32(1); } else { uint32_t tmp = le32_to_cpu(desc_virtual->num_pages); desc_virtual->num_pages = cpu_to_le32(tmp + 1); } prev_pfn = pfn; --num_pages; ++pages; } (++desc_virtual)->ppn = cpu_to_le32(0); desc_virtual->num_pages = cpu_to_le32(0); } if (likely(page_virtual != NULL)) kunmap_atomic(page_virtual, KM_USER0); return 0; out_err: list_for_each_entry_safe(page, next, desc_pages, lru) { list_del_init(&page->lru); __free_page(page); }
/* * This is a little more tricky than the file -> pipe splicing. There are * basically three cases: * * - Destination page already exists in the address space and there * are users of it. For that case we have no other option that * copying the data. Tough luck. * - Destination page already exists in the address space, but there * are no users of it. Make sure it's uptodate, then drop it. Fall * through to last case. * - Destination page does not exist, we can add the pipe page to * the page cache and avoid the copy. * * If asked to move pages to the output file (SPLICE_F_MOVE is set in * sd->flags), we attempt to migrate pages from the pipe to the output * file address space page cache. This is possible if no one else has * the pipe page referenced outside of the pipe and page cache. If * SPLICE_F_MOVE isn't set, or we cannot move the page, we simply create * a new page in the output file page cache and fill/dirty that. */ static int pipe_to_file(struct pipe_inode_info *pipe, struct pipe_buffer *buf, struct splice_desc *sd) { struct file *file = sd->file; struct address_space *mapping = file->f_mapping; unsigned int offset, this_len; struct page *page; pgoff_t index; int ret; /* * make sure the data in this buffer is uptodate */ ret = buf->ops->pin(pipe, buf); if (unlikely(ret)) return ret; index = sd->pos >> PAGE_CACHE_SHIFT; offset = sd->pos & ~PAGE_CACHE_MASK; this_len = sd->len; if (this_len + offset > PAGE_CACHE_SIZE) this_len = PAGE_CACHE_SIZE - offset; find_page: page = find_lock_page(mapping, index); if (!page) { ret = -ENOMEM; page = page_cache_alloc_cold(mapping); if (unlikely(!page)) goto out_ret; /* * This will also lock the page */ ret = add_to_page_cache_lru(page, mapping, index, GFP_KERNEL); if (unlikely(ret)) goto out; } ret = mapping->a_ops->prepare_write(file, page, offset, offset+this_len); if (unlikely(ret)) { loff_t isize = i_size_read(mapping->host); if (ret != AOP_TRUNCATED_PAGE) unlock_page(page); page_cache_release(page); if (ret == AOP_TRUNCATED_PAGE) goto find_page; /* * prepare_write() may have instantiated a few blocks * outside i_size. Trim these off again. */ if (sd->pos + this_len > isize) vmtruncate(mapping->host, isize); goto out_ret; } if (buf->page != page) { /* * Careful, ->map() uses KM_USER0! */ char *src = buf->ops->map(pipe, buf, 1); char *dst = kmap_atomic(page, KM_USER1); memcpy(dst + offset, src + buf->offset, this_len); flush_dcache_page(page); kunmap_atomic(dst, KM_USER1); buf->ops->unmap(pipe, buf, src); } ret = mapping->a_ops->commit_write(file, page, offset, offset+this_len); if (ret) { if (ret == AOP_TRUNCATED_PAGE) { page_cache_release(page); goto find_page; } if (ret < 0) goto out; /* * Partial write has happened, so 'ret' already initialized by * number of bytes written, Where is nothing we have to do here. */ } else ret = this_len; /* * Return the number of bytes written and mark page as * accessed, we are now done! */ mark_page_accessed(page); balance_dirty_pages_ratelimited(mapping); out: page_cache_release(page); unlock_page(page); out_ret: return ret; }
static int copy_user_bh(struct page *to, struct inode *inode, struct buffer_head *bh, unsigned long vaddr) { struct blk_dax_ctl dax = { .sector = to_sector(bh, inode), .size = bh->b_size, }; struct block_device *bdev = bh->b_bdev; void *vto; if (dax_map_atomic(bdev, &dax) < 0) return PTR_ERR(dax.addr); vto = kmap_atomic(to); copy_user_page(vto, (void __force *)dax.addr, vaddr, to); kunmap_atomic(vto); dax_unmap_atomic(bdev, &dax); return 0; } #define NO_SECTOR -1 #define DAX_PMD_INDEX(page_index) (page_index & (PMD_MASK >> PAGE_CACHE_SHIFT)) static int dax_radix_entry(struct address_space *mapping, pgoff_t index, sector_t sector, bool pmd_entry, bool dirty) { struct radix_tree_root *page_tree = &mapping->page_tree; pgoff_t pmd_index = DAX_PMD_INDEX(index); int type, error = 0; void *entry; WARN_ON_ONCE(pmd_entry && !dirty); __mark_inode_dirty(mapping->host, I_DIRTY_PAGES); spin_lock_irq(&mapping->tree_lock); entry = radix_tree_lookup(page_tree, pmd_index); if (entry && RADIX_DAX_TYPE(entry) == RADIX_DAX_PMD) { index = pmd_index; goto dirty; } entry = radix_tree_lookup(page_tree, index); if (entry) { type = RADIX_DAX_TYPE(entry); if (WARN_ON_ONCE(type != RADIX_DAX_PTE && type != RADIX_DAX_PMD)) { error = -EIO; goto unlock; } if (!pmd_entry || type == RADIX_DAX_PMD) goto dirty; /* * We only insert dirty PMD entries into the radix tree. This * means we don't need to worry about removing a dirty PTE * entry and inserting a clean PMD entry, thus reducing the * range we would flush with a follow-up fsync/msync call. */ radix_tree_delete(&mapping->page_tree, index); mapping->nrexceptional--; } if (sector == NO_SECTOR) { /* * This can happen during correct operation if our pfn_mkwrite * fault raced against a hole punch operation. If this * happens the pte that was hole punched will have been * unmapped and the radix tree entry will have been removed by * the time we are called, but the call will still happen. We * will return all the way up to wp_pfn_shared(), where the * pte_same() check will fail, eventually causing page fault * to be retried by the CPU. */ goto unlock; } error = radix_tree_insert(page_tree, index, RADIX_DAX_ENTRY(sector, pmd_entry)); if (error) goto unlock; mapping->nrexceptional++; dirty: if (dirty) radix_tree_tag_set(page_tree, index, PAGECACHE_TAG_DIRTY); unlock: spin_unlock_irq(&mapping->tree_lock); return error; } static int dax_writeback_one(struct block_device *bdev, struct address_space *mapping, pgoff_t index, void *entry) { struct radix_tree_root *page_tree = &mapping->page_tree; int type = RADIX_DAX_TYPE(entry); struct radix_tree_node *node; struct blk_dax_ctl dax; void **slot; int ret = 0; spin_lock_irq(&mapping->tree_lock); /* * Regular page slots are stabilized by the page lock even * without the tree itself locked. These unlocked entries * need verification under the tree lock. */ if (!__radix_tree_lookup(page_tree, index, &node, &slot)) goto unlock; if (*slot != entry) goto unlock; /* another fsync thread may have already written back this entry */ if (!radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE)) goto unlock; if (WARN_ON_ONCE(type != RADIX_DAX_PTE && type != RADIX_DAX_PMD)) { ret = -EIO; goto unlock; } dax.sector = RADIX_DAX_SECTOR(entry); dax.size = (type == RADIX_DAX_PMD ? PMD_SIZE : PAGE_SIZE); spin_unlock_irq(&mapping->tree_lock); /* * We cannot hold tree_lock while calling dax_map_atomic() because it * eventually calls cond_resched(). */ ret = dax_map_atomic(bdev, &dax); if (ret < 0) return ret; if (WARN_ON_ONCE(ret < dax.size)) { ret = -EIO; goto unmap; } wb_cache_pmem(dax.addr, dax.size); spin_lock_irq(&mapping->tree_lock); radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_TOWRITE); spin_unlock_irq(&mapping->tree_lock); unmap: dax_unmap_atomic(bdev, &dax); return ret; unlock: spin_unlock_irq(&mapping->tree_lock); return ret; } /* * Flush the mapping to the persistent domain within the byte range of [start, * end]. This is required by data integrity operations to ensure file data is * on persistent storage prior to completion of the operation. */ int dax_writeback_mapping_range(struct address_space *mapping, loff_t start, loff_t end) { struct inode *inode = mapping->host; struct block_device *bdev = inode->i_sb->s_bdev; pgoff_t start_index, end_index, pmd_index; pgoff_t indices[PAGEVEC_SIZE]; struct pagevec pvec; bool done = false; int i, ret = 0; void *entry; if (WARN_ON_ONCE(inode->i_blkbits != PAGE_SHIFT)) return -EIO; start_index = start >> PAGE_CACHE_SHIFT; end_index = end >> PAGE_CACHE_SHIFT; pmd_index = DAX_PMD_INDEX(start_index); rcu_read_lock(); entry = radix_tree_lookup(&mapping->page_tree, pmd_index); rcu_read_unlock(); /* see if the start of our range is covered by a PMD entry */ if (entry && RADIX_DAX_TYPE(entry) == RADIX_DAX_PMD) start_index = pmd_index; tag_pages_for_writeback(mapping, start_index, end_index); pagevec_init(&pvec, 0); while (!done) { pvec.nr = find_get_entries_tag(mapping, start_index, PAGECACHE_TAG_TOWRITE, PAGEVEC_SIZE, pvec.pages, indices); if (pvec.nr == 0) break; for (i = 0; i < pvec.nr; i++) { if (indices[i] > end_index) { done = true; break; } ret = dax_writeback_one(bdev, mapping, indices[i], pvec.pages[i]); if (ret < 0) return ret; } } wmb_pmem(); return 0; } EXPORT_SYMBOL_GPL(dax_writeback_mapping_range); static int dax_insert_mapping(struct inode *inode, struct buffer_head *bh, struct vm_area_struct *vma, struct vm_fault *vmf) { unsigned long vaddr = (unsigned long)vmf->virtual_address; struct address_space *mapping = inode->i_mapping; struct block_device *bdev = bh->b_bdev; struct blk_dax_ctl dax = { .sector = to_sector(bh, inode), .size = bh->b_size, }; pgoff_t size; int error; i_mmap_lock_read(mapping); /* * Check truncate didn't happen while we were allocating a block. * If it did, this block may or may not be still allocated to the * file. We can't tell the filesystem to free it because we can't * take i_mutex here. In the worst case, the file still has blocks * allocated past the end of the file. */ size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT; if (unlikely(vmf->pgoff >= size)) { error = -EIO; goto out; } if (dax_map_atomic(bdev, &dax) < 0) { error = PTR_ERR(dax.addr); goto out; } if (buffer_unwritten(bh) || buffer_new(bh)) { clear_pmem(dax.addr, PAGE_SIZE); wmb_pmem(); } dax_unmap_atomic(bdev, &dax); error = dax_radix_entry(mapping, vmf->pgoff, dax.sector, false, vmf->flags & FAULT_FLAG_WRITE); if (error) goto out; error = vm_insert_mixed(vma, vaddr, dax.pfn); out: i_mmap_unlock_read(mapping); return error; } /** * __dax_fault - handle a page fault on a DAX file * @vma: The virtual memory area where the fault occurred * @vmf: The description of the fault * @get_block: The filesystem method used to translate file offsets to blocks * @complete_unwritten: The filesystem method used to convert unwritten blocks * to written so the data written to them is exposed. This is required for * required by write faults for filesystems that will return unwritten * extent mappings from @get_block, but it is optional for reads as * dax_insert_mapping() will always zero unwritten blocks. If the fs does * not support unwritten extents, the it should pass NULL. * * When a page fault occurs, filesystems may call this helper in their * fault handler for DAX files. __dax_fault() assumes the caller has done all * the necessary locking for the page fault to proceed successfully. */ int __dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf, get_block_t get_block, dax_iodone_t complete_unwritten) { struct file *file = vma->vm_file; struct address_space *mapping = file->f_mapping; struct inode *inode = mapping->host; struct page *page; struct buffer_head bh; unsigned long vaddr = (unsigned long)vmf->virtual_address; unsigned blkbits = inode->i_blkbits; sector_t block; pgoff_t size; int error; int major = 0; size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT; if (vmf->pgoff >= size) return VM_FAULT_SIGBUS; memset(&bh, 0, sizeof(bh)); block = (sector_t)vmf->pgoff << (PAGE_SHIFT - blkbits); bh.b_bdev = inode->i_sb->s_bdev; bh.b_size = PAGE_SIZE; repeat: page = find_get_page(mapping, vmf->pgoff); if (page) { if (!lock_page_or_retry(page, vma->vm_mm, vmf->flags)) { page_cache_release(page); return VM_FAULT_RETRY; } if (unlikely(page->mapping != mapping)) { unlock_page(page); page_cache_release(page); goto repeat; } size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT; if (unlikely(vmf->pgoff >= size)) { /* * We have a struct page covering a hole in the file * from a read fault and we've raced with a truncate */ error = -EIO; goto unlock_page; } } error = get_block(inode, block, &bh, 0); if (!error && (bh.b_size < PAGE_SIZE)) error = -EIO; /* fs corruption? */ if (error) goto unlock_page; if (!buffer_mapped(&bh) && !buffer_unwritten(&bh) && !vmf->cow_page) { if (vmf->flags & FAULT_FLAG_WRITE) { error = get_block(inode, block, &bh, 1); count_vm_event(PGMAJFAULT); mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT); major = VM_FAULT_MAJOR; if (!error && (bh.b_size < PAGE_SIZE)) error = -EIO; if (error) goto unlock_page; } else { return dax_load_hole(mapping, page, vmf); } } if (vmf->cow_page) { struct page *new_page = vmf->cow_page; if (buffer_written(&bh)) error = copy_user_bh(new_page, inode, &bh, vaddr); else clear_user_highpage(new_page, vaddr); if (error) goto unlock_page; vmf->page = page; if (!page) { i_mmap_lock_read(mapping); /* Check we didn't race with truncate */ size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT; if (vmf->pgoff >= size) { i_mmap_unlock_read(mapping); error = -EIO; goto out; } } return VM_FAULT_LOCKED; } /* Check we didn't race with a read fault installing a new page */ if (!page && major) page = find_lock_page(mapping, vmf->pgoff); if (page) { unmap_mapping_range(mapping, vmf->pgoff << PAGE_SHIFT, PAGE_CACHE_SIZE, 0); delete_from_page_cache(page); unlock_page(page); page_cache_release(page); page = NULL; } /* * If we successfully insert the new mapping over an unwritten extent, * we need to ensure we convert the unwritten extent. If there is an * error inserting the mapping, the filesystem needs to leave it as * unwritten to prevent exposure of the stale underlying data to * userspace, but we still need to call the completion function so * the private resources on the mapping buffer can be released. We * indicate what the callback should do via the uptodate variable, same * as for normal BH based IO completions. */ error = dax_insert_mapping(inode, &bh, vma, vmf); if (buffer_unwritten(&bh)) { if (complete_unwritten) complete_unwritten(&bh, !error); else WARN_ON_ONCE(!(vmf->flags & FAULT_FLAG_WRITE)); } out: if (error == -ENOMEM) return VM_FAULT_OOM | major; /* -EBUSY is fine, somebody else faulted on the same PTE */ if ((error < 0) && (error != -EBUSY)) return VM_FAULT_SIGBUS | major; return VM_FAULT_NOPAGE | major; unlock_page: if (page) { unlock_page(page); page_cache_release(page); } goto out; }
static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index, int offset) { int ret = 0; size_t clen; unsigned long handle; struct page *page; unsigned char *user_mem, *cmem, *src, *uncmem = NULL; page = bvec->bv_page; src = zram->compress_buffer; if (is_partial_io(bvec)) { /* * This is a partial IO. We need to read the full page * before to write the changes. */ uncmem = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!uncmem) { pr_info("Error allocating temp memory!\n"); ret = -ENOMEM; goto out; } ret = zram_decompress_page(zram, uncmem, index); if (ret) goto out; } /* * System overwrites unused sectors. Free memory associated * with this sector now. */ if (zram->table[index].handle || zram_test_flag(zram, index, ZRAM_ZERO)) zram_free_page(zram, index); user_mem = kmap_atomic(page); if (is_partial_io(bvec)) { memcpy(uncmem + offset, user_mem + bvec->bv_offset, bvec->bv_len); kunmap_atomic(user_mem); user_mem = NULL; } else { uncmem = user_mem; } if (page_zero_filled(uncmem)) { if (!is_partial_io(bvec)) kunmap_atomic(user_mem); zram_stat_inc(&zram->stats.pages_zero); zram_set_flag(zram, index, ZRAM_ZERO); ret = 0; goto out; } ret = zram_comp_op(ZRAM_COMPOP_COMPRESS, uncmem, PAGE_SIZE, src, &clen); if (!is_partial_io(bvec)) { kunmap_atomic(user_mem); user_mem = NULL; uncmem = NULL; } if (unlikely(ret != 0)) { pr_err("Compression failed! err=%d\n", ret); goto out; } if (unlikely(clen > max_zpage_size)) { zram_stat_inc(&zram->stats.bad_compress); clen = PAGE_SIZE; src = NULL; if (is_partial_io(bvec)) src = uncmem; } handle = zs_malloc(zram->mem_pool, clen); if (!handle) { pr_info("Error allocating memory for compressed " "page: %u, size=%zu\n", index, clen); ret = -ENOMEM; goto out; } cmem = zs_map_object(zram->mem_pool, handle, ZS_MM_WO); if ((clen == PAGE_SIZE) && !is_partial_io(bvec)) src = kmap_atomic(page); memcpy(cmem, src, clen); if ((clen == PAGE_SIZE) && !is_partial_io(bvec)) kunmap_atomic(src); zs_unmap_object(zram->mem_pool, handle); zram->table[index].handle = handle; zram->table[index].size = clen; /* Update stats */ zram_stat64_add(zram, &zram->stats.compr_size, clen); zram_stat_inc(&zram->stats.pages_stored); if (clen <= PAGE_SIZE / 2) zram_stat_inc(&zram->stats.good_compress); out: if (is_partial_io(bvec)) kfree(uncmem); if (ret) zram_stat64_inc(zram, &zram->stats.failed_writes); return ret; }
/* * Receive write data frame. */ void ft_recv_write_data(struct ft_cmd *cmd, struct fc_frame *fp) { struct se_cmd *se_cmd = &cmd->se_cmd; struct fc_seq *seq = cmd->seq; struct fc_exch *ep; struct fc_lport *lport; struct fc_frame_header *fh; struct scatterlist *sg = NULL; u32 mem_off = 0; u32 rel_off; size_t frame_len; size_t mem_len = 0; size_t tlen; struct page *page = NULL; void *page_addr; void *from; void *to; u32 f_ctl; void *buf; fh = fc_frame_header_get(fp); if (!(ntoh24(fh->fh_f_ctl) & FC_FC_REL_OFF)) goto drop; f_ctl = ntoh24(fh->fh_f_ctl); ep = fc_seq_exch(seq); lport = ep->lp; if (cmd->was_ddp_setup) { BUG_ON(!ep); BUG_ON(!lport); /* * Since DDP (Large Rx offload) was setup for this request, * payload is expected to be copied directly to user buffers. */ buf = fc_frame_payload_get(fp, 1); if (buf) pr_err("%s: xid 0x%x, f_ctl 0x%x, cmd->sg %p, " "cmd->sg_cnt 0x%x. DDP was setup" " hence not expected to receive frame with " "payload, Frame will be dropped if" "'Sequence Initiative' bit in f_ctl is" "not set\n", __func__, ep->xid, f_ctl, cmd->sg, cmd->sg_cnt); /* * Invalidate HW DDP context if it was setup for respective * command. Invalidation of HW DDP context is requited in both * situation (success and error). */ ft_invl_hw_context(cmd); /* * If "Sequence Initiative (TSI)" bit set in f_ctl, means last * write data frame is received successfully where payload is * posted directly to user buffer and only the last frame's * header is posted in receive queue. * * If "Sequence Initiative (TSI)" bit is not set, means error * condition w.r.t. DDP, hence drop the packet and let explict * ABORTS from other end of exchange timer trigger the recovery. */ if (f_ctl & FC_FC_SEQ_INIT) goto last_frame; else goto drop; } rel_off = ntohl(fh->fh_parm_offset); frame_len = fr_len(fp); if (frame_len <= sizeof(*fh)) goto drop; frame_len -= sizeof(*fh); from = fc_frame_payload_get(fp, 0); if (rel_off >= se_cmd->data_length) goto drop; if (frame_len + rel_off > se_cmd->data_length) frame_len = se_cmd->data_length - rel_off; /* * Setup to use first mem list entry, unless no data. */ BUG_ON(frame_len && !se_cmd->t_data_sg); if (frame_len) { sg = se_cmd->t_data_sg; mem_len = sg->length; mem_off = sg->offset; page = sg_page(sg); } while (frame_len) { if (!mem_len) { sg = sg_next(sg); mem_len = sg->length; mem_off = sg->offset; page = sg_page(sg); } if (rel_off >= mem_len) { rel_off -= mem_len; mem_len = 0; continue; } mem_off += rel_off; mem_len -= rel_off; rel_off = 0; tlen = min(mem_len, frame_len); to = kmap_atomic(page + (mem_off >> PAGE_SHIFT)); page_addr = to; to += mem_off & ~PAGE_MASK; tlen = min(tlen, (size_t)(PAGE_SIZE - (mem_off & ~PAGE_MASK))); memcpy(to, from, tlen); kunmap_atomic(page_addr); from += tlen; frame_len -= tlen; mem_off += tlen; mem_len -= tlen; cmd->write_data_len += tlen; } last_frame: if (cmd->write_data_len == se_cmd->data_length) transport_generic_handle_data(se_cmd); drop: fc_frame_free(fp); }
static int squashfs_symlink_readpage(struct file *file, struct page *page) { struct inode *inode = page->mapping->host; struct super_block *sb = inode->i_sb; struct squashfs_sb_info *msblk = sb->s_fs_info; int index = page->index << PAGE_CACHE_SHIFT; u64 block = squashfs_i(inode)->start; int offset = squashfs_i(inode)->offset; int length = min_t(int, i_size_read(inode) - index, PAGE_CACHE_SIZE); int bytes, copied; void *pageaddr; struct squashfs_cache_entry *entry; TRACE("Entered squashfs_symlink_readpage, page index %ld, start block " "%llx, offset %x\n", page->index, block, offset); /* * Skip index bytes into symlink metadata. */ if (index) { bytes = squashfs_read_metadata(sb, NULL, &block, &offset, index); if (bytes < 0) { ERROR("Unable to read symlink [%llx:%x]\n", squashfs_i(inode)->start, squashfs_i(inode)->offset); goto error_out; } } /* * Read length bytes from symlink metadata. Squashfs_read_metadata * is not used here because it can sleep and we want to use * kmap_atomic to map the page. Instead call the underlying * squashfs_cache_get routine. As length bytes may overlap metadata * blocks, we may need to call squashfs_cache_get multiple times. */ for (bytes = 0; bytes < length; offset = 0, bytes += copied) { entry = squashfs_cache_get(sb, msblk->block_cache, block, 0); if (entry->error) { ERROR("Unable to read symlink [%llx:%x]\n", squashfs_i(inode)->start, squashfs_i(inode)->offset); squashfs_cache_put(entry); goto error_out; } pageaddr = kmap_atomic(page, KM_USER0); copied = squashfs_copy_data(pageaddr + bytes, entry, offset, length - bytes); if (copied == length - bytes) memset(pageaddr + length, 0, PAGE_CACHE_SIZE - length); else block = entry->next_index; kunmap_atomic(pageaddr, KM_USER0); squashfs_cache_put(entry); } flush_dcache_page(page); SetPageUptodate(page); unlock_page(page); return 0; error_out: SetPageError(page); unlock_page(page); return 0; }
static int gfs2_write_begin(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, unsigned flags, struct page **pagep, void **fsdata) { struct gfs2_inode *ip = GFS2_I(mapping->host); struct gfs2_sbd *sdp = GFS2_SB(mapping->host); struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode); unsigned int data_blocks = 0, ind_blocks = 0, rblocks; unsigned requested = 0; int alloc_required; int error = 0; pgoff_t index = pos >> PAGE_CACHE_SHIFT; unsigned from = pos & (PAGE_CACHE_SIZE - 1); struct page *page; gfs2_holder_init(ip->i_gl, LM_ST_EXCLUSIVE, 0, &ip->i_gh); error = gfs2_glock_nq(&ip->i_gh); if (unlikely(error)) goto out_uninit; if (&ip->i_inode == sdp->sd_rindex) { error = gfs2_glock_nq_init(m_ip->i_gl, LM_ST_EXCLUSIVE, GL_NOCACHE, &m_ip->i_gh); if (unlikely(error)) { gfs2_glock_dq(&ip->i_gh); goto out_uninit; } } alloc_required = gfs2_write_alloc_required(ip, pos, len); if (alloc_required || gfs2_is_jdata(ip)) gfs2_write_calc_reserv(ip, len, &data_blocks, &ind_blocks); if (alloc_required) { struct gfs2_alloc_parms ap = { .aflags = 0, }; error = gfs2_quota_lock_check(ip); if (error) goto out_unlock; requested = data_blocks + ind_blocks; ap.target = requested; error = gfs2_inplace_reserve(ip, &ap); if (error) goto out_qunlock; } rblocks = RES_DINODE + ind_blocks; if (gfs2_is_jdata(ip)) rblocks += data_blocks ? data_blocks : 1; if (ind_blocks || data_blocks) rblocks += RES_STATFS + RES_QUOTA; if (&ip->i_inode == sdp->sd_rindex) rblocks += 2 * RES_STATFS; if (alloc_required) rblocks += gfs2_rg_blocks(ip, requested); error = gfs2_trans_begin(sdp, rblocks, PAGE_CACHE_SIZE/sdp->sd_sb.sb_bsize); if (error) goto out_trans_fail; error = -ENOMEM; flags |= AOP_FLAG_NOFS; page = grab_cache_page_write_begin(mapping, index, flags); *pagep = page; if (unlikely(!page)) goto out_endtrans; if (gfs2_is_stuffed(ip)) { error = 0; if (pos + len > sdp->sd_sb.sb_bsize - sizeof(struct gfs2_dinode)) { error = gfs2_unstuff_dinode(ip, page); if (error == 0) goto prepare_write; } else if (!PageUptodate(page)) { error = stuffed_readpage(ip, page); } goto out; } prepare_write: error = __block_write_begin(page, from, len, gfs2_block_map); out: if (error == 0) return 0; unlock_page(page); page_cache_release(page); gfs2_trans_end(sdp); if (pos + len > ip->i_inode.i_size) gfs2_trim_blocks(&ip->i_inode); goto out_trans_fail; out_endtrans: gfs2_trans_end(sdp); out_trans_fail: if (alloc_required) { gfs2_inplace_release(ip); out_qunlock: gfs2_quota_unlock(ip); } out_unlock: if (&ip->i_inode == sdp->sd_rindex) { gfs2_glock_dq(&m_ip->i_gh); gfs2_holder_uninit(&m_ip->i_gh); } gfs2_glock_dq(&ip->i_gh); out_uninit: gfs2_holder_uninit(&ip->i_gh); return error; } /** * adjust_fs_space - Adjusts the free space available due to gfs2_grow * @inode: the rindex inode */ static void adjust_fs_space(struct inode *inode) { struct gfs2_sbd *sdp = inode->i_sb->s_fs_info; struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode); struct gfs2_inode *l_ip = GFS2_I(sdp->sd_sc_inode); struct gfs2_statfs_change_host *m_sc = &sdp->sd_statfs_master; struct gfs2_statfs_change_host *l_sc = &sdp->sd_statfs_local; struct buffer_head *m_bh, *l_bh; u64 fs_total, new_free; /* Total up the file system space, according to the latest rindex. */ fs_total = gfs2_ri_total(sdp); if (gfs2_meta_inode_buffer(m_ip, &m_bh) != 0) return; spin_lock(&sdp->sd_statfs_spin); gfs2_statfs_change_in(m_sc, m_bh->b_data + sizeof(struct gfs2_dinode)); if (fs_total > (m_sc->sc_total + l_sc->sc_total)) new_free = fs_total - (m_sc->sc_total + l_sc->sc_total); else new_free = 0; spin_unlock(&sdp->sd_statfs_spin); fs_warn(sdp, "File system extended by %llu blocks.\n", (unsigned long long)new_free); gfs2_statfs_change(sdp, new_free, new_free, 0); if (gfs2_meta_inode_buffer(l_ip, &l_bh) != 0) goto out; update_statfs(sdp, m_bh, l_bh); brelse(l_bh); out: brelse(m_bh); } /** * gfs2_stuffed_write_end - Write end for stuffed files * @inode: The inode * @dibh: The buffer_head containing the on-disk inode * @pos: The file position * @len: The length of the write * @copied: How much was actually copied by the VFS * @page: The page * * This copies the data from the page into the inode block after * the inode data structure itself. * * Returns: errno */ static int gfs2_stuffed_write_end(struct inode *inode, struct buffer_head *dibh, loff_t pos, unsigned len, unsigned copied, struct page *page) { struct gfs2_inode *ip = GFS2_I(inode); struct gfs2_sbd *sdp = GFS2_SB(inode); struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode); u64 to = pos + copied; void *kaddr; unsigned char *buf = dibh->b_data + sizeof(struct gfs2_dinode); BUG_ON((pos + len) > (dibh->b_size - sizeof(struct gfs2_dinode))); kaddr = kmap_atomic(page); memcpy(buf + pos, kaddr + pos, copied); memset(kaddr + pos + copied, 0, len - copied); flush_dcache_page(page); kunmap_atomic(kaddr); if (!PageUptodate(page)) SetPageUptodate(page); unlock_page(page); page_cache_release(page); if (copied) { if (inode->i_size < to) i_size_write(inode, to); mark_inode_dirty(inode); } if (inode == sdp->sd_rindex) { adjust_fs_space(inode); sdp->sd_rindex_uptodate = 0; } brelse(dibh); gfs2_trans_end(sdp); if (inode == sdp->sd_rindex) { gfs2_glock_dq(&m_ip->i_gh); gfs2_holder_uninit(&m_ip->i_gh); } gfs2_glock_dq(&ip->i_gh); gfs2_holder_uninit(&ip->i_gh); return copied; } /** * gfs2_write_end * @file: The file to write to * @mapping: The address space to write to * @pos: The file position * @len: The length of the data * @copied: * @page: The page that has been written * @fsdata: The fsdata (unused in GFS2) * * The main write_end function for GFS2. We have a separate one for * stuffed files as they are slightly different, otherwise we just * put our locking around the VFS provided functions. * * Returns: errno */ static int gfs2_write_end(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, unsigned copied, struct page *page, void *fsdata) { struct inode *inode = page->mapping->host; struct gfs2_inode *ip = GFS2_I(inode); struct gfs2_sbd *sdp = GFS2_SB(inode); struct gfs2_inode *m_ip = GFS2_I(sdp->sd_statfs_inode); struct buffer_head *dibh; unsigned int from = pos & (PAGE_CACHE_SIZE - 1); unsigned int to = from + len; int ret; struct gfs2_trans *tr = current->journal_info; BUG_ON(!tr); BUG_ON(gfs2_glock_is_locked_by_me(ip->i_gl) == NULL); ret = gfs2_meta_inode_buffer(ip, &dibh); if (unlikely(ret)) { unlock_page(page); page_cache_release(page); goto failed; } if (gfs2_is_stuffed(ip)) return gfs2_stuffed_write_end(inode, dibh, pos, len, copied, page); if (!gfs2_is_writeback(ip)) gfs2_page_add_databufs(ip, page, from, to); ret = generic_write_end(file, mapping, pos, len, copied, page, fsdata); if (tr->tr_num_buf_new) __mark_inode_dirty(inode, I_DIRTY_DATASYNC); else gfs2_trans_add_meta(ip->i_gl, dibh); if (inode == sdp->sd_rindex) { adjust_fs_space(inode); sdp->sd_rindex_uptodate = 0; } brelse(dibh); failed: gfs2_trans_end(sdp); gfs2_inplace_release(ip); if (ip->i_res->rs_qa_qd_num) gfs2_quota_unlock(ip); if (inode == sdp->sd_rindex) { gfs2_glock_dq(&m_ip->i_gh); gfs2_holder_uninit(&m_ip->i_gh); } gfs2_glock_dq(&ip->i_gh); gfs2_holder_uninit(&ip->i_gh); return ret; } /** * gfs2_set_page_dirty - Page dirtying function * @page: The page to dirty * * Returns: 1 if it dirtyed the page, or 0 otherwise */ static int gfs2_set_page_dirty(struct page *page) { SetPageChecked(page); return __set_page_dirty_buffers(page); }
/* * Decode the result of a readdir call. * We're not really decoding anymore, we just leave the buffer untouched * and only check that it is syntactically correct. * The real decoding happens in nfs_decode_entry below, called directly * from nfs_readdir for each entry. */ static int nfs_xdr_readdirres(struct rpc_rqst *req, __be32 *p, void *dummy) { struct xdr_buf *rcvbuf = &req->rq_rcv_buf; struct kvec *iov = rcvbuf->head; struct page **page; size_t hdrlen; unsigned int pglen, recvd; u32 len; int status, nr = 0; __be32 *end, *entry, *kaddr; if ((status = ntohl(*p++))) return nfs_stat_to_errno(status); hdrlen = (u8 *) p - (u8 *) iov->iov_base; if (iov->iov_len < hdrlen) { dprintk("NFS: READDIR reply header overflowed:" "length %Zu > %Zu\n", hdrlen, iov->iov_len); return -errno_NFSERR_IO; } else if (iov->iov_len != hdrlen) { dprintk("NFS: READDIR header is short. iovec will be shifted.\n"); xdr_shift_buf(rcvbuf, iov->iov_len - hdrlen); } pglen = rcvbuf->page_len; recvd = rcvbuf->len - hdrlen; if (pglen > recvd) pglen = recvd; page = rcvbuf->pages; kaddr = p = kmap_atomic(*page, KM_USER0); end = (__be32 *)((char *)p + pglen); entry = p; /* Make sure the packet actually has a value_follows and EOF entry */ if ((entry + 1) > end) goto short_pkt; for (; *p++; nr++) { if (p + 2 > end) goto short_pkt; p++; /* fileid */ len = ntohl(*p++); p += XDR_QUADLEN(len) + 1; /* name plus cookie */ if (len > NFS2_MAXNAMLEN) { dprintk("NFS: giant filename in readdir (len 0x%x)!\n", len); goto err_unmap; } if (p + 2 > end) goto short_pkt; entry = p; } /* * Apparently some server sends responses that are a valid size, but * contain no entries, and have value_follows==0 and EOF==0. For * those, just set the EOF marker. */ if (!nr && entry[1] == 0) { dprintk("NFS: readdir reply truncated!\n"); entry[1] = 1; } out: kunmap_atomic(kaddr, KM_USER0); return nr; short_pkt: /* * When we get a short packet there are 2 possibilities. We can * return an error, or fix up the response to look like a valid * response and return what we have so far. If there are no * entries and the packet was short, then return -EIO. If there * are valid entries in the response, return them and pretend that * the call was successful, but incomplete. The caller can retry the * readdir starting at the last cookie. */ entry[0] = entry[1] = 0; if (!nr) nr = -errno_NFSERR_IO; goto out; err_unmap: nr = -errno_NFSERR_IO; goto out; }
/* * bitmap_init_from_disk -- called at bitmap_create time to initialize * the in-memory bitmap from the on-disk bitmap -- also, sets up the * memory mapping of the bitmap file * Special cases: * if there's no bitmap file, or if the bitmap file had been * previously kicked from the array, we mark all the bits as * 1's in order to cause a full resync. * * We ignore all bits for sectors that end earlier than 'start'. * This is used when reading an out-of-date bitmap... */ static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start) { unsigned long i, chunks, index, oldindex, bit; struct page *page = NULL, *oldpage = NULL; unsigned long num_pages, bit_cnt = 0; struct file *file; unsigned long bytes, offset; int outofdate; int ret = -ENOSPC; void *paddr; chunks = bitmap->chunks; file = bitmap->file; BUG_ON(!file && !bitmap->offset); #ifdef INJECT_FAULTS_3 outofdate = 1; #else outofdate = bitmap->flags & BITMAP_STALE; #endif if (outofdate) printk(KERN_INFO "%s: bitmap file is out of date, doing full " "recovery\n", bmname(bitmap)); bytes = (chunks + 7) / 8; num_pages = (bytes + sizeof(bitmap_super_t) + PAGE_SIZE - 1) / PAGE_SIZE; if (file && i_size_read(file->f_mapping->host) < bytes + sizeof(bitmap_super_t)) { printk(KERN_INFO "%s: bitmap file too short %lu < %lu\n", bmname(bitmap), (unsigned long) i_size_read(file->f_mapping->host), bytes + sizeof(bitmap_super_t)); goto err; } ret = -ENOMEM; bitmap->filemap = kmalloc(sizeof(struct page *) * num_pages, GFP_KERNEL); if (!bitmap->filemap) goto err; /* We need 4 bits per page, rounded up to a multiple of sizeof(unsigned long) */ bitmap->filemap_attr = kzalloc( roundup( DIV_ROUND_UP(num_pages*4, 8), sizeof(unsigned long)), GFP_KERNEL); if (!bitmap->filemap_attr) goto err; oldindex = ~0L; for (i = 0; i < chunks; i++) { int b; index = file_page_index(i); bit = file_page_offset(i); if (index != oldindex) { /* this is a new page, read it in */ int count; /* unmap the old page, we're done with it */ if (index == num_pages-1) count = bytes + sizeof(bitmap_super_t) - index * PAGE_SIZE; else count = PAGE_SIZE; if (index == 0) { /* * if we're here then the superblock page * contains some bits (PAGE_SIZE != sizeof sb) * we've already read it in, so just use it */ page = bitmap->sb_page; offset = sizeof(bitmap_super_t); } else if (file) { page = read_page(file, index, bitmap, count); offset = 0; } else { page = read_sb_page(bitmap->mddev, bitmap->offset, index); offset = 0; } if (IS_ERR(page)) { /* read error */ ret = PTR_ERR(page); goto err; } oldindex = index; oldpage = page; if (outofdate) { /* * if bitmap is out of date, dirty the * whole page and write it out */ paddr = kmap_atomic(page, KM_USER0); memset(paddr + offset, 0xff, PAGE_SIZE - offset); kunmap_atomic(paddr, KM_USER0); write_page(bitmap, page, 1); ret = -EIO; if (bitmap->flags & BITMAP_WRITE_ERROR) { /* release, page not in filemap yet */ put_page(page); goto err; } } bitmap->filemap[bitmap->file_pages++] = page; bitmap->last_page_size = count; } paddr = kmap_atomic(page, KM_USER0); if (bitmap->flags & BITMAP_HOSTENDIAN) b = test_bit(bit, paddr); else b = ext2_test_bit(bit, paddr); kunmap_atomic(paddr, KM_USER0); if (b) { /* if the disk bit is set, set the memory bit */ bitmap_set_memory_bits(bitmap, i << CHUNK_BLOCK_SHIFT(bitmap), ((i+1) << (CHUNK_BLOCK_SHIFT(bitmap)) >= start) ); bit_cnt++; set_page_attr(bitmap, page, BITMAP_PAGE_CLEAN); } } /* everything went OK */ ret = 0; bitmap_mask_state(bitmap, BITMAP_STALE, MASK_UNSET); if (bit_cnt) { /* Kick recovery if any bits were set */ set_bit(MD_RECOVERY_NEEDED, &bitmap->mddev->recovery); md_wakeup_thread(bitmap->mddev->thread); } printk(KERN_INFO "%s: bitmap initialized from disk: " "read %lu/%lu pages, set %lu bits\n", bmname(bitmap), bitmap->file_pages, num_pages, bit_cnt); return 0; err: printk(KERN_INFO "%s: bitmap initialisation failed: %d\n", bmname(bitmap), ret); return ret; }
/* * Attempts to free an entry by adding a page to the swap cache, * decompressing the entry data into the page, and issuing a * bio write to write the page back to the swap device. * * This can be thought of as a "resumed writeback" of the page * to the swap device. We are basically resuming the same swap * writeback path that was intercepted with the frontswap_store() * in the first place. After the page has been decompressed into * the swap cache, the compressed version stored by zswap can be * freed. */ static int zswap_writeback_entry(struct zbud_pool *pool, unsigned long handle) { struct zswap_header *zhdr; swp_entry_t swpentry; struct zswap_tree *tree; pgoff_t offset; struct zswap_entry *entry; struct page *page; u8 *src, *dst; unsigned int dlen; int ret; struct writeback_control wbc = { .sync_mode = WB_SYNC_NONE, }; /* extract swpentry from data */ zhdr = zbud_map(pool, handle); swpentry = zhdr->swpentry; /* here */ zbud_unmap(pool, handle); tree = zswap_trees[swp_type(swpentry)]; offset = swp_offset(swpentry); BUG_ON(pool != tree->pool); /* find and ref zswap entry */ spin_lock(&tree->lock); entry = zswap_entry_find_get(&tree->rbroot, offset); if (!entry) { /* entry was invalidated */ spin_unlock(&tree->lock); return 0; } spin_unlock(&tree->lock); BUG_ON(offset != entry->offset); /* try to allocate swap cache page */ switch (zswap_get_swap_cache_page(swpentry, &page)) { case ZSWAP_SWAPCACHE_FAIL: /* no memory or invalidate happened */ ret = -ENOMEM; goto fail; case ZSWAP_SWAPCACHE_EXIST: /* page is already in the swap cache, ignore for now */ page_cache_release(page); ret = -EEXIST; goto fail; case ZSWAP_SWAPCACHE_NEW: /* page is locked */ /* decompress */ dlen = PAGE_SIZE; src = (u8 *)zbud_map(tree->pool, entry->handle) + sizeof(struct zswap_header); dst = kmap_atomic(page); ret = zswap_comp_op(ZSWAP_COMPOP_DECOMPRESS, src, entry->length, dst, &dlen); kunmap_atomic(dst); zbud_unmap(tree->pool, entry->handle); BUG_ON(ret); BUG_ON(dlen != PAGE_SIZE); /* page is up to date */ SetPageUptodate(page); } /* move it to the tail of the inactive list after end_writeback */ SetPageReclaim(page); /* start writeback */ __swap_writepage(page, &wbc, end_swap_bio_write); page_cache_release(page); zswap_written_back_pages++; spin_lock(&tree->lock); /* drop local reference */ zswap_entry_put(tree, entry); /* * There are two possible situations for entry here: * (1) refcount is 1(normal case), entry is valid and on the tree * (2) refcount is 0, entry is freed and not on the tree * because invalidate happened during writeback * search the tree and free the entry if find entry */ if (entry == zswap_rb_search(&tree->rbroot, offset)) zswap_entry_put(tree, entry); spin_unlock(&tree->lock); goto end; /* * if we get here due to ZSWAP_SWAPCACHE_EXIST * a load may happening concurrently * it is safe and okay to not free the entry * if we free the entry in the following put * it it either okay to return !0 */ fail: spin_lock(&tree->lock); zswap_entry_put(tree, entry); spin_unlock(&tree->lock); end: return ret; }
static ssize_t nilfs_cpfile_do_get_ssinfo(struct inode *cpfile, __u64 *cnop, void *buf, unsigned cisz, size_t nci) { struct buffer_head *bh; struct nilfs_cpfile_header *header; struct nilfs_checkpoint *cp; struct nilfs_cpinfo *ci = buf; __u64 curr = *cnop, next; unsigned long curr_blkoff, next_blkoff; void *kaddr; int n = 0, ret; down_read(&NILFS_MDT(cpfile)->mi_sem); if (curr == 0) { ret = nilfs_cpfile_get_header_block(cpfile, &bh); if (ret < 0) goto out; kaddr = kmap_atomic(bh->b_page); header = nilfs_cpfile_block_get_header(cpfile, bh, kaddr); curr = le64_to_cpu(header->ch_snapshot_list.ssl_next); kunmap_atomic(kaddr); brelse(bh); if (curr == 0) { ret = 0; goto out; } } else if (unlikely(curr == ~(__u64)0)) { ret = 0; goto out; } curr_blkoff = nilfs_cpfile_get_blkoff(cpfile, curr); ret = nilfs_cpfile_get_checkpoint_block(cpfile, curr, 0, &bh); if (unlikely(ret < 0)) { if (ret == -ENOENT) ret = 0; /* No snapshots (started from a hole block) */ goto out; } kaddr = kmap_atomic(bh->b_page); while (n < nci) { cp = nilfs_cpfile_block_get_checkpoint(cpfile, curr, bh, kaddr); curr = ~(__u64)0; /* Terminator */ if (unlikely(nilfs_checkpoint_invalid(cp) || !nilfs_checkpoint_snapshot(cp))) break; nilfs_cpfile_checkpoint_to_cpinfo(cpfile, cp, ci); ci = (void *)ci + cisz; n++; next = le64_to_cpu(cp->cp_snapshot_list.ssl_next); if (next == 0) break; /* reach end of the snapshot list */ next_blkoff = nilfs_cpfile_get_blkoff(cpfile, next); if (curr_blkoff != next_blkoff) { kunmap_atomic(kaddr); brelse(bh); ret = nilfs_cpfile_get_checkpoint_block(cpfile, next, 0, &bh); if (unlikely(ret < 0)) { WARN_ON(ret == -ENOENT); goto out; } kaddr = kmap_atomic(bh->b_page); } curr = next; curr_blkoff = next_blkoff; } kunmap_atomic(kaddr); brelse(bh); *cnop = curr; ret = n; out: up_read(&NILFS_MDT(cpfile)->mi_sem); return ret; }
/* attempts to compress and store an single page */ static int zswap_frontswap_store(unsigned type, pgoff_t offset, struct page *page) { struct zswap_tree *tree = zswap_trees[type]; struct zswap_entry *entry, *dupentry; int ret; unsigned int dlen = PAGE_SIZE, len; unsigned long handle; char *buf; u8 *src, *dst; struct zswap_header *zhdr; if (!tree) { ret = -ENODEV; goto reject; } /* reclaim space if needed */ if (zswap_is_full()) { zswap_pool_limit_hit++; if (zbud_reclaim_page(tree->pool, 8)) { zswap_reject_reclaim_fail++; ret = -ENOMEM; goto reject; } } /* allocate entry */ entry = zswap_entry_cache_alloc(GFP_KERNEL); if (!entry) { zswap_reject_kmemcache_fail++; ret = -ENOMEM; goto reject; } /* compress */ dst = get_cpu_var(zswap_dstmem); src = kmap_atomic(page); ret = zswap_comp_op(ZSWAP_COMPOP_COMPRESS, src, PAGE_SIZE, dst, &dlen); kunmap_atomic(src); if (ret) { ret = -EINVAL; goto freepage; } /* store */ len = dlen + sizeof(struct zswap_header); ret = zbud_alloc(tree->pool, len, __GFP_NORETRY | __GFP_NOWARN, &handle); if (ret == -ENOSPC) { zswap_reject_compress_poor++; goto freepage; } if (ret) { zswap_reject_alloc_fail++; goto freepage; } zhdr = zbud_map(tree->pool, handle); zhdr->swpentry = swp_entry(type, offset); buf = (u8 *)(zhdr + 1); memcpy(buf, dst, dlen); zbud_unmap(tree->pool, handle); put_cpu_var(zswap_dstmem); /* populate entry */ entry->offset = offset; entry->handle = handle; entry->length = dlen; /* map */ spin_lock(&tree->lock); do { ret = zswap_rb_insert(&tree->rbroot, entry, &dupentry); if (ret == -EEXIST) { zswap_duplicate_entry++; /* remove from rbtree */ zswap_rb_erase(&tree->rbroot, dupentry); zswap_entry_put(tree, dupentry); } } while (ret == -EEXIST); spin_unlock(&tree->lock); /* update stats */ atomic_inc(&zswap_stored_pages); zswap_pool_pages = zbud_get_pool_size(tree->pool); return 0; freepage: put_cpu_var(zswap_dstmem); zswap_entry_cache_free(entry); reject: return ret; }
static int nilfs_cpfile_clear_snapshot(struct inode *cpfile, __u64 cno) { struct buffer_head *header_bh, *next_bh, *prev_bh, *cp_bh; struct nilfs_cpfile_header *header; struct nilfs_checkpoint *cp; struct nilfs_snapshot_list *list; __u64 next, prev; void *kaddr; int ret; if (cno == 0) return -ENOENT; /* checkpoint number 0 is invalid */ down_write(&NILFS_MDT(cpfile)->mi_sem); ret = nilfs_cpfile_get_checkpoint_block(cpfile, cno, 0, &cp_bh); if (ret < 0) goto out_sem; kaddr = kmap_atomic(cp_bh->b_page); cp = nilfs_cpfile_block_get_checkpoint(cpfile, cno, cp_bh, kaddr); if (nilfs_checkpoint_invalid(cp)) { ret = -ENOENT; kunmap_atomic(kaddr); goto out_cp; } if (!nilfs_checkpoint_snapshot(cp)) { ret = 0; kunmap_atomic(kaddr); goto out_cp; } list = &cp->cp_snapshot_list; next = le64_to_cpu(list->ssl_next); prev = le64_to_cpu(list->ssl_prev); kunmap_atomic(kaddr); ret = nilfs_cpfile_get_header_block(cpfile, &header_bh); if (ret < 0) goto out_cp; if (next != 0) { ret = nilfs_cpfile_get_checkpoint_block(cpfile, next, 0, &next_bh); if (ret < 0) goto out_header; } else { next_bh = header_bh; get_bh(next_bh); } if (prev != 0) { ret = nilfs_cpfile_get_checkpoint_block(cpfile, prev, 0, &prev_bh); if (ret < 0) goto out_next; } else { prev_bh = header_bh; get_bh(prev_bh); } kaddr = kmap_atomic(next_bh->b_page); list = nilfs_cpfile_block_get_snapshot_list( cpfile, next, next_bh, kaddr); list->ssl_prev = cpu_to_le64(prev); kunmap_atomic(kaddr); kaddr = kmap_atomic(prev_bh->b_page); list = nilfs_cpfile_block_get_snapshot_list( cpfile, prev, prev_bh, kaddr); list->ssl_next = cpu_to_le64(next); kunmap_atomic(kaddr); kaddr = kmap_atomic(cp_bh->b_page); cp = nilfs_cpfile_block_get_checkpoint(cpfile, cno, cp_bh, kaddr); cp->cp_snapshot_list.ssl_next = cpu_to_le64(0); cp->cp_snapshot_list.ssl_prev = cpu_to_le64(0); nilfs_checkpoint_clear_snapshot(cp); kunmap_atomic(kaddr); kaddr = kmap_atomic(header_bh->b_page); header = nilfs_cpfile_block_get_header(cpfile, header_bh, kaddr); le64_add_cpu(&header->ch_nsnapshots, -1); kunmap_atomic(kaddr); mark_buffer_dirty(next_bh); mark_buffer_dirty(prev_bh); mark_buffer_dirty(cp_bh); mark_buffer_dirty(header_bh); nilfs_mdt_mark_dirty(cpfile); brelse(prev_bh); out_next: brelse(next_bh); out_header: brelse(header_bh); out_cp: brelse(cp_bh); out_sem: up_write(&NILFS_MDT(cpfile)->mi_sem); return ret; }
static int msmsdcc_pio_irq(int irq, void *dev_id) { struct msmsdcc_host *host = dev_id; void __iomem *base = host->base; uint32_t status; status = readl(base + MMCISTATUS); #if IRQ_DEBUG msmsdcc_print_status(host, "irq1-r", status); #endif do { unsigned long flags; unsigned int remain, len; char *buffer; if (!(status & (MCI_TXFIFOHALFEMPTY | MCI_RXDATAAVLBL))) break; /* Map the current scatter buffer */ local_irq_save(flags); buffer = kmap_atomic(sg_page(host->pio.sg), KM_BIO_SRC_IRQ) + host->pio.sg->offset; buffer += host->pio.sg_off; remain = host->pio.sg->length - host->pio.sg_off; len = 0; if (status & MCI_RXACTIVE) len = msmsdcc_pio_read(host, buffer, remain); if (status & MCI_TXACTIVE) len = msmsdcc_pio_write(host, buffer, remain, status); /* Unmap the buffer */ kunmap_atomic(buffer, KM_BIO_SRC_IRQ); local_irq_restore(flags); host->pio.sg_off += len; host->curr.xfer_remain -= len; host->curr.data_xfered += len; remain -= len; if (remain) /* Done with this page? */ break; /* Nope */ if (status & MCI_RXACTIVE && host->curr.user_pages) flush_dcache_page(sg_page(host->pio.sg)); if (!--host->pio.sg_len) { memset(&host->pio, 0, sizeof(host->pio)); break; } /* Advance to next sg */ host->pio.sg++; host->pio.sg_off = 0; status = readl(base + MMCISTATUS); } while (1); if (status & MCI_RXACTIVE && host->curr.xfer_remain < MCI_FIFOSIZE) writel(MCI_RXDATAAVLBLMASK, base + MMCIMASK1); if (!host->curr.xfer_remain) writel(0, base + MMCIMASK1); return IRQ_HANDLED; }
static void put_ptr_atomic(void *ptr, enum km_type type) { kunmap_atomic(ptr, type); }
static void __bm_unmap(unsigned long *p_addr, const enum km_type km) { kunmap_atomic(p_addr, km); };