/**
* nilfs_copy_buffer -- copy buffer data and flags
* @dbh: destination buffer
* @sbh: source buffer
*/
void nilfs_copy_buffer(struct buffer_head *dbh, struct buffer_head *sbh)
{
void *kaddr0, *kaddr1;
unsigned long bits;
struct page *spage = sbh->b_page, *dpage = dbh->b_page;
struct buffer_head *bh;
kaddr0 = kmap_atomic(spage);
kaddr1 = kmap_atomic(dpage);
memcpy(kaddr1 + bh_offset(dbh), kaddr0 + bh_offset(sbh), sbh->b_size);
kunmap_atomic(kaddr1);
kunmap_atomic(kaddr0);
dbh->b_state = sbh->b_state & NILFS_BUFFER_INHERENT_BITS;
dbh->b_blocknr = sbh->b_blocknr;
dbh->b_bdev = sbh->b_bdev;
bh = dbh;
bits = sbh->b_state & (BIT(BH_Uptodate) | BIT(BH_Mapped));
while ((bh = bh->b_this_page) != dbh) {
lock_buffer(bh);
bits &= bh->b_state;
unlock_buffer(bh);
}
if (bits & BIT(BH_Uptodate))
SetPageUptodate(dpage);
else
ClearPageUptodate(dpage);
if (bits & BIT(BH_Mapped))
SetPageMappedToDisk(dpage);
else
ClearPageMappedToDisk(dpage);
}
static void ext4_finish_bio(struct bio *bio)
{
int i;
struct bio_vec *bvec;
bio_for_each_segment_all(bvec, bio, i) {
struct page *page = bvec->bv_page;
#ifdef CONFIG_EXT4_FS_ENCRYPTION
struct page *data_page = NULL;
#endif
struct buffer_head *bh, *head;
unsigned bio_start = bvec->bv_offset;
unsigned bio_end = bio_start + bvec->bv_len;
unsigned under_io = 0;
unsigned long flags;
if (!page)
continue;
#ifdef CONFIG_EXT4_FS_ENCRYPTION
if (!page->mapping) {
/* The bounce data pages are unmapped. */
data_page = page;
fscrypt_pullback_bio_page(&page, false);
}
#endif
if (bio->bi_error) {
SetPageError(page);
mapping_set_error(page->mapping, -EIO);
}
bh = head = page_buffers(page);
/*
* We check all buffers in the page under BH_Uptodate_Lock
* to avoid races with other end io clearing async_write flags
*/
local_irq_save(flags);
bit_spin_lock(BH_Uptodate_Lock, &head->b_state);
do {
if (bh_offset(bh) < bio_start ||
bh_offset(bh) + bh->b_size > bio_end) {
if (buffer_async_write(bh))
under_io++;
continue;
}
clear_buffer_async_write(bh);
if (bio->bi_error)
buffer_io_error(bh);
} while ((bh = bh->b_this_page) != head);
bit_spin_unlock(BH_Uptodate_Lock, &head->b_state);
local_irq_restore(flags);
if (!under_io) {
#ifdef CONFIG_EXT4_FS_ENCRYPTION
if (data_page)
fscrypt_restore_control_page(data_page);
#endif
end_page_writeback(page);
}
}
}
int blockio(int rw, struct sb *sb, struct buffer_head *buffer, block_t block)
{
struct bio_vec vec = {
.bv_page = buffer->b_page,
.bv_offset = bh_offset(buffer),
.bv_len = sb->blocksize,
};
return syncio(rw, sb_dev(sb), block << sb->blockbits, 1, &vec);
}
/*
* bufvec based I/O. This takes the bufvec has contiguous range, and
* will submit the count of buffers to block (physical address).
*
* If there was I/O error, it would be handled in ->bi_end_bio()
* completion.
*/
int blockio_vec(int rw, struct bufvec *bufvec, block_t block, unsigned count)
{
return bufvec_io(rw, bufvec, block, count);
}
void hexdump(void *data, unsigned size)
{
print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS, 16, 1, data, size, 1);
}
static inline struct nilfs_cpfile_header *
nilfs_cpfile_block_get_header(const struct inode *cpfile,
struct buffer_head *bh,
void *kaddr)
{
return kaddr + bh_offset(bh);
}
/**
* nilfs_palloc_abort_alloc_entry - cancel allocation of a persistent object
* @inode: inode of metadata file using this allocator
* @req: nilfs_palloc_req structure exchanged for the allocation
*/
void nilfs_palloc_abort_alloc_entry(struct inode *inode,
struct nilfs_palloc_req *req)
{
struct nilfs_palloc_group_desc *desc;
void *desc_kaddr, *bitmap_kaddr;
unsigned char *bitmap;
unsigned long group, group_offset;
spinlock_t *lock;
group = nilfs_palloc_group(inode, req->pr_entry_nr, &group_offset);
desc_kaddr = kmap(req->pr_desc_bh->b_page);
desc = nilfs_palloc_block_get_group_desc(inode, group,
req->pr_desc_bh, desc_kaddr);
bitmap_kaddr = kmap(req->pr_bitmap_bh->b_page);
bitmap = bitmap_kaddr + bh_offset(req->pr_bitmap_bh);
lock = nilfs_mdt_bgl_lock(inode, group);
if (!nilfs_clear_bit_atomic(lock, group_offset, bitmap))
nilfs_msg(inode->i_sb, KERN_WARNING,
"%s (ino=%lu): entry number %llu already freed",
__func__, inode->i_ino,
(unsigned long long)req->pr_entry_nr);
else
nilfs_palloc_group_desc_add_entries(desc, lock, 1);
kunmap(req->pr_bitmap_bh->b_page);
kunmap(req->pr_desc_bh->b_page);
brelse(req->pr_bitmap_bh);
brelse(req->pr_desc_bh);
req->pr_entry_nr = 0;
req->pr_bitmap_bh = NULL;
req->pr_desc_bh = NULL;
}
static int
nilfs_mdt_insert_new_block(struct inode *inode, unsigned long block,
struct buffer_head *bh,
void (*init_block)(struct inode *,
struct buffer_head *, void *))
{
struct nilfs_inode_info *ii = NILFS_I(inode);
void *kaddr;
int ret;
/* Caller exclude read accesses using page lock */
/* set_buffer_new(bh); */
bh->b_blocknr = 0;
ret = nilfs_bmap_insert(ii->i_bmap, block, (unsigned long)bh);
if (unlikely(ret))
return ret;
set_buffer_mapped(bh);
kaddr = kmap_atomic(bh->b_page, KM_USER0);
memset(kaddr + bh_offset(bh), 0, 1 << inode->i_blkbits);
if (init_block)
init_block(inode, bh, kaddr);
flush_dcache_page(bh->b_page);
kunmap_atomic(kaddr, KM_USER0);
set_buffer_uptodate(bh);
nilfs_mark_buffer_dirty(bh);
nilfs_mdt_mark_dirty(inode);
return 0;
}
/**
* nilfs_palloc_abort_alloc_entry - cancel allocation of a persistent object
* @inode: inode of metadata file using this allocator
* @req: nilfs_palloc_req structure exchanged for the allocation
*/
void nilfs_palloc_abort_alloc_entry(struct inode *inode,
struct nilfs_palloc_req *req)
{
struct nilfs_palloc_group_desc *desc;
void *desc_kaddr, *bitmap_kaddr;
unsigned char *bitmap;
unsigned long group, group_offset;
group = nilfs_palloc_group(inode, req->pr_entry_nr, &group_offset);
desc_kaddr = kmap(req->pr_desc_bh->b_page);
desc = nilfs_palloc_block_get_group_desc(inode, group,
req->pr_desc_bh, desc_kaddr);
bitmap_kaddr = kmap(req->pr_bitmap_bh->b_page);
bitmap = bitmap_kaddr + bh_offset(req->pr_bitmap_bh);
if (!nilfs_clear_bit_atomic(nilfs_mdt_bgl_lock(inode, group),
group_offset, bitmap))
printk(KERN_WARNING "%s: entry number %llu already freed\n",
__func__, (unsigned long long)req->pr_entry_nr);
nilfs_palloc_group_desc_add_entries(inode, group, desc, 1);
kunmap(req->pr_bitmap_bh->b_page);
kunmap(req->pr_desc_bh->b_page);
brelse(req->pr_bitmap_bh);
brelse(req->pr_desc_bh);
req->pr_entry_nr = 0;
req->pr_bitmap_bh = NULL;
req->pr_desc_bh = NULL;
}
int nilfs_sufile_get_stat(struct inode *sufile, struct nilfs_sustat *sustat)
{
struct buffer_head *header_bh;
struct nilfs_sufile_header *header;
struct the_nilfs *nilfs = sufile->i_sb->s_fs_info;
void *kaddr;
int ret;
down_read(&NILFS_MDT(sufile)->mi_sem);
ret = nilfs_sufile_get_header_block(sufile, &header_bh);
if (ret < 0)
goto out_sem;
kaddr = kmap_atomic(header_bh->b_page);
header = kaddr + bh_offset(header_bh);
sustat->ss_nsegs = nilfs_sufile_get_nsegments(sufile);
sustat->ss_ncleansegs = le64_to_cpu(header->sh_ncleansegs);
sustat->ss_ndirtysegs = le64_to_cpu(header->sh_ndirtysegs);
sustat->ss_ctime = nilfs->ns_ctime;
sustat->ss_nongc_ctime = nilfs->ns_nongc_ctime;
spin_lock(&nilfs->ns_last_segment_lock);
sustat->ss_prot_seq = nilfs->ns_prot_seq;
spin_unlock(&nilfs->ns_last_segment_lock);
kunmap_atomic(kaddr);
brelse(header_bh);
out_sem:
up_read(&NILFS_MDT(sufile)->mi_sem);
return ret;
}
static int io_submit_init(struct ext4_io_submit *io,
struct inode *inode,
struct writeback_control *wbc,
struct buffer_head *bh)
{
ext4_io_end_t *io_end;
struct page *page = bh->b_page;
int nvecs = bio_get_nr_vecs(bh->b_bdev);
struct bio *bio;
io_end = ext4_init_io_end(inode, GFP_NOFS);
if (!io_end)
return -ENOMEM;
bio = bio_alloc(GFP_NOIO, min(nvecs, BIO_MAX_PAGES));
bio->bi_sector = bh->b_blocknr * (bh->b_size >> 9);
bio->bi_bdev = bh->b_bdev;
bio->bi_private = io->io_end = io_end;
bio->bi_end_io = ext4_end_bio;
io_end->offset = (page->index << PAGE_CACHE_SHIFT) + bh_offset(bh);
io->io_bio = bio;
io->io_op = (wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : WRITE);
io->io_next_block = bh->b_blocknr;
return 0;
}
/**
* nilfs_palloc_block_get_group_desc - get kernel address of a group descriptor
* @inode: inode of metadata file using this allocator
* @group: group number
* @bh: buffer head of the buffer storing the group descriptor block
* @kaddr: kernel address mapped for the page including the buffer
*/
static struct nilfs_palloc_group_desc *
nilfs_palloc_block_get_group_desc(const struct inode *inode,
unsigned long group,
const struct buffer_head *bh, void *kaddr)
{
return (struct nilfs_palloc_group_desc *)(kaddr + bh_offset(bh)) +
group % nilfs_palloc_groups_per_desc_block(inode);
}
static struct nilfs_checkpoint *
nilfs_cpfile_block_get_checkpoint(const struct inode *cpfile, __u64 cno,
struct buffer_head *bh,
void *kaddr)
{
return kaddr + bh_offset(bh) + nilfs_cpfile_get_offset(cpfile, cno) *
NILFS_MDT(cpfile)->mi_entry_size;
}
static struct nilfs_segment_usage *
nilfs_sufile_block_get_segment_usage(const struct inode *sufile, __u64 segnum,
struct buffer_head *bh, void *kaddr)
{
return kaddr + bh_offset(bh) +
nilfs_sufile_get_offset(sufile, segnum) *
NILFS_MDT(sufile)->mi_entry_size;
}
static void ext4_finish_bio(struct bio *bio)
{
int i;
int error = !test_bit(BIO_UPTODATE, &bio->bi_flags);
struct bio_vec *bvec;
bio_for_each_segment_all(bvec, bio, i) {
struct page *page = bvec->bv_page;
struct buffer_head *bh, *head;
unsigned bio_start = bvec->bv_offset;
unsigned bio_end = bio_start + bvec->bv_len;
unsigned under_io = 0;
unsigned long flags;
if (!page)
continue;
if (error) {
SetPageError(page);
set_bit(AS_EIO, &page->mapping->flags);
}
bh = head = page_buffers(page);
/*
* We check all buffers in the page under BH_Uptodate_Lock
* to avoid races with other end io clearing async_write flags
*/
local_irq_save(flags);
bit_spin_lock(BH_Uptodate_Lock, &head->b_state);
do {
if (bh_offset(bh) < bio_start ||
bh_offset(bh) + bh->b_size > bio_end) {
if (buffer_async_write(bh))
under_io++;
continue;
}
clear_buffer_async_write(bh);
if (error)
buffer_io_error(bh);
} while ((bh = bh->b_this_page) != head);
bit_spin_unlock(BH_Uptodate_Lock, &head->b_state);
local_irq_restore(flags);
if (!under_io)
end_page_writeback(page);
}
}
/**
* "Copied from drivers/ide/ide-dma.c"
* sgiioc4_ide_build_sglist - map IDE scatter gather for DMA I/O
* @hwif: the interface to build the DMA table for
* @rq: the request holding the sg list
* @ddir: data direction
*
* Perform the PCI mapping magic neccessary to access the source
* or target buffers of a request via PCI DMA. The lower layers
* of the kernel provide the neccessary cache management so that
* we can operate in a portable fashion.
*
* This code is identical to ide_build_sglist in ide-dma.c
* however that it not exported and even if it were would create
* dependancy problems for modular drivers.
*/
static int
sgiioc4_ide_build_sglist(ide_hwif_t * hwif, struct request *rq, int ddir)
{
struct buffer_head *bh;
struct scatterlist *sg = hwif->sg_table;
unsigned long lastdataend = ~0UL;
int nents = 0;
if (hwif->sg_dma_active)
BUG();
bh = rq->bh;
do {
int contig = 0;
if (bh->b_page) {
if (bh_phys(bh) == lastdataend)
contig = 1;
} else {
if ((unsigned long) bh->b_data == lastdataend)
contig = 1;
}
if (contig) {
sg[nents - 1].length += bh->b_size;
lastdataend += bh->b_size;
continue;
}
if (nents >= PRD_ENTRIES)
return 0;
memset(&sg[nents], 0, sizeof (*sg));
if (bh->b_page) {
sg[nents].page = bh->b_page;
sg[nents].offset = bh_offset(bh);
lastdataend = bh_phys(bh) + bh->b_size;
} else {
if ((unsigned long) bh->b_data < PAGE_SIZE)
BUG();
sg[nents].address = bh->b_data;
lastdataend = (unsigned long) bh->b_data + bh->b_size;
}
sg[nents].length = bh->b_size;
nents++;
} while ((bh = bh->b_reqnext) != NULL);
if (nents == 0)
BUG();
hwif->sg_dma_direction = ddir;
return pci_map_sg(hwif->pci_dev, sg, nents, ddir);
}
/**
* nilfs_palloc_freev - deallocate a set of persistent objects
* @inode: inode of metadata file using this allocator
* @entry_nrs: array of entry numbers to be deallocated
* @nitems: number of entries stored in @entry_nrs
*/
int nilfs_palloc_freev(struct inode *inode, __u64 *entry_nrs, size_t nitems)
{
struct buffer_head *desc_bh, *bitmap_bh;
struct nilfs_palloc_group_desc *desc;
unsigned char *bitmap;
void *desc_kaddr, *bitmap_kaddr;
unsigned long group, group_offset;
int i, j, n, ret;
for (i = 0; i < nitems; i = j) {
group = nilfs_palloc_group(inode, entry_nrs[i], &group_offset);
ret = nilfs_palloc_get_desc_block(inode, group, 0, &desc_bh);
if (ret < 0)
return ret;
ret = nilfs_palloc_get_bitmap_block(inode, group, 0,
&bitmap_bh);
if (ret < 0) {
brelse(desc_bh);
return ret;
}
desc_kaddr = kmap(desc_bh->b_page);
desc = nilfs_palloc_block_get_group_desc(
inode, group, desc_bh, desc_kaddr);
bitmap_kaddr = kmap(bitmap_bh->b_page);
bitmap = bitmap_kaddr + bh_offset(bitmap_bh);
for (j = i, n = 0;
(j < nitems) && nilfs_palloc_group_is_in(inode, group,
entry_nrs[j]);
j++) {
nilfs_palloc_group(inode, entry_nrs[j], &group_offset);
if (!nilfs_clear_bit_atomic(
nilfs_mdt_bgl_lock(inode, group),
group_offset, bitmap)) {
printk(KERN_WARNING
"%s: entry number %llu already freed\n",
__func__,
(unsigned long long)entry_nrs[j]);
} else {
n++;
}
}
nilfs_palloc_group_desc_add_entries(inode, group, desc, n);
kunmap(bitmap_bh->b_page);
kunmap(desc_bh->b_page);
nilfs_mdt_mark_buffer_dirty(desc_bh);
nilfs_mdt_mark_buffer_dirty(bitmap_bh);
nilfs_mdt_mark_dirty(inode);
brelse(bitmap_bh);
brelse(desc_bh);
}
return 0;
}
/**
* nilfs_palloc_block_get_entry - get kernel address of an entry
* @inode: inode of metadata file using this allocator
* @nr: serial number of the entry (e.g. inode number)
* @bh: buffer head of the buffer storing the entry block
* @kaddr: kernel address mapped for the page including the buffer
*/
void *nilfs_palloc_block_get_entry(const struct inode *inode, __u64 nr,
const struct buffer_head *bh, void *kaddr)
{
unsigned long entry_offset, group_offset;
nilfs_palloc_group(inode, nr, &group_offset);
entry_offset = group_offset % NILFS_MDT(inode)->mi_entries_per_block;
return kaddr + bh_offset(bh) +
entry_offset * NILFS_MDT(inode)->mi_entry_size;
}
/**
* nilfs_palloc_desc_block_init - initialize buffer of a group descriptor block
* @inode: inode of metadata file
* @bh: buffer head of the buffer to be initialized
* @kaddr: kernel address mapped for the page including the buffer
*/
static void nilfs_palloc_desc_block_init(struct inode *inode,
struct buffer_head *bh, void *kaddr)
{
struct nilfs_palloc_group_desc *desc = kaddr + bh_offset(bh);
unsigned long n = nilfs_palloc_groups_per_desc_block(inode);
__le32 nfrees;
nfrees = cpu_to_le32(nilfs_palloc_entries_per_group(inode));
while (n-- > 0) {
desc->pg_nfrees = nfrees;
desc++;
}
}
static void nilfs_cpfile_block_init(struct inode *cpfile,
struct buffer_head *bh,
void *kaddr)
{
struct nilfs_checkpoint *cp = kaddr + bh_offset(bh);
size_t cpsz = NILFS_MDT(cpfile)->mi_entry_size;
int n = nilfs_cpfile_checkpoints_per_block(cpfile);
while (n-- > 0) {
nilfs_checkpoint_set_invalid(cp);
cp = (void *)cp + cpsz;
}
}
static unsigned int
nilfs_cpfile_block_add_valid_checkpoints(const struct inode *cpfile,
struct buffer_head *bh,
void *kaddr,
unsigned int n)
{
struct nilfs_checkpoint *cp = kaddr + bh_offset(bh);
unsigned int count;
count = le32_to_cpu(cp->cp_checkpoints_count) + n;
cp->cp_checkpoints_count = cpu_to_le32(count);
return count;
}
static int io_submit_add_bh(struct ext4_io_submit *io,
struct ext4_io_page *io_page,
struct inode *inode,
struct writeback_control *wbc,
struct buffer_head *bh)
{
ext4_io_end_t *io_end;
int ret;
if (buffer_new(bh)) {
clear_buffer_new(bh);
unmap_underlying_metadata(bh->b_bdev, bh->b_blocknr);
}
if (!buffer_mapped(bh) || buffer_delay(bh)) {
if (!buffer_mapped(bh))
clear_buffer_dirty(bh);
if (io->io_bio)
ext4_io_submit(io);
return 0;
}
if (io->io_bio && bh->b_blocknr != io->io_next_block) {
submit_and_retry:
ext4_io_submit(io);
}
if (io->io_bio == NULL) {
ret = io_submit_init(io, inode, wbc, bh);
if (ret)
return ret;
}
io_end = io->io_end;
if ((io_end->num_io_pages >= MAX_IO_PAGES) &&
(io_end->pages[io_end->num_io_pages-1] != io_page))
goto submit_and_retry;
if (buffer_uninit(bh) && !(io_end->flag & EXT4_IO_END_UNWRITTEN)) {
io_end->flag |= EXT4_IO_END_UNWRITTEN;
atomic_inc(&EXT4_I(inode)->i_aiodio_unwritten);
}
io->io_end->size += bh->b_size;
io->io_next_block++;
ret = bio_add_page(io->io_bio, bh->b_page, bh->b_size, bh_offset(bh));
if (ret != bh->b_size)
goto submit_and_retry;
if ((io_end->num_io_pages == 0) ||
(io_end->pages[io_end->num_io_pages-1] != io_page)) {
io_end->pages[io_end->num_io_pages++] = io_page;
atomic_inc(&io_page->p_count);
}
return 0;
}
int nilfs_sufile_resize(struct inode *sufile, __u64 newnsegs)
{
struct the_nilfs *nilfs = sufile->i_sb->s_fs_info;
struct buffer_head *header_bh;
struct nilfs_sufile_header *header;
struct nilfs_sufile_info *sui = NILFS_SUI(sufile);
void *kaddr;
unsigned long nsegs, nrsvsegs;
int ret = 0;
down_write(&NILFS_MDT(sufile)->mi_sem);
nsegs = nilfs_sufile_get_nsegments(sufile);
if (nsegs == newnsegs)
goto out;
ret = -ENOSPC;
nrsvsegs = nilfs_nrsvsegs(nilfs, newnsegs);
if (newnsegs < nsegs && nsegs - newnsegs + nrsvsegs > sui->ncleansegs)
goto out;
ret = nilfs_sufile_get_header_block(sufile, &header_bh);
if (ret < 0)
goto out;
if (newnsegs > nsegs) {
sui->ncleansegs += newnsegs - nsegs;
} else {
ret = nilfs_sufile_truncate_range(sufile, newnsegs, nsegs - 1);
if (ret < 0)
goto out_header;
sui->ncleansegs -= nsegs - newnsegs;
}
kaddr = kmap_atomic(header_bh->b_page);
header = kaddr + bh_offset(header_bh);
header->sh_ncleansegs = cpu_to_le64(sui->ncleansegs);
kunmap_atomic(kaddr);
mark_buffer_dirty(header_bh);
nilfs_mdt_mark_dirty(sufile);
nilfs_set_nsegments(nilfs, newnsegs);
out_header:
brelse(header_bh);
out:
up_write(&NILFS_MDT(sufile)->mi_sem);
return ret;
}
static void nilfs_sufile_mod_counter(struct buffer_head *header_bh,
u64 ncleanadd, u64 ndirtyadd)
{
struct nilfs_sufile_header *header;
void *kaddr;
kaddr = kmap_atomic(header_bh->b_page);
header = kaddr + bh_offset(header_bh);
le64_add_cpu(&header->sh_ncleansegs, ncleanadd);
le64_add_cpu(&header->sh_ndirtysegs, ndirtyadd);
kunmap_atomic(kaddr);
mark_buffer_dirty(header_bh);
}
int nilfs_sufile_read(struct super_block *sb, size_t susize,
struct nilfs_inode *raw_inode, struct inode **inodep)
{
struct inode *sufile;
struct nilfs_sufile_info *sui;
struct buffer_head *header_bh;
struct nilfs_sufile_header *header;
void *kaddr;
int err;
sufile = nilfs_iget_locked(sb, NULL, NILFS_SUFILE_INO);
if (unlikely(!sufile))
return -ENOMEM;
if (!(sufile->i_state & I_NEW))
goto out;
err = nilfs_mdt_init(sufile, NILFS_MDT_GFP, sizeof(*sui));
if (err)
goto failed;
nilfs_mdt_set_entry_size(sufile, susize,
sizeof(struct nilfs_sufile_header));
err = nilfs_read_inode_common(sufile, raw_inode);
if (err)
goto failed;
err = nilfs_sufile_get_header_block(sufile, &header_bh);
if (err)
goto failed;
sui = NILFS_SUI(sufile);
kaddr = kmap_atomic(header_bh->b_page);
header = kaddr + bh_offset(header_bh);
sui->ncleansegs = le64_to_cpu(header->sh_ncleansegs);
kunmap_atomic(kaddr);
brelse(header_bh);
sui->allocmax = nilfs_sufile_get_nsegments(sufile) - 1;
sui->allocmin = 0;
unlock_new_inode(sufile);
out:
*inodep = sufile;
return 0;
failed:
iget_failed(sufile);
return err;
}
static int nilfs_recovery_copy_block(struct the_nilfs *nilfs,
struct nilfs_recovery_block *rb,
struct page *page)
{
struct buffer_head *bh_org;
void *kaddr;
bh_org = __bread(nilfs->ns_bdev, rb->blocknr, nilfs->ns_blocksize);
if (unlikely(!bh_org))
return -EIO;
kaddr = kmap_atomic(page);
memcpy(kaddr + bh_offset(bh_org), bh_org->b_data, bh_org->b_size);
kunmap_atomic(kaddr);
brelse(bh_org);
return 0;
}
static int nilfs_recovery_copy_block(struct nilfs_sb_info *sbi,
struct nilfs_recovery_block *rb,
struct page *page)
{
struct buffer_head *bh_org;
void *kaddr;
bh_org = sb_bread(sbi->s_super, rb->blocknr);
if (unlikely(!bh_org))
return -EIO;
kaddr = kmap_atomic(page, KM_USER0);
memcpy(kaddr + bh_offset(bh_org), bh_org->b_data, bh_org->b_size);
kunmap_atomic(kaddr, KM_USER0);
brelse(bh_org);
return 0;
}
int nilfs_sufile_read(struct inode *sufile, struct nilfs_inode *raw_inode)
{
struct nilfs_sufile_info *sui = NILFS_SUI(sufile);
struct buffer_head *header_bh;
struct nilfs_sufile_header *header;
void *kaddr;
int ret;
ret = nilfs_read_inode_common(sufile, raw_inode);
if (ret < 0)
return ret;
ret = nilfs_sufile_get_header_block(sufile, &header_bh);
if (!ret) {
kaddr = kmap_atomic(header_bh->b_page, KM_USER0);
header = kaddr + bh_offset(header_bh);
sui->ncleansegs = le64_to_cpu(header->sh_ncleansegs);
kunmap_atomic(kaddr, KM_USER0);
brelse(header_bh);
}
return ret;
}
int nilfs_sufile_alloc(struct inode *sufile, __u64 *segnump)
{
struct buffer_head *header_bh, *su_bh;
struct nilfs_sufile_header *header;
struct nilfs_segment_usage *su;
struct nilfs_sufile_info *sui = NILFS_SUI(sufile);
size_t susz = NILFS_MDT(sufile)->mi_entry_size;
__u64 segnum, maxsegnum, last_alloc;
void *kaddr;
unsigned long nsegments, ncleansegs, nsus, cnt;
int ret, j;
down_write(&NILFS_MDT(sufile)->mi_sem);
ret = nilfs_sufile_get_header_block(sufile, &header_bh);
if (ret < 0)
goto out_sem;
kaddr = kmap_atomic(header_bh->b_page);
header = kaddr + bh_offset(header_bh);
ncleansegs = le64_to_cpu(header->sh_ncleansegs);
last_alloc = le64_to_cpu(header->sh_last_alloc);
kunmap_atomic(kaddr);
nsegments = nilfs_sufile_get_nsegments(sufile);
maxsegnum = sui->allocmax;
segnum = last_alloc + 1;
if (segnum < sui->allocmin || segnum > sui->allocmax)
segnum = sui->allocmin;
for (cnt = 0; cnt < nsegments; cnt += nsus) {
if (segnum > maxsegnum) {
if (cnt < sui->allocmax - sui->allocmin + 1) {
segnum = sui->allocmin;
maxsegnum = last_alloc;
} else if (segnum > sui->allocmin &&
sui->allocmax + 1 < nsegments) {
segnum = sui->allocmax + 1;
maxsegnum = nsegments - 1;
} else if (sui->allocmin > 0) {
segnum = 0;
maxsegnum = sui->allocmin - 1;
} else {
break;
}
}
ret = nilfs_sufile_get_segment_usage_block(sufile, segnum, 1,
&su_bh);
if (ret < 0)
goto out_header;
kaddr = kmap_atomic(su_bh->b_page);
su = nilfs_sufile_block_get_segment_usage(
sufile, segnum, su_bh, kaddr);
nsus = nilfs_sufile_segment_usages_in_block(
sufile, segnum, maxsegnum);
for (j = 0; j < nsus; j++, su = (void *)su + susz, segnum++) {
if (!nilfs_segment_usage_clean(su))
continue;
nilfs_segment_usage_set_dirty(su);
kunmap_atomic(kaddr);
kaddr = kmap_atomic(header_bh->b_page);
header = kaddr + bh_offset(header_bh);
le64_add_cpu(&header->sh_ncleansegs, -1);
le64_add_cpu(&header->sh_ndirtysegs, 1);
header->sh_last_alloc = cpu_to_le64(segnum);
kunmap_atomic(kaddr);
sui->ncleansegs--;
mark_buffer_dirty(header_bh);
mark_buffer_dirty(su_bh);
nilfs_mdt_mark_dirty(sufile);
brelse(su_bh);
*segnump = segnum;
goto out_header;
}
kunmap_atomic(kaddr);
brelse(su_bh);
}
ret = -ENOSPC;
out_header:
brelse(header_bh);
out_sem:
up_write(&NILFS_MDT(sufile)->mi_sem);
return ret;
}
/**
* nilfs_palloc_freev - deallocate a set of persistent objects
* @inode: inode of metadata file using this allocator
* @entry_nrs: array of entry numbers to be deallocated
* @nitems: number of entries stored in @entry_nrs
*/
int nilfs_palloc_freev(struct inode *inode, __u64 *entry_nrs, size_t nitems)
{
struct buffer_head *desc_bh, *bitmap_bh;
struct nilfs_palloc_group_desc *desc;
unsigned char *bitmap;
void *desc_kaddr, *bitmap_kaddr;
unsigned long group, group_offset;
__u64 group_min_nr, last_nrs[8];
const unsigned long epg = nilfs_palloc_entries_per_group(inode);
const unsigned int epb = NILFS_MDT(inode)->mi_entries_per_block;
unsigned int entry_start, end, pos;
spinlock_t *lock;
int i, j, k, ret;
u32 nfree;
for (i = 0; i < nitems; i = j) {
int change_group = false;
int nempties = 0, n = 0;
group = nilfs_palloc_group(inode, entry_nrs[i], &group_offset);
ret = nilfs_palloc_get_desc_block(inode, group, 0, &desc_bh);
if (ret < 0)
return ret;
ret = nilfs_palloc_get_bitmap_block(inode, group, 0,
&bitmap_bh);
if (ret < 0) {
brelse(desc_bh);
return ret;
}
/* Get the first entry number of the group */
group_min_nr = (__u64)group * epg;
bitmap_kaddr = kmap(bitmap_bh->b_page);
bitmap = bitmap_kaddr + bh_offset(bitmap_bh);
lock = nilfs_mdt_bgl_lock(inode, group);
j = i;
entry_start = rounddown(group_offset, epb);
do {
if (!nilfs_clear_bit_atomic(lock, group_offset,
bitmap)) {
nilfs_msg(inode->i_sb, KERN_WARNING,
"%s (ino=%lu): entry number %llu already freed",
__func__, inode->i_ino,
(unsigned long long)entry_nrs[j]);
} else {
n++;
}
j++;
if (j >= nitems || entry_nrs[j] < group_min_nr ||
entry_nrs[j] >= group_min_nr + epg) {
change_group = true;
} else {
group_offset = entry_nrs[j] - group_min_nr;
if (group_offset >= entry_start &&
group_offset < entry_start + epb) {
/* This entry is in the same block */
continue;
}
}
/* Test if the entry block is empty or not */
end = entry_start + epb;
pos = nilfs_find_next_bit(bitmap, end, entry_start);
if (pos >= end) {
last_nrs[nempties++] = entry_nrs[j - 1];
if (nempties >= ARRAY_SIZE(last_nrs))
break;
}
if (change_group)
break;
/* Go on to the next entry block */
entry_start = rounddown(group_offset, epb);
} while (true);
kunmap(bitmap_bh->b_page);
mark_buffer_dirty(bitmap_bh);
brelse(bitmap_bh);
for (k = 0; k < nempties; k++) {
ret = nilfs_palloc_delete_entry_block(inode,
last_nrs[k]);
if (ret && ret != -ENOENT)
nilfs_msg(inode->i_sb, KERN_WARNING,
"error %d deleting block that object (entry=%llu, ino=%lu) belongs to",
ret, (unsigned long long)last_nrs[k],
inode->i_ino);
}
desc_kaddr = kmap_atomic(desc_bh->b_page);
desc = nilfs_palloc_block_get_group_desc(
inode, group, desc_bh, desc_kaddr);
nfree = nilfs_palloc_group_desc_add_entries(desc, lock, n);
kunmap_atomic(desc_kaddr);
mark_buffer_dirty(desc_bh);
nilfs_mdt_mark_dirty(inode);
brelse(desc_bh);
if (nfree == nilfs_palloc_entries_per_group(inode)) {
ret = nilfs_palloc_delete_bitmap_block(inode, group);
if (ret && ret != -ENOENT)
nilfs_msg(inode->i_sb, KERN_WARNING,
"error %d deleting bitmap block of group=%lu, ino=%lu",
ret, group, inode->i_ino);
}
}
return 0;
}
/**
* nilfs_sufile_read - read or get sufile inode
* @sb: super block instance
* @susize: size of a segment usage entry
* @raw_inode: on-disk sufile inode
* @inodep: buffer to store the inode
*/
int nilfs_sufile_read(struct super_block *sb, size_t susize,
struct nilfs_inode *raw_inode, struct inode **inodep)
{
struct inode *sufile;
struct nilfs_sufile_info *sui;
struct buffer_head *header_bh;
struct nilfs_sufile_header *header;
void *kaddr;
int err;
if (susize > sb->s_blocksize) {
printk(KERN_ERR
"NILFS: too large segment usage size: %zu bytes.\n",
susize);
return -EINVAL;
} else if (susize < NILFS_MIN_SEGMENT_USAGE_SIZE) {
printk(KERN_ERR
"NILFS: too small segment usage size: %zu bytes.\n",
susize);
return -EINVAL;
}
sufile = nilfs_iget_locked(sb, NULL, NILFS_SUFILE_INO);
if (unlikely(!sufile))
return -ENOMEM;
if (!(sufile->i_state & I_NEW))
goto out;
err = nilfs_mdt_init(sufile, NILFS_MDT_GFP, sizeof(*sui));
if (err)
goto failed;
nilfs_mdt_set_entry_size(sufile, susize,
sizeof(struct nilfs_sufile_header));
err = nilfs_read_inode_common(sufile, raw_inode);
if (err)
goto failed;
err = nilfs_sufile_get_header_block(sufile, &header_bh);
if (err)
goto failed;
sui = NILFS_SUI(sufile);
kaddr = kmap_atomic(header_bh->b_page, KM_USER0);
header = kaddr + bh_offset(header_bh);
sui->ncleansegs = le64_to_cpu(header->sh_ncleansegs);
kunmap_atomic(kaddr, KM_USER0);
brelse(header_bh);
sui->allocmax = nilfs_sufile_get_nsegments(sufile) - 1;
sui->allocmin = 0;
unlock_new_inode(sufile);
out:
*inodep = sufile;
return 0;
failed:
iget_failed(sufile);
return err;
}
/**
* nilfs_palloc_prepare_alloc_entry - prepare to allocate a persistent object
* @inode: inode of metadata file using this allocator
* @req: nilfs_palloc_req structure exchanged for the allocation
*/
int nilfs_palloc_prepare_alloc_entry(struct inode *inode,
struct nilfs_palloc_req *req)
{
struct buffer_head *desc_bh, *bitmap_bh;
struct nilfs_palloc_group_desc *desc;
unsigned char *bitmap;
void *desc_kaddr, *bitmap_kaddr;
unsigned long group, maxgroup, ngroups;
unsigned long group_offset, maxgroup_offset;
unsigned long n, entries_per_group;
unsigned long i, j;
spinlock_t *lock;
int pos, ret;
ngroups = nilfs_palloc_groups_count(inode);
maxgroup = ngroups - 1;
group = nilfs_palloc_group(inode, req->pr_entry_nr, &group_offset);
entries_per_group = nilfs_palloc_entries_per_group(inode);
for (i = 0; i < ngroups; i += n) {
if (group >= ngroups) {
/* wrap around */
group = 0;
maxgroup = nilfs_palloc_group(inode, req->pr_entry_nr,
&maxgroup_offset) - 1;
}
ret = nilfs_palloc_get_desc_block(inode, group, 1, &desc_bh);
if (ret < 0)
return ret;
desc_kaddr = kmap(desc_bh->b_page);
desc = nilfs_palloc_block_get_group_desc(
inode, group, desc_bh, desc_kaddr);
n = nilfs_palloc_rest_groups_in_desc_block(inode, group,
maxgroup);
for (j = 0; j < n; j++, desc++, group++) {
lock = nilfs_mdt_bgl_lock(inode, group);
if (nilfs_palloc_group_desc_nfrees(desc, lock) > 0) {
ret = nilfs_palloc_get_bitmap_block(
inode, group, 1, &bitmap_bh);
if (ret < 0)
goto out_desc;
bitmap_kaddr = kmap(bitmap_bh->b_page);
bitmap = bitmap_kaddr + bh_offset(bitmap_bh);
pos = nilfs_palloc_find_available_slot(
bitmap, group_offset,
entries_per_group, lock);
if (pos >= 0) {
/* found a free entry */
nilfs_palloc_group_desc_add_entries(
desc, lock, -1);
req->pr_entry_nr =
entries_per_group * group + pos;
kunmap(desc_bh->b_page);
kunmap(bitmap_bh->b_page);
req->pr_desc_bh = desc_bh;
req->pr_bitmap_bh = bitmap_bh;
return 0;
}
kunmap(bitmap_bh->b_page);
brelse(bitmap_bh);
}
group_offset = 0;
}
kunmap(desc_bh->b_page);
brelse(desc_bh);
}
/* no entries left */
return -ENOSPC;
out_desc:
kunmap(desc_bh->b_page);
brelse(desc_bh);
return ret;
}
