static int gfs2_read_super(struct gfs2_sbd *sdp, sector_t sector, int silent) { struct super_block *sb = sdp->sd_vfs; struct gfs2_sb *p; struct page *page; struct bio *bio; page = alloc_page(GFP_NOFS); if (unlikely(!page)) return -ENOBUFS; ClearPageUptodate(page); ClearPageDirty(page); lock_page(page); bio = bio_alloc(GFP_NOFS, 1); bio->bi_sector = sector * (sb->s_blocksize >> 9); bio->bi_bdev = sb->s_bdev; bio_add_page(bio, page, PAGE_SIZE, 0); bio->bi_end_io = end_bio_io_page; bio->bi_private = page; submit_bio(READ_SYNC | REQ_META, bio); wait_on_page_locked(page); bio_put(bio); if (!PageUptodate(page)) { __free_page(page); return -EIO; } p = kmap(page); gfs2_sb_in(sdp, p); kunmap(page); __free_page(page); return gfs2_check_sb(sdp, silent); }
/** * submit - submit BIO request * @writing: READ or WRITE. * @dev: The block device we're using. * @first_block: The first sector we're using. * @page: The page being used for I/O. * @free_group: If writing, the group that was used in allocating the page * and which will be used in freeing the page from the completion * routine. * * Based on Patrick Mochell's pmdisk code from long ago: "Straight from the * textbook - allocate and initialize the bio. If we're writing, make sure * the page is marked as dirty. Then submit it and carry on." * * If we're just testing the speed of our own code, we fake having done all * the hard work and all toi_end_bio immediately. **/ static int submit(int writing, struct block_device *dev, sector_t first_block, struct page *page, int free_group) { struct bio *bio = NULL; int cur_outstanding_io, result; /* * Shouldn't throttle if reading - can deadlock in the single * threaded case as pages are only freed when we use the * readahead. */ if (writing) { result = throttle_if_needed(MEMORY_ONLY | THROTTLE_WAIT); if (result) return result; } while (!bio) { bio = bio_alloc(TOI_ATOMIC_GFP, 1); if (!bio) { set_free_mem_throttle(); do_bio_wait(1); } } bio->bi_bdev = dev; bio->bi_sector = first_block; bio->bi_private = (void *)((unsigned long)free_group); bio->bi_end_io = toi_end_bio; bio->bi_flags |= (1 << BIO_TOI); if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) { printk(KERN_DEBUG "ERROR: adding page to bio at %lld\n", (unsigned long long)first_block); bio_put(bio); return -EFAULT; } bio_get(bio); cur_outstanding_io = atomic_add_return(1, &toi_io_in_progress); if (writing) { if (cur_outstanding_io > max_outstanding_writes) max_outstanding_writes = cur_outstanding_io; } else { if (cur_outstanding_io > max_outstanding_reads) max_outstanding_reads = cur_outstanding_io; } /* Still read the header! */ if (unlikely(test_action_state(TOI_TEST_BIO) && writing)) { /* Fake having done the hard work */ set_bit(BIO_UPTODATE, &bio->bi_flags); toi_end_bio(bio, 0); } else submit_bio(writing | REQ_SYNC, bio); return 0; }
static int submit(int rw, pgoff_t page_off, void * page) { int error = 0; struct bio * bio; bio = bio_alloc(GFP_ATOMIC,1); if (!bio) return -ENOMEM; bio->bi_sector = page_off * (PAGE_SIZE >> 9); bio_get(bio); bio->bi_bdev = resume_bdev; bio->bi_end_io = end_io; if (bio_add_page(bio, virt_to_page(page), PAGE_SIZE, 0) < PAGE_SIZE) { printk("pmdisk: ERROR: adding page to bio at %ld\n",page_off); error = -EFAULT; goto Done; } if (rw == WRITE) bio_set_pages_dirty(bio); start_io(); submit_bio(rw | (1 << BIO_RW_SYNC), bio); wait_io(); Done: bio_put(bio); return error; }
static struct bio *get_failover_bio(struct path *path, unsigned data_size) { struct bio *bio; struct page *page; bio = bio_alloc(GFP_ATOMIC, 1); if (!bio) { DMERR("dm-emc: get_failover_bio: bio_alloc() failed."); return NULL; } bio->bi_rw |= (1 << BIO_RW); bio->bi_bdev = path->dev->bdev; bio->bi_sector = 0; bio->bi_private = path; bio->bi_end_io = emc_endio; page = alloc_page(GFP_ATOMIC); if (!page) { DMERR("dm-emc: get_failover_bio: alloc_page() failed."); bio_put(bio); return NULL; } if (bio_add_page(bio, page, data_size, 0) != data_size) { DMERR("dm-emc: get_failover_bio: alloc_page() failed."); __free_page(page); bio_put(bio); return NULL; } return bio; }
static int test_bio_add_page(void) { int res = 0 , i = 0 , offset = 0; unsigned long addr = 0; struct page *ppage = NULL; for (i = 0 ; i < 128 ; i ++) { addr = get_zeroed_page(GFP_KERNEL); if(addr == 0) { printk("tbio: get free page failed %ld\n" , addr); return -1; } ppage = virt_to_page(addr); if ( ppage == NULL ) { printk ("tbio: covert virture page to page struct failed\n"); return -1; } res = bio_add_page(tbiop , ppage , PAGE_SIZE , offset ); if (res <0 ) { printk("bio_add_page :res %d\n",res); return -1; } offset += res; // printk ("tbio: bio_add_page : %d\n", res); } return 0; }
static int xixfs_raw_submit( struct block_device *bdev, sector_t startsector, int32 size, int32 sectorsize, PXIX_BUF xbuf, int32 rw) { struct bio * bio; int ret = 0; bio = bio_alloc(GFP_NOIO, 1); bio->bi_sector = startsector; bio->bi_bdev = bdev; bio_add_page(bio, xbuf->xix_page, size, xbuf->xixcore_buffer.xcb_offset); bio->bi_private = (void *)xbuf; bio->bi_end_io = end_bio_xbuf_io_async; bio->bi_rw = rw; bio_get(bio); submit_bio(rw, bio); if(bio_flagged(bio, BIO_EOPNOTSUPP)) ret = -EOPNOTSUPP; bio_put(bio); return ret; }
static int test_bio_add_page(void) { int ret = 0, i = 0, offset = 0; unsigned long addr = 0; struct page *ppage = NULL; for (i = 0; i < 128; i++) { addr = get_zeroed_page(GFP_KERNEL); if (addr == 0) { prk_err("get free page failed %ld", addr); ret = -1; break; } ppage = virt_to_page(addr); if (ppage == NULL) { prk_err("covert virture page to page struct failed"); ret = -1; break; } ret = bio_add_page(tbiop, ppage, PAGE_SIZE, offset); if (ret < 0) { prk_err("bio_add_page failed"); break; } offset += ret; } return ret; }
int hfsplus_submit_bio(struct block_device *bdev, sector_t sector, void *data, int rw) { DECLARE_COMPLETION_ONSTACK(wait); struct bio *bio; bio = bio_alloc(GFP_NOIO, 1); bio->bi_sector = sector; bio->bi_bdev = bdev; bio->bi_end_io = hfsplus_end_io_sync; bio->bi_private = &wait; /* * We always submit one sector at a time, so bio_add_page must not fail. */ if (bio_add_page(bio, virt_to_page(data), HFSPLUS_SECTOR_SIZE, offset_in_page(data)) != HFSPLUS_SECTOR_SIZE) BUG(); submit_bio(rw, bio); wait_for_completion(&wait); if (!bio_flagged(bio, BIO_UPTODATE)) return -EIO; return 0; }
/** * Allocate a bio with pages. * * @size size in bytes. * * You must set bi_bdev, bi_opf, bi_iter by yourself. * bi_iter.bi_size will be set to the specified size if size is not 0. */ struct bio* bio_alloc_with_pages(uint size, struct block_device *bdev, gfp_t gfp_mask) { struct bio *bio; uint i, nr_pages, remaining; nr_pages = (size + PAGE_SIZE - 1) / PAGE_SIZE; bio = bio_alloc(gfp_mask, nr_pages); if (!bio) return NULL; bio->bi_bdev = bdev; /* required to bio_add_page(). */ remaining = size; for (i = 0; i < nr_pages; i++) { uint len0, len1; struct page *page = alloc_page_inc(gfp_mask); if (!page) goto err; len0 = min_t(uint, PAGE_SIZE, remaining); len1 = bio_add_page(bio, page, len0, 0); ASSERT(len0 == len1); remaining -= len0; } ASSERT(remaining == 0); ASSERT(bio->bi_iter.bi_size == size); return bio; err: bio_put_with_pages(bio); return NULL; }
int fscrypt_zeroout_range(const struct inode *inode, pgoff_t lblk, sector_t pblk, unsigned int len) { struct fscrypt_ctx *ctx; struct page *ciphertext_page = NULL; struct bio *bio; int ret, err = 0; BUG_ON(inode->i_sb->s_blocksize != PAGE_SIZE); ctx = fscrypt_get_ctx(inode, GFP_NOFS); if (IS_ERR(ctx)) return PTR_ERR(ctx); ciphertext_page = fscrypt_alloc_bounce_page(ctx, GFP_NOWAIT); if (IS_ERR(ciphertext_page)) { err = PTR_ERR(ciphertext_page); goto errout; } while (len--) { err = fscrypt_do_page_crypto(inode, FS_ENCRYPT, lblk, ZERO_PAGE(0), ciphertext_page, PAGE_SIZE, 0, GFP_NOFS); if (err) goto errout; bio = bio_alloc(GFP_NOWAIT, 1); if (!bio) { err = -ENOMEM; goto errout; } bio_set_dev(bio, inode->i_sb->s_bdev); bio->bi_iter.bi_sector = pblk << (inode->i_sb->s_blocksize_bits - 9); bio_set_op_attrs(bio, REQ_OP_WRITE, 0); ret = bio_add_page(bio, ciphertext_page, inode->i_sb->s_blocksize, 0); if (ret != inode->i_sb->s_blocksize) { /* should never happen! */ WARN_ON(1); bio_put(bio); err = -EIO; goto errout; } err = submit_bio_wait(bio); if (err == 0 && bio->bi_status) err = -EIO; bio_put(bio); if (err) goto errout; lblk++; pblk++; } err = 0; errout: fscrypt_release_ctx(ctx); return err; }
STATIC int _drbd_md_sync_page_io(struct drbd_conf *mdev, struct drbd_backing_dev *bdev, struct page *page, sector_t sector, int rw, int size) { struct bio *bio; struct drbd_md_io md_io; int ok; md_io.mdev = mdev; init_completion(&md_io.event); md_io.error = 0; if ((rw & WRITE) && !test_bit(MD_NO_BARRIER, &mdev->flags)) rw |= DRBD_REQ_FUA | DRBD_REQ_FLUSH; rw |= DRBD_REQ_UNPLUG | DRBD_REQ_SYNC; #ifndef REQ_FLUSH /* < 2.6.36, "barrier" semantic may fail with EOPNOTSUPP */ retry: #endif bio = bio_alloc(GFP_NOIO, 1); bio->bi_bdev = bdev->md_bdev; bio->bi_sector = sector; ok = (bio_add_page(bio, page, size, 0) == size); if (!ok) goto out; bio->bi_private = &md_io; bio->bi_end_io = drbd_md_io_complete; bio->bi_rw = rw; trace_drbd_bio(mdev, "Md", bio, 0, NULL); if (drbd_insert_fault(mdev, (rw & WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD)) bio_endio(bio, -EIO); else submit_bio(rw, bio); wait_for_completion(&md_io.event); ok = bio_flagged(bio, BIO_UPTODATE) && md_io.error == 0; #ifndef REQ_FLUSH /* check for unsupported barrier op. * would rather check on EOPNOTSUPP, but that is not reliable. * don't try again for ANY return value != 0 */ if (unlikely((bio->bi_rw & DRBD_REQ_HARDBARRIER) && !ok)) { /* Try again with no barrier */ dev_warn(DEV, "Barriers not supported on meta data device - disabling\n"); set_bit(MD_NO_BARRIER, &mdev->flags); rw &= ~DRBD_REQ_HARDBARRIER; bio_put(bio); goto retry; } #endif out: bio_put(bio); return ok; }
/* * hfsplus_submit_bio - Perfrom block I/O * @sb: super block of volume for I/O * @sector: block to read or write, for blocks of HFSPLUS_SECTOR_SIZE bytes * @buf: buffer for I/O * @data: output pointer for location of requested data * @rw: direction of I/O * * The unit of I/O is hfsplus_min_io_size(sb), which may be bigger than * HFSPLUS_SECTOR_SIZE, and @buf must be sized accordingly. On reads * @data will return a pointer to the start of the requested sector, * which may not be the same location as @buf. * * If @sector is not aligned to the bdev logical block size it will * be rounded down. For writes this means that @buf should contain data * that starts at the rounded-down address. As long as the data was * read using hfsplus_submit_bio() and the same buffer is used things * will work correctly. */ int hfsplus_submit_bio(struct super_block *sb, sector_t sector, void *buf, void **data, int rw) { DECLARE_COMPLETION_ONSTACK(wait); struct bio *bio; int ret = 0; u64 io_size; loff_t start; int offset; /* * Align sector to hardware sector size and find offset. We * assume that io_size is a power of two, which _should_ * be true. */ io_size = hfsplus_min_io_size(sb); start = (loff_t)sector << HFSPLUS_SECTOR_SHIFT; offset = start & (io_size - 1); sector &= ~((io_size >> HFSPLUS_SECTOR_SHIFT) - 1); bio = bio_alloc(GFP_NOIO, 1); bio->bi_sector = sector; bio->bi_bdev = sb->s_bdev; bio->bi_end_io = hfsplus_end_io_sync; bio->bi_private = &wait; if (!(rw & WRITE) && data) *data = (u8 *)buf + offset; while (io_size > 0) { unsigned int page_offset = offset_in_page(buf); unsigned int len = min_t(unsigned int, PAGE_SIZE - page_offset, io_size); ret = bio_add_page(bio, virt_to_page(buf), len, page_offset); if (ret != len) { ret = -EIO; goto out; } io_size -= len; buf = (u8 *)buf + len; } submit_bio(rw, bio); wait_for_completion(&wait); if (!bio_flagged(bio, BIO_UPTODATE)) ret = -EIO; out: bio_put(bio); return ret < 0 ? ret : 0; }
int ext4_encrypted_zeroout(struct inode *inode, struct ext4_extent *ex) { struct ext4_crypto_ctx *ctx; struct page *ciphertext_page = NULL; struct bio *bio; ext4_lblk_t lblk = ex->ee_block; ext4_fsblk_t pblk = ext4_ext_pblock(ex); unsigned int len = ext4_ext_get_actual_len(ex); int err = 0; BUG_ON(inode->i_sb->s_blocksize != PAGE_CACHE_SIZE); ctx = ext4_get_crypto_ctx(inode); if (IS_ERR(ctx)) return PTR_ERR(ctx); ciphertext_page = alloc_bounce_page(ctx); if (IS_ERR(ciphertext_page)) { err = PTR_ERR(ciphertext_page); goto errout; } while (len--) { err = ext4_page_crypto(ctx, inode, EXT4_ENCRYPT, lblk, ZERO_PAGE(0), ciphertext_page); if (err) goto errout; bio = bio_alloc(GFP_KERNEL, 1); if (!bio) { err = -ENOMEM; goto errout; } bio->bi_bdev = inode->i_sb->s_bdev; bio->bi_sector = pblk; err = bio_add_page(bio, ciphertext_page, inode->i_sb->s_blocksize, 0); if (err) { bio_put(bio); goto errout; } err = submit_bio_wait(WRITE, bio); bio_put(bio); if (err) goto errout; } err = 0; errout: ext4_release_crypto_ctx(ctx); return err; }
static int io_submit_add_bh(struct ext4_io_submit *io, struct ext4_io_page *io_page, struct inode *inode, struct writeback_control *wbc, struct buffer_head *bh) { ext4_io_end_t *io_end; int ret; if (buffer_new(bh)) { clear_buffer_new(bh); unmap_underlying_metadata(bh->b_bdev, bh->b_blocknr); } if (!buffer_mapped(bh) || buffer_delay(bh)) { if (!buffer_mapped(bh)) clear_buffer_dirty(bh); if (io->io_bio) ext4_io_submit(io); return 0; } if (io->io_bio && bh->b_blocknr != io->io_next_block) { submit_and_retry: ext4_io_submit(io); } if (io->io_bio == NULL) { ret = io_submit_init(io, inode, wbc, bh); if (ret) return ret; } io_end = io->io_end; if ((io_end->num_io_pages >= MAX_IO_PAGES) && (io_end->pages[io_end->num_io_pages-1] != io_page)) goto submit_and_retry; if (buffer_uninit(bh) && !(io_end->flag & EXT4_IO_END_UNWRITTEN)) { io_end->flag |= EXT4_IO_END_UNWRITTEN; atomic_inc(&EXT4_I(inode)->i_aiodio_unwritten); } io->io_end->size += bh->b_size; io->io_next_block++; ret = bio_add_page(io->io_bio, bh->b_page, bh->b_size, bh_offset(bh)); if (ret != bh->b_size) goto submit_and_retry; if ((io_end->num_io_pages == 0) || (io_end->pages[io_end->num_io_pages-1] != io_page)) { io_end->pages[io_end->num_io_pages++] = io_page; atomic_inc(&io_page->p_count); } return 0; }
/*----------------------------------------------------------------- * IO routines that accept a list of pages. *---------------------------------------------------------------*/ static void do_region(int rw, unsigned region, struct dm_io_region *where, struct dpages *dp, struct io *io) { struct bio *bio; struct page *page; unsigned long len; unsigned offset; unsigned num_bvecs; sector_t remaining = where->count; /* * where->count may be zero if rw holds a flush and we need to * send a zero-sized flush. */ do { /* * Allocate a suitably sized-bio. */ num_bvecs = dm_sector_div_up(remaining, (PAGE_SIZE >> SECTOR_SHIFT)); num_bvecs = min_t(int, bio_get_nr_vecs(where->bdev), num_bvecs); bio = bio_alloc_bioset(GFP_NOIO, num_bvecs, io->client->bios); if (!bio) { printk(KERN_WARNING "%s : %s() failed\n", __FILE__, __func__); BUG_ON(1); } bio->bi_sector = where->sector + (where->count - remaining); bio->bi_bdev = where->bdev; bio->bi_end_io = endio; bio->bi_destructor = dm_bio_destructor; store_io_and_region_in_bio(bio, io, region); /* * Try and add as many pages as possible. */ while (remaining) { dp->get_page(dp, &page, &len, &offset); len = min(len, to_bytes(remaining)); if (!bio_add_page(bio, page, len, offset)) break; offset = 0; remaining -= to_sector(len); dp->next_page(dp); } atomic_inc(&io->count); submit_bio(rw, bio); } while (remaining); }
struct bio * _do_readpage_aio(struct page *page) { DD("_do_readpage_aio."); if(page == NULL) { return NULL; } struct bio *bio = make_bio(); bio_add_page(bio, page, BIO_READ); return bio; }
static int write_metadata(struct log_writes_c *lc, void *entry, size_t entrylen, void *data, size_t datalen, sector_t sector) { struct bio *bio; struct page *page; void *ptr; size_t ret; bio = bio_alloc(GFP_KERNEL, 1); if (!bio) { DMERR("Couldn't alloc log bio"); goto error; } bio->bi_iter.bi_size = 0; bio->bi_iter.bi_sector = sector; bio->bi_bdev = lc->logdev->bdev; bio->bi_end_io = log_end_io; bio->bi_private = lc; bio_set_op_attrs(bio, REQ_OP_WRITE, 0); page = alloc_page(GFP_KERNEL); if (!page) { DMERR("Couldn't alloc log page"); bio_put(bio); goto error; } ptr = kmap_atomic(page); memcpy(ptr, entry, entrylen); if (datalen) memcpy(ptr + entrylen, data, datalen); memset(ptr + entrylen + datalen, 0, lc->sectorsize - entrylen - datalen); kunmap_atomic(ptr); ret = bio_add_page(bio, page, lc->sectorsize, 0); if (ret != lc->sectorsize) { DMERR("Couldn't add page to the log block"); goto error_bio; } submit_bio(bio); return 0; error_bio: bio_put(bio); __free_page(page); error: put_io_block(lc); return -1; }
static struct bio* kr_create_bio(KrDevice* dev, struct page* page, int sector) { struct bio* bio = bio_alloc(GFP_NOIO, 1); if (!bio) return NULL; /* setup bio. */ bio->bi_bdev = dev->bdev; bio->bi_sector = sector; bio_add_page(bio, page, KR_BLOCK_SIZE, 0); return bio; }
/* * Generate a new unfragmented bio with the given size * This should never violate the device limitations * May return a smaller bio when running out of pages, indicated by * *out_of_pages set to 1. */ static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned size, unsigned *out_of_pages) { struct crypt_config *cc = io->cc; struct bio *clone; unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; gfp_t gfp_mask = GFP_NOIO | __GFP_HIGHMEM; unsigned i, len; struct page *page; clone = bio_alloc_bioset(GFP_NOIO, nr_iovecs, cc->bs); if (!clone) return NULL; clone_init(io, clone); *out_of_pages = 0; for (i = 0; i < nr_iovecs; i++) { page = mempool_alloc(cc->page_pool, gfp_mask); if (!page) { *out_of_pages = 1; break; } /* * If additional pages cannot be allocated without waiting, * return a partially-allocated bio. The caller will then try * to allocate more bios while submitting this partial bio. */ gfp_mask = (gfp_mask | __GFP_NOWARN) & ~__GFP_WAIT; len = (size > PAGE_SIZE) ? PAGE_SIZE : size; if (!bio_add_page(clone, page, len, 0)) { mempool_free(page, cc->page_pool); break; } size -= len; } if (!clone->bi_size) { bio_put(clone); return NULL; } return clone; }
static int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, sector_t nr_sects, gfp_t gfp_mask) { int ret; struct bio *bio; struct bio_batch bb; unsigned int sz; DECLARE_COMPLETION_ONSTACK(wait); atomic_set(&bb.done, 1); bb.error = 0; bb.wait = &wait; ret = 0; while (nr_sects != 0) { bio = bio_alloc(gfp_mask, min(nr_sects, (sector_t)BIO_MAX_PAGES)); if (!bio) { ret = -ENOMEM; break; } bio->bi_iter.bi_sector = sector; bio->bi_bdev = bdev; bio->bi_end_io = bio_batch_end_io; bio->bi_private = &bb; while (nr_sects != 0) { sz = min((sector_t) PAGE_SIZE >> 9 , nr_sects); ret = bio_add_page(bio, ZERO_PAGE(0), sz << 9, 0); nr_sects -= ret >> 9; sector += ret >> 9; if (ret < (sz << 9)) break; } ret = 0; atomic_inc(&bb.done); submit_bio(WRITE, bio); } /* Wait for bios in-flight */ if (!atomic_dec_and_test(&bb.done)) wait_for_completion_io(&wait); if (bb.error) return bb.error; return ret; }
static struct bio *get_swap_bio(gfp_t gfp_flags, struct page *page, bio_end_io_t end_io) { struct bio *bio; bio = bio_alloc(gfp_flags, 1); if (bio) { bio->bi_iter.bi_sector = map_swap_page(page, &bio->bi_bdev); bio->bi_iter.bi_sector <<= PAGE_SHIFT - 9; bio->bi_end_io = end_io; bio_add_page(bio, page, PAGE_SIZE, 0); BUG_ON(bio->bi_iter.bi_size != PAGE_SIZE); } return bio; }
static void do_region(int rw, unsigned region, struct dm_io_region *where, struct dpages *dp, struct io *io) { struct bio *bio; struct page *page; unsigned long len; unsigned offset; unsigned num_bvecs; sector_t remaining = where->count; struct request_queue *q = bdev_get_queue(where->bdev); sector_t discard_sectors; do { if (rw & REQ_DISCARD) num_bvecs = 1; else num_bvecs = min_t(int, bio_get_nr_vecs(where->bdev), dm_sector_div_up(remaining, (PAGE_SIZE >> SECTOR_SHIFT))); bio = bio_alloc_bioset(GFP_NOIO, num_bvecs, io->client->bios); bio->bi_sector = where->sector + (where->count - remaining); bio->bi_bdev = where->bdev; bio->bi_end_io = endio; bio->bi_destructor = dm_bio_destructor; store_io_and_region_in_bio(bio, io, region); if (rw & REQ_DISCARD) { discard_sectors = min_t(sector_t, q->limits.max_discard_sectors, remaining); bio->bi_size = discard_sectors << SECTOR_SHIFT; remaining -= discard_sectors; } else while (remaining) { dp->get_page(dp, &page, &len, &offset); len = min(len, to_bytes(remaining)); if (!bio_add_page(bio, page, len, offset)) break; offset = 0; remaining -= to_sector(len); dp->next_page(dp); } atomic_inc(&io->count); submit_bio(rw, bio); } while (remaining); }
/* * Fill the locked page with data located in the block address. * Return unlocked page. */ int f2fs_submit_page_bio(struct f2fs_io_info *fio) { struct bio *bio; struct page *page = fio->encrypted_page ? fio->encrypted_page : fio->page; trace_f2fs_submit_page_bio(page, fio); f2fs_trace_ios(fio, 0); /* Allocate a new bio */ bio = __bio_alloc(fio->sbi, fio->blk_addr, 1, is_read_io(fio->rw)); if (bio_add_page(bio, page, PAGE_CACHE_SIZE, 0) < PAGE_CACHE_SIZE) { bio_put(bio); return -EFAULT; } submit_bio(fio->rw, bio); return 0; }
void f2fs_submit_page_mbio(struct f2fs_io_info *fio) { struct f2fs_sb_info *sbi = fio->sbi; enum page_type btype = PAGE_TYPE_OF_BIO(fio->type); struct f2fs_bio_info *io; bool is_read = is_read_io(fio->rw); struct page *bio_page; io = is_read ? &sbi->read_io : &sbi->write_io[btype]; verify_block_addr(sbi, fio->blk_addr); down_write(&io->io_rwsem); if (!is_read) inc_page_count(sbi, F2FS_WRITEBACK); if (io->bio && (io->last_block_in_bio != fio->blk_addr - 1 || io->fio.rw != fio->rw)) __submit_merged_bio(io); alloc_new: if (io->bio == NULL) { int bio_blocks = MAX_BIO_BLOCKS(sbi); io->bio = __bio_alloc(sbi, fio->blk_addr, bio_blocks, is_read); io->fio = *fio; } bio_page = fio->encrypted_page ? fio->encrypted_page : fio->page; if (bio_add_page(io->bio, bio_page, PAGE_CACHE_SIZE, 0) < PAGE_CACHE_SIZE) { __submit_merged_bio(io); goto alloc_new; } io->last_block_in_bio = fio->blk_addr; f2fs_trace_ios(fio, 0); up_write(&io->io_rwsem); trace_f2fs_submit_page_mbio(fio->page, fio); }
/** * blkmtd_add_page - add a page to the current BIO * @bio: bio to add to (NULL to alloc initial bio) * @blkdev: block device * @page: page to add * @pagecnt: pages left to add * * Adds a page to the current bio, allocating it if necessary. If it cannot be * added, the current bio is written out and a new one is allocated. Returns * the new bio to add or NULL on error */ static struct bio *blkmtd_add_page(struct bio *bio, struct block_device *blkdev, struct page *page, int pagecnt) { retry: if(!bio) { bio = bio_alloc(GFP_KERNEL, pagecnt); if(!bio) return NULL; bio->bi_sector = page->index << (PAGE_SHIFT-9); bio->bi_bdev = blkdev; } if(bio_add_page(bio, page, PAGE_SIZE, 0) != PAGE_SIZE) { blkmtd_write_out(bio); bio = NULL; goto retry; } return bio; }
static int _drbd_md_sync_page_io(struct drbd_conf *mdev, struct drbd_backing_dev *bdev, struct page *page, sector_t sector, int rw, int size) { struct bio *bio; struct drbd_md_io md_io; int ok; md_io.mdev = mdev; init_completion(&md_io.event); md_io.error = 0; if ((rw & WRITE) && !test_bit(MD_NO_FUA, &mdev->flags)) rw |= REQ_FUA; rw |= REQ_UNPLUG | REQ_SYNC; bio = bio_alloc(GFP_NOIO, 1); bio->bi_bdev = bdev->md_bdev; bio->bi_sector = sector; ok = (bio_add_page(bio, page, size, 0) == size); if (!ok) goto out; bio->bi_private = &md_io; bio->bi_end_io = drbd_md_io_complete; bio->bi_rw = rw; if (FAULT_ACTIVE(mdev, (rw & WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD)) bio_endio(bio, -EIO); else submit_bio(rw, bio); wait_for_completion(&md_io.event); ok = bio_flagged(bio, BIO_UPTODATE) && md_io.error == 0; out: bio_put(bio); return ok; }
/* read one page from the block device */ static int blkmtd_readpage(struct blkmtd_dev *dev, struct page *page) { struct bio *bio; struct completion event; int err = -ENOMEM; if(PageUptodate(page)) { DEBUG(2, "blkmtd: readpage page %ld is already upto date\n", page->index); unlock_page(page); return 0; } ClearPageUptodate(page); ClearPageError(page); bio = bio_alloc(GFP_KERNEL, 1); if(bio) { init_completion(&event); bio->bi_bdev = dev->blkdev; bio->bi_sector = page->index << (PAGE_SHIFT-9); bio->bi_private = &event; bio->bi_end_io = bi_read_complete; if(bio_add_page(bio, page, PAGE_SIZE, 0) == PAGE_SIZE) { submit_bio(READ_SYNC, bio); wait_for_completion(&event); err = test_bit(BIO_UPTODATE, &bio->bi_flags) ? 0 : -EIO; bio_put(bio); } } if(err) SetPageError(page); else SetPageUptodate(page); flush_dcache_page(page); unlock_page(page); return err; }
/** * submit - submit BIO request. * @rw: READ or WRITE. * @off physical offset of page. * @page: page we're reading or writing. * @bio_chain: list of pending biod (for async reading) * * Straight from the textbook - allocate and initialize the bio. * If we're reading, make sure the page is marked as dirty. * Then submit it and, if @bio_chain == NULL, wait. */ static int submit(int rw, struct block_device *bdev, sector_t sector, struct page *page, struct bio **bio_chain) { const int bio_rw = rw | REQ_SYNC; struct bio *bio; bio = bio_alloc(__GFP_WAIT | __GFP_HIGH, 1); bio->bi_sector = sector; bio->bi_bdev = bdev; bio->bi_end_io = end_swap_bio_read; if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) { printk(KERN_ERR "PM: Adding page to bio failed at %llu\n", (unsigned long long)sector); bio_put(bio); return -EFAULT; } lock_page(page); bio_get(bio); if (bio_chain == NULL) { submit_bio(bio_rw, bio); wait_on_page_locked(page); if (rw == READ) bio_set_pages_dirty(bio); bio_put(bio); } else { if (rw == READ) get_page(page); /* These pages are freed later */ bio->bi_private = *bio_chain; *bio_chain = bio; submit_bio(bio_rw, bio); } return 0; }
int ext4_mpage_readpages(struct address_space *mapping, struct list_head *pages, struct page *page, unsigned nr_pages, bool is_readahead) { struct bio *bio = NULL; sector_t last_block_in_bio = 0; struct inode *inode = mapping->host; const unsigned blkbits = inode->i_blkbits; const unsigned blocks_per_page = PAGE_SIZE >> blkbits; const unsigned blocksize = 1 << blkbits; sector_t block_in_file; sector_t last_block; sector_t last_block_in_file; sector_t blocks[MAX_BUF_PER_PAGE]; unsigned page_block; struct block_device *bdev = inode->i_sb->s_bdev; int length; unsigned relative_block = 0; struct ext4_map_blocks map; map.m_pblk = 0; map.m_lblk = 0; map.m_len = 0; map.m_flags = 0; for (; nr_pages; nr_pages--) { int fully_mapped = 1; unsigned first_hole = blocks_per_page; prefetchw(&page->flags); if (pages) { page = lru_to_page(pages); list_del(&page->lru); if (add_to_page_cache_lru(page, mapping, page->index, readahead_gfp_mask(mapping))) goto next_page; } if (page_has_buffers(page)) goto confused; block_in_file = (sector_t)page->index << (PAGE_SHIFT - blkbits); last_block = block_in_file + nr_pages * blocks_per_page; last_block_in_file = (i_size_read(inode) + blocksize - 1) >> blkbits; if (last_block > last_block_in_file) last_block = last_block_in_file; page_block = 0; /* * Map blocks using the previous result first. */ if ((map.m_flags & EXT4_MAP_MAPPED) && block_in_file > map.m_lblk && block_in_file < (map.m_lblk + map.m_len)) { unsigned map_offset = block_in_file - map.m_lblk; unsigned last = map.m_len - map_offset; for (relative_block = 0; ; relative_block++) { if (relative_block == last) { /* needed? */ map.m_flags &= ~EXT4_MAP_MAPPED; break; } if (page_block == blocks_per_page) break; blocks[page_block] = map.m_pblk + map_offset + relative_block; page_block++; block_in_file++; } } /* * Then do more ext4_map_blocks() calls until we are * done with this page. */ while (page_block < blocks_per_page) { if (block_in_file < last_block) { map.m_lblk = block_in_file; map.m_len = last_block - block_in_file; if (ext4_map_blocks(NULL, inode, &map, 0) < 0) { set_error_page: SetPageError(page); zero_user_segment(page, 0, PAGE_SIZE); unlock_page(page); goto next_page; } } if ((map.m_flags & EXT4_MAP_MAPPED) == 0) { fully_mapped = 0; if (first_hole == blocks_per_page) first_hole = page_block; page_block++; block_in_file++; continue; } if (first_hole != blocks_per_page) goto confused; /* hole -> non-hole */ /* Contiguous blocks? */ if (page_block && blocks[page_block-1] != map.m_pblk-1) goto confused; for (relative_block = 0; ; relative_block++) { if (relative_block == map.m_len) { /* needed? */ map.m_flags &= ~EXT4_MAP_MAPPED; break; } else if (page_block == blocks_per_page) break; blocks[page_block] = map.m_pblk+relative_block; page_block++; block_in_file++; } } if (first_hole != blocks_per_page) { zero_user_segment(page, first_hole << blkbits, PAGE_SIZE); if (first_hole == 0) { SetPageUptodate(page); unlock_page(page); goto next_page; } } else if (fully_mapped) { SetPageMappedToDisk(page); } if (fully_mapped && blocks_per_page == 1 && !PageUptodate(page) && cleancache_get_page(page) == 0) { SetPageUptodate(page); goto confused; } /* * This page will go to BIO. Do we need to send this * BIO off first? */ if (bio && (last_block_in_bio != blocks[0] - 1)) { submit_and_realloc: submit_bio(bio); bio = NULL; } if (bio == NULL) { struct fscrypt_ctx *ctx = NULL; if (IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode)) { ctx = fscrypt_get_ctx(inode, GFP_NOFS); if (IS_ERR(ctx)) goto set_error_page; } bio = bio_alloc(GFP_KERNEL, min_t(int, nr_pages, BIO_MAX_PAGES)); if (!bio) { if (ctx) fscrypt_release_ctx(ctx); goto set_error_page; } bio_set_dev(bio, bdev); bio->bi_iter.bi_sector = blocks[0] << (blkbits - 9); bio->bi_end_io = mpage_end_io; bio->bi_private = ctx; bio_set_op_attrs(bio, REQ_OP_READ, is_readahead ? REQ_RAHEAD : 0); } length = first_hole << blkbits; if (bio_add_page(bio, page, length, 0) < length) goto submit_and_realloc; if (((map.m_flags & EXT4_MAP_BOUNDARY) && (relative_block == map.m_len)) || (first_hole != blocks_per_page)) { submit_bio(bio); bio = NULL; } else last_block_in_bio = blocks[blocks_per_page - 1]; goto next_page; confused: if (bio) { submit_bio(bio); bio = NULL; } if (!PageUptodate(page)) block_read_full_page(page, ext4_get_block); else unlock_page(page); next_page: if (pages) put_page(page); } BUG_ON(pages && !list_empty(pages)); if (bio) submit_bio(bio); return 0; }
static int __mpage_writepage(struct page *page, struct writeback_control *wbc, void *data) { struct mpage_data *mpd = data; struct bio *bio = mpd->bio; struct address_space *mapping = page->mapping; struct inode *inode = page->mapping->host; const unsigned blkbits = inode->i_blkbits; unsigned long end_index; const unsigned blocks_per_page = PAGE_CACHE_SIZE >> blkbits; sector_t last_block; sector_t block_in_file; sector_t blocks[MAX_BUF_PER_PAGE]; unsigned page_block; unsigned first_unmapped = blocks_per_page; struct block_device *bdev = NULL; int boundary = 0; sector_t boundary_block = 0; struct block_device *boundary_bdev = NULL; int length; struct buffer_head map_bh; loff_t i_size = i_size_read(inode); int ret = 0; int wr = (wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : WRITE); if (page_has_buffers(page)) { struct buffer_head *head = page_buffers(page); struct buffer_head *bh = head; /* If they're all mapped and dirty, do it */ page_block = 0; do { BUG_ON(buffer_locked(bh)); if (!buffer_mapped(bh)) { /* * unmapped dirty buffers are created by * __set_page_dirty_buffers -> mmapped data */ if (buffer_dirty(bh)) goto confused; if (first_unmapped == blocks_per_page) first_unmapped = page_block; continue; } if (first_unmapped != blocks_per_page) goto confused; /* hole -> non-hole */ if (!buffer_dirty(bh) || !buffer_uptodate(bh)) goto confused; if (page_block) { if (bh->b_blocknr != blocks[page_block-1] + 1) goto confused; } blocks[page_block++] = bh->b_blocknr; boundary = buffer_boundary(bh); if (boundary) { boundary_block = bh->b_blocknr; boundary_bdev = bh->b_bdev; } bdev = bh->b_bdev; } while ((bh = bh->b_this_page) != head); if (first_unmapped) goto page_is_mapped; /* * Page has buffers, but they are all unmapped. The page was * created by pagein or read over a hole which was handled by * block_read_full_page(). If this address_space is also * using mpage_readpages then this can rarely happen. */ goto confused; } /* * The page has no buffers: map it to disk */ BUG_ON(!PageUptodate(page)); block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits); last_block = (i_size - 1) >> blkbits; map_bh.b_page = page; for (page_block = 0; page_block < blocks_per_page; ) { map_bh.b_state = 0; map_bh.b_size = 1 << blkbits; if (mpd->get_block(inode, block_in_file, &map_bh, 1)) goto confused; if (buffer_new(&map_bh)) unmap_underlying_metadata(map_bh.b_bdev, map_bh.b_blocknr); if (buffer_boundary(&map_bh)) { boundary_block = map_bh.b_blocknr; boundary_bdev = map_bh.b_bdev; } if (page_block) { if (map_bh.b_blocknr != blocks[page_block-1] + 1) goto confused; } blocks[page_block++] = map_bh.b_blocknr; boundary = buffer_boundary(&map_bh); bdev = map_bh.b_bdev; if (block_in_file == last_block) break; block_in_file++; } BUG_ON(page_block == 0); first_unmapped = page_block; page_is_mapped: end_index = i_size >> PAGE_CACHE_SHIFT; if (page->index >= end_index) { /* * The page straddles i_size. It must be zeroed out on each * and every writepage invocation because it may be mmapped. * "A file is mapped in multiples of the page size. For a file * that is not a multiple of the page size, the remaining memory * is zeroed when mapped, and writes to that region are not * written out to the file." */ unsigned offset = i_size & (PAGE_CACHE_SIZE - 1); if (page->index > end_index || !offset) goto confused; zero_user_segment(page, offset, PAGE_CACHE_SIZE); } /* * This page will go to BIO. Do we need to send this BIO off first? */ if (bio && mpd->last_block_in_bio != blocks[0] - 1) bio = mpage_bio_submit(wr, bio); alloc_new: if (bio == NULL) { if (first_unmapped == blocks_per_page) { if (!bdev_write_page(bdev, blocks[0] << (blkbits - 9), page, wbc)) { clean_buffers(page, first_unmapped); goto out; } } bio = mpage_alloc(bdev, blocks[0] << (blkbits - 9), BIO_MAX_PAGES, GFP_NOFS|__GFP_HIGH); if (bio == NULL) goto confused; wbc_init_bio(wbc, bio); } /* * Must try to add the page before marking the buffer clean or * the confused fail path above (OOM) will be very confused when * it finds all bh marked clean (i.e. it will not write anything) */ wbc_account_io(wbc, page, PAGE_SIZE); length = first_unmapped << blkbits; if (bio_add_page(bio, page, length, 0) < length) { bio = mpage_bio_submit(wr, bio); goto alloc_new; } clean_buffers(page, first_unmapped); BUG_ON(PageWriteback(page)); set_page_writeback(page); unlock_page(page); if (boundary || (first_unmapped != blocks_per_page)) { bio = mpage_bio_submit(wr, bio); if (boundary_block) { write_boundary_block(boundary_bdev, boundary_block, 1 << blkbits); } } else { mpd->last_block_in_bio = blocks[blocks_per_page - 1]; } goto out; confused: if (bio) bio = mpage_bio_submit(wr, bio); if (mpd->use_writepage) { ret = mapping->a_ops->writepage(page, wbc); } else { ret = -EAGAIN; goto out; } /* * The caller has a ref on the inode, so *mapping is stable */ mapping_set_error(mapping, ret); out: mpd->bio = bio; return ret; }