static ssize_t ocfs2_file_aio_read(struct kiocb *iocb, const struct iovec *iov, unsigned long nr_segs, loff_t pos) { int ret = 0, rw_level = -1, have_alloc_sem = 0, lock_level = 0; struct file *filp = iocb->ki_filp; struct inode *inode = filp->f_path.dentry->d_inode; mlog_entry("(0x%p, %u, '%.*s')\n", filp, (unsigned int)nr_segs, filp->f_path.dentry->d_name.len, filp->f_path.dentry->d_name.name); if (!inode) { ret = -EINVAL; mlog_errno(ret); goto bail; } /* * buffered reads protect themselves in ->readpage(). O_DIRECT reads * need locks to protect pending reads from racing with truncate. */ if (filp->f_flags & O_DIRECT) { down_read(&inode->i_alloc_sem); have_alloc_sem = 1; ret = ocfs2_rw_lock(inode, 0); if (ret < 0) { mlog_errno(ret); goto bail; } rw_level = 0; /* communicate with ocfs2_dio_end_io */ ocfs2_iocb_set_rw_locked(iocb, rw_level); } /* * We're fine letting folks race truncates and extending * writes with read across the cluster, just like they can * locally. Hence no rw_lock during read. * * Take and drop the meta data lock to update inode fields * like i_size. This allows the checks down below * generic_file_aio_read() a chance of actually working. */ ret = ocfs2_meta_lock_atime(inode, filp->f_vfsmnt, &lock_level); if (ret < 0) { mlog_errno(ret); goto bail; } ocfs2_meta_unlock(inode, lock_level); ret = generic_file_aio_read(iocb, iov, nr_segs, iocb->ki_pos); if (ret == -EINVAL) mlog(ML_ERROR, "generic_file_aio_read returned -EINVAL\n"); /* buffered aio wouldn't have proper lock coverage today */ BUG_ON(ret == -EIOCBQUEUED && !(filp->f_flags & O_DIRECT)); /* see ocfs2_file_aio_write */ if (ret == -EIOCBQUEUED || !ocfs2_iocb_is_rw_locked(iocb)) { rw_level = -1; have_alloc_sem = 0; } bail: if (have_alloc_sem) up_read(&inode->i_alloc_sem); if (rw_level != -1) ocfs2_rw_unlock(inode, rw_level); mlog_exit(ret); return ret; }
/* * ocfs2_dio_end_io is called by the dio core when a dio is finished. We're * particularly interested in the aio/dio case. Like the core uses * i_alloc_sem, we use the rw_lock DLM lock to protect io on one node from * truncation on another. */ static void ocfs2_dio_end_io(struct kiocb *iocb, loff_t offset, ssize_t bytes, void *private) { struct inode *inode = iocb->ki_filp->f_path.dentry->d_inode; int level; /* this io's submitter should not have unlocked this before we could */ BUG_ON(!ocfs2_iocb_is_rw_locked(iocb)); ocfs2_iocb_clear_rw_locked(iocb); level = ocfs2_iocb_rw_locked_level(iocb); if (!level) up_read(&inode->i_alloc_sem); ocfs2_rw_unlock(inode, level); } /* * ocfs2_invalidatepage() and ocfs2_releasepage() are shamelessly stolen * from ext3. PageChecked() bits have been removed as OCFS2 does not * do journalled data. */ static void ocfs2_invalidatepage(struct page *page, unsigned long offset)
static ssize_t ocfs2_file_aio_write(struct kiocb *iocb, const struct iovec *iov, unsigned long nr_segs, loff_t pos) { int ret, direct_io, appending, rw_level, have_alloc_sem = 0; int can_do_direct, sync = 0; ssize_t written = 0; size_t ocount; /* original count */ size_t count; /* after file limit checks */ loff_t *ppos = &iocb->ki_pos; struct file *file = iocb->ki_filp; struct inode *inode = file->f_path.dentry->d_inode; mlog_entry("(0x%p, %u, '%.*s')\n", file, (unsigned int)nr_segs, file->f_path.dentry->d_name.len, file->f_path.dentry->d_name.name); if (iocb->ki_left == 0) return 0; ret = generic_segment_checks(iov, &nr_segs, &ocount, VERIFY_READ); if (ret) return ret; count = ocount; vfs_check_frozen(inode->i_sb, SB_FREEZE_WRITE); appending = file->f_flags & O_APPEND ? 1 : 0; direct_io = file->f_flags & O_DIRECT ? 1 : 0; mutex_lock(&inode->i_mutex); relock: /* to match setattr's i_mutex -> i_alloc_sem -> rw_lock ordering */ if (direct_io) { down_read(&inode->i_alloc_sem); have_alloc_sem = 1; } /* concurrent O_DIRECT writes are allowed */ rw_level = !direct_io; ret = ocfs2_rw_lock(inode, rw_level); if (ret < 0) { mlog_errno(ret); goto out_sems; } can_do_direct = direct_io; ret = ocfs2_prepare_inode_for_write(file->f_path.dentry, ppos, iocb->ki_left, appending, &can_do_direct); if (ret < 0) { mlog_errno(ret); goto out; } /* * We can't complete the direct I/O as requested, fall back to * buffered I/O. */ if (direct_io && !can_do_direct) { ocfs2_rw_unlock(inode, rw_level); up_read(&inode->i_alloc_sem); have_alloc_sem = 0; rw_level = -1; direct_io = 0; sync = 1; goto relock; } if (!sync && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) sync = 1; /* * XXX: Is it ok to execute these checks a second time? */ ret = generic_write_checks(file, ppos, &count, S_ISBLK(inode->i_mode)); if (ret) goto out; /* * Set pos so that sync_page_range_nolock() below understands * where to start from. We might've moved it around via the * calls above. The range we want to actually sync starts from * *ppos here. * */ pos = *ppos; /* communicate with ocfs2_dio_end_io */ ocfs2_iocb_set_rw_locked(iocb, rw_level); if (direct_io) { written = generic_file_direct_write(iocb, iov, &nr_segs, *ppos, ppos, count, ocount); if (written < 0) { ret = written; goto out_dio; } } else { written = ocfs2_file_buffered_write(file, ppos, iov, nr_segs, count, written); if (written < 0) { ret = written; if (ret != -EFAULT || ret != -ENOSPC) mlog_errno(ret); goto out; } } out_dio: /* buffered aio wouldn't have proper lock coverage today */ BUG_ON(ret == -EIOCBQUEUED && !(file->f_flags & O_DIRECT)); /* * deep in g_f_a_w_n()->ocfs2_direct_IO we pass in a ocfs2_dio_end_io * function pointer which is called when o_direct io completes so that * it can unlock our rw lock. (it's the clustered equivalent of * i_alloc_sem; protects truncate from racing with pending ios). * Unfortunately there are error cases which call end_io and others * that don't. so we don't have to unlock the rw_lock if either an * async dio is going to do it in the future or an end_io after an * error has already done it. */ if (ret == -EIOCBQUEUED || !ocfs2_iocb_is_rw_locked(iocb)) { rw_level = -1; have_alloc_sem = 0; } out: if (rw_level != -1) ocfs2_rw_unlock(inode, rw_level); out_sems: if (have_alloc_sem) up_read(&inode->i_alloc_sem); if (written > 0 && sync) { ssize_t err; err = sync_page_range_nolock(inode, file->f_mapping, pos, count); if (err < 0) written = err; } mutex_unlock(&inode->i_mutex); mlog_exit(ret); return written ? written : ret; }