static int aio_ref_get(struct aio_output *output, struct mref_object *mref)
{
loff_t total_size;
if (unlikely(!output->brick->power.led_on))
return -EBADFD;
if (unlikely(!output->mf)) {
MARS_ERR("brick is not switched on\n");
return -EILSEQ;
}
if (unlikely(mref->ref_len <= 0)) {
MARS_ERR("bad ref_len=%d\n", mref->ref_len);
return -EILSEQ;
}
total_size = get_total_size(output);
if (unlikely(total_size < 0)) {
return total_size;
}
mref->ref_total_size = total_size;
if (mref->ref_initialized) {
_mref_get(mref);
return mref->ref_len;
}
/* Buffered IO.
*/
if (!mref->ref_data) {
struct aio_mref_aspect *mref_a = aio_mref_get_aspect(output->brick, mref);
if (unlikely(!mref_a)) {
MARS_ERR("bad mref_a\n");
return -EILSEQ;
}
if (unlikely(mref->ref_len <= 0)) {
MARS_ERR("bad ref_len = %d\n", mref->ref_len);
return -ENOMEM;
}
mref->ref_data = brick_block_alloc(mref->ref_pos, (mref_a->alloc_len = mref->ref_len));
if (unlikely(!mref->ref_data)) {
MARS_ERR("ENOMEM %d bytes\n", mref->ref_len);
return -ENOMEM;
}
#if 0 // ???
mref->ref_flags = 0;
#endif
mref_a->do_dealloc = true;
atomic_inc(&output->total_alloc_count);
atomic_inc(&output->alloc_count);
}
_mref_get_first(mref);
return mref->ref_len;
}
static void _usebuf_endio(struct generic_callback *cb)
{
struct usebuf_mref_aspect *mref_a = cb->cb_private;
struct mref_object *mref;
struct usebuf_mref_aspect *sub_mref_a;
struct mref_object *sub_mref;
LAST_CALLBACK(cb);
CHECK_PTR(mref_a, done);
mref = mref_a->object;
CHECK_PTR(mref, done);
sub_mref_a = mref_a->sub_mref_a;
CHECK_PTR(sub_mref_a, done);
sub_mref = sub_mref_a->object;
CHECK_PTR(sub_mref, done);
//MARS_INF("HALLO %p %p len = %d may_write = %d rw = %d flags = %d\n", mref, sub_mref, sub_mref->ref_len, sub_mref->ref_may_write, sub_mref->ref_rw, sub_mref->ref_flags);
if (mref->ref_data != sub_mref->ref_data && cb->cb_error >= 0) {
if (sub_mref->ref_may_write == 0) {
if (sub_mref->ref_flags & MREF_UPTODATE) {
_usebuf_copy(mref, sub_mref, 0);
mref->ref_flags |= MREF_UPTODATE;
}
#ifndef FAKE_ALL
} else if (sub_mref->ref_rw == 0) {
MARS_IO("re-kick %p\n", sub_mref);
sub_mref->ref_rw = 1;
_usebuf_copy(mref, sub_mref, 1);
mref->ref_flags |= MREF_UPTODATE;
GENERIC_INPUT_CALL(mref_a->input, mref_io, sub_mref);
return;
#endif
}
}
#if 1
if (mref_a->yyy++ > 0)
MARS_ERR("yyy = %d\n", mref_a->yyy - 1);
if (cb->cb_error < 0)
MARS_ERR("error = %d\n", cb->cb_error);
#endif
CHECKED_CALLBACK(mref, cb->cb_error, done);
if (!_mref_put(mref))
return;
#if 1
_mref_put(sub_mref);
#endif
usebuf_free_mref(mref);
done:;
}
static int client_switch(struct client_brick *brick)
{
struct client_output *output = brick->outputs[0];
int status = 0;
if (brick->power.button) {
if (brick->power.led_on)
goto done;
mars_power_led_off((void*)brick, false);
if (!output->sender.thread) {
brick->connection_state = 1;
output->sender.thread = brick_thread_create(sender_thread, output, "mars_sender%d", thread_count++);
if (unlikely(!output->sender.thread)) {
MARS_ERR("cannot start sender thread\n");
status = -ENOENT;
goto done;
}
}
if (output->sender.thread) {
mars_power_led_on((void*)brick, true);
}
} else {
if (brick->power.led_off)
goto done;
mars_power_led_on((void*)brick, false);
_kill_thread(&output->sender, "sender");
brick->connection_state = 0;
if (!output->sender.thread) {
mars_power_led_off((void*)brick, !output->sender.thread);
}
}
done:
return status;
}
static
void aio_sync_all(struct aio_output *output, struct list_head *tmp_list)
{
unsigned long long latency;
int err;
output->fdsync_active = true;
atomic_inc(&output->total_fdsync_count);
latency = TIME_STATS(
&timings[2],
err = aio_sync(output->mf->mf_filp)
);
threshold_check(&aio_sync_threshold, latency);
output->fdsync_active = false;
wake_up_interruptible_all(&output->fdsync_event);
if (err < 0) {
MARS_ERR("FDSYNC error %d\n", err);
}
/* Signal completion for the whole list.
* No locking needed, it's on the stack.
*/
_complete_all(tmp_list, output, err);
}
static int aio_output_destruct(struct aio_output *output)
{
if (unlikely(output->fd >= 0)) {
MARS_ERR("active fd = %d detected\n", output->fd);
}
return 0;
}
static int aio_output_destruct(struct aio_output *output)
{
CHECK_HEAD_EMPTY(&output->dirty_anchor);
if (unlikely(output->fd >= 0)) {
MARS_ERR("active fd = %d detected\n", output->fd);
}
return 0;
}
int __init init_mars(void)
{
MARS_INF("init_mars()\n");
set_fake();
#ifdef MARS_TRACING
{
int flags = O_CREAT | O_TRUNC | O_RDWR | O_LARGEFILE;
int prot = 0600;
mm_segment_t oldfs;
oldfs = get_fs();
set_fs(get_ds());
mars_log_file = filp_open("/mars/trace.csv", flags, prot);
set_fs(oldfs);
if (IS_ERR(mars_log_file)) {
MARS_ERR("cannot create trace logfile, status = %ld\n", PTR_ERR(mars_log_file));
mars_log_file = NULL;
}
}
#endif
mars_tfm = crypto_alloc_hash("md5", 0, CRYPTO_ALG_ASYNC);
if (!mars_tfm) {
MARS_ERR("cannot alloc crypto hash\n");
return -ENOMEM;
}
if (IS_ERR(mars_tfm)) {
MARS_ERR("alloc crypto hash failed, status = %d\n", (int)PTR_ERR(mars_tfm));
return PTR_ERR(mars_tfm);
}
#if 0
if (crypto_tfm_alg_type(crypto_hash_tfm(mars_tfm)) != CRYPTO_ALG_TYPE_DIGEST) {
MARS_ERR("bad crypto hash type\n");
return -EINVAL;
}
#endif
mars_digest_size = crypto_hash_digestsize(mars_tfm);
MARS_INF("digest_size = %d\n", mars_digest_size);
return 0;
}
static
loff_t get_total_size(struct aio_output *output)
{
struct file *file;
struct inode *inode;
loff_t min;
file = output->mf->mf_filp;
if (unlikely(!file)) {
MARS_ERR("file is not open\n");
return -EILSEQ;
}
if (unlikely(!file->f_mapping)) {
MARS_ERR("file %p has no mapping\n", file);
return -EILSEQ;
}
inode = file->f_mapping->host;
if (unlikely(!inode)) {
MARS_ERR("file %p has no inode\n", file);
return -EILSEQ;
}
min = i_size_read(inode);
/* Workaround for races in the page cache.
* It appears that concurrent reads and writes seem to
* result in inconsistent reads in some very rare cases, due to
* races. Sometimes, the inode claims that the file has been already
* appended by a write operation, but the data has not actually hit
* the page cache, such that a concurrent read gets NULL blocks.
*/
if (!output->brick->is_static_device) {
loff_t max = 0;
mf_get_dirty(output->mf, &min, &max, 0, 99);
}
return min;
}
static
int lamport_sysctl_handler(
struct ctl_table *table,
int write,
void __user *buffer,
size_t *length,
loff_t *ppos)
{
ssize_t res = 0;
size_t len = *length;
MARS_DBG("write = %d len = %ld pos = %lld\n", write, len, *ppos);
if (!len || *ppos > 0) {
goto done;
}
if (write) {
return -EINVAL;
} else {
int my_len = 128;
char *tmp = brick_string_alloc(my_len);
struct timespec know = CURRENT_TIME;
struct timespec lnow;
get_lamport(&lnow);
res = scnprintf(tmp, my_len,
"CURRENT_TIME=%ld.%09ld\n"
"lamport_now=%ld.%09ld\n",
know.tv_sec, know.tv_nsec,
lnow.tv_sec, lnow.tv_nsec
);
if (copy_to_user(buffer, tmp, res)) {
MARS_ERR("write %ld bytes at %p failed\n", res, buffer);
res = -EFAULT;
}
brick_string_free(tmp);
}
done:
MARS_DBG("res = %ld\n", res);
*length = res;
if (res >= 0) {
*ppos += res;
return 0;
}
return res;
}
static
void _complete(struct aio_output *output, struct aio_mref_aspect *mref_a, int err)
{
struct mref_object *mref;
CHECK_PTR(mref_a, fatal);
mref = mref_a->object;
CHECK_PTR(mref, fatal);
mars_trace(mref, "aio_endio");
if (err < 0) {
MARS_ERR("IO error %d at pos=%lld len=%d (mref=%p ref_data=%p)\n", err, mref->ref_pos, mref->ref_len, mref, mref->ref_data);
} else {
mref_checksum(mref);
mref->ref_flags |= MREF_UPTODATE;
}
CHECKED_CALLBACK(mref, err, err_found);
done:
if (mref->ref_rw) {
atomic_dec(&output->write_count);
} else {
atomic_dec(&output->read_count);
}
mf_remove_dirty(output->mf, &mref_a->di);
aio_ref_put(output, mref);
atomic_dec(&output->work_count);
atomic_dec(&mars_global_io_flying);
return;
err_found:
MARS_FAT("giving up...\n");
goto done;
fatal:
MARS_FAT("bad pointer, giving up...\n");
}
static int client_output_construct(struct client_output *output)
{
int i;
output->hash_table = brick_block_alloc(0, PAGE_SIZE);
if (unlikely(!output->hash_table)) {
MARS_ERR("cannot allocate hash table\n");
return -ENOMEM;
}
for (i = 0; i < CLIENT_HASH_MAX; i++) {
INIT_LIST_HEAD(&output->hash_table[i]);
}
spin_lock_init(&output->lock);
INIT_LIST_HEAD(&output->mref_list);
INIT_LIST_HEAD(&output->wait_list);
init_waitqueue_head(&output->event);
init_waitqueue_head(&output->sender.run_event);
init_waitqueue_head(&output->receiver.run_event);
init_waitqueue_head(&output->info_event);
return 0;
}
static void client_ref_io(struct client_output *output, struct mref_object *mref)
{
struct client_mref_aspect *mref_a;
int error = -EINVAL;
mref_a = client_mref_get_aspect(output->brick, mref);
if (unlikely(!mref_a)) {
goto error;
}
while (output->brick->max_flying > 0 && atomic_read(&output->fly_count) > output->brick->max_flying) {
MARS_IO("sleeping request pos = %lld len = %d rw = %d (flying = %d)\n", mref->ref_pos, mref->ref_len, mref->ref_rw, atomic_read(&output->fly_count));
#ifdef IO_DEBUGGING
brick_msleep(3000);
#else
brick_msleep(1000 * 2 / HZ);
#endif
}
atomic_inc(&mars_global_io_flying);
atomic_inc(&output->fly_count);
_mref_get(mref);
mref_a->submit_jiffies = jiffies;
_hash_insert(output, mref_a);
MARS_IO("added request id = %d pos = %lld len = %d rw = %d (flying = %d)\n", mref->ref_id, mref->ref_pos, mref->ref_len, mref->ref_rw, atomic_read(&output->fly_count));
wake_up_interruptible(&output->event);
return;
error:
MARS_ERR("IO error = %d\n", error);
SIMPLE_CALLBACK(mref, error);
client_ref_put(output, mref);
}
static int aio_event_thread(void *data)
{
struct aio_threadinfo *tinfo = data;
struct aio_output *output = tinfo->output;
struct aio_threadinfo *other = &output->tinfo[2];
int err = -ENOMEM;
MARS_DBG("event thread has started.\n");
//set_user_nice(current, -20);
use_fake_mm();
if (!current->mm)
goto err;
err = aio_start_thread(output, &output->tinfo[2], aio_sync_thread, 'y');
if (unlikely(err < 0))
goto err;
while (!brick_thread_should_stop() || atomic_read(&tinfo->queued_sum) > 0) {
mm_segment_t oldfs;
int count;
int i;
struct timespec timeout = {
.tv_sec = 1,
};
struct io_event events[MARS_MAX_AIO_READ];
oldfs = get_fs();
set_fs(get_ds());
/* TODO: don't timeout upon termination.
* Probably we should submit a dummy request.
*/
count = sys_io_getevents(output->ctxp, 1, MARS_MAX_AIO_READ, events, &timeout);
set_fs(oldfs);
if (likely(count > 0)) {
atomic_sub(count, &output->submit_count);
}
for (i = 0; i < count; i++) {
struct aio_mref_aspect *mref_a = (void*)events[i].data;
struct mref_object *mref;
int err = events[i].res;
if (!mref_a) {
continue; // this was a dummy request
}
mref = mref_a->object;
MARS_IO("AIO done %p pos = %lld len = %d rw = %d\n", mref, mref->ref_pos, mref->ref_len, mref->ref_rw);
mapfree_set(output->mf, mref->ref_pos, mref->ref_pos + mref->ref_len);
if (output->brick->o_fdsync
&& err >= 0
&& mref->ref_rw != READ
&& !mref->ref_skip_sync
&& !mref_a->resubmit++) {
// workaround for non-implemented AIO FSYNC operation
if (output->mf &&
output->mf->mf_filp &&
output->mf->mf_filp->f_op &&
!output->mf->mf_filp->f_op->aio_fsync) {
mars_trace(mref, "aio_fsync");
_enqueue(other, mref_a, mref->ref_prio, true);
continue;
}
err = aio_submit(output, mref_a, true);
if (likely(err >= 0))
continue;
}
_complete(output, mref_a, err);
}
}
err = 0;
err:
MARS_DBG("event thread has stopped, err = %d\n", err);
aio_stop_thread(output, 2, false);
unuse_fake_mm();
tinfo->terminated = true;
wake_up_interruptible_all(&tinfo->terminate_event);
return err;
}
#if 1
/* This should go to fs/open.c (as long as vfs_submit() is not implemented)
*/
#include <linux/fdtable.h>
void fd_uninstall(unsigned int fd)
{
struct files_struct *files = current->files;
struct fdtable *fdt;
MARS_DBG("fd = %d\n", fd);
if (unlikely(fd < 0)) {
MARS_ERR("bad fd = %d\n", fd);
return;
}
spin_lock(&files->file_lock);
fdt = files_fdtable(files);
rcu_assign_pointer(fdt->fd[fd], NULL);
spin_unlock(&files->file_lock);
}
EXPORT_SYMBOL(fd_uninstall);
#endif
static
atomic_t ioctx_count = ATOMIC_INIT(0);
static
void _destroy_ioctx(struct aio_output *output)
{
if (unlikely(!output))
goto done;
aio_stop_thread(output, 1, true);
use_fake_mm();
if (likely(output->ctxp)) {
mm_segment_t oldfs;
int err;
MARS_DBG("ioctx count = %d destroying %p\n", atomic_read(&ioctx_count), (void*)output->ctxp);
oldfs = get_fs();
set_fs(get_ds());
err = sys_io_destroy(output->ctxp);
set_fs(oldfs);
atomic_dec(&ioctx_count);
MARS_DBG("ioctx count = %d status = %d\n", atomic_read(&ioctx_count), err);
output->ctxp = 0;
}
if (likely(output->fd >= 0)) {
MARS_DBG("destroying fd %d\n", output->fd);
fd_uninstall(output->fd);
put_unused_fd(output->fd);
output->fd = -1;
}
done:
if (likely(current->mm)) {
unuse_fake_mm();
}
}
static
int _create_ioctx(struct aio_output *output)
{
struct file *file;
mm_segment_t oldfs;
int err = -EINVAL;
CHECK_PTR_NULL(output, done);
CHECK_PTR_NULL(output->mf, done);
file = output->mf->mf_filp;
CHECK_PTR_NULL(file, done);
/* TODO: this is provisionary. We only need it for sys_io_submit()
* which uses userspace concepts like file handles.
* This should be accompanied by a future kernelsapce vfs_submit() or
* do_submit() which currently does not exist :(
*/
err = get_unused_fd();
MARS_DBG("file %p '%s' new fd = %d\n", file, output->mf->mf_name, err);
if (unlikely(err < 0)) {
MARS_ERR("cannot get fd, err=%d\n", err);
goto done;
}
output->fd = err;
fd_install(err, file);
use_fake_mm();
err = -ENOMEM;
if (unlikely(!current->mm)) {
MARS_ERR("cannot fake mm\n");
goto done;
}
MARS_DBG("ioctx count = %d old = %p\n", atomic_read(&ioctx_count), (void*)output->ctxp);
output->ctxp = 0;
oldfs = get_fs();
set_fs(get_ds());
err = sys_io_setup(MARS_MAX_AIO, &output->ctxp);
set_fs(oldfs);
if (likely(output->ctxp))
atomic_inc(&ioctx_count);
MARS_DBG("ioctx count = %d new = %p status = %d\n", atomic_read(&ioctx_count), (void*)output->ctxp, err);
if (unlikely(err < 0)) {
MARS_ERR("io_setup failed, err=%d\n", err);
goto done;
}
err = aio_start_thread(output, &output->tinfo[1], aio_event_thread, 'e');
if (unlikely(err < 0)) {
MARS_ERR("could not start event thread\n");
goto done;
}
done:
if (likely(current->mm)) {
unuse_fake_mm();
}
return err;
}
static int aio_submit_thread(void *data)
{
struct aio_threadinfo *tinfo = data;
struct aio_output *output = tinfo->output;
struct file *file;
int err = -EINVAL;
MARS_DBG("submit thread has started.\n");
file = output->mf->mf_filp;
use_fake_mm();
while (!brick_thread_should_stop() || atomic_read(&output->read_count) + atomic_read(&output->write_count) + atomic_read(&tinfo->queued_sum) > 0) {
struct aio_mref_aspect *mref_a;
struct mref_object *mref;
int sleeptime;
int status;
wait_event_interruptible_timeout(
tinfo->event,
atomic_read(&tinfo->queued_sum) > 0,
HZ / 4);
mref_a = _dequeue(tinfo);
if (!mref_a) {
continue;
}
mref = mref_a->object;
status = -EINVAL;
CHECK_PTR(mref, error);
mapfree_set(output->mf, mref->ref_pos, -1);
if (mref->ref_rw) {
insert_dirty(output, mref_a);
}
// check for reads exactly at EOF (special case)
if (mref->ref_pos == mref->ref_total_size &&
!mref->ref_rw &&
mref->ref_timeout > 0) {
loff_t total_size = i_size_read(file->f_mapping->host);
loff_t len = total_size - mref->ref_pos;
if (len > 0) {
mref->ref_total_size = total_size;
mref->ref_len = len;
} else {
if (!mref_a->start_jiffies) {
mref_a->start_jiffies = jiffies;
}
if ((long long)jiffies - mref_a->start_jiffies <= mref->ref_timeout) {
if (atomic_read(&tinfo->queued_sum) <= 0) {
atomic_inc(&output->total_msleep_count);
brick_msleep(1000 * 4 / HZ);
}
_enqueue(tinfo, mref_a, MARS_PRIO_LOW, true);
continue;
}
MARS_DBG("ENODATA %lld\n", len);
_complete(output, mref_a, -ENODATA);
continue;
}
}
sleeptime = 1;
for (;;) {
status = aio_submit(output, mref_a, false);
if (likely(status != -EAGAIN)) {
break;
}
atomic_inc(&output->total_delay_count);
brick_msleep(sleeptime);
if (sleeptime < 100) {
sleeptime++;
}
}
error:
if (unlikely(status < 0)) {
MARS_IO("submit_count = %d status = %d\n", atomic_read(&output->submit_count), status);
_complete_mref(output, mref, status);
}
}
MARS_DBG("submit thread has stopped, status = %d.\n", err);
if (likely(current->mm)) {
unuse_fake_mm();
}
tinfo->terminated = true;
wake_up_interruptible_all(&tinfo->terminate_event);
return err;
}
static int aio_get_info(struct aio_output *output, struct mars_info *info)
{
struct file *file;
loff_t min;
loff_t max;
if (unlikely(!output ||
!output->mf ||
!(file = output->mf->mf_filp) ||
!file->f_mapping ||
!file->f_mapping->host))
return -EINVAL;
info->tf_align = 1;
info->tf_min_size = 1;
/* Workaround for races in the page cache.
*
* It appears that concurrent reads and writes seem to
* result in inconsistent reads in some very rare cases, due to
* races. Sometimes, the inode claims that the file has been already
* appended by a write operation, but the data has not actually hit
* the page cache, such that a concurrent read gets NULL blocks.
*/
min = i_size_read(file->f_mapping->host);
max = 0;
if (!output->brick->is_static_device) {
get_dirty(output, &min, &max);
}
info->current_size = min;
MARS_DBG("determined file size = %lld\n", info->current_size);
return 0;
}
//////////////// informational / statistics ///////////////
static noinline
char *aio_statistics(struct aio_brick *brick, int verbose)
{
struct aio_output *output = brick->outputs[0];
char *res = brick_string_alloc(4096);
char *sync = NULL;
int pos = 0;
if (!res)
return NULL;
pos += report_timing(&timings[0], res + pos, 4096 - pos);
pos += report_timing(&timings[1], res + pos, 4096 - pos);
pos += report_timing(&timings[2], res + pos, 4096 - pos);
snprintf(res + pos, 4096 - pos,
"total "
"reads = %d "
"writes = %d "
"allocs = %d "
"submits = %d "
"again = %d "
"delays = %d "
"msleeps = %d "
"fdsyncs = %d "
"fdsync_waits = %d "
"map_free = %d | "
"flying reads = %d "
"writes = %d "
"allocs = %d "
"submits = %d "
"q0 = %d "
"q1 = %d "
"q2 = %d "
"| total "
"q0 = %d "
"q1 = %d "
"q2 = %d "
"%s\n",
atomic_read(&output->total_read_count),
atomic_read(&output->total_write_count),
atomic_read(&output->total_alloc_count),
atomic_read(&output->total_submit_count),
atomic_read(&output->total_again_count),
atomic_read(&output->total_delay_count),
atomic_read(&output->total_msleep_count),
atomic_read(&output->total_fdsync_count),
atomic_read(&output->total_fdsync_wait_count),
atomic_read(&output->total_mapfree_count),
atomic_read(&output->read_count),
atomic_read(&output->write_count),
atomic_read(&output->alloc_count),
atomic_read(&output->submit_count),
atomic_read(&output->tinfo[0].queued_sum),
atomic_read(&output->tinfo[1].queued_sum),
atomic_read(&output->tinfo[2].queued_sum),
atomic_read(&output->tinfo[0].total_enqueue_count),
atomic_read(&output->tinfo[1].total_enqueue_count),
atomic_read(&output->tinfo[2].total_enqueue_count),
sync ? sync : "");
if (sync)
brick_string_free(sync);
return res;
}
static noinline
void aio_reset_statistics(struct aio_brick *brick)
{
struct aio_output *output = brick->outputs[0];
int i;
atomic_set(&output->total_read_count, 0);
atomic_set(&output->total_write_count, 0);
atomic_set(&output->total_alloc_count, 0);
atomic_set(&output->total_submit_count, 0);
atomic_set(&output->total_again_count, 0);
atomic_set(&output->total_delay_count, 0);
atomic_set(&output->total_msleep_count, 0);
atomic_set(&output->total_fdsync_count, 0);
atomic_set(&output->total_fdsync_wait_count, 0);
atomic_set(&output->total_mapfree_count, 0);
for (i = 0; i < 3; i++) {
struct aio_threadinfo *tinfo = &output->tinfo[i];
atomic_set(&tinfo->total_enqueue_count, 0);
}
}
//////////////// object / aspect constructors / destructors ///////////////
static int aio_mref_aspect_init_fn(struct generic_aspect *_ini)
{
struct aio_mref_aspect *ini = (void*)_ini;
INIT_LIST_HEAD(&ini->io_head);
INIT_LIST_HEAD(&ini->dirty_head);
return 0;
}
static void aio_mref_aspect_exit_fn(struct generic_aspect *_ini)
{
struct aio_mref_aspect *ini = (void*)_ini;
CHECK_HEAD_EMPTY(&ini->dirty_head);
CHECK_HEAD_EMPTY(&ini->io_head);
}
MARS_MAKE_STATICS(aio);
////////////////////// brick constructors / destructors ////////////////////
static int aio_brick_construct(struct aio_brick *brick)
{
return 0;
}
static int aio_switch(struct aio_brick *brick)
{
static int index;
struct aio_output *output = brick->outputs[0];
const char *path = output->brick->brick_path;
int flags = O_RDWR | O_LARGEFILE;
int status = 0;
MARS_DBG("power.button = %d\n", brick->power.button);
if (!brick->power.button)
goto cleanup;
if (brick->power.led_on || output->mf)
goto done;
mars_power_led_off((void*)brick, false);
if (brick->o_creat) {
flags |= O_CREAT;
MARS_DBG("using O_CREAT on %s\n", path);
}
if (brick->o_direct) {
flags |= O_DIRECT;
MARS_DBG("using O_DIRECT on %s\n", path);
}
output->mf = mapfree_get(path, flags);
if (unlikely(!output->mf)) {
MARS_ERR("could not open file = '%s' flags = %d\n", path, flags);
status = -ENOENT;
goto err;
}
output->index = ++index;
status = _create_ioctx(output);
if (unlikely(status < 0)) {
MARS_ERR("could not create ioctx, status = %d\n", status);
goto err;
}
status = aio_start_thread(output, &output->tinfo[0], aio_submit_thread, 's');
if (unlikely(status < 0)) {
MARS_ERR("could not start theads, status = %d\n", status);
goto err;
}
MARS_DBG("opened file '%s'\n", path);
mars_power_led_on((void*)brick, true);
done:
return 0;
err:
MARS_ERR("status = %d\n", status);
cleanup:
if (brick->power.led_off) {
goto done;
}
mars_power_led_on((void*)brick, false);
aio_stop_thread(output, 0, false);
_destroy_ioctx(output);
mars_power_led_off((void*)brick,
(output->tinfo[0].thread == NULL &&
output->tinfo[1].thread == NULL &&
output->tinfo[2].thread == NULL));
MARS_DBG("switch off led_off = %d status = %d\n", brick->power.led_off, status);
if (brick->power.led_off) {
if (output->mf) {
MARS_DBG("closing file = '%s'\n", output->mf->mf_name);
mapfree_put(output->mf);
output->mf = NULL;
}
}
return status;
}
static int aio_output_construct(struct aio_output *output)
{
INIT_LIST_HEAD(&output->dirty_anchor);
spin_lock_init(&output->dirty_lock);
init_waitqueue_head(&output->fdsync_event);
output->fd = -1;
return 0;
}
static int aio_ref_get(struct aio_output *output, struct mref_object *mref)
{
struct file *file;
struct inode *inode;
loff_t total_size;
if (unlikely(!output->mf)) {
MARS_ERR("brick is not switched on\n");
return -EILSEQ;
}
if (unlikely(mref->ref_len <= 0)) {
MARS_ERR("bad ref_len=%d\n", mref->ref_len);
return -EILSEQ;
}
if (mref->ref_initialized) {
_mref_get(mref);
return mref->ref_len;
}
file = output->mf->mf_filp;
if (unlikely(!file)) {
MARS_ERR("file is not open\n");
return -EILSEQ;
}
if (unlikely(!file->f_mapping)) {
MARS_ERR("file %p has no mapping\n", file);
return -EILSEQ;
}
inode = file->f_mapping->host;
if (unlikely(!inode)) {
MARS_ERR("file %p has no inode\n", file);
return -EILSEQ;
}
total_size = i_size_read(inode);
mref->ref_total_size = total_size;
/* Only check reads.
* Writes behind EOF are always allowed (sparse files)
*/
if (!mref->ref_may_write) {
loff_t len = total_size - mref->ref_pos;
if (unlikely(len <= 0)) {
/* Special case: allow reads starting _exactly_ at EOF when a timeout is specified.
*/
if (len < 0 || mref->ref_timeout <= 0) {
MARS_DBG("ENODATA %lld\n", len);
return -ENODATA;
}
}
// Shorten below EOF, but allow special case
if (mref->ref_len > len && len > 0) {
mref->ref_len = len;
}
}
/* Buffered IO.
*/
if (!mref->ref_data) {
struct aio_mref_aspect *mref_a = aio_mref_get_aspect(output->brick, mref);
if (unlikely(!mref_a)) {
MARS_ERR("bad mref_a\n");
return -EILSEQ;
}
if (unlikely(mref->ref_len <= 0)) {
MARS_ERR("bad ref_len = %d\n", mref->ref_len);
return -ENOMEM;
}
mref->ref_data = brick_block_alloc(mref->ref_pos, (mref_a->alloc_len = mref->ref_len));
if (unlikely(!mref->ref_data)) {
MARS_ERR("ENOMEM %d bytes\n", mref->ref_len);
return -ENOMEM;
}
#if 0 // ???
mref->ref_flags = 0;
#endif
mref_a->do_dealloc = true;
atomic_inc(&output->total_alloc_count);
atomic_inc(&output->alloc_count);
}
_mref_get_first(mref);
return mref->ref_len;
}
static int aio_submit(struct aio_output *output, struct aio_mref_aspect *mref_a, bool use_fdsync)
{
struct mref_object *mref = mref_a->object;
mm_segment_t oldfs;
int res;
struct iocb iocb = {
.aio_data = (__u64)mref_a,
.aio_lio_opcode = use_fdsync ? IOCB_CMD_FDSYNC : (mref->ref_rw != 0 ? IOCB_CMD_PWRITE : IOCB_CMD_PREAD),
.aio_fildes = output->fd,
.aio_buf = (unsigned long)mref->ref_data,
.aio_nbytes = mref->ref_len,
.aio_offset = mref->ref_pos,
// .aio_reqprio = something(mref->ref_prio) field exists, but not yet implemented in kernelspace :(
};
struct iocb *iocbp = &iocb;
unsigned long long latency;
mars_trace(mref, "aio_submit");
if (unlikely(output->fd < 0)) {
MARS_ERR("bad fd = %d\n", output->fd);
res = -EBADF;
goto done;
}
oldfs = get_fs();
set_fs(get_ds());
latency = TIME_STATS(&timings[mref->ref_rw & 1], res = sys_io_submit(output->ctxp, 1, &iocbp));
set_fs(oldfs);
threshold_check(&aio_submit_threshold, latency);
atomic_inc(&output->total_submit_count);
if (likely(res >= 0)) {
atomic_inc(&output->submit_count);
} else if (likely(res == -EAGAIN)) {
atomic_inc(&output->total_again_count);
} else {
MARS_ERR("error = %d\n", res);
}
done:
return res;
}
static int aio_submit_dummy(struct aio_output *output)
{
mm_segment_t oldfs;
int res;
int dummy;
struct iocb iocb = {
.aio_buf = (__u64)&dummy,
};
struct iocb *iocbp = &iocb;
oldfs = get_fs();
set_fs(get_ds());
res = sys_io_submit(output->ctxp, 1, &iocbp);
set_fs(oldfs);
if (likely(res >= 0)) {
atomic_inc(&output->submit_count);
}
return res;
}
static
int aio_start_thread(
struct aio_output *output,
struct aio_threadinfo *tinfo,
int(*fn)(void*),
char class)
{
int j;
for (j = 0; j < MARS_PRIO_NR; j++) {
INIT_LIST_HEAD(&tinfo->mref_list[j]);
}
tinfo->output = output;
spin_lock_init(&tinfo->lock);
init_waitqueue_head(&tinfo->event);
init_waitqueue_head(&tinfo->terminate_event);
tinfo->terminated = false;
tinfo->thread = brick_thread_create(fn, tinfo, "mars_aio_%c%d", class, output->index);
if (unlikely(!tinfo->thread)) {
MARS_ERR("cannot create thread\n");
return -ENOENT;
}
return 0;
}
static
void aio_stop_thread(struct aio_output *output, int i, bool do_submit_dummy)
{
struct aio_threadinfo *tinfo = &output->tinfo[i];
if (tinfo->thread) {
MARS_DBG("stopping thread %d ...\n", i);
brick_thread_stop_nowait(tinfo->thread);
// workaround for waking up the receiver thread. TODO: check whether signal handlong could do better.
if (do_submit_dummy) {
MARS_DBG("submitting dummy for wakeup %d...\n", i);
use_fake_mm();
aio_submit_dummy(output);
if (likely(current->mm)) {
unuse_fake_mm();
}
}
// wait for termination
MARS_DBG("waiting for thread %d ...\n", i);
wait_event_interruptible_timeout(
tinfo->terminate_event,
tinfo->terminated,
(60 - i * 2) * HZ);
if (likely(tinfo->terminated)) {
brick_thread_stop(tinfo->thread);
} else {
MARS_ERR("thread %d did not terminate - leaving a zombie\n", i);
}
}
}
static
int aio_sync(struct file *file)
{
int err;
switch (aio_sync_mode) {
case 1:
#if defined(S_BIAS) || (defined(RHEL_MAJOR) && (RHEL_MAJOR < 7))
err = vfs_fsync_range(file, file->f_path.dentry, 0, LLONG_MAX, 1);
#else
err = vfs_fsync_range(file, 0, LLONG_MAX, 1);
#endif
break;
case 2:
#if defined(S_BIAS) || (defined(RHEL_MAJOR) && (RHEL_MAJOR < 7))
err = vfs_fsync_range(file, file->f_path.dentry, 0, LLONG_MAX, 0);
#else
err = vfs_fsync_range(file, 0, LLONG_MAX, 0);
#endif
break;
default:
err = filemap_write_and_wait_range(file->f_mapping, 0, LLONG_MAX);
}
return err;
}
static
int receiver_thread(void *data)
{
struct client_output *output = data;
int status = 0;
while (!brick_thread_should_stop()) {
struct mars_cmd cmd = {};
struct list_head *tmp;
struct client_mref_aspect *mref_a = NULL;
struct mref_object *mref = NULL;
unsigned long flags;
status = mars_recv_struct(&output->socket, &cmd, mars_cmd_meta);
MARS_IO("got cmd = %d status = %d\n", cmd.cmd_code, status);
if (status < 0)
goto done;
switch (cmd.cmd_code & CMD_FLAG_MASK) {
case CMD_NOTIFY:
mars_trigger();
break;
case CMD_CONNECT:
if (cmd.cmd_int1 < 0) {
status = cmd.cmd_int1;
MARS_ERR("at remote side: brick connect failed, remote status = %d\n", status);
goto done;
}
break;
case CMD_CB:
{
int hash_index = cmd.cmd_int1 % CLIENT_HASH_MAX;
traced_lock(&output->lock, flags);
for (tmp = output->hash_table[hash_index].next; tmp != &output->hash_table[hash_index]; tmp = tmp->next) {
struct mref_object *tmp_mref;
mref_a = container_of(tmp, struct client_mref_aspect, hash_head);
tmp_mref = mref_a->object;
if (unlikely(!tmp_mref)) {
traced_unlock(&output->lock, flags);
MARS_ERR("bad internal mref pointer\n");
status = -EBADR;
goto done;
}
if (tmp_mref->ref_id == cmd.cmd_int1) {
mref = tmp_mref;
list_del_init(&mref_a->hash_head);
list_del_init(&mref_a->io_head);
break;
}
}
traced_unlock(&output->lock, flags);
if (unlikely(!mref)) {
MARS_WRN("got unknown id = %d for callback\n", cmd.cmd_int1);
status = -EBADR;
goto done;
}
MARS_IO("got callback id = %d, old pos = %lld len = %d rw = %d\n", mref->ref_id, mref->ref_pos, mref->ref_len, mref->ref_rw);
status = mars_recv_cb(&output->socket, mref, &cmd);
MARS_IO("new status = %d, pos = %lld len = %d rw = %d\n", status, mref->ref_pos, mref->ref_len, mref->ref_rw);
if (unlikely(status < 0)) {
MARS_WRN("interrupted data transfer during callback, status = %d\n", status);
_hash_insert(output, mref_a);
goto done;
}
SIMPLE_CALLBACK(mref, 0);
client_ref_put(output, mref);
atomic_dec(&output->fly_count);
atomic_dec(&mars_global_io_flying);
break;
}
case CMD_GETINFO:
status = mars_recv_struct(&output->socket, &output->info, mars_info_meta);
if (status < 0) {
MARS_WRN("got bad info from remote side, status = %d\n", status);
goto done;
}
output->got_info = true;
wake_up_interruptible(&output->info_event);
break;
default:
MARS_ERR("got bad command %d from remote side, terminating.\n", cmd.cmd_code);
status = -EBADR;
goto done;
}
done:
brick_string_free(cmd.cmd_str1);
if (unlikely(status < 0)) {
if (!output->recv_error) {
MARS_DBG("signalling status = %d\n", status);
output->recv_error = status;
}
wake_up_interruptible(&output->event);
brick_msleep(100);
}
}
if (status < 0) {
MARS_WRN("receiver thread terminated with status = %d, recv_error = %d\n", status, output->recv_error);
}
mars_shutdown_socket(&output->socket);
wake_up_interruptible(&output->receiver.run_event);
return status;
}
static int _connect(struct client_output *output, const char *str)
{
struct sockaddr_storage sockaddr = {};
int status;
if (unlikely(!output->path)) {
output->path = brick_strdup(str);
status = -ENOMEM;
if (!output->path) {
MARS_DBG("no mem\n");
goto done;
}
status = -EINVAL;
output->host = strchr(output->path, '@');
if (!output->host) {
brick_string_free(output->path);
output->path = NULL;
MARS_ERR("parameter string '%s' contains no remote specifier with '@'-syntax\n", str);
goto done;
}
*output->host++ = '\0';
}
if (unlikely(output->receiver.thread)) {
MARS_WRN("receiver thread unexpectedly not dead\n");
_kill_thread(&output->receiver, "receiver");
}
status = mars_create_sockaddr(&sockaddr, output->host);
if (unlikely(status < 0)) {
MARS_DBG("no sockaddr, status = %d\n", status);
goto done;
}
status = mars_create_socket(&output->socket, &sockaddr, false);
if (unlikely(status < 0)) {
MARS_DBG("no socket, status = %d\n", status);
goto really_done;
}
output->socket.s_shutdown_on_err = true;
output->receiver.thread = brick_thread_create(receiver_thread, output, "mars_receiver%d", thread_count++);
if (unlikely(!output->receiver.thread)) {
MARS_ERR("cannot start receiver thread, status = %d\n", status);
status = -ENOENT;
goto done;
}
{
struct mars_cmd cmd = {
.cmd_code = CMD_CONNECT,
.cmd_str1 = output->path,
};
status = mars_send_struct(&output->socket, &cmd, mars_cmd_meta);
if (unlikely(status < 0)) {
MARS_DBG("send of connect failed, status = %d\n", status);
goto done;
}
}
if (status >= 0) {
status = _request_info(output);
}
done:
if (status < 0) {
MARS_INF("cannot connect to remote host '%s' (status = %d) -- retrying\n", output->host ? output->host : "NULL", status);
_kill_socket(output);
}
really_done:
return status;
}
////////////////// own brick / input / output operations //////////////////
static int client_get_info(struct client_output *output, struct mars_info *info)
{
int status;
output->got_info = false;
output->get_info = true;
wake_up_interruptible(&output->event);
wait_event_interruptible_timeout(output->info_event, output->got_info, 60 * HZ);
status = -EIO;
if (output->got_info && info) {
memcpy(info, &output->info, sizeof(*info));
status = 0;
}
//done:
return status;
}
static void usebuf_ref_io(struct usebuf_output *output, struct mref_object *mref)
{
struct usebuf_input *input = output->brick->inputs[0];
struct usebuf_mref_aspect *mref_a;
struct usebuf_mref_aspect *sub_mref_a;
struct mref_object *sub_mref;
int error = -EILSEQ;
might_sleep();
_mref_check(mref);
mref_a = usebuf_mref_get_aspect(output->brick, mref);
if (unlikely(!mref_a)) {
MARS_FAT("cannot get aspect\n");
goto err;
}
sub_mref_a = mref_a->sub_mref_a;
if (!sub_mref_a) {
MARS_FAT("sub_mref is missing\n");
goto err;
}
sub_mref = sub_mref_a->object;
if (!sub_mref) {
MARS_FAT("sub_mref is missing\n");
goto err;
}
if (mref->ref_rw != 0 && sub_mref->ref_may_write == 0) {
MARS_ERR("mref_may_write was not set before\n");
goto err;
}
_mref_get(mref);
sub_mref->ref_rw = mref->ref_rw;
sub_mref->ref_len = mref->ref_len;
mref_a->input = input;
/* Optimization: when buffered IO is used and buffer is already
* uptodate, skip real IO operation.
*/
if (mref->ref_rw != 0) {
#ifdef DIRECT_WRITE
sub_mref->ref_rw = 1;
#else // normal case
sub_mref->ref_rw = 0;
if (sub_mref->ref_flags & MREF_UPTODATE) {
sub_mref->ref_rw = 1;
}
#endif
} else if (sub_mref->ref_flags & MREF_UPTODATE) {
MARS_IO("direct _usebuf_endio\n");
_usebuf_endio(sub_mref->object_cb);
return;
}
if (mref->ref_data != sub_mref->ref_data) {
if (sub_mref->ref_rw != 0) {
_usebuf_copy(mref, sub_mref, 1);
mref->ref_flags |= MREF_UPTODATE;
}
}
#ifdef FAKE_ALL
_usebuf_endio(sub_mref->ref_cb);
return;
#endif
GENERIC_INPUT_CALL(input, mref_io, sub_mref);
return;
err:
SIMPLE_CALLBACK(mref, error);
return;
}
static int usebuf_ref_get(struct usebuf_output *output, struct mref_object *mref)
{
struct usebuf_input *input = output->brick->inputs[0];
struct usebuf_mref_aspect *mref_a;
struct usebuf_mref_aspect *sub_mref_a;
struct mref_object *sub_mref;
int status = 0;
might_sleep();
mref_a = usebuf_mref_get_aspect(output->brick, mref);
if (unlikely(!mref_a)) {
MARS_FAT("cannot get aspect\n");
return -EILSEQ;
}
sub_mref_a = mref_a->sub_mref_a;
if (!sub_mref_a) {
sub_mref = usebuf_alloc_mref(output->brick);
if (unlikely(!sub_mref)) {
MARS_FAT("cannot get sub_mref\n");
return -ENOMEM;
}
sub_mref_a = usebuf_mref_get_aspect(output->brick, sub_mref);
if (unlikely(!sub_mref_a)) {
MARS_FAT("cannot get aspect\n");
return -EILSEQ;
}
mref_a->sub_mref_a = sub_mref_a;
sub_mref->ref_pos = mref->ref_pos;
sub_mref->ref_len = mref->ref_len;
sub_mref->ref_may_write = mref->ref_may_write;
#ifdef DIRECT_IO // shortcut solely for testing: do direct IO
if (!mref->ref_data)
MARS_ERR("NULL.......\n");
sub_mref->ref_data = mref->ref_data;
#else // normal case: buffered IO
sub_mref->ref_data = NULL;
#endif
SETUP_CALLBACK(sub_mref, _usebuf_endio, mref_a);
mref->ref_flags = 0;
} else {
sub_mref = sub_mref_a->object;
#if 1
MARS_ERR("please do not use this broken feature\n");
#endif
}
status = GENERIC_INPUT_CALL(input, mref_get, sub_mref);
if (status < 0) {
return status;
}
mref->ref_len = sub_mref->ref_len;
//MARS_INF("GOT %p %p flags = %d\n", mref, sub_mref, sub_mref->ref_flags);
if (!mref->ref_data) {
MARS_INF("uiiiiiiiiiii\n");
mref->ref_data = sub_mref->ref_data;
}
_mref_get(mref);
return status;
}
static
int trigger_sysctl_handler(
struct ctl_table *table,
int write,
void __user *buffer,
size_t *length,
loff_t *ppos)
{
ssize_t res = 0;
size_t len = *length;
MARS_DBG("write = %d len = %ld pos = %lld\n", write, len, *ppos);
if (!len || *ppos > 0) {
goto done;
}
if (write) {
char tmp[8] = {};
int code = 0;
res = len; // fake consumption of all data
if (len > 7)
len = 7;
if (!copy_from_user(tmp, buffer, len)) {
sscanf(tmp, "%d", &code);
if (code > 0) {
mars_trigger();
}
if (code > 1) {
mars_remote_trigger();
}
}
} else {
char *answer = "MARS module not operational\n";
char *tmp = NULL;
int mylen;
if (mars_info) {
answer = "internal error while determining mars_info\n";
tmp = mars_info();
if (tmp)
answer = tmp;
}
mylen = strlen(answer);
if (len > mylen)
len = mylen;
res = len;
if (copy_to_user(buffer, answer, len)) {
MARS_ERR("write %ld bytes at %p failed\n", len, buffer);
res = -EFAULT;
}
brick_string_free(tmp);
}
done:
MARS_DBG("res = %ld\n", res);
*length = res;
if (res >= 0) {
*ppos += res;
return 0;
}
return res;
}
