static void aio_ref_io(struct aio_output *output, struct mref_object *mref)
{
struct aio_threadinfo *tinfo = &output->tinfo[0];
struct aio_mref_aspect *mref_a;
int err = -EINVAL;
_mref_check(mref);
if (unlikely(!output->brick->power.led_on)) {
SIMPLE_CALLBACK(mref, -EBADFD);
return;
}
_mref_get(mref);
atomic_inc(&mars_global_io_flying);
atomic_inc(&output->work_count);
// statistics
if (mref->ref_rw) {
atomic_inc(&output->total_write_count);
atomic_inc(&output->write_count);
} else {
atomic_inc(&output->total_read_count);
atomic_inc(&output->read_count);
}
if (unlikely(!output->mf || !output->mf->mf_filp)) {
goto done;
}
mapfree_set(output->mf, mref->ref_pos, -1);
MARS_IO("AIO rw=%d pos=%lld len=%d data=%p\n", mref->ref_rw, mref->ref_pos, mref->ref_len, mref->ref_data);
mref_a = aio_mref_get_aspect(output->brick, mref);
if (unlikely(!mref_a)) {
goto done;
}
_enqueue(tinfo, mref_a, mref->ref_prio, true);
return;
done:
_complete_mref(output, mref, err);
}
static void client_ref_io(struct client_output *output, struct mref_object *mref)
{
struct client_mref_aspect *mref_a;
int error = -EINVAL;
mref_a = client_mref_get_aspect(output->brick, mref);
if (unlikely(!mref_a)) {
goto error;
}
while (output->brick->max_flying > 0 && atomic_read(&output->fly_count) > output->brick->max_flying) {
MARS_IO("sleeping request pos = %lld len = %d rw = %d (flying = %d)\n", mref->ref_pos, mref->ref_len, mref->ref_rw, atomic_read(&output->fly_count));
#ifdef IO_DEBUGGING
brick_msleep(3000);
#else
brick_msleep(1000 * 2 / HZ);
#endif
}
atomic_inc(&mars_global_io_flying);
atomic_inc(&output->fly_count);
_mref_get(mref);
mref_a->submit_jiffies = jiffies;
_hash_insert(output, mref_a);
MARS_IO("added request id = %d pos = %lld len = %d rw = %d (flying = %d)\n", mref->ref_id, mref->ref_pos, mref->ref_len, mref->ref_rw, atomic_read(&output->fly_count));
wake_up_interruptible(&output->event);
return;
error:
MARS_ERR("IO error = %d\n", error);
SIMPLE_CALLBACK(mref, error);
client_ref_put(output, mref);
}
static void usebuf_ref_io(struct usebuf_output *output, struct mref_object *mref)
{
struct usebuf_input *input = output->brick->inputs[0];
struct usebuf_mref_aspect *mref_a;
struct usebuf_mref_aspect *sub_mref_a;
struct mref_object *sub_mref;
int error = -EILSEQ;
might_sleep();
_mref_check(mref);
mref_a = usebuf_mref_get_aspect(output->brick, mref);
if (unlikely(!mref_a)) {
MARS_FAT("cannot get aspect\n");
goto err;
}
sub_mref_a = mref_a->sub_mref_a;
if (!sub_mref_a) {
MARS_FAT("sub_mref is missing\n");
goto err;
}
sub_mref = sub_mref_a->object;
if (!sub_mref) {
MARS_FAT("sub_mref is missing\n");
goto err;
}
if (mref->ref_rw != 0 && sub_mref->ref_may_write == 0) {
MARS_ERR("mref_may_write was not set before\n");
goto err;
}
_mref_get(mref);
sub_mref->ref_rw = mref->ref_rw;
sub_mref->ref_len = mref->ref_len;
mref_a->input = input;
/* Optimization: when buffered IO is used and buffer is already
* uptodate, skip real IO operation.
*/
if (mref->ref_rw != 0) {
#ifdef DIRECT_WRITE
sub_mref->ref_rw = 1;
#else // normal case
sub_mref->ref_rw = 0;
if (sub_mref->ref_flags & MREF_UPTODATE) {
sub_mref->ref_rw = 1;
}
#endif
} else if (sub_mref->ref_flags & MREF_UPTODATE) {
MARS_IO("direct _usebuf_endio\n");
_usebuf_endio(sub_mref->object_cb);
return;
}
if (mref->ref_data != sub_mref->ref_data) {
if (sub_mref->ref_rw != 0) {
_usebuf_copy(mref, sub_mref, 1);
mref->ref_flags |= MREF_UPTODATE;
}
}
#ifdef FAKE_ALL
_usebuf_endio(sub_mref->ref_cb);
return;
#endif
GENERIC_INPUT_CALL(input, mref_io, sub_mref);
return;
err:
SIMPLE_CALLBACK(mref, error);
return;
}
static
void _do_timeout(struct client_output *output, struct list_head *anchor, bool force)
{
struct client_brick *brick = output->brick;
struct list_head *tmp;
struct list_head *next;
LIST_HEAD(tmp_list);
int rounds = 0;
long io_timeout = brick->io_timeout;
unsigned long flags;
if (io_timeout <= 0)
io_timeout = global_net_io_timeout;
if (!mars_net_is_alive)
force = true;
if (!force && io_timeout <= 0)
return;
io_timeout *= HZ;
traced_lock(&output->lock, flags);
for (tmp = anchor->next, next = tmp->next; tmp != anchor; tmp = next, next = tmp->next) {
struct client_mref_aspect *mref_a;
mref_a = container_of(tmp, struct client_mref_aspect, io_head);
if (!force &&
!time_is_before_jiffies(mref_a->submit_jiffies + io_timeout)) {
continue;
}
list_del_init(&mref_a->hash_head);
list_del_init(&mref_a->io_head);
list_add_tail(&mref_a->tmp_head, &tmp_list);
}
traced_unlock(&output->lock, flags);
while (!list_empty(&tmp_list)) {
struct client_mref_aspect *mref_a;
struct mref_object *mref;
tmp = tmp_list.next;
list_del_init(tmp);
mref_a = container_of(tmp, struct client_mref_aspect, tmp_head);
mref = mref_a->object;
if (!rounds++) {
MARS_WRN("timeout after %ld: signalling IO error at pos = %lld len = %d\n",
io_timeout,
mref->ref_pos,
mref->ref_len);
}
atomic_inc(&output->timeout_count);
SIMPLE_CALLBACK(mref, -ENOTCONN);
client_ref_put(output, mref);
atomic_dec(&output->fly_count);
atomic_dec(&mars_global_io_flying);
}
}
static
int receiver_thread(void *data)
{
struct client_output *output = data;
int status = 0;
while (!brick_thread_should_stop()) {
struct mars_cmd cmd = {};
struct list_head *tmp;
struct client_mref_aspect *mref_a = NULL;
struct mref_object *mref = NULL;
unsigned long flags;
status = mars_recv_struct(&output->socket, &cmd, mars_cmd_meta);
MARS_IO("got cmd = %d status = %d\n", cmd.cmd_code, status);
if (status < 0)
goto done;
switch (cmd.cmd_code & CMD_FLAG_MASK) {
case CMD_NOTIFY:
mars_trigger();
break;
case CMD_CONNECT:
if (cmd.cmd_int1 < 0) {
status = cmd.cmd_int1;
MARS_ERR("at remote side: brick connect failed, remote status = %d\n", status);
goto done;
}
break;
case CMD_CB:
{
int hash_index = cmd.cmd_int1 % CLIENT_HASH_MAX;
traced_lock(&output->lock, flags);
for (tmp = output->hash_table[hash_index].next; tmp != &output->hash_table[hash_index]; tmp = tmp->next) {
struct mref_object *tmp_mref;
mref_a = container_of(tmp, struct client_mref_aspect, hash_head);
tmp_mref = mref_a->object;
if (unlikely(!tmp_mref)) {
traced_unlock(&output->lock, flags);
MARS_ERR("bad internal mref pointer\n");
status = -EBADR;
goto done;
}
if (tmp_mref->ref_id == cmd.cmd_int1) {
mref = tmp_mref;
list_del_init(&mref_a->hash_head);
list_del_init(&mref_a->io_head);
break;
}
}
traced_unlock(&output->lock, flags);
if (unlikely(!mref)) {
MARS_WRN("got unknown id = %d for callback\n", cmd.cmd_int1);
status = -EBADR;
goto done;
}
MARS_IO("got callback id = %d, old pos = %lld len = %d rw = %d\n", mref->ref_id, mref->ref_pos, mref->ref_len, mref->ref_rw);
status = mars_recv_cb(&output->socket, mref, &cmd);
MARS_IO("new status = %d, pos = %lld len = %d rw = %d\n", status, mref->ref_pos, mref->ref_len, mref->ref_rw);
if (unlikely(status < 0)) {
MARS_WRN("interrupted data transfer during callback, status = %d\n", status);
_hash_insert(output, mref_a);
goto done;
}
SIMPLE_CALLBACK(mref, 0);
client_ref_put(output, mref);
atomic_dec(&output->fly_count);
atomic_dec(&mars_global_io_flying);
break;
}
case CMD_GETINFO:
status = mars_recv_struct(&output->socket, &output->info, mars_info_meta);
if (status < 0) {
MARS_WRN("got bad info from remote side, status = %d\n", status);
goto done;
}
output->got_info = true;
wake_up_interruptible(&output->info_event);
break;
default:
MARS_ERR("got bad command %d from remote side, terminating.\n", cmd.cmd_code);
status = -EBADR;
goto done;
}
done:
brick_string_free(cmd.cmd_str1);
if (unlikely(status < 0)) {
if (!output->recv_error) {
MARS_DBG("signalling status = %d\n", status);
output->recv_error = status;
}
wake_up_interruptible(&output->event);
brick_msleep(100);
}
}
if (status < 0) {
MARS_WRN("receiver thread terminated with status = %d, recv_error = %d\n", status, output->recv_error);
}
mars_shutdown_socket(&output->socket);
wake_up_interruptible(&output->receiver.run_event);
return status;
}