void coroutine_fn qemu_co_queue_wait(CoQueue *queue)
{
Coroutine *self = qemu_coroutine_self();
QSIMPLEQ_INSERT_TAIL(&queue->entries, self, co_queue_next);
qemu_coroutine_yield();
assert(qemu_in_coroutine());
}
static void test_in_coroutine(void)
{
Coroutine *coroutine;
g_assert(!qemu_in_coroutine());
coroutine = qemu_coroutine_create(verify_in_coroutine);
qemu_coroutine_enter(coroutine, NULL);
}
ssize_t nbd_wr_syncv(QIOChannel *ioc,
struct iovec *iov,
size_t niov,
size_t length,
bool do_read)
{
ssize_t done = 0;
Error *local_err = NULL;
struct iovec *local_iov = g_new(struct iovec, niov);
struct iovec *local_iov_head = local_iov;
unsigned int nlocal_iov = niov;
nlocal_iov = iov_copy(local_iov, nlocal_iov, iov, niov, 0, length);
while (nlocal_iov > 0) {
ssize_t len;
if (do_read) {
len = qio_channel_readv(ioc, local_iov, nlocal_iov, &local_err);
} else {
len = qio_channel_writev(ioc, local_iov, nlocal_iov, &local_err);
}
if (len == QIO_CHANNEL_ERR_BLOCK) {
if (qemu_in_coroutine()) {
/* XXX figure out if we can create a variant on
* qio_channel_yield() that works with AIO contexts
* and consider using that in this branch */
qemu_coroutine_yield();
} else if (done) {
/* XXX this is needed by nbd_reply_ready. */
qio_channel_wait(ioc,
do_read ? G_IO_IN : G_IO_OUT);
} else {
return -EAGAIN;
}
continue;
}
if (len < 0) {
TRACE("I/O error: %s", error_get_pretty(local_err));
error_free(local_err);
/* XXX handle Error objects */
done = -EIO;
goto cleanup;
}
if (do_read && len == 0) {
break;
}
iov_discard_front(&local_iov, &nlocal_iov, len);
done += len;
}
cleanup:
g_free(local_iov_head);
return done;
}
void coroutine_fn yield_until_fd_readable(int fd)
{
FDYieldUntilData data;
assert(qemu_in_coroutine());
data.co = qemu_coroutine_self();
data.fd = fd;
qemu_set_fd_handler(fd, fd_coroutine_enter, NULL, &data);
qemu_coroutine_yield();
}
static int nbd_co_send_request(BlockDriverState *bs,
struct nbd_request *request,
QEMUIOVector *qiov, int offset)
{
NbdClientSession *s = nbd_get_client_session(bs);
AioContext *aio_context;
int rc, ret, i;
qemu_co_mutex_lock(&s->send_mutex);
for (i = 0; i < MAX_NBD_REQUESTS; i++) {
if (s->recv_coroutine[i] == NULL) {
s->recv_coroutine[i] = qemu_coroutine_self();
break;
}
}
g_assert(qemu_in_coroutine());
assert(i < MAX_NBD_REQUESTS);
request->handle = INDEX_TO_HANDLE(s, i);
if (!s->ioc) {
qemu_co_mutex_unlock(&s->send_mutex);
return -EPIPE;
}
s->send_coroutine = qemu_coroutine_self();
aio_context = bdrv_get_aio_context(bs);
aio_set_fd_handler(aio_context, s->sioc->fd, false,
nbd_reply_ready, nbd_restart_write, bs);
if (qiov) {
qio_channel_set_cork(s->ioc, true);
rc = nbd_send_request(s->ioc, request);
if (rc >= 0) {
ret = nbd_wr_syncv(s->ioc, qiov->iov, qiov->niov,
offset, request->len, 0);
if (ret != request->len) {
rc = -EIO;
}
}
qio_channel_set_cork(s->ioc, false);
} else {
rc = nbd_send_request(s->ioc, request);
}
aio_set_fd_handler(aio_context, s->sioc->fd, false,
nbd_reply_ready, NULL, bs);
s->send_coroutine = NULL;
qemu_co_mutex_unlock(&s->send_mutex);
return rc;
}
void coroutine_fn qemu_co_mutex_unlock(CoMutex *mutex)
{
Coroutine *self = qemu_coroutine_self();
trace_qemu_co_mutex_unlock_entry(mutex, self);
assert(mutex->locked == true);
assert(qemu_in_coroutine());
mutex->locked = false;
qemu_co_queue_next(&mutex->queue);
trace_qemu_co_mutex_unlock_return(mutex, self);
}
void qemu_co_rwlock_unlock(CoRwlock *lock)
{
assert(qemu_in_coroutine());
if (lock->writer) {
lock->writer = false;
qemu_co_queue_restart_all(&lock->queue);
} else {
lock->reader--;
assert(lock->reader >= 0);
/* Wakeup only one waiting writer */
if (!lock->reader) {
qemu_co_queue_next(&lock->queue);
}
}
}
void coroutine_fn qio_channel_yield(QIOChannel *ioc,
GIOCondition condition)
{
QIOChannelYieldData data;
assert(qemu_in_coroutine());
data.ioc = ioc;
data.co = qemu_coroutine_self();
qio_channel_add_watch(ioc,
condition,
qio_channel_yield_enter,
&data,
NULL);
qemu_coroutine_yield();
}
void coroutine_fn qio_channel_yield(QIOChannel *ioc,
GIOCondition condition)
{
assert(qemu_in_coroutine());
if (condition == G_IO_IN) {
assert(!ioc->read_coroutine);
ioc->read_coroutine = qemu_coroutine_self();
} else if (condition == G_IO_OUT) {
assert(!ioc->write_coroutine);
ioc->write_coroutine = qemu_coroutine_self();
} else {
abort();
}
qio_channel_set_aio_fd_handlers(ioc);
qemu_coroutine_yield();
}
int qio_channel_readv_all_eof(QIOChannel *ioc,
const struct iovec *iov,
size_t niov,
Error **errp)
{
int ret = -1;
struct iovec *local_iov = g_new(struct iovec, niov);
struct iovec *local_iov_head = local_iov;
unsigned int nlocal_iov = niov;
bool partial = false;
nlocal_iov = iov_copy(local_iov, nlocal_iov,
iov, niov,
0, iov_size(iov, niov));
while (nlocal_iov > 0) {
ssize_t len;
len = qio_channel_readv(ioc, local_iov, nlocal_iov, errp);
if (len == QIO_CHANNEL_ERR_BLOCK) {
if (qemu_in_coroutine()) {
qio_channel_yield(ioc, G_IO_IN);
} else {
qio_channel_wait(ioc, G_IO_IN);
}
continue;
} else if (len < 0) {
goto cleanup;
} else if (len == 0) {
if (partial) {
error_setg(errp,
"Unexpected end-of-file before all bytes were read");
} else {
ret = 0;
}
goto cleanup;
}
partial = true;
iov_discard_front(&local_iov, &nlocal_iov, len);
}
ret = 1;
cleanup:
g_free(local_iov_head);
return ret;
}
static int nbd_co_send_request(BlockDriverState *bs,
NBDRequest *request,
QEMUIOVector *qiov)
{
NBDClientSession *s = nbd_get_client_session(bs);
int rc, ret, i;
qemu_co_mutex_lock(&s->send_mutex);
while (s->in_flight == MAX_NBD_REQUESTS) {
qemu_co_queue_wait(&s->free_sema, &s->send_mutex);
}
s->in_flight++;
for (i = 0; i < MAX_NBD_REQUESTS; i++) {
if (s->recv_coroutine[i] == NULL) {
s->recv_coroutine[i] = qemu_coroutine_self();
break;
}
}
g_assert(qemu_in_coroutine());
assert(i < MAX_NBD_REQUESTS);
request->handle = INDEX_TO_HANDLE(s, i);
if (!s->ioc) {
qemu_co_mutex_unlock(&s->send_mutex);
return -EPIPE;
}
if (qiov) {
qio_channel_set_cork(s->ioc, true);
rc = nbd_send_request(s->ioc, request);
if (rc >= 0) {
ret = nbd_rwv(s->ioc, qiov->iov, qiov->niov, request->len, false,
NULL);
if (ret != request->len) {
rc = -EIO;
}
}
qio_channel_set_cork(s->ioc, false);
} else {
rc = nbd_send_request(s->ioc, request);
}
qemu_co_mutex_unlock(&s->send_mutex);
return rc;
}
ssize_t nbd_wr_sync(int fd, void *buffer, size_t size, bool do_read)
{
size_t offset = 0;
int err;
if (qemu_in_coroutine()) {
if (do_read) {
return qemu_co_recv(fd, buffer, size);
} else {
return qemu_co_send(fd, buffer, size);
}
}
while (offset < size) {
ssize_t len;
if (do_read) {
len = qemu_recv(fd, buffer + offset, size - offset, 0);
} else {
len = send(fd, buffer + offset, size - offset, 0);
}
if (len < 0) {
err = socket_error();
/* recoverable error */
if (err == EINTR || (offset > 0 && err == EAGAIN)) {
continue;
}
/* unrecoverable error */
return -err;
}
/* eof */
if (len == 0) {
break;
}
offset += len;
}
return offset;
}
int qio_channel_writev_all(QIOChannel *ioc,
const struct iovec *iov,
size_t niov,
Error **errp)
{
int ret = -1;
struct iovec *local_iov = g_new(struct iovec, niov);
struct iovec *local_iov_head = local_iov;
unsigned int nlocal_iov = niov;
nlocal_iov = iov_copy(local_iov, nlocal_iov,
iov, niov,
0, iov_size(iov, niov));
while (nlocal_iov > 0) {
ssize_t len;
len = qio_channel_writev(ioc, local_iov, nlocal_iov, errp);
if (len == QIO_CHANNEL_ERR_BLOCK) {
if (qemu_in_coroutine()) {
qio_channel_yield(ioc, G_IO_OUT);
} else {
qio_channel_wait(ioc, G_IO_OUT);
}
continue;
}
if (len < 0) {
goto cleanup;
}
iov_discard_front(&local_iov, &nlocal_iov, len);
}
ret = 0;
cleanup:
g_free(local_iov_head);
return ret;
}
bool coroutine_fn qemu_co_queue_next(CoQueue *queue)
{
assert(qemu_in_coroutine());
return qemu_co_queue_do_restart(queue, true);
}
void coroutine_fn qemu_co_queue_restart_all(CoQueue *queue)
{
assert(qemu_in_coroutine());
qemu_co_queue_do_restart(queue, false);
}
static void coroutine_fn verify_in_coroutine(void *opaque)
{
g_assert(qemu_in_coroutine());
}
static int nbd_co_send_request(BlockDriverState *bs,
NBDRequest *request,
QEMUIOVector *qiov)
{
NBDClientSession *s = nbd_get_client_session(bs);
int rc, i;
qemu_co_mutex_lock(&s->send_mutex);
while (s->in_flight == MAX_NBD_REQUESTS) {
qemu_co_queue_wait(&s->free_sema, &s->send_mutex);
}
s->in_flight++;
for (i = 0; i < MAX_NBD_REQUESTS; i++) {
if (s->requests[i].coroutine == NULL) {
break;
}
}
g_assert(qemu_in_coroutine());
assert(i < MAX_NBD_REQUESTS);
s->requests[i].coroutine = qemu_coroutine_self();
s->requests[i].receiving = false;
request->handle = INDEX_TO_HANDLE(s, i);
if (s->quit) {
rc = -EIO;
goto err;
}
if (!s->ioc) {
rc = -EPIPE;
goto err;
}
if (qiov) {
qio_channel_set_cork(s->ioc, true);
rc = nbd_send_request(s->ioc, request);
if (rc >= 0 && !s->quit) {
assert(request->len == iov_size(qiov->iov, qiov->niov));
if (qio_channel_writev_all(s->ioc, qiov->iov, qiov->niov,
NULL) < 0) {
rc = -EIO;
}
}
qio_channel_set_cork(s->ioc, false);
} else {
rc = nbd_send_request(s->ioc, request);
}
err:
if (rc < 0) {
s->quit = true;
s->requests[i].coroutine = NULL;
s->in_flight--;
qemu_co_queue_next(&s->free_sema);
}
qemu_co_mutex_unlock(&s->send_mutex);
return rc;
}