long qibwrite(struct queue *q, struct block *b)
{
int n, dowakeup;
dowakeup = 0;
n = BLEN(b);
spin_lock_irqsave(&q->lock);
QDEBUG checkb(b, "qibwrite");
if (q->bfirst)
q->blast->next = b;
else
q->bfirst = b;
q->blast = b;
q->len += BALLOC(b);
q->dlen += n;
if (q->state & Qstarve) {
q->state &= ~Qstarve;
dowakeup = 1;
}
spin_unlock_irqsave(&q->lock);
if (dowakeup) {
if (q->kick)
q->kick(q->arg);
rendez_wakeup(&q->rr);
qwake_cb(q, FDTAP_FILT_READABLE);
}
return n;
}
/*
* Mark a queue as closed. No further IO is permitted.
* All blocks are released.
*/
void qclose(struct queue *q)
{
struct block *bfirst;
if (q == NULL)
return;
/* mark it */
spin_lock_irqsave(&q->lock);
q->state |= Qclosed;
q->state &= ~(Qflow | Qstarve | Qdropoverflow);
q->err[0] = 0;
bfirst = q->bfirst;
q->bfirst = 0;
q->len = 0;
q->dlen = 0;
spin_unlock_irqsave(&q->lock);
/* free queued blocks */
freeblist(bfirst);
/* wake up readers/writers */
rendez_wakeup(&q->rr);
rendez_wakeup(&q->wr);
qwake_cb(q, FDTAP_FILT_HANGUP);
}
/*
* flow control, get producer going again
* called with q ilocked
*/
static void qwakeup_iunlock(struct queue *q)
{
int dowakeup = 0;
/*
* if writer flow controlled, restart
*
* This used to be
* q->len < q->limit/2
* but it slows down tcp too much for certain write sizes.
* I really don't understand it completely. It may be
* due to the queue draining so fast that the transmission
* stalls waiting for the app to produce more data. - presotto
*/
if ((q->state & Qflow) && q->len < q->limit) {
q->state &= ~Qflow;
dowakeup = 1;
}
spin_unlock_irqsave(&q->lock);
/* wakeup flow controlled writers */
if (dowakeup) {
if (q->kick)
q->kick(q->arg);
rendez_wakeup(&q->wr);
}
qwake_cb(q, FDTAP_FILT_WRITABLE);
}
int qpass(struct queue *q, struct block *b)
{
int dlen, len, dowakeup;
/* sync with qread */
dowakeup = 0;
spin_lock_irqsave(&q->lock);
if (q->len >= q->limit) {
freeblist(b);
spin_unlock_irqsave(&q->lock);
return -1;
}
if (q->state & Qclosed) {
len = blocklen(b);
freeblist(b);
spin_unlock_irqsave(&q->lock);
return len;
}
/* add buffer to queue */
if (q->bfirst)
q->blast->next = b;
else
q->bfirst = b;
len = BALLOC(b);
dlen = BLEN(b);
QDEBUG checkb(b, "qpass");
while (b->next) {
b = b->next;
QDEBUG checkb(b, "qpass");
len += BALLOC(b);
dlen += BLEN(b);
}
q->blast = b;
q->len += len;
q->dlen += dlen;
if (q->len >= q->limit / 2)
q->state |= Qflow;
if (q->state & Qstarve) {
q->state &= ~Qstarve;
dowakeup = 1;
}
spin_unlock_irqsave(&q->lock);
if (dowakeup) {
rendez_wakeup(&q->rr);
qwake_cb(q, FDTAP_FILT_READABLE);
}
return len;
}
/* Mark a queue as closed. Wakeup any readers. Don't remove queued blocks.
*
* msg will be the errstr received by any waiters (qread, qbread, etc). If
* there is no message, which is what also happens during a natural qclose(),
* those waiters will simply return 0. qwriters will always error() on a
* closed/hungup queue. */
void qhangup(struct queue *q, char *msg)
{
/* mark it */
spin_lock_irqsave(&q->lock);
q->state |= Qclosed;
if (msg == 0 || *msg == 0)
q->err[0] = 0;
else
strlcpy(q->err, msg, ERRMAX);
spin_unlock_irqsave(&q->lock);
/* wake up readers/writers */
rendez_wakeup(&q->rr);
rendez_wakeup(&q->wr);
qwake_cb(q, FDTAP_FILT_HANGUP);
}
/*
* flush the output queue
*/
void qflush(struct queue *q)
{
struct block *bfirst;
/* mark it */
spin_lock_irqsave(&q->lock);
bfirst = q->bfirst;
q->bfirst = 0;
q->len = 0;
q->dlen = 0;
spin_unlock_irqsave(&q->lock);
/* free queued blocks */
freeblist(bfirst);
/* wake up writers */
rendez_wakeup(&q->wr);
qwake_cb(q, FDTAP_FILT_WRITABLE);
}
/*
* flow control, get producer going again
* called with q ilocked
*/
static void qwakeup_iunlock(struct queue *q)
{
int dowakeup = 0;
/* if writer flow controlled, restart */
if ((q->state & Qflow) && q->len < q->limit / 2) {
q->state &= ~Qflow;
dowakeup = 1;
}
spin_unlock_irqsave(&q->lock);
/* wakeup flow controlled writers */
if (dowakeup) {
if (q->kick)
q->kick(q->arg);
rendez_wakeup(&q->wr);
qwake_cb(q, FDTAP_FILT_WRITABLE);
}
}
/*
* get next block from a queue, return null if nothing there
*/
struct block *qget(struct queue *q)
{
int dowakeup;
struct block *b;
/* sync with qwrite */
spin_lock_irqsave(&q->lock);
b = q->bfirst;
if (b == NULL) {
q->state |= Qstarve;
spin_unlock_irqsave(&q->lock);
return NULL;
}
q->bfirst = b->next;
b->next = 0;
q->len -= BALLOC(b);
q->dlen -= BLEN(b);
QDEBUG checkb(b, "qget");
/* if writer flow controlled, restart */
if ((q->state & Qflow) && q->len < q->limit / 2) {
q->state &= ~Qflow;
dowakeup = 1;
} else
dowakeup = 0;
spin_unlock_irqsave(&q->lock);
if (dowakeup) {
rendez_wakeup(&q->wr);
/* We only send the writable event on wakeup, which is edge triggered */
qwake_cb(q, FDTAP_FILT_WRITABLE);
}
return b;
}
/* Adds block (which can be a list of blocks) to the queue, subject to
* qio_flags. Returns the length written on success or -1 on non-throwable
* error. Adjust qio_flags to control the value-added features!. */
static ssize_t __qbwrite(struct queue *q, struct block *b, int qio_flags)
{
ssize_t ret;
bool dowakeup = FALSE;
bool was_empty;
if (q->bypass) {
ret = blocklen(b);
(*q->bypass) (q->arg, b);
return ret;
}
spin_lock_irqsave(&q->lock);
was_empty = q->len == 0;
if (q->state & Qclosed) {
spin_unlock_irqsave(&q->lock);
freeblist(b);
if (!(qio_flags & QIO_CAN_ERR_SLEEP))
return -1;
if (q->err[0])
error(EFAIL, q->err);
else
error(EFAIL, "connection closed");
}
if ((qio_flags & QIO_LIMIT) && (q->len >= q->limit)) {
/* drop overflow takes priority over regular non-blocking */
if ((qio_flags & QIO_DROP_OVERFLOW) || (q->state & Qdropoverflow)) {
spin_unlock_irqsave(&q->lock);
freeb(b);
return -1;
}
/* People shouldn't set NON_BLOCK without CAN_ERR, but we can be nice
* and catch it. */
if ((qio_flags & QIO_CAN_ERR_SLEEP) && (qio_flags & QIO_NON_BLOCK)) {
spin_unlock_irqsave(&q->lock);
freeb(b);
error(EAGAIN, "queue full");
}
}
ret = enqueue_blist(q, b);
QDEBUG checkb(b, "__qbwrite");
/* make sure other end gets awakened */
if (q->state & Qstarve) {
q->state &= ~Qstarve;
dowakeup = TRUE;
}
spin_unlock_irqsave(&q->lock);
/* TODO: not sure if the usage of a kick is mutually exclusive with a
* wakeup, meaning that actual users either want a kick or have qreaders. */
if (q->kick && (dowakeup || (q->state & Qkick)))
q->kick(q->arg);
if (dowakeup)
rendez_wakeup(&q->rr);
if (was_empty)
qwake_cb(q, FDTAP_FILT_READABLE);
/*
* flow control, wait for queue to get below the limit
* before allowing the process to continue and queue
* more. We do this here so that postnote can only
* interrupt us after the data has been queued. This
* means that things like 9p flushes and ssl messages
* will not be disrupted by software interrupts.
*
* Note - this is moderately dangerous since a process
* that keeps getting interrupted and rewriting will
* queue infinite crud.
*/
if ((qio_flags & QIO_CAN_ERR_SLEEP) &&
!(q->state & Qdropoverflow) && !(qio_flags & QIO_NON_BLOCK)) {
/* This is a racy peek at the q status. If we accidentally block, we
* set Qflow, so someone should wake us. If we accidentally don't
* block, we just returned to the user and let them slip a block past
* flow control. */
while (!qnotfull(q)) {
spin_lock_irqsave(&q->lock);
q->state |= Qflow;
spin_unlock_irqsave(&q->lock);
rendez_sleep(&q->wr, qnotfull, q);
}
}
return ret;
}
int qproduce(struct queue *q, void *vp, int len)
{
struct block *b;
int dowakeup;
uint8_t *p = vp;
/* sync with qread */
dowakeup = 0;
spin_lock_irqsave(&q->lock);
/* no waiting receivers, room in buffer? */
if (q->len >= q->limit) {
q->state |= Qflow;
spin_unlock_irqsave(&q->lock);
return -1;
}
/* save in buffer */
/* use Qcoalesce here to save storage */
// TODO: Consider removing the Qcoalesce flag and force a coalescing
// strategy by default.
b = q->blast;
if ((q->state & Qcoalesce) == 0 || q->bfirst == NULL
|| b->lim - b->wp < len) {
/* need a new block */
b = iallocb(len);
if (b == 0) {
spin_unlock_irqsave(&q->lock);
return 0;
}
if (q->bfirst)
q->blast->next = b;
else
q->bfirst = b;
q->blast = b;
/* b->next = 0; done by iallocb() */
q->len += BALLOC(b);
}
PANIC_EXTRA(b);
memmove(b->wp, p, len);
producecnt += len;
b->wp += len;
q->dlen += len;
QDEBUG checkb(b, "qproduce");
if (q->state & Qstarve) {
q->state &= ~Qstarve;
dowakeup = 1;
}
if (q->len >= q->limit)
q->state |= Qflow;
spin_unlock_irqsave(&q->lock);
if (dowakeup) {
rendez_wakeup(&q->rr);
qwake_cb(q, FDTAP_FILT_READABLE);
}
return len;
}
/*
* Interrupt level copy out of a queue, return # bytes copied.
*/
int qconsume(struct queue *q, void *vp, int len)
{
struct block *b;
int n, dowakeup;
uint8_t *p = vp;
struct block *tofree = NULL;
/* sync with qwrite */
spin_lock_irqsave(&q->lock);
for (;;) {
b = q->bfirst;
if (b == 0) {
q->state |= Qstarve;
spin_unlock_irqsave(&q->lock);
return -1;
}
QDEBUG checkb(b, "qconsume 1");
n = BLEN(b);
if (n > 0)
break;
q->bfirst = b->next;
q->len -= BALLOC(b);
/* remember to free this */
b->next = tofree;
tofree = b;
};
PANIC_EXTRA(b);
if (n < len)
len = n;
memmove(p, b->rp, len);
consumecnt += n;
b->rp += len;
q->dlen -= len;
/* discard the block if we're done with it */
if ((q->state & Qmsg) || len == n) {
q->bfirst = b->next;
b->next = 0;
q->len -= BALLOC(b);
q->dlen -= BLEN(b);
/* remember to free this */
b->next = tofree;
tofree = b;
}
/* if writer flow controlled, restart */
if ((q->state & Qflow) && q->len < q->limit / 2) {
q->state &= ~Qflow;
dowakeup = 1;
} else
dowakeup = 0;
spin_unlock_irqsave(&q->lock);
if (dowakeup) {
rendez_wakeup(&q->wr);
qwake_cb(q, FDTAP_FILT_WRITABLE);
}
if (tofree != NULL)
freeblist(tofree);
return len;
}
/*
* throw away the next 'len' bytes in the queue
* returning the number actually discarded
*/
int qdiscard(struct queue *q, int len)
{
struct block *b;
int dowakeup, n, sofar, body_amt, extra_amt;
struct extra_bdata *ebd;
spin_lock_irqsave(&q->lock);
for (sofar = 0; sofar < len; sofar += n) {
b = q->bfirst;
if (b == NULL)
break;
QDEBUG checkb(b, "qdiscard");
n = BLEN(b);
if (n <= len - sofar) {
q->bfirst = b->next;
b->next = 0;
q->len -= BALLOC(b);
q->dlen -= BLEN(b);
freeb(b);
} else {
n = len - sofar;
q->dlen -= n;
/* partial block removal */
body_amt = MIN(BHLEN(b), n);
b->rp += body_amt;
extra_amt = n - body_amt;
/* reduce q->len by the amount we remove from the extras. The
* header will always be accounted for above, during block removal.
* */
q->len -= extra_amt;
for (int i = 0; (i < b->nr_extra_bufs) && extra_amt; i++) {
ebd = &b->extra_data[i];
if (!ebd->base || !ebd->len)
continue;
if (extra_amt >= ebd->len) {
/* remove the entire entry, note the kfree release */
b->extra_len -= ebd->len;
extra_amt -= ebd->len;
kfree((void*)ebd->base);
ebd->base = ebd->off = ebd->len = 0;
continue;
}
ebd->off += extra_amt;
ebd->len -= extra_amt;
b->extra_len -= extra_amt;
extra_amt = 0;
}
}
}
/*
* if writer flow controlled, restart
*
* This used to be
* q->len < q->limit/2
* but it slows down tcp too much for certain write sizes.
* I really don't understand it completely. It may be
* due to the queue draining so fast that the transmission
* stalls waiting for the app to produce more data. - presotto
*/
if ((q->state & Qflow) && q->len < q->limit) {
q->state &= ~Qflow;
dowakeup = 1;
} else
dowakeup = 0;
spin_unlock_irqsave(&q->lock);
if (dowakeup) {
rendez_wakeup(&q->wr);
qwake_cb(q, FDTAP_FILT_WRITABLE);
}
return sofar;
}
/*
* add a block to a queue obeying flow control
*/
long qbwrite(struct queue *q, struct block *b)
{
ERRSTACK(1);
int n, dowakeup;
volatile bool should_free_b = TRUE;
n = BLEN(b);
if (q->bypass) {
(*q->bypass) (q->arg, b);
return n;
}
dowakeup = 0;
qlock(&q->wlock);
if (waserror()) {
if (b != NULL && should_free_b)
freeb(b);
qunlock(&q->wlock);
nexterror();
}
spin_lock_irqsave(&q->lock);
/* give up if the queue is closed */
if (q->state & Qclosed) {
spin_unlock_irqsave(&q->lock);
error(EFAIL, q->err);
}
/* if nonblocking, don't queue over the limit */
if (q->len >= q->limit) {
/* drop overflow takes priority over regular non-blocking */
if (q->state & Qdropoverflow) {
spin_unlock_irqsave(&q->lock);
freeb(b);
dropcnt += n;
qunlock(&q->wlock);
poperror();
return n;
}
if (q->state & Qnonblock) {
spin_unlock_irqsave(&q->lock);
freeb(b);
error(EAGAIN, "queue full");
}
}
/* queue the block */
should_free_b = FALSE;
if (q->bfirst)
q->blast->next = b;
else
q->bfirst = b;
q->blast = b;
b->next = 0;
q->len += BALLOC(b);
q->dlen += n;
QDEBUG checkb(b, "qbwrite");
b = NULL;
/* make sure other end gets awakened */
if (q->state & Qstarve) {
q->state &= ~Qstarve;
dowakeup = 1;
}
spin_unlock_irqsave(&q->lock);
/* get output going again */
if (q->kick && (dowakeup || (q->state & Qkick)))
q->kick(q->arg);
/* wakeup anyone consuming at the other end */
if (dowakeup) {
rendez_wakeup(&q->rr);
qwake_cb(q, FDTAP_FILT_READABLE);
}
/*
* flow control, wait for queue to get below the limit
* before allowing the process to continue and queue
* more. We do this here so that postnote can only
* interrupt us after the data has been queued. This
* means that things like 9p flushes and ssl messages
* will not be disrupted by software interrupts.
*
* Note - this is moderately dangerous since a process
* that keeps getting interrupted and rewriting will
* queue infinite crud.
*/
for (;;) {
if ((q->state & (Qdropoverflow | Qnonblock)) || qnotfull(q))
break;
spin_lock_irqsave(&q->lock);
q->state |= Qflow;
spin_unlock_irqsave(&q->lock);
rendez_sleep(&q->wr, qnotfull, q);
}
qunlock(&q->wlock);
poperror();
return n;
}
