/*
* Mark a queue as closed. No further IO is permitted.
* All blocks are released.
*/
void qclose(struct queue *q)
{
struct block *bfirst;
if (q == NULL)
return;
/* mark it */
spin_lock_irqsave(&q->lock);
q->state |= Qclosed;
q->state &= ~(Qflow | Qstarve | Qdropoverflow);
q->err[0] = 0;
bfirst = q->bfirst;
q->bfirst = 0;
q->len = 0;
q->dlen = 0;
spin_unlock_irqsave(&q->lock);
/* free queued blocks */
freeblist(bfirst);
/* wake up readers/writers */
rendez_wakeup(&q->rr);
rendez_wakeup(&q->wr);
qwake_cb(q, FDTAP_FILT_HANGUP);
}
/*
* flow control, get producer going again
* called with q ilocked
*/
static void qwakeup_iunlock(struct queue *q)
{
int dowakeup = 0;
/*
* if writer flow controlled, restart
*
* This used to be
* q->len < q->limit/2
* but it slows down tcp too much for certain write sizes.
* I really don't understand it completely. It may be
* due to the queue draining so fast that the transmission
* stalls waiting for the app to produce more data. - presotto
*/
if ((q->state & Qflow) && q->len < q->limit) {
q->state &= ~Qflow;
dowakeup = 1;
}
spin_unlock_irqsave(&q->lock);
/* wakeup flow controlled writers */
if (dowakeup) {
if (q->kick)
q->kick(q->arg);
rendez_wakeup(&q->wr);
}
qwake_cb(q, FDTAP_FILT_WRITABLE);
}
/* Mark a queue as closed. Wakeup any readers. Don't remove queued blocks.
*
* msg will be the errstr received by any waiters (qread, qbread, etc). If
* there is no message, which is what also happens during a natural qclose(),
* those waiters will simply return 0. qwriters will always error() on a
* closed/hungup queue. */
void qhangup(struct queue *q, char *msg)
{
/* mark it */
spin_lock_irqsave(&q->lock);
q->state |= Qclosed;
if (msg == 0 || *msg == 0)
q->err[0] = 0;
else
strlcpy(q->err, msg, ERRMAX);
spin_unlock_irqsave(&q->lock);
/* wake up readers/writers */
rendez_wakeup(&q->rr);
rendez_wakeup(&q->wr);
qwake_cb(q, FDTAP_FILT_HANGUP);
}
long qibwrite(struct queue *q, struct block *b)
{
int n, dowakeup;
dowakeup = 0;
n = BLEN(b);
spin_lock_irqsave(&q->lock);
QDEBUG checkb(b, "qibwrite");
if (q->bfirst)
q->blast->next = b;
else
q->bfirst = b;
q->blast = b;
q->len += BALLOC(b);
q->dlen += n;
if (q->state & Qstarve) {
q->state &= ~Qstarve;
dowakeup = 1;
}
spin_unlock_irqsave(&q->lock);
if (dowakeup) {
if (q->kick)
q->kick(q->arg);
rendez_wakeup(&q->rr);
qwake_cb(q, FDTAP_FILT_READABLE);
}
return n;
}
int qpass(struct queue *q, struct block *b)
{
int dlen, len, dowakeup;
/* sync with qread */
dowakeup = 0;
spin_lock_irqsave(&q->lock);
if (q->len >= q->limit) {
freeblist(b);
spin_unlock_irqsave(&q->lock);
return -1;
}
if (q->state & Qclosed) {
len = blocklen(b);
freeblist(b);
spin_unlock_irqsave(&q->lock);
return len;
}
/* add buffer to queue */
if (q->bfirst)
q->blast->next = b;
else
q->bfirst = b;
len = BALLOC(b);
dlen = BLEN(b);
QDEBUG checkb(b, "qpass");
while (b->next) {
b = b->next;
QDEBUG checkb(b, "qpass");
len += BALLOC(b);
dlen += BLEN(b);
}
q->blast = b;
q->len += len;
q->dlen += dlen;
if (q->len >= q->limit / 2)
q->state |= Qflow;
if (q->state & Qstarve) {
q->state &= ~Qstarve;
dowakeup = 1;
}
spin_unlock_irqsave(&q->lock);
if (dowakeup) {
rendez_wakeup(&q->rr);
qwake_cb(q, FDTAP_FILT_READABLE);
}
return len;
}
void netlog(struct Fs *f, int mask, char *fmt, ...)
{
char buf[256], *t, *fp;
int i, n;
va_list arg;
struct timespec ts_now;
if (!(f->alog->logmask & mask))
return;
if (f->alog->opens == 0)
return;
/* Same style as trace_printk */
if (likely(__proc_global_info.tsc_freq))
ts_now = tsc2timespec(read_tsc());
n = snprintf(buf, sizeof(buf), "[%lu.%09lu]: ",
ts_now.tv_sec, ts_now.tv_nsec);
va_start(arg, fmt);
n += vsnprintf(buf + n, sizeof(buf) - n, fmt, arg);
va_end(arg);
spin_lock(&f->alog->lock);
i = f->alog->len + n - Nlog;
if (i > 0) {
f->alog->len -= i;
f->alog->rptr += i;
if (f->alog->rptr >= f->alog->end)
f->alog->rptr = f->alog->buf + (f->alog->rptr -
f->alog->end);
}
t = f->alog->rptr + f->alog->len;
fp = buf;
f->alog->len += n;
while (n-- > 0) {
if (t >= f->alog->end)
t = f->alog->buf + (t - f->alog->end);
*t++ = *fp++;
}
spin_unlock(&f->alog->lock);
rendez_wakeup(&f->alog->r);
}
/*
* flush the output queue
*/
void qflush(struct queue *q)
{
struct block *bfirst;
/* mark it */
spin_lock_irqsave(&q->lock);
bfirst = q->bfirst;
q->bfirst = 0;
q->len = 0;
q->dlen = 0;
spin_unlock_irqsave(&q->lock);
/* free queued blocks */
freeblist(bfirst);
/* wake up writers */
rendez_wakeup(&q->wr);
qwake_cb(q, FDTAP_FILT_WRITABLE);
}
/*
* flow control, get producer going again
* called with q ilocked
*/
static void qwakeup_iunlock(struct queue *q)
{
int dowakeup = 0;
/* if writer flow controlled, restart */
if ((q->state & Qflow) && q->len < q->limit / 2) {
q->state &= ~Qflow;
dowakeup = 1;
}
spin_unlock_irqsave(&q->lock);
/* wakeup flow controlled writers */
if (dowakeup) {
if (q->kick)
q->kick(q->arg);
rendez_wakeup(&q->wr);
qwake_cb(q, FDTAP_FILT_WRITABLE);
}
}
static int isdbgkey(Rune r)
{
static int ctrlt;
Dbgkey *dp;
int echoctrlt = ctrlt;
/*
* ^t hack BUG
*/
if (dbg.on || (ctrlt >= 2)) {
if (r == 0x14 || r == 0x05) {
ctrlt++;
return 0;
}
if (dp = finddbgkey(r)) {
if (dp->i || ctrlt > 2)
dp->f(r);
else {
dbg.work = dp;
rendez_wakeup(&dbg);
}
ctrlt = 0;
return 1;
}
ctrlt = 0;
} else if (r == 0x14) {
ctrlt++;
return 1;
} else
ctrlt = 0;
if (echoctrlt) {
char buf[3];
buf[0] = 0x14;
while (--echoctrlt >= 0) {
echo(buf[0], buf, 1);
qproduce(kbdq, buf, 1);
}
}
return 0;
}
/*
* get next block from a queue, return null if nothing there
*/
struct block *qget(struct queue *q)
{
int dowakeup;
struct block *b;
/* sync with qwrite */
spin_lock_irqsave(&q->lock);
b = q->bfirst;
if (b == NULL) {
q->state |= Qstarve;
spin_unlock_irqsave(&q->lock);
return NULL;
}
q->bfirst = b->next;
b->next = 0;
q->len -= BALLOC(b);
q->dlen -= BLEN(b);
QDEBUG checkb(b, "qget");
/* if writer flow controlled, restart */
if ((q->state & Qflow) && q->len < q->limit / 2) {
q->state &= ~Qflow;
dowakeup = 1;
} else
dowakeup = 0;
spin_unlock_irqsave(&q->lock);
if (dowakeup) {
rendez_wakeup(&q->wr);
/* We only send the writable event on wakeup, which is edge triggered */
qwake_cb(q, FDTAP_FILT_WRITABLE);
}
return b;
}
void netlog(struct Fs *f, int mask, char *fmt, ...)
{
char buf[128], *t, *fp;
int i, n;
va_list arg;
if (!(f->alog->logmask & mask))
return;
if (f->alog->opens == 0)
return;
va_start(arg, fmt);
n = vsnprintf(buf, sizeof(buf), fmt, arg);
va_end(arg);
spin_lock(&f->alog->lock);
i = f->alog->len + n - Nlog;
if (i > 0) {
f->alog->len -= i;
f->alog->rptr += i;
if (f->alog->rptr >= f->alog->end)
f->alog->rptr = f->alog->buf + (f->alog->rptr - f->alog->end);
}
t = f->alog->rptr + f->alog->len;
fp = buf;
f->alog->len += n;
while (n-- > 0) {
if (t >= f->alog->end)
t = f->alog->buf + (t - f->alog->end);
*t++ = *fp++;
}
spin_unlock(&f->alog->lock);
rendez_wakeup(&f->alog->r);
}
/*
* create a new arp entry for an ip address.
*/
static struct arpent *newarp6(struct arp *arp, uint8_t *ip, struct Ipifc *ifc,
int addrxt)
{
unsigned int t;
struct block *next, *xp;
struct arpent *a, *e, *f, **l;
struct medium *m = ifc->m;
int empty;
/* find oldest entry */
e = &arp->cache[NCACHE];
a = arp->cache;
t = a->utime;
for (f = a; f < e; f++) {
if (f->utime < t) {
t = f->utime;
a = f;
}
}
/* dump waiting packets */
xp = a->hold;
a->hold = NULL;
if (isv4(a->ip)) {
while (xp) {
next = xp->list;
freeblist(xp);
xp = next;
}
} else {
/* queue icmp unreachable for rxmitproc later, w/o arp lock */
if (xp) {
if (arp->dropl == NULL)
arp->dropf = xp;
else
arp->dropl->list = xp;
for (next = xp->list; next; next = next->list)
xp = next;
arp->dropl = xp;
rendez_wakeup(&arp->rxmtq);
}
}
/* take out of current chain */
l = &arp->hash[haship(a->ip)];
for (f = *l; f; f = f->hash) {
if (f == a) {
*l = a->hash;
break;
}
l = &f->hash;
}
/* insert into new chain */
l = &arp->hash[haship(ip)];
a->hash = *l;
*l = a;
memmove(a->ip, ip, sizeof(a->ip));
a->utime = NOW;
a->ctime = 0; /* somewhat of a "last sent time". 0, to trigger a send. */
a->type = m;
a->rtime = NOW + ReTransTimer;
a->rxtsrem = MAX_MULTICAST_SOLICIT;
a->ifc = ifc;
a->ifcid = ifc->ifcid;
/* put to the end of re-transmit chain; addrxt is 0 when isv4(a->ip) */
if (!ipismulticast(a->ip) && addrxt) {
l = &arp->rxmt;
empty = (*l == NULL);
for (f = *l; f; f = f->nextrxt) {
if (f == a) {
*l = a->nextrxt;
break;
}
l = &f->nextrxt;
}
for (f = *l; f; f = f->nextrxt) {
l = &f->nextrxt;
}
*l = a;
if (empty)
rendez_wakeup(&arp->rxmtq);
}
a->nextrxt = NULL;
return a;
}
/* Force a wakeup of all waiters on the rv, including non-timeout users. For
* those, they will just wake up, see the condition is still false (probably)
* and go back to sleep. */
static void rendez_alarm_handler(struct alarm_waiter *awaiter)
{
struct rendez *rv = (struct rendez*)awaiter->data;
rendez_wakeup(rv);
}
/*
* add a block to a queue obeying flow control
*/
long qbwrite(struct queue *q, struct block *b)
{
ERRSTACK(1);
int n, dowakeup;
volatile bool should_free_b = TRUE;
n = BLEN(b);
if (q->bypass) {
(*q->bypass) (q->arg, b);
return n;
}
dowakeup = 0;
qlock(&q->wlock);
if (waserror()) {
if (b != NULL && should_free_b)
freeb(b);
qunlock(&q->wlock);
nexterror();
}
spin_lock_irqsave(&q->lock);
/* give up if the queue is closed */
if (q->state & Qclosed) {
spin_unlock_irqsave(&q->lock);
error(EFAIL, q->err);
}
/* if nonblocking, don't queue over the limit */
if (q->len >= q->limit) {
/* drop overflow takes priority over regular non-blocking */
if (q->state & Qdropoverflow) {
spin_unlock_irqsave(&q->lock);
freeb(b);
dropcnt += n;
qunlock(&q->wlock);
poperror();
return n;
}
if (q->state & Qnonblock) {
spin_unlock_irqsave(&q->lock);
freeb(b);
error(EAGAIN, "queue full");
}
}
/* queue the block */
should_free_b = FALSE;
if (q->bfirst)
q->blast->next = b;
else
q->bfirst = b;
q->blast = b;
b->next = 0;
q->len += BALLOC(b);
q->dlen += n;
QDEBUG checkb(b, "qbwrite");
b = NULL;
/* make sure other end gets awakened */
if (q->state & Qstarve) {
q->state &= ~Qstarve;
dowakeup = 1;
}
spin_unlock_irqsave(&q->lock);
/* get output going again */
if (q->kick && (dowakeup || (q->state & Qkick)))
q->kick(q->arg);
/* wakeup anyone consuming at the other end */
if (dowakeup) {
rendez_wakeup(&q->rr);
qwake_cb(q, FDTAP_FILT_READABLE);
}
/*
* flow control, wait for queue to get below the limit
* before allowing the process to continue and queue
* more. We do this here so that postnote can only
* interrupt us after the data has been queued. This
* means that things like 9p flushes and ssl messages
* will not be disrupted by software interrupts.
*
* Note - this is moderately dangerous since a process
* that keeps getting interrupted and rewriting will
* queue infinite crud.
*/
for (;;) {
if ((q->state & (Qdropoverflow | Qnonblock)) || qnotfull(q))
break;
spin_lock_irqsave(&q->lock);
q->state |= Qflow;
spin_unlock_irqsave(&q->lock);
rendez_sleep(&q->wr, qnotfull, q);
}
qunlock(&q->wlock);
poperror();
return n;
}
/*
* throw away the next 'len' bytes in the queue
* returning the number actually discarded
*/
int qdiscard(struct queue *q, int len)
{
struct block *b;
int dowakeup, n, sofar, body_amt, extra_amt;
struct extra_bdata *ebd;
spin_lock_irqsave(&q->lock);
for (sofar = 0; sofar < len; sofar += n) {
b = q->bfirst;
if (b == NULL)
break;
QDEBUG checkb(b, "qdiscard");
n = BLEN(b);
if (n <= len - sofar) {
q->bfirst = b->next;
b->next = 0;
q->len -= BALLOC(b);
q->dlen -= BLEN(b);
freeb(b);
} else {
n = len - sofar;
q->dlen -= n;
/* partial block removal */
body_amt = MIN(BHLEN(b), n);
b->rp += body_amt;
extra_amt = n - body_amt;
/* reduce q->len by the amount we remove from the extras. The
* header will always be accounted for above, during block removal.
* */
q->len -= extra_amt;
for (int i = 0; (i < b->nr_extra_bufs) && extra_amt; i++) {
ebd = &b->extra_data[i];
if (!ebd->base || !ebd->len)
continue;
if (extra_amt >= ebd->len) {
/* remove the entire entry, note the kfree release */
b->extra_len -= ebd->len;
extra_amt -= ebd->len;
kfree((void*)ebd->base);
ebd->base = ebd->off = ebd->len = 0;
continue;
}
ebd->off += extra_amt;
ebd->len -= extra_amt;
b->extra_len -= extra_amt;
extra_amt = 0;
}
}
}
/*
* if writer flow controlled, restart
*
* This used to be
* q->len < q->limit/2
* but it slows down tcp too much for certain write sizes.
* I really don't understand it completely. It may be
* due to the queue draining so fast that the transmission
* stalls waiting for the app to produce more data. - presotto
*/
if ((q->state & Qflow) && q->len < q->limit) {
q->state &= ~Qflow;
dowakeup = 1;
} else
dowakeup = 0;
spin_unlock_irqsave(&q->lock);
if (dowakeup) {
rendez_wakeup(&q->wr);
qwake_cb(q, FDTAP_FILT_WRITABLE);
}
return sofar;
}
/*
* Interrupt level copy out of a queue, return # bytes copied.
*/
int qconsume(struct queue *q, void *vp, int len)
{
struct block *b;
int n, dowakeup;
uint8_t *p = vp;
struct block *tofree = NULL;
/* sync with qwrite */
spin_lock_irqsave(&q->lock);
for (;;) {
b = q->bfirst;
if (b == 0) {
q->state |= Qstarve;
spin_unlock_irqsave(&q->lock);
return -1;
}
QDEBUG checkb(b, "qconsume 1");
n = BLEN(b);
if (n > 0)
break;
q->bfirst = b->next;
q->len -= BALLOC(b);
/* remember to free this */
b->next = tofree;
tofree = b;
};
PANIC_EXTRA(b);
if (n < len)
len = n;
memmove(p, b->rp, len);
consumecnt += n;
b->rp += len;
q->dlen -= len;
/* discard the block if we're done with it */
if ((q->state & Qmsg) || len == n) {
q->bfirst = b->next;
b->next = 0;
q->len -= BALLOC(b);
q->dlen -= BLEN(b);
/* remember to free this */
b->next = tofree;
tofree = b;
}
/* if writer flow controlled, restart */
if ((q->state & Qflow) && q->len < q->limit / 2) {
q->state &= ~Qflow;
dowakeup = 1;
} else
dowakeup = 0;
spin_unlock_irqsave(&q->lock);
if (dowakeup) {
rendez_wakeup(&q->wr);
qwake_cb(q, FDTAP_FILT_WRITABLE);
}
if (tofree != NULL)
freeblist(tofree);
return len;
}
/* Adds block (which can be a list of blocks) to the queue, subject to
* qio_flags. Returns the length written on success or -1 on non-throwable
* error. Adjust qio_flags to control the value-added features!. */
static ssize_t __qbwrite(struct queue *q, struct block *b, int qio_flags)
{
ssize_t ret;
bool dowakeup = FALSE;
bool was_empty;
if (q->bypass) {
ret = blocklen(b);
(*q->bypass) (q->arg, b);
return ret;
}
spin_lock_irqsave(&q->lock);
was_empty = q->len == 0;
if (q->state & Qclosed) {
spin_unlock_irqsave(&q->lock);
freeblist(b);
if (!(qio_flags & QIO_CAN_ERR_SLEEP))
return -1;
if (q->err[0])
error(EFAIL, q->err);
else
error(EFAIL, "connection closed");
}
if ((qio_flags & QIO_LIMIT) && (q->len >= q->limit)) {
/* drop overflow takes priority over regular non-blocking */
if ((qio_flags & QIO_DROP_OVERFLOW) || (q->state & Qdropoverflow)) {
spin_unlock_irqsave(&q->lock);
freeb(b);
return -1;
}
/* People shouldn't set NON_BLOCK without CAN_ERR, but we can be nice
* and catch it. */
if ((qio_flags & QIO_CAN_ERR_SLEEP) && (qio_flags & QIO_NON_BLOCK)) {
spin_unlock_irqsave(&q->lock);
freeb(b);
error(EAGAIN, "queue full");
}
}
ret = enqueue_blist(q, b);
QDEBUG checkb(b, "__qbwrite");
/* make sure other end gets awakened */
if (q->state & Qstarve) {
q->state &= ~Qstarve;
dowakeup = TRUE;
}
spin_unlock_irqsave(&q->lock);
/* TODO: not sure if the usage of a kick is mutually exclusive with a
* wakeup, meaning that actual users either want a kick or have qreaders. */
if (q->kick && (dowakeup || (q->state & Qkick)))
q->kick(q->arg);
if (dowakeup)
rendez_wakeup(&q->rr);
if (was_empty)
qwake_cb(q, FDTAP_FILT_READABLE);
/*
* flow control, wait for queue to get below the limit
* before allowing the process to continue and queue
* more. We do this here so that postnote can only
* interrupt us after the data has been queued. This
* means that things like 9p flushes and ssl messages
* will not be disrupted by software interrupts.
*
* Note - this is moderately dangerous since a process
* that keeps getting interrupted and rewriting will
* queue infinite crud.
*/
if ((qio_flags & QIO_CAN_ERR_SLEEP) &&
!(q->state & Qdropoverflow) && !(qio_flags & QIO_NON_BLOCK)) {
/* This is a racy peek at the q status. If we accidentally block, we
* set Qflow, so someone should wake us. If we accidentally don't
* block, we just returned to the user and let them slip a block past
* flow control. */
while (!qnotfull(q)) {
spin_lock_irqsave(&q->lock);
q->state |= Qflow;
spin_unlock_irqsave(&q->lock);
rendez_sleep(&q->wr, qnotfull, q);
}
}
return ret;
}
int qproduce(struct queue *q, void *vp, int len)
{
struct block *b;
int dowakeup;
uint8_t *p = vp;
/* sync with qread */
dowakeup = 0;
spin_lock_irqsave(&q->lock);
/* no waiting receivers, room in buffer? */
if (q->len >= q->limit) {
q->state |= Qflow;
spin_unlock_irqsave(&q->lock);
return -1;
}
/* save in buffer */
/* use Qcoalesce here to save storage */
// TODO: Consider removing the Qcoalesce flag and force a coalescing
// strategy by default.
b = q->blast;
if ((q->state & Qcoalesce) == 0 || q->bfirst == NULL
|| b->lim - b->wp < len) {
/* need a new block */
b = iallocb(len);
if (b == 0) {
spin_unlock_irqsave(&q->lock);
return 0;
}
if (q->bfirst)
q->blast->next = b;
else
q->bfirst = b;
q->blast = b;
/* b->next = 0; done by iallocb() */
q->len += BALLOC(b);
}
PANIC_EXTRA(b);
memmove(b->wp, p, len);
producecnt += len;
b->wp += len;
q->dlen += len;
QDEBUG checkb(b, "qproduce");
if (q->state & Qstarve) {
q->state &= ~Qstarve;
dowakeup = 1;
}
if (q->len >= q->limit)
q->state |= Qflow;
spin_unlock_irqsave(&q->lock);
if (dowakeup) {
rendez_wakeup(&q->rr);
qwake_cb(q, FDTAP_FILT_READABLE);
}
return len;
}
