static void
alq_shutdown(struct alq *alq)
{
ALQ_LOCK(alq);
/* Stop any new writers. */
alq->aq_flags |= AQ_SHUTDOWN;
/*
* If the ALQ isn't active but has unwritten data (possible if
* the ALQ_NOACTIVATE flag has been used), explicitly activate the
* ALQ here so that the pending data gets flushed by the ald_daemon.
*/
if (!(alq->aq_flags & AQ_ACTIVE) && HAS_PENDING_DATA(alq)) {
alq->aq_flags |= AQ_ACTIVE;
ALQ_UNLOCK(alq);
ALD_LOCK();
ald_activate(alq);
ALD_UNLOCK();
ALQ_LOCK(alq);
}
/* Drain IO */
while (alq->aq_flags & AQ_ACTIVE) {
alq->aq_flags |= AQ_WANTED;
msleep_spin(alq, &alq->aq_mtx, "aldclose", 0);
}
ALQ_UNLOCK(alq);
vn_close(alq->aq_vp, FWRITE, alq->aq_cred,
curthread);
crfree(alq->aq_cred);
}
void
alq_flush(struct alq *alq)
{
int needwakeup = 0;
ALD_LOCK();
ALQ_LOCK(alq);
/*
* Pull the lever iff there is data to flush and we're
* not already in the middle of a flush operation.
*/
if (HAS_PENDING_DATA(alq) && !(alq->aq_flags & AQ_FLUSHING)) {
if (alq->aq_flags & AQ_ACTIVE)
ald_deactivate(alq);
ALD_UNLOCK();
needwakeup = alq_doio(alq);
} else
ALD_UNLOCK();
ALQ_UNLOCK(alq);
if (needwakeup)
wakeup_one(alq);
}
struct ale *
alq_get(struct alq *alq, int waitok)
{
struct ale *ale;
struct ale *aln;
ale = NULL;
ALQ_LOCK(alq);
/* Loop until we get an entry or we're shutting down */
while ((alq->aq_flags & AQ_SHUTDOWN) == 0 &&
(ale = alq->aq_entfree) == NULL &&
(waitok & ALQ_WAITOK)) {
alq->aq_flags |= AQ_WANTED;
msleep_spin(alq, &alq->aq_mtx, "alqget", 0);
}
if (ale != NULL) {
aln = ale->ae_next;
if ((aln->ae_flags & AE_VALID) == 0)
alq->aq_entfree = aln;
else
alq->aq_entfree = NULL;
} else
ALQ_UNLOCK(alq);
return (ale);
}
static void
ald_daemon(void)
{
int needwakeup;
struct alq *alq;
ald_thread = FIRST_THREAD_IN_PROC(ald_proc);
EVENTHANDLER_REGISTER(shutdown_pre_sync, ald_shutdown, NULL,
SHUTDOWN_PRI_FIRST);
ALD_LOCK();
for (;;) {
while ((alq = LIST_FIRST(&ald_active)) == NULL)
msleep(&ald_active, &ald_mtx, PWAIT, "aldslp", 0);
ALQ_LOCK(alq);
ald_deactivate(alq);
ALD_UNLOCK();
needwakeup = alq_doio(alq);
ALQ_UNLOCK(alq);
if (needwakeup)
wakeup(alq);
ALD_LOCK();
}
}
void
alq_post_flags(struct alq *alq, struct ale *ale, int flags)
{
int activate;
void *waitchan;
activate = 0;
if (ale->ae_bytesused > 0) {
if (!(alq->aq_flags & AQ_ACTIVE) &&
!(flags & ALQ_NOACTIVATE)) {
alq->aq_flags |= AQ_ACTIVE;
activate = 1;
}
alq->aq_writehead += ale->ae_bytesused;
alq->aq_freebytes -= ale->ae_bytesused;
/* Wrap aq_writehead if we filled to the end of the buffer. */
if (alq->aq_writehead == alq->aq_buflen)
alq->aq_writehead = 0;
KASSERT((alq->aq_writehead >= 0 &&
alq->aq_writehead < alq->aq_buflen),
("%s: aq_writehead < 0 || aq_writehead >= aq_buflen",
__func__));
KASSERT((HAS_PENDING_DATA(alq)), ("%s: queue empty!", __func__));
}
/*
* If there are waiters, we need to signal the waiting threads after we
* complete our work. The alq ptr is used as a wait channel for threads
* requiring resources to be freed up. In the AQ_ORDERED case, threads
* are not allowed to concurrently compete for resources in the
* alq_getn() while loop, so we use a different wait channel in this case.
*/
if (alq->aq_waiters > 0) {
if (alq->aq_flags & AQ_ORDERED)
waitchan = &alq->aq_waiters;
else
waitchan = alq;
} else
waitchan = NULL;
ALQ_UNLOCK(alq);
if (activate) {
ALD_LOCK();
ald_activate(alq);
ALD_UNLOCK();
}
/* NB: We rely on wakeup_one waking threads in a FIFO manner. */
if (waitchan != NULL)
wakeup_one(waitchan);
}
void
alq_flush(struct alq *alq)
{
int needwakeup = 0;
ALD_LOCK();
ALQ_LOCK(alq);
if (alq->aq_flags & AQ_ACTIVE) {
ald_deactivate(alq);
ALD_UNLOCK();
needwakeup = alq_doio(alq);
} else
ALD_UNLOCK();
ALQ_UNLOCK(alq);
if (needwakeup)
wakeup(alq);
}
static void
alq_shutdown(struct alq *alq)
{
ALQ_LOCK(alq);
/* Stop any new writers. */
alq->aq_flags |= AQ_SHUTDOWN;
/* Drain IO */
while (alq->aq_flags & (AQ_FLUSHING|AQ_ACTIVE)) {
alq->aq_flags |= AQ_WANTED;
msleep_spin(alq, &alq->aq_mtx, "aldclose", 0);
}
ALQ_UNLOCK(alq);
vn_close(alq->aq_vp, FWRITE, alq->aq_cred,
curthread);
crfree(alq->aq_cred);
}
void
alq_post(struct alq *alq, struct ale *ale)
{
int activate;
ale->ae_flags |= AE_VALID;
if (alq->aq_entvalid == NULL)
alq->aq_entvalid = ale;
if ((alq->aq_flags & AQ_ACTIVE) == 0) {
alq->aq_flags |= AQ_ACTIVE;
activate = 1;
} else
activate = 0;
ALQ_UNLOCK(alq);
if (activate) {
ALD_LOCK();
ald_activate(alq);
ALD_UNLOCK();
}
}
static void
ald_daemon(void)
{
int needwakeup;
struct alq *alq;
ald_thread = FIRST_THREAD_IN_PROC(ald_proc);
alq_eventhandler_tag = EVENTHANDLER_REGISTER(shutdown_pre_sync,
ald_shutdown, NULL, SHUTDOWN_PRI_FIRST);
ALD_LOCK();
for (;;) {
while ((alq = BSD_LIST_FIRST(&ald_active)) == NULL &&
!ald_shutingdown)
mtx_sleep(&ald_active, &ald_mtx, PWAIT, "aldslp", 0);
/* Don't shutdown until all active ALQs are flushed. */
if (ald_shutingdown && alq == NULL) {
ALD_UNLOCK();
break;
}
ALQ_LOCK(alq);
ald_deactivate(alq);
ALD_UNLOCK();
needwakeup = alq_doio(alq);
ALQ_UNLOCK(alq);
if (needwakeup)
wakeup_one(alq);
ALD_LOCK();
}
kproc_exit(0);
}
/*
* Retrieve a pointer for the ALQ to write directly into, avoiding bcopy.
*/
struct ale *
alq_getn(struct alq *alq, int len, int flags)
{
int contigbytes;
void *waitchan;
KASSERT((len > 0 && len <= alq->aq_buflen),
("%s: len <= 0 || len > alq->aq_buflen", __func__));
waitchan = NULL;
ALQ_LOCK(alq);
/*
* Determine the number of free contiguous bytes.
* We ensure elsewhere that if aq_writehead == aq_writetail because
* the buffer is empty, they will both be set to 0 and therefore
* aq_freebytes == aq_buflen and is fully contiguous.
* If they are equal and the buffer is not empty, aq_freebytes will
* be 0 indicating the buffer is full.
*/
if (alq->aq_writehead <= alq->aq_writetail)
contigbytes = alq->aq_freebytes;
else {
contigbytes = alq->aq_buflen - alq->aq_writehead;
if (contigbytes < len) {
/*
* Insufficient space at end of buffer to handle a
* contiguous write. Wrap early if there's space at
* the beginning. This will leave a hole at the end
* of the buffer which we will have to skip over when
* flushing the buffer to disk.
*/
if (alq->aq_writetail >= len || flags & ALQ_WAITOK) {
/* Keep track of # bytes left blank. */
alq->aq_wrapearly = contigbytes;
/* Do the wrap and adjust counters. */
contigbytes = alq->aq_freebytes =
alq->aq_writetail;
alq->aq_writehead = 0;
}
}
}
/*
* Return a NULL ALE if:
* - The message is larger than our underlying buffer.
* - The ALQ is being shutdown.
* - There is insufficient free space in our underlying buffer
* to accept the message and the user can't wait for space.
* - There is insufficient free space in our underlying buffer
* to accept the message and the alq is inactive due to prior
* use of the ALQ_NOACTIVATE flag (which would lead to deadlock).
*/
if (len > alq->aq_buflen ||
alq->aq_flags & AQ_SHUTDOWN ||
(((flags & ALQ_NOWAIT) || (!(alq->aq_flags & AQ_ACTIVE) &&
HAS_PENDING_DATA(alq))) && contigbytes < len)) {
ALQ_UNLOCK(alq);
return (NULL);
}
/*
* If we want ordered writes and there is already at least one thread
* waiting for resources to become available, sleep until we're woken.
*/
if (alq->aq_flags & AQ_ORDERED && alq->aq_waiters > 0) {
KASSERT(!(flags & ALQ_NOWAIT),
("%s: ALQ_NOWAIT set but incorrectly ignored!", __func__));
alq->aq_waiters++;
msleep_spin(&alq->aq_waiters, &alq->aq_mtx, "alqgnord", 0);
alq->aq_waiters--;
}
/*
* (ALQ_WAITOK && contigbytes < len) or contigbytes >= len, either enter
* while loop and sleep until we have enough contiguous free bytes
* (former) or skip (latter). If AQ_ORDERED is set, only 1 thread at a
* time will be in this loop. Otherwise, multiple threads may be
* sleeping here competing for ALQ resources.
*/
while (contigbytes < len && !(alq->aq_flags & AQ_SHUTDOWN)) {
KASSERT(!(flags & ALQ_NOWAIT),
("%s: ALQ_NOWAIT set but incorrectly ignored!", __func__));
alq->aq_flags |= AQ_WANTED;
alq->aq_waiters++;
if (waitchan)
wakeup(waitchan);
msleep_spin(alq, &alq->aq_mtx, "alqgnres", 0);
alq->aq_waiters--;
if (alq->aq_writehead <= alq->aq_writetail)
contigbytes = alq->aq_freebytes;
else
contigbytes = alq->aq_buflen - alq->aq_writehead;
/*
* If we're the first thread to wake after an AQ_WANTED wakeup
* but there isn't enough free space for us, we're going to loop
* and sleep again. If there are other threads waiting in this
* loop, schedule a wakeup so that they can see if the space
* they require is available.
*/
if (alq->aq_waiters > 0 && !(alq->aq_flags & AQ_ORDERED) &&
contigbytes < len && !(alq->aq_flags & AQ_WANTED))
waitchan = alq;
else
waitchan = NULL;
}
/*
* If there are waiters, we need to signal the waiting threads after we
* complete our work. The alq ptr is used as a wait channel for threads
* requiring resources to be freed up. In the AQ_ORDERED case, threads
* are not allowed to concurrently compete for resources in the above
* while loop, so we use a different wait channel in this case.
*/
if (alq->aq_waiters > 0) {
if (alq->aq_flags & AQ_ORDERED)
waitchan = &alq->aq_waiters;
else
waitchan = alq;
} else
waitchan = NULL;
/* Bail if we're shutting down. */
if (alq->aq_flags & AQ_SHUTDOWN) {
ALQ_UNLOCK(alq);
if (waitchan != NULL)
wakeup_one(waitchan);
return (NULL);
}
/*
* If we are here, we have a contiguous number of bytes >= len
* available in our buffer starting at aq_writehead.
*/
alq->aq_getpost.ae_data = alq->aq_entbuf + alq->aq_writehead;
alq->aq_getpost.ae_bytesused = len;
return (&alq->aq_getpost);
}
/*
* Copy a new entry into the queue. If the operation would block either
* wait or return an error depending on the value of waitok.
*/
int
alq_writen(struct alq *alq, void *data, int len, int flags)
{
int activate, copy, ret;
void *waitchan;
KASSERT((len > 0 && len <= alq->aq_buflen),
("%s: len <= 0 || len > aq_buflen", __func__));
activate = ret = 0;
copy = len;
waitchan = NULL;
ALQ_LOCK(alq);
/*
* Fail to perform the write and return EWOULDBLOCK if:
* - The message is larger than our underlying buffer.
* - The ALQ is being shutdown.
* - There is insufficient free space in our underlying buffer
* to accept the message and the user can't wait for space.
* - There is insufficient free space in our underlying buffer
* to accept the message and the alq is inactive due to prior
* use of the ALQ_NOACTIVATE flag (which would lead to deadlock).
*/
if (len > alq->aq_buflen ||
alq->aq_flags & AQ_SHUTDOWN ||
(((flags & ALQ_NOWAIT) || (!(alq->aq_flags & AQ_ACTIVE) &&
HAS_PENDING_DATA(alq))) && alq->aq_freebytes < len)) {
ALQ_UNLOCK(alq);
return (EWOULDBLOCK);
}
/*
* If we want ordered writes and there is already at least one thread
* waiting for resources to become available, sleep until we're woken.
*/
if (alq->aq_flags & AQ_ORDERED && alq->aq_waiters > 0) {
KASSERT(!(flags & ALQ_NOWAIT),
("%s: ALQ_NOWAIT set but incorrectly ignored!", __func__));
alq->aq_waiters++;
msleep_spin(&alq->aq_waiters, &alq->aq_mtx, "alqwnord", 0);
alq->aq_waiters--;
}
/*
* (ALQ_WAITOK && aq_freebytes < len) or aq_freebytes >= len, either
* enter while loop and sleep until we have enough free bytes (former)
* or skip (latter). If AQ_ORDERED is set, only 1 thread at a time will
* be in this loop. Otherwise, multiple threads may be sleeping here
* competing for ALQ resources.
*/
while (alq->aq_freebytes < len && !(alq->aq_flags & AQ_SHUTDOWN)) {
KASSERT(!(flags & ALQ_NOWAIT),
("%s: ALQ_NOWAIT set but incorrectly ignored!", __func__));
alq->aq_flags |= AQ_WANTED;
alq->aq_waiters++;
if (waitchan)
wakeup(waitchan);
msleep_spin(alq, &alq->aq_mtx, "alqwnres", 0);
alq->aq_waiters--;
/*
* If we're the first thread to wake after an AQ_WANTED wakeup
* but there isn't enough free space for us, we're going to loop
* and sleep again. If there are other threads waiting in this
* loop, schedule a wakeup so that they can see if the space
* they require is available.
*/
if (alq->aq_waiters > 0 && !(alq->aq_flags & AQ_ORDERED) &&
alq->aq_freebytes < len && !(alq->aq_flags & AQ_WANTED))
waitchan = alq;
else
waitchan = NULL;
}
/*
* If there are waiters, we need to signal the waiting threads after we
* complete our work. The alq ptr is used as a wait channel for threads
* requiring resources to be freed up. In the AQ_ORDERED case, threads
* are not allowed to concurrently compete for resources in the above
* while loop, so we use a different wait channel in this case.
*/
if (alq->aq_waiters > 0) {
if (alq->aq_flags & AQ_ORDERED)
waitchan = &alq->aq_waiters;
else
waitchan = alq;
} else
waitchan = NULL;
/* Bail if we're shutting down. */
if (alq->aq_flags & AQ_SHUTDOWN) {
ret = EWOULDBLOCK;
goto unlock;
}
/*
* If we need to wrap the buffer to accommodate the write,
* we'll need 2 calls to bcopy.
*/
if ((alq->aq_buflen - alq->aq_writehead) < len)
copy = alq->aq_buflen - alq->aq_writehead;
/* Copy message (or part thereof if wrap required) to the buffer. */
bcopy(data, alq->aq_entbuf + alq->aq_writehead, copy);
alq->aq_writehead += copy;
if (alq->aq_writehead >= alq->aq_buflen) {
KASSERT((alq->aq_writehead == alq->aq_buflen),
("%s: alq->aq_writehead (%d) > alq->aq_buflen (%d)",
__func__,
alq->aq_writehead,
alq->aq_buflen));
alq->aq_writehead = 0;
}
if (copy != len) {
/*
* Wrap the buffer by copying the remainder of our message
* to the start of the buffer and resetting aq_writehead.
*/
bcopy(((uint8_t *)data)+copy, alq->aq_entbuf, len - copy);
alq->aq_writehead = len - copy;
}
KASSERT((alq->aq_writehead >= 0 && alq->aq_writehead < alq->aq_buflen),
("%s: aq_writehead < 0 || aq_writehead >= aq_buflen", __func__));
alq->aq_freebytes -= len;
if (!(alq->aq_flags & AQ_ACTIVE) && !(flags & ALQ_NOACTIVATE)) {
alq->aq_flags |= AQ_ACTIVE;
activate = 1;
}
KASSERT((HAS_PENDING_DATA(alq)), ("%s: queue empty!", __func__));
unlock:
ALQ_UNLOCK(alq);
if (activate) {
ALD_LOCK();
ald_activate(alq);
ALD_UNLOCK();
}
/* NB: We rely on wakeup_one waking threads in a FIFO manner. */
if (waitchan != NULL)
wakeup_one(waitchan);
return (ret);
}
/*
* Flush all pending data to disk. This operation will block.
*/
static int
alq_doio(struct alq *alq)
{
struct thread *td;
struct mount *mp;
struct vnode *vp;
struct uio auio;
struct iovec aiov[2];
int totlen;
int iov;
int vfslocked;
int wrapearly;
KASSERT((HAS_PENDING_DATA(alq)), ("%s: queue empty!", __func__));
vp = alq->aq_vp;
td = curthread;
totlen = 0;
iov = 1;
wrapearly = alq->aq_wrapearly;
bzero(&aiov, sizeof(aiov));
bzero(&auio, sizeof(auio));
/* Start the write from the location of our buffer tail pointer. */
aiov[0].iov_base = alq->aq_entbuf + alq->aq_writetail;
if (alq->aq_writetail < alq->aq_writehead) {
/* Buffer not wrapped. */
totlen = aiov[0].iov_len = alq->aq_writehead - alq->aq_writetail;
} else if (alq->aq_writehead == 0) {
/* Buffer not wrapped (special case to avoid an empty iov). */
totlen = aiov[0].iov_len = alq->aq_buflen - alq->aq_writetail -
wrapearly;
} else {
/*
* Buffer wrapped, requires 2 aiov entries:
* - first is from writetail to end of buffer
* - second is from start of buffer to writehead
*/
aiov[0].iov_len = alq->aq_buflen - alq->aq_writetail -
wrapearly;
iov++;
aiov[1].iov_base = alq->aq_entbuf;
aiov[1].iov_len = alq->aq_writehead;
totlen = aiov[0].iov_len + aiov[1].iov_len;
}
alq->aq_flags |= AQ_FLUSHING;
ALQ_UNLOCK(alq);
auio.uio_iov = &aiov[0];
auio.uio_offset = 0;
auio.uio_segflg = UIO_SYSSPACE;
auio.uio_rw = UIO_WRITE;
auio.uio_iovcnt = iov;
auio.uio_resid = totlen;
auio.uio_td = td;
/*
* Do all of the junk required to write now.
*/
vfslocked = VFS_LOCK_GIANT(vp->v_mount);
vn_start_write(vp, &mp, V_WAIT);
vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
/*
* XXX: VOP_WRITE error checks are ignored.
*/
#ifdef MAC
if (mac_vnode_check_write(alq->aq_cred, NOCRED, vp) == 0)
#endif
VOP_WRITE(vp, &auio, IO_UNIT | IO_APPEND, alq->aq_cred);
VOP_UNLOCK(vp, 0);
vn_finished_write(mp);
VFS_UNLOCK_GIANT(vfslocked);
ALQ_LOCK(alq);
alq->aq_flags &= ~AQ_FLUSHING;
/* Adjust writetail as required, taking into account wrapping. */
alq->aq_writetail = (alq->aq_writetail + totlen + wrapearly) %
alq->aq_buflen;
alq->aq_freebytes += totlen + wrapearly;
/*
* If we just flushed part of the buffer which wrapped, reset the
* wrapearly indicator.
*/
if (wrapearly)
alq->aq_wrapearly = 0;
/*
* If we just flushed the buffer completely, reset indexes to 0 to
* minimise buffer wraps.
* This is also required to ensure alq_getn() can't wedge itself.
*/
if (!HAS_PENDING_DATA(alq))
alq->aq_writehead = alq->aq_writetail = 0;
KASSERT((alq->aq_writetail >= 0 && alq->aq_writetail < alq->aq_buflen),
("%s: aq_writetail < 0 || aq_writetail >= aq_buflen", __func__));
if (alq->aq_flags & AQ_WANTED) {
alq->aq_flags &= ~AQ_WANTED;
return (1);
}
return(0);
}
/*
* Flush all pending data to disk. This operation will block.
*/
static int
alq_doio(struct alq *alq)
{
struct thread *td;
struct mount *mp;
struct vnode *vp;
struct uio auio;
struct iovec aiov[2];
struct ale *ale;
struct ale *alstart;
int totlen;
int iov;
int vfslocked;
vp = alq->aq_vp;
td = curthread;
totlen = 0;
iov = 0;
alstart = ale = alq->aq_entvalid;
alq->aq_entvalid = NULL;
bzero(&aiov, sizeof(aiov));
bzero(&auio, sizeof(auio));
do {
if (aiov[iov].iov_base == NULL)
aiov[iov].iov_base = ale->ae_data;
aiov[iov].iov_len += alq->aq_entlen;
totlen += alq->aq_entlen;
/* Check to see if we're wrapping the buffer */
if (ale->ae_data + alq->aq_entlen != ale->ae_next->ae_data)
iov++;
ale->ae_flags &= ~AE_VALID;
ale = ale->ae_next;
} while (ale->ae_flags & AE_VALID);
alq->aq_flags |= AQ_FLUSHING;
ALQ_UNLOCK(alq);
if (iov == 2 || aiov[iov].iov_base == NULL)
iov--;
auio.uio_iov = &aiov[0];
auio.uio_offset = 0;
auio.uio_segflg = UIO_SYSSPACE;
auio.uio_rw = UIO_WRITE;
auio.uio_iovcnt = iov + 1;
auio.uio_resid = totlen;
auio.uio_td = td;
/*
* Do all of the junk required to write now.
*/
vfslocked = VFS_LOCK_GIANT(vp->v_mount);
vn_start_write(vp, &mp, V_WAIT);
vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
/*
* XXX: VOP_WRITE error checks are ignored.
*/
#ifdef MAC
if (mac_vnode_check_write(alq->aq_cred, NOCRED, vp) == 0)
#endif
VOP_WRITE(vp, &auio, IO_UNIT | IO_APPEND, alq->aq_cred);
VOP_UNLOCK(vp, 0);
vn_finished_write(mp);
VFS_UNLOCK_GIANT(vfslocked);
ALQ_LOCK(alq);
alq->aq_flags &= ~AQ_FLUSHING;
if (alq->aq_entfree == NULL)
alq->aq_entfree = alstart;
if (alq->aq_flags & AQ_WANTED) {
alq->aq_flags &= ~AQ_WANTED;
return (1);
}
return(0);
}
