/*
* ehci_intr:
*
* EHCI (EHCI) interrupt handling routine.
*/
uint_t
ehci_intr(caddr_t arg1, caddr_t arg2)
{
uint_t intr;
ehci_state_t *ehcip = (ehci_state_t *)arg1;
USB_DPRINTF_L3(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
"ehci_intr: Interrupt occurred, arg1 0x%p arg2 0x%p", arg1, arg2);
/* Get the ehci global mutex */
mutex_enter(&ehcip->ehci_int_mutex);
/*
* Now process the actual ehci interrupt events that caused
* invocation of this ehci interrupt handler.
*/
intr = (Get_OpReg(ehci_status) & Get_OpReg(ehci_interrupt));
/* Update kstat values */
ehci_do_intrs_stats(ehcip, intr);
/*
* We could have gotten a spurious interrupts. If so, do not
* claim it. This is quite possible on some architectures
* where more than one PCI slots share the IRQs. If so, the
* associated driver's interrupt routine may get called even
* if the interrupt is not meant for them.
*
* By unclaiming the interrupt, the other driver gets chance
* to service its interrupt.
*/
if (!intr) {
mutex_exit(&ehcip->ehci_int_mutex);
return (DDI_INTR_UNCLAIMED);
}
/* Acknowledge the interrupt */
Set_OpReg(ehci_status, intr);
if (ehcip->ehci_hc_soft_state == EHCI_CTLR_ERROR_STATE) {
mutex_exit(&ehcip->ehci_int_mutex);
return (DDI_INTR_CLAIMED);
}
USB_DPRINTF_L3(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
"Interrupt status 0x%x", intr);
/*
* If necessary broadcast that an interrupt has occured. This
* is only necessary during controller init.
*/
if (ehcip->ehci_flags & EHCI_CV_INTR) {
ehcip->ehci_flags &= ~EHCI_CV_INTR;
cv_broadcast(&ehcip->ehci_async_schedule_advance_cv);
}
/* Check for Frame List Rollover */
if (intr & EHCI_INTR_FRAME_LIST_ROLLOVER) {
USB_DPRINTF_L3(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
"ehci_intr: Frame List Rollover");
ehci_handle_frame_list_rollover(ehcip);
/* VIA VT6202 looses EHCI_INTR_USB interrupts, workaround. */
if ((ehcip->ehci_vendor_id == PCI_VENDOR_VIA) &&
(ehci_vt62x2_workaround & EHCI_VIA_LOST_INTERRUPTS)) {
ehcip->ehci_missed_intr_sts |= EHCI_INTR_USB;
}
}
/* Check for Advance on Asynchronous Schedule */
if (intr & EHCI_INTR_ASYNC_ADVANCE) {
USB_DPRINTF_L3(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
"ehci_intr: Asynchronous Schedule Advance Notification");
/* Disable async list advance interrupt */
Set_OpReg(ehci_interrupt,
(Get_OpReg(ehci_interrupt) & ~EHCI_INTR_ASYNC_ADVANCE));
/*
* Call cv_broadcast on every this interrupt to wakeup
* all the threads that are waiting the async list advance
* event.
*/
cv_broadcast(&ehcip->ehci_async_schedule_advance_cv);
}
/* Always process completed itds */
ehci_traverse_active_isoc_list(ehcip);
/*
* Check for any USB transaction completion notification. Also
* process any missed USB transaction completion interrupts.
*/
if ((intr & EHCI_INTR_USB) || (intr & EHCI_INTR_USB_ERROR) ||
(ehcip->ehci_missed_intr_sts & EHCI_INTR_USB) ||
(ehcip->ehci_missed_intr_sts & EHCI_INTR_USB_ERROR)) {
USB_DPRINTF_L3(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
"ehci_intr: USB Transaction Completion Notification");
/* Clear missed interrupts */
if (ehcip->ehci_missed_intr_sts) {
ehcip->ehci_missed_intr_sts = 0;
}
/* Process completed qtds */
ehci_traverse_active_qtd_list(ehcip);
}
/* Process endpoint reclamation list */
if (ehcip->ehci_reclaim_list) {
ehci_handle_endpoint_reclaimation(ehcip);
}
/* Check for Host System Error */
if (intr & EHCI_INTR_HOST_SYSTEM_ERROR) {
USB_DPRINTF_L2(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
"ehci_intr: Unrecoverable error");
ehci_handle_ue(ehcip);
}
/*
* Read interrupt status register to make sure that any PIO
* store to clear the ISR has made it on the PCI bus before
* returning from its interrupt handler.
*/
(void) Get_OpReg(ehci_status);
/* Release the ehci global mutex */
mutex_exit(&ehcip->ehci_int_mutex);
USB_DPRINTF_L3(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
"Interrupt handling completed");
return (DDI_INTR_CLAIMED);
}
int
drm_queue_vblank_event(struct drm_device *dev, int crtc,
union drm_wait_vblank *vblwait, struct drm_file *file_priv)
{
struct drm_pending_vblank_event *vev;
struct timeval now;
u_int seq;
vev = drm_calloc(1, sizeof(*vev));
if (vev == NULL)
return (ENOMEM);
vev->event.base.type = DRM_EVENT_VBLANK;
vev->event.base.length = sizeof(vev->event);
vev->event.user_data = vblwait->request.signal;
vev->base.event = &vev->event.base;
vev->base.file_priv = file_priv;
vev->base.destroy = (void (*) (struct drm_pending_event *))drm_free;
microtime(&now);
mtx_enter(&dev->event_lock);
if (file_priv->event_space < sizeof(vev->event)) {
mtx_leave(&dev->event_lock);
drm_free(vev);
return (ENOMEM);
}
seq = drm_vblank_count(dev, crtc);
file_priv->event_space -= sizeof(vev->event);
DPRINTF("%s: queueing event %d on crtc %d\n", __func__, seq, crtc);
if ((vblwait->request.type & _DRM_VBLANK_NEXTONMISS) &&
(seq - vblwait->request.sequence) <= (1 << 23)) {
vblwait->request.sequence = seq + 1;
vblwait->reply.sequence = vblwait->request.sequence;
}
vev->event.sequence = vblwait->request.sequence;
if ((seq - vblwait->request.sequence) <= (1 << 23)) {
vev->event.tv_sec = now.tv_sec;
vev->event.tv_usec = now.tv_usec;
DPRINTF("%s: already passed, dequeuing: crtc %d, value %d\n",
__func__, crtc, seq);
drm_vblank_put(dev, crtc);
TAILQ_INSERT_TAIL(&file_priv->evlist, &vev->base, link);
#if !defined(__NetBSD__)
wakeup(&file_priv->evlist);
#else /* !defined(__NetBSD__) */
cv_broadcast(&file_priv->evlist_condvar);
#endif /* !defined(__NetBSD__) */
selwakeup(&file_priv->rsel);
} else {
TAILQ_INSERT_TAIL(&dev->vblank->vb_crtcs[crtc].vbl_events,
&vev->base, link);
}
mtx_leave(&dev->event_lock);
return (0);
}
static int
traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp,
const blkptr_t *bp, const zbookmark_phys_t *zb)
{
zbookmark_phys_t czb;
int err = 0;
arc_buf_t *buf = NULL;
prefetch_data_t *pd = td->td_pfd;
boolean_t hard = td->td_flags & TRAVERSE_HARD;
switch (resume_skip_check(td, dnp, zb)) {
case RESUME_SKIP_ALL:
return (0);
case RESUME_SKIP_CHILDREN:
goto post;
case RESUME_SKIP_NONE:
break;
default:
ASSERT(0);
}
if (bp->blk_birth == 0) {
/*
* Since this block has a birth time of 0 it must be one of
* two things: a hole created before the
* SPA_FEATURE_HOLE_BIRTH feature was enabled, or a hole
* which has always been a hole in an object.
*
* If a file is written sparsely, then the unwritten parts of
* the file were "always holes" -- that is, they have been
* holes since this object was allocated. However, we (and
* our callers) can not necessarily tell when an object was
* allocated. Therefore, if it's possible that this object
* was freed and then its object number reused, we need to
* visit all the holes with birth==0.
*
* If it isn't possible that the object number was reused,
* then if SPA_FEATURE_HOLE_BIRTH was enabled before we wrote
* all the blocks we will visit as part of this traversal,
* then this hole must have always existed, so we can skip
* it. We visit blocks born after (exclusive) td_min_txg.
*
* Note that the meta-dnode cannot be reallocated.
*/
if (!send_holes_without_birth_time &&
(!td->td_realloc_possible ||
zb->zb_object == DMU_META_DNODE_OBJECT) &&
td->td_hole_birth_enabled_txg <= td->td_min_txg)
return (0);
} else if (bp->blk_birth <= td->td_min_txg) {
return (0);
}
if (pd != NULL && !pd->pd_exited && prefetch_needed(pd, bp)) {
uint64_t size = BP_GET_LSIZE(bp);
mutex_enter(&pd->pd_mtx);
ASSERT(pd->pd_bytes_fetched >= 0);
while (pd->pd_bytes_fetched < size && !pd->pd_exited)
cv_wait(&pd->pd_cv, &pd->pd_mtx);
pd->pd_bytes_fetched -= size;
cv_broadcast(&pd->pd_cv);
mutex_exit(&pd->pd_mtx);
}
if (BP_IS_HOLE(bp)) {
err = td->td_func(td->td_spa, NULL, bp, zb, dnp, td->td_arg);
if (err != 0)
goto post;
return (0);
}
if (td->td_flags & TRAVERSE_PRE) {
err = td->td_func(td->td_spa, NULL, bp, zb, dnp,
td->td_arg);
if (err == TRAVERSE_VISIT_NO_CHILDREN)
return (0);
if (err != 0)
goto post;
}
if (BP_GET_LEVEL(bp) > 0) {
arc_flags_t flags = ARC_FLAG_WAIT;
int i;
blkptr_t *cbp;
int epb = BP_GET_LSIZE(bp) >> SPA_BLKPTRSHIFT;
err = arc_read(NULL, td->td_spa, bp, arc_getbuf_func, &buf,
ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL, &flags, zb);
if (err != 0)
goto post;
cbp = buf->b_data;
for (i = 0; i < epb; i++) {
SET_BOOKMARK(&czb, zb->zb_objset, zb->zb_object,
zb->zb_level - 1,
zb->zb_blkid * epb + i);
traverse_prefetch_metadata(td, &cbp[i], &czb);
}
/* recursively visitbp() blocks below this */
for (i = 0; i < epb; i++) {
SET_BOOKMARK(&czb, zb->zb_objset, zb->zb_object,
zb->zb_level - 1,
zb->zb_blkid * epb + i);
err = traverse_visitbp(td, dnp, &cbp[i], &czb);
if (err != 0)
break;
}
} else if (BP_GET_TYPE(bp) == DMU_OT_DNODE) {
/*
* Release the buffer, with no I/O implied.
*/
void
brelse(struct buf *bp)
{
struct buf **backp;
uint_t index;
kmutex_t *hmp;
struct buf *dp;
struct hbuf *hp;
ASSERT(SEMA_HELD(&bp->b_sem));
/*
* Clear the retry write flag if the buffer was written without
* error. The presence of B_DELWRI means the buffer has not yet
* been written and the presence of B_ERROR means that an error
* is still occurring.
*/
if ((bp->b_flags & (B_ERROR | B_DELWRI | B_RETRYWRI)) == B_RETRYWRI) {
bp->b_flags &= ~B_RETRYWRI;
}
/* Check for anomalous conditions */
if (bp->b_flags & (B_ERROR|B_NOCACHE)) {
if (bp->b_flags & B_NOCACHE) {
/* Don't add to the freelist. Destroy it now */
kmem_free(bp->b_un.b_addr, bp->b_bufsize);
sema_destroy(&bp->b_sem);
sema_destroy(&bp->b_io);
kmem_free(bp, sizeof (struct buf));
return;
}
/*
* If a write failed and we are supposed to retry write,
* don't toss the buffer. Keep it around and mark it
* delayed write in the hopes that it will eventually
* get flushed (and still keep the system running.)
*/
if ((bp->b_flags & (B_READ | B_RETRYWRI)) == B_RETRYWRI) {
bp->b_flags |= B_DELWRI;
/* keep fsflush from trying continuously to flush */
bp->b_start = ddi_get_lbolt();
} else
bp->b_flags |= B_AGE|B_STALE;
bp->b_flags &= ~B_ERROR;
bp->b_error = 0;
}
/*
* If delayed write is set then put in on the delayed
* write list instead of the free buffer list.
*/
index = bio_bhash(bp->b_edev, bp->b_blkno);
hmp = &hbuf[index].b_lock;
mutex_enter(hmp);
hp = &hbuf[index];
dp = (struct buf *)hp;
/*
* Make sure that the number of entries on this list are
* Zero <= count <= total # buffers
*/
ASSERT(hp->b_length >= 0);
ASSERT(hp->b_length < nbuf);
hp->b_length++; /* We are adding this buffer */
if (bp->b_flags & B_DELWRI) {
/*
* This buffer goes on the delayed write buffer list
*/
dp = (struct buf *)&dwbuf[index];
}
ASSERT(bp->b_bufsize > 0);
ASSERT(bp->b_bcount > 0);
ASSERT(bp->b_un.b_addr != NULL);
if (bp->b_flags & B_AGE) {
backp = &dp->av_forw;
(*backp)->av_back = bp;
bp->av_forw = *backp;
*backp = bp;
bp->av_back = dp;
} else {
backp = &dp->av_back;
(*backp)->av_forw = bp;
bp->av_back = *backp;
*backp = bp;
bp->av_forw = dp;
}
mutex_exit(hmp);
if (bfreelist.b_flags & B_WANTED) {
/*
* Should come here very very rarely.
*/
mutex_enter(&bfree_lock);
if (bfreelist.b_flags & B_WANTED) {
bfreelist.b_flags &= ~B_WANTED;
cv_broadcast(&bio_mem_cv);
}
mutex_exit(&bfree_lock);
}
bp->b_flags &= ~(B_WANTED|B_BUSY|B_ASYNC);
/*
* Don't let anyone get the buffer off the freelist before we
* release our hold on it.
*/
sema_v(&bp->b_sem);
}
/*
* dm2s_event_handler - Mailbox event handler.
*/
void
dm2s_event_handler(scf_event_t event, void *arg)
{
dm2s_t *dm2sp = (dm2s_t *)arg;
queue_t *rq;
ASSERT(dm2sp != NULL);
mutex_enter(&dm2sp->ms_lock);
if (!(dm2sp->ms_state & DM2S_MB_INITED)) {
/*
* Ignore all events if the state flag indicates that the
* mailbox not initialized, this may happen during the close.
*/
mutex_exit(&dm2sp->ms_lock);
DPRINTF(DBG_MBOX,
("Event(0x%X) received - Mailbox not inited\n", event));
return;
}
switch (event) {
case SCF_MB_CONN_OK:
/*
* Now the mailbox is ready to use, lets wake up
* any one waiting for this event.
*/
dm2sp->ms_state |= DM2S_MB_CONN;
cv_broadcast(&dm2sp->ms_wait);
DPRINTF(DBG_MBOX, ("Event received = CONN_OK\n"));
break;
case SCF_MB_MSG_DATA:
if (!DM2S_MBOX_READY(dm2sp)) {
DPRINTF(DBG_MBOX,
("Event(MSG_DATA) received - Mailbox not READY\n"));
break;
}
/*
* A message is available in the mailbox.
* Lets enable the read service procedure
* to receive this message.
*/
if (dm2sp->ms_rq != NULL) {
qenable(dm2sp->ms_rq);
}
DPRINTF(DBG_MBOX, ("Event received = MSG_DATA\n"));
break;
case SCF_MB_SPACE:
if (!DM2S_MBOX_READY(dm2sp)) {
DPRINTF(DBG_MBOX,
("Event(MB_SPACE) received - Mailbox not READY\n"));
break;
}
/*
* Now the mailbox is ready to transmit, lets
* schedule the write service procedure.
*/
if (dm2sp->ms_wq != NULL) {
qenable(dm2sp->ms_wq);
}
DPRINTF(DBG_MBOX, ("Event received = MB_SPACE\n"));
break;
case SCF_MB_DISC_ERROR:
dm2sp->ms_state |= DM2S_MB_DISC;
if (dm2sp->ms_state & DM2S_MB_CONN) {
/*
* If it was previously connected,
* then send a hangup message.
*/
rq = dm2sp->ms_rq;
if (rq != NULL) {
mutex_exit(&dm2sp->ms_lock);
/*
* Send a hangup message to indicate
* disconnect event.
*/
(void) putctl(rq, M_HANGUP);
DTRACE_PROBE1(dm2s_hangup, dm2s_t, dm2sp);
mutex_enter(&dm2sp->ms_lock);
}
} else {
/*
* Signal if the open is waiting for a
* connection.
*/
cv_broadcast(&dm2sp->ms_wait);
}
DPRINTF(DBG_MBOX, ("Event received = DISC_ERROR\n"));
break;
default:
cmn_err(CE_WARN, "Unexpected event received\n");
break;
}
mutex_exit(&dm2sp->ms_lock);
}
static int /* ERRNO if error, 0 if ok */
syscall_stage_response(
sam_mount_t *mp, /* pointer to mount entry. */
sam_fsstage_arg_t *stage, /* pointer to syscall stage response */
rval_t *rvp, /* returned value pointer */
cred_t *credp) /* credentials pointer. */
{
sam_handle_t *fhandle;
sam_node_t *ip;
vnode_t *vp;
int error;
int opened = 0;
rvp->r_val1 = 0;
fhandle = (sam_handle_t *)&stage->handle;
/* Is this inode still waiting for the stage? */
if ((error = sam_find_ino(mp->mi.m_vfsp, IG_EXISTS,
&fhandle->id, &ip))) {
return (ECANCELED);
}
vp = SAM_ITOV(ip);
if (stage->ret_err == EEXIST) {
/*
* Outstanding stage request already exists, wake up anyone
* waiting
*/
mutex_enter(&ip->rm_mutex);
if (ip->rm_wait) cv_broadcast(&ip->rm_cv);
mutex_exit(&ip->rm_mutex);
sam_rele_ino(ip);
return (0);
}
RW_LOCK_OS(&ip->inode_rwl, RW_WRITER);
if (ip->stage_pid > 0) {
/*
* Another stage is in progress. This can happen if the incore
* inode is released after a nowait stage for copy 1, then stage
* for copy 2.
*/
RW_UNLOCK_OS(&ip->inode_rwl, RW_WRITER);
sam_rele_ino(ip);
return (ECANCELED);
}
ip->flags.b.staging = 1; /* Reset for acquired incore inode */
ip->copy = 0;
ip->stage_off = fhandle->stage_off;
ip->stage_len = fhandle->stage_len;
if (stage->ret_err == 0) {
if ((!ip->di.status.b.offline &&
!(ip->di.rm.ui.flags & RM_DATA_VERIFY)) ||
!ip->di.arch_status ||
ip->stage_err) {
error = ECANCELED;
TRACE(T_SAM_IOCTL_STCAN, vp,
fhandle->id.ino, ip->stage_err,
ip->flags.bits);
} else {
offset_t cur_size = ip->size;
error = 0;
if (!fhandle->flags.b.stage_wait) {
/*
* Allocate file now for nowait stage.
*
* Assert
* permissions based on the uid of the handle
* structure, not the stager. Root can force
* staging, but users have to live with quotas.
*/
if (ip->di.blocks == 0) {
error =
sam_set_unit(ip->mp, &(ip->di));
}
if (error == 0) {
error = sam_map_block(ip, ip->stage_off,
ip->stage_len,
SAM_WRITE_BLOCK, NULL, credp);
ip->size = cur_size;
}
}
if (error == 0) {
/*
* Simulate the open because we don't
* have the path.
*/
if ((error = sam_get_fd(vp,
&rvp->r_val1)) == 0) {
if (vp->v_type == VREG) {
atomic_add_32(&vp->v_rdcnt, 1);
atomic_add_32(&vp->v_wrcnt, 1);
}
ip->stage_pid = SAM_CUR_PID;
ip->no_opens++;
opened = 1;
if (stage->directio) {
sam_set_directio(ip,
DIRECTIO_ON);
}
}
}
if (error) {
ip->stage_err = (short)error;
TRACE(T_SAM_IOCTL_SERROR, vp, fhandle->id.ino,
(sam_tr_t)ip->size, error);
error = ECANCELED;
} else {
TRACE(T_SAM_IOCTL_STAGE, vp, fhandle->id.ino,
(sam_tr_t)ip->stage_off,
(uint_t)ip->stage_len);
}
}
/*
* Issue close to clean up the inode. Stage daemon won't get an
* fd.
*/
if (error) {
(void) sam_close_stage(ip, credp);
/* tell stage daemon a close not needed */
rvp->r_val1 = -1;
}
} else {
/*
* Set stage_err from stagerd..
*/
ip->stage_err = stage->ret_err;
TRACE(T_SAM_IOCTL_STERR, vp, fhandle->id.ino, fhandle->id.gen,
stage->ret_err);
/*
* Issue close to clean up the inode. Daemon knows not to close.
*/
(void) sam_close_stage(ip, credp);
error = 0;
}
RW_UNLOCK_OS(&ip->inode_rwl, RW_WRITER);
if (opened) {
VN_RELE(SAM_ITOV(ip));
} else {
sam_rele_ino(ip);
}
return (error);
}
static int
traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp,
arc_buf_t *pbuf, blkptr_t *bp, const zbookmark_t *zb)
{
zbookmark_t czb;
int err = 0, lasterr = 0;
arc_buf_t *buf = NULL;
prefetch_data_t *pd = td->td_pfd;
boolean_t hard = td->td_flags & TRAVERSE_HARD;
boolean_t pause = B_FALSE;
switch (resume_skip_check(td, dnp, zb)) {
case RESUME_SKIP_ALL:
return (0);
case RESUME_SKIP_CHILDREN:
goto post;
case RESUME_SKIP_NONE:
break;
default:
ASSERT(0);
}
if (BP_IS_HOLE(bp)) {
err = td->td_func(td->td_spa, NULL, NULL, pbuf, zb, dnp,
td->td_arg);
return (err);
}
if (bp->blk_birth <= td->td_min_txg)
return (0);
if (pd && !pd->pd_exited &&
((pd->pd_flags & TRAVERSE_PREFETCH_DATA) ||
BP_GET_TYPE(bp) == DMU_OT_DNODE || BP_GET_LEVEL(bp) > 0)) {
mutex_enter(&pd->pd_mtx);
ASSERT(pd->pd_blks_fetched >= 0);
while (pd->pd_blks_fetched == 0 && !pd->pd_exited)
cv_wait(&pd->pd_cv, &pd->pd_mtx);
pd->pd_blks_fetched--;
cv_broadcast(&pd->pd_cv);
mutex_exit(&pd->pd_mtx);
}
if (td->td_flags & TRAVERSE_PRE) {
err = td->td_func(td->td_spa, NULL, bp, pbuf, zb, dnp,
td->td_arg);
if (err == TRAVERSE_VISIT_NO_CHILDREN)
return (0);
if (err == ERESTART)
pause = B_TRUE; /* handle pausing at a common point */
if (err != 0)
goto post;
}
if (BP_GET_LEVEL(bp) > 0) {
uint32_t flags = ARC_WAIT;
int i;
blkptr_t *cbp;
int epb = BP_GET_LSIZE(bp) >> SPA_BLKPTRSHIFT;
err = dsl_read(NULL, td->td_spa, bp, pbuf,
arc_getbuf_func, &buf,
ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL, &flags, zb);
if (err)
return (err);
/* recursively visitbp() blocks below this */
cbp = buf->b_data;
for (i = 0; i < epb; i++, cbp++) {
SET_BOOKMARK(&czb, zb->zb_objset, zb->zb_object,
zb->zb_level - 1,
zb->zb_blkid * epb + i);
err = traverse_visitbp(td, dnp, buf, cbp, &czb);
if (err) {
if (!hard)
break;
lasterr = err;
}
}
} else if (BP_GET_TYPE(bp) == DMU_OT_DNODE) {
static void
rtems_bsd_sim_set_state_and_notify(struct cam_sim *sim, enum bsd_sim_state state)
{
sim->state = state;
cv_broadcast(&sim->state_changed);
}
/*
* This routine is used to notify the framework when a provider is being
* removed. Hardware providers call this routine in their detach routines.
* Software providers call this routine in their _fini() routine.
*/
int
crypto_unregister_provider(crypto_kcf_provider_handle_t handle)
{
uint_t mech_idx;
kcf_provider_desc_t *desc;
kcf_prov_state_t saved_state;
/* lookup provider descriptor */
if ((desc = kcf_prov_tab_lookup((crypto_provider_id_t)handle)) == NULL)
return (CRYPTO_UNKNOWN_PROVIDER);
mutex_enter(&desc->pd_lock);
/*
* Check if any other thread is disabling or removing
* this provider. We return if this is the case.
*/
if (desc->pd_state >= KCF_PROV_DISABLED) {
mutex_exit(&desc->pd_lock);
/* Release reference held by kcf_prov_tab_lookup(). */
KCF_PROV_REFRELE(desc);
return (CRYPTO_BUSY);
}
saved_state = desc->pd_state;
desc->pd_state = KCF_PROV_REMOVED;
if (saved_state == KCF_PROV_BUSY) {
/*
* The per-provider taskq threads may be waiting. We
* signal them so that they can start failing requests.
*/
cv_broadcast(&desc->pd_resume_cv);
}
if (desc->pd_prov_type == CRYPTO_SW_PROVIDER) {
/*
* Check if this provider is currently being used.
* pd_irefcnt is the number of holds from the internal
* structures. We add one to account for the above lookup.
*/
if (desc->pd_refcnt > desc->pd_irefcnt + 1) {
desc->pd_state = saved_state;
mutex_exit(&desc->pd_lock);
/* Release reference held by kcf_prov_tab_lookup(). */
KCF_PROV_REFRELE(desc);
/*
* The administrator presumably will stop the clients
* thus removing the holds, when they get the busy
* return value. Any retry will succeed then.
*/
return (CRYPTO_BUSY);
}
}
mutex_exit(&desc->pd_lock);
if (desc->pd_prov_type != CRYPTO_SW_PROVIDER) {
remove_provider(desc);
}
if (desc->pd_prov_type != CRYPTO_LOGICAL_PROVIDER) {
/* remove the provider from the mechanisms tables */
for (mech_idx = 0; mech_idx < desc->pd_mech_list_count;
mech_idx++) {
kcf_remove_mech_provider(
desc->pd_mechanisms[mech_idx].cm_mech_name, desc);
}
}
/* remove provider from providers table */
if (kcf_prov_tab_rem_provider((crypto_provider_id_t)handle) !=
CRYPTO_SUCCESS) {
/* Release reference held by kcf_prov_tab_lookup(). */
KCF_PROV_REFRELE(desc);
return (CRYPTO_UNKNOWN_PROVIDER);
}
delete_kstat(desc);
if (desc->pd_prov_type == CRYPTO_SW_PROVIDER) {
/* Release reference held by kcf_prov_tab_lookup(). */
KCF_PROV_REFRELE(desc);
/*
* Wait till the existing requests complete.
*/
mutex_enter(&desc->pd_lock);
while (desc->pd_state != KCF_PROV_FREED)
cv_wait(&desc->pd_remove_cv, &desc->pd_lock);
mutex_exit(&desc->pd_lock);
} else {
/*
* Wait until requests that have been sent to the provider
* complete.
*/
mutex_enter(&desc->pd_lock);
while (desc->pd_irefcnt > 0)
cv_wait(&desc->pd_remove_cv, &desc->pd_lock);
mutex_exit(&desc->pd_lock);
}
kcf_do_notify(desc, B_FALSE);
if (desc->pd_prov_type == CRYPTO_SW_PROVIDER) {
/*
* This is the only place where kcf_free_provider_desc()
* is called directly. KCF_PROV_REFRELE() should free the
* structure in all other places.
*/
ASSERT(desc->pd_state == KCF_PROV_FREED &&
desc->pd_refcnt == 0);
kcf_free_provider_desc(desc);
} else {
KCF_PROV_REFRELE(desc);
}
return (CRYPTO_SUCCESS);
}
/*
* Return value:
* 1 - exitlwps() failed, call (or continue) lwp_exit()
* 0 - restarting init. Return through system call path
*/
int
proc_exit(int why, int what)
{
kthread_t *t = curthread;
klwp_t *lwp = ttolwp(t);
proc_t *p = ttoproc(t);
zone_t *z = p->p_zone;
timeout_id_t tmp_id;
int rv;
proc_t *q;
task_t *tk;
vnode_t *exec_vp, *execdir_vp, *cdir, *rdir;
sigqueue_t *sqp;
lwpdir_t *lwpdir;
uint_t lwpdir_sz;
tidhash_t *tidhash;
uint_t tidhash_sz;
ret_tidhash_t *ret_tidhash;
refstr_t *cwd;
hrtime_t hrutime, hrstime;
int evaporate;
/*
* Stop and discard the process's lwps except for the current one,
* unless some other lwp beat us to it. If exitlwps() fails then
* return and the calling lwp will call (or continue in) lwp_exit().
*/
proc_is_exiting(p);
if (exitlwps(0) != 0)
return (1);
mutex_enter(&p->p_lock);
if (p->p_ttime > 0) {
/*
* Account any remaining ticks charged to this process
* on its way out.
*/
(void) task_cpu_time_incr(p->p_task, p->p_ttime);
p->p_ttime = 0;
}
mutex_exit(&p->p_lock);
DTRACE_PROC(lwp__exit);
DTRACE_PROC1(exit, int, why);
/*
* Will perform any brand specific proc exit processing, since this
* is always the last lwp, will also perform lwp_exit and free brand
* data
*/
if (PROC_IS_BRANDED(p)) {
lwp_detach_brand_hdlrs(lwp);
brand_clearbrand(p, B_FALSE);
}
/*
* Don't let init exit unless zone_start_init() failed its exec, or
* we are shutting down the zone or the machine.
*
* Since we are single threaded, we don't need to lock the
* following accesses to zone_proc_initpid.
*/
if (p->p_pid == z->zone_proc_initpid) {
if (z->zone_boot_err == 0 &&
zone_status_get(z) < ZONE_IS_SHUTTING_DOWN &&
zone_status_get(global_zone) < ZONE_IS_SHUTTING_DOWN &&
z->zone_restart_init == B_TRUE &&
restart_init(what, why) == 0)
return (0);
/*
* Since we didn't or couldn't restart init, we clear
* the zone's init state and proceed with exit
* processing.
*/
z->zone_proc_initpid = -1;
}
lwp_pcb_exit();
/*
* Allocate a sigqueue now, before we grab locks.
* It will be given to sigcld(), below.
* Special case: If we will be making the process disappear
* without a trace because it is either:
* * an exiting SSYS process, or
* * a posix_spawn() vfork child who requests it,
* we don't bother to allocate a useless sigqueue.
*/
evaporate = (p->p_flag & SSYS) || ((p->p_flag & SVFORK) &&
why == CLD_EXITED && what == _EVAPORATE);
if (!evaporate)
sqp = kmem_zalloc(sizeof (sigqueue_t), KM_SLEEP);
/*
* revoke any doors created by the process.
*/
if (p->p_door_list)
door_exit();
/*
* Release schedctl data structures.
*/
if (p->p_pagep)
schedctl_proc_cleanup();
/*
* make sure all pending kaio has completed.
*/
if (p->p_aio)
aio_cleanup_exit();
/*
* discard the lwpchan cache.
*/
if (p->p_lcp != NULL)
lwpchan_destroy_cache(0);
/*
* Clean up any DTrace helper actions or probes for the process.
*/
if (p->p_dtrace_helpers != NULL) {
ASSERT(dtrace_helpers_cleanup != NULL);
(*dtrace_helpers_cleanup)();
}
/* untimeout the realtime timers */
if (p->p_itimer != NULL)
timer_exit();
if ((tmp_id = p->p_alarmid) != 0) {
p->p_alarmid = 0;
(void) untimeout(tmp_id);
}
/*
* Remove any fpollinfo_t's for this (last) thread from our file
* descriptors so closeall() can ASSERT() that they're all gone.
*/
pollcleanup();
if (p->p_rprof_cyclic != CYCLIC_NONE) {
mutex_enter(&cpu_lock);
cyclic_remove(p->p_rprof_cyclic);
mutex_exit(&cpu_lock);
}
mutex_enter(&p->p_lock);
/*
* Clean up any DTrace probes associated with this process.
*/
if (p->p_dtrace_probes) {
ASSERT(dtrace_fasttrap_exit_ptr != NULL);
dtrace_fasttrap_exit_ptr(p);
}
while ((tmp_id = p->p_itimerid) != 0) {
p->p_itimerid = 0;
mutex_exit(&p->p_lock);
(void) untimeout(tmp_id);
mutex_enter(&p->p_lock);
}
lwp_cleanup();
/*
* We are about to exit; prevent our resource associations from
* being changed.
*/
pool_barrier_enter();
/*
* Block the process against /proc now that we have really
* acquired p->p_lock (to manipulate p_tlist at least).
*/
prbarrier(p);
sigfillset(&p->p_ignore);
sigemptyset(&p->p_siginfo);
sigemptyset(&p->p_sig);
sigemptyset(&p->p_extsig);
sigemptyset(&t->t_sig);
sigemptyset(&t->t_extsig);
sigemptyset(&p->p_sigmask);
sigdelq(p, t, 0);
lwp->lwp_cursig = 0;
lwp->lwp_extsig = 0;
p->p_flag &= ~(SKILLED | SEXTKILLED);
if (lwp->lwp_curinfo) {
siginfofree(lwp->lwp_curinfo);
lwp->lwp_curinfo = NULL;
}
t->t_proc_flag |= TP_LWPEXIT;
ASSERT(p->p_lwpcnt == 1 && p->p_zombcnt == 0);
prlwpexit(t); /* notify /proc */
lwp_hash_out(p, t->t_tid);
prexit(p);
p->p_lwpcnt = 0;
p->p_tlist = NULL;
sigqfree(p);
term_mstate(t);
p->p_mterm = gethrtime();
exec_vp = p->p_exec;
execdir_vp = p->p_execdir;
p->p_exec = NULLVP;
p->p_execdir = NULLVP;
mutex_exit(&p->p_lock);
pr_free_watched_pages(p);
closeall(P_FINFO(p));
/* Free the controlling tty. (freectty() always assumes curproc.) */
ASSERT(p == curproc);
(void) freectty(B_TRUE);
#if defined(__sparc)
if (p->p_utraps != NULL)
utrap_free(p);
#endif
if (p->p_semacct) /* IPC semaphore exit */
semexit(p);
rv = wstat(why, what);
acct(rv & 0xff);
exacct_commit_proc(p, rv);
/*
* Release any resources associated with C2 auditing
*/
if (AU_AUDITING()) {
/*
* audit exit system call
*/
audit_exit(why, what);
}
/*
* Free address space.
*/
relvm();
if (exec_vp) {
/*
* Close this executable which has been opened when the process
* was created by getproc().
*/
(void) VOP_CLOSE(exec_vp, FREAD, 1, (offset_t)0, CRED(), NULL);
VN_RELE(exec_vp);
}
if (execdir_vp)
VN_RELE(execdir_vp);
/*
* Release held contracts.
*/
contract_exit(p);
/*
* Depart our encapsulating process contract.
*/
if ((p->p_flag & SSYS) == 0) {
ASSERT(p->p_ct_process);
contract_process_exit(p->p_ct_process, p, rv);
}
/*
* Remove pool association, and block if requested by pool_do_bind.
*/
mutex_enter(&p->p_lock);
ASSERT(p->p_pool->pool_ref > 0);
atomic_add_32(&p->p_pool->pool_ref, -1);
p->p_pool = pool_default;
/*
* Now that our address space has been freed and all other threads
* in this process have exited, set the PEXITED pool flag. This
* tells the pools subsystems to ignore this process if it was
* requested to rebind this process to a new pool.
*/
p->p_poolflag |= PEXITED;
pool_barrier_exit();
mutex_exit(&p->p_lock);
mutex_enter(&pidlock);
/*
* Delete this process from the newstate list of its parent. We
* will put it in the right place in the sigcld in the end.
*/
delete_ns(p->p_parent, p);
/*
* Reassign the orphans to the next of kin.
* Don't rearrange init's orphanage.
*/
if ((q = p->p_orphan) != NULL && p != proc_init) {
proc_t *nokp = p->p_nextofkin;
for (;;) {
q->p_nextofkin = nokp;
if (q->p_nextorph == NULL)
break;
q = q->p_nextorph;
}
q->p_nextorph = nokp->p_orphan;
nokp->p_orphan = p->p_orphan;
p->p_orphan = NULL;
}
/*
* Reassign the children to init.
* Don't try to assign init's children to init.
*/
if ((q = p->p_child) != NULL && p != proc_init) {
struct proc *np;
struct proc *initp = proc_init;
boolean_t setzonetop = B_FALSE;
if (!INGLOBALZONE(curproc))
setzonetop = B_TRUE;
pgdetach(p);
do {
np = q->p_sibling;
/*
* Delete it from its current parent new state
* list and add it to init new state list
*/
delete_ns(q->p_parent, q);
q->p_ppid = 1;
q->p_pidflag &= ~(CLDNOSIGCHLD | CLDWAITPID);
if (setzonetop) {
mutex_enter(&q->p_lock);
q->p_flag |= SZONETOP;
mutex_exit(&q->p_lock);
}
q->p_parent = initp;
/*
* Since q will be the first child,
* it will not have a previous sibling.
*/
q->p_psibling = NULL;
if (initp->p_child) {
initp->p_child->p_psibling = q;
}
q->p_sibling = initp->p_child;
initp->p_child = q;
if (q->p_proc_flag & P_PR_PTRACE) {
mutex_enter(&q->p_lock);
sigtoproc(q, NULL, SIGKILL);
mutex_exit(&q->p_lock);
}
/*
* sigcld() will add the child to parents
* newstate list.
*/
if (q->p_stat == SZOMB)
sigcld(q, NULL);
} while ((q = np) != NULL);
p->p_child = NULL;
ASSERT(p->p_child_ns == NULL);
}
TRACE_1(TR_FAC_PROC, TR_PROC_EXIT, "proc_exit: %p", p);
mutex_enter(&p->p_lock);
CL_EXIT(curthread); /* tell the scheduler that curthread is exiting */
/*
* Have our task accummulate our resource usage data before they
* become contaminated by p_cacct etc., and before we renounce
* membership of the task.
*
* We do this regardless of whether or not task accounting is active.
* This is to avoid having nonsense data reported for this task if
* task accounting is subsequently enabled. The overhead is minimal;
* by this point, this process has accounted for the usage of all its
* LWPs. We nonetheless do the work here, and under the protection of
* pidlock, so that the movement of the process's usage to the task
* happens at the same time as the removal of the process from the
* task, from the point of view of exacct_snapshot_task_usage().
*/
exacct_update_task_mstate(p);
hrutime = mstate_aggr_state(p, LMS_USER);
hrstime = mstate_aggr_state(p, LMS_SYSTEM);
p->p_utime = (clock_t)NSEC_TO_TICK(hrutime) + p->p_cutime;
p->p_stime = (clock_t)NSEC_TO_TICK(hrstime) + p->p_cstime;
p->p_acct[LMS_USER] += p->p_cacct[LMS_USER];
p->p_acct[LMS_SYSTEM] += p->p_cacct[LMS_SYSTEM];
p->p_acct[LMS_TRAP] += p->p_cacct[LMS_TRAP];
p->p_acct[LMS_TFAULT] += p->p_cacct[LMS_TFAULT];
p->p_acct[LMS_DFAULT] += p->p_cacct[LMS_DFAULT];
p->p_acct[LMS_KFAULT] += p->p_cacct[LMS_KFAULT];
p->p_acct[LMS_USER_LOCK] += p->p_cacct[LMS_USER_LOCK];
p->p_acct[LMS_SLEEP] += p->p_cacct[LMS_SLEEP];
p->p_acct[LMS_WAIT_CPU] += p->p_cacct[LMS_WAIT_CPU];
p->p_acct[LMS_STOPPED] += p->p_cacct[LMS_STOPPED];
p->p_ru.minflt += p->p_cru.minflt;
p->p_ru.majflt += p->p_cru.majflt;
p->p_ru.nswap += p->p_cru.nswap;
p->p_ru.inblock += p->p_cru.inblock;
p->p_ru.oublock += p->p_cru.oublock;
p->p_ru.msgsnd += p->p_cru.msgsnd;
p->p_ru.msgrcv += p->p_cru.msgrcv;
p->p_ru.nsignals += p->p_cru.nsignals;
p->p_ru.nvcsw += p->p_cru.nvcsw;
p->p_ru.nivcsw += p->p_cru.nivcsw;
p->p_ru.sysc += p->p_cru.sysc;
p->p_ru.ioch += p->p_cru.ioch;
p->p_stat = SZOMB;
p->p_proc_flag &= ~P_PR_PTRACE;
p->p_wdata = what;
p->p_wcode = (char)why;
cdir = PTOU(p)->u_cdir;
rdir = PTOU(p)->u_rdir;
cwd = PTOU(p)->u_cwd;
ASSERT(cdir != NULL || p->p_parent == &p0);
/*
* Release resource controls, as they are no longer enforceable.
*/
rctl_set_free(p->p_rctls);
/*
* Decrement tk_nlwps counter for our task.max-lwps resource control.
* An extended accounting record, if that facility is active, is
* scheduled to be written. We cannot give up task and project
* membership at this point because that would allow zombies to escape
* from the max-processes resource controls. Zombies stay in their
* current task and project until the process table slot is released
* in freeproc().
*/
tk = p->p_task;
mutex_enter(&p->p_zone->zone_nlwps_lock);
tk->tk_nlwps--;
tk->tk_proj->kpj_nlwps--;
p->p_zone->zone_nlwps--;
mutex_exit(&p->p_zone->zone_nlwps_lock);
/*
* Clear the lwp directory and the lwpid hash table
* now that /proc can't bother us any more.
* We free the memory below, after dropping p->p_lock.
*/
lwpdir = p->p_lwpdir;
lwpdir_sz = p->p_lwpdir_sz;
tidhash = p->p_tidhash;
tidhash_sz = p->p_tidhash_sz;
ret_tidhash = p->p_ret_tidhash;
p->p_lwpdir = NULL;
p->p_lwpfree = NULL;
p->p_lwpdir_sz = 0;
p->p_tidhash = NULL;
p->p_tidhash_sz = 0;
p->p_ret_tidhash = NULL;
/*
* If the process has context ops installed, call the exit routine
* on behalf of this last remaining thread. Normally exitpctx() is
* called during thread_exit() or lwp_exit(), but because this is the
* last thread in the process, we must call it here. By the time
* thread_exit() is called (below), the association with the relevant
* process has been lost.
*
* We also free the context here.
*/
if (p->p_pctx) {
kpreempt_disable();
exitpctx(p);
kpreempt_enable();
freepctx(p, 0);
}
/*
* curthread's proc pointer is changed to point to the 'sched'
* process for the corresponding zone, except in the case when
* the exiting process is in fact a zsched instance, in which
* case the proc pointer is set to p0. We do so, so that the
* process still points at the right zone when we call the VN_RELE()
* below.
*
* This is because curthread's original proc pointer can be freed as
* soon as the child sends a SIGCLD to its parent. We use zsched so
* that for user processes, even in the final moments of death, the
* process is still associated with its zone.
*/
if (p != t->t_procp->p_zone->zone_zsched)
t->t_procp = t->t_procp->p_zone->zone_zsched;
else
t->t_procp = &p0;
mutex_exit(&p->p_lock);
if (!evaporate) {
p->p_pidflag &= ~CLDPEND;
sigcld(p, sqp);
} else {
/*
* Do what sigcld() would do if the disposition
* of the SIGCHLD signal were set to be ignored.
*/
cv_broadcast(&p->p_srwchan_cv);
freeproc(p);
}
mutex_exit(&pidlock);
/*
* We don't release u_cdir and u_rdir until SZOMB is set.
* This protects us against dofusers().
*/
if (cdir)
VN_RELE(cdir);
if (rdir)
VN_RELE(rdir);
if (cwd)
refstr_rele(cwd);
/*
* task_rele() may ultimately cause the zone to go away (or
* may cause the last user process in a zone to go away, which
* signals zsched to go away). So prior to this call, we must
* no longer point at zsched.
*/
t->t_procp = &p0;
kmem_free(lwpdir, lwpdir_sz * sizeof (lwpdir_t));
kmem_free(tidhash, tidhash_sz * sizeof (tidhash_t));
while (ret_tidhash != NULL) {
ret_tidhash_t *next = ret_tidhash->rth_next;
kmem_free(ret_tidhash->rth_tidhash,
ret_tidhash->rth_tidhash_sz * sizeof (tidhash_t));
kmem_free(ret_tidhash, sizeof (*ret_tidhash));
ret_tidhash = next;
}
thread_exit();
/* NOTREACHED */
}
/*------------------------------------------------------------------------*
* usb_process
*
* This function is the USB process dispatcher.
*------------------------------------------------------------------------*/
static void
usb_process(void *arg)
{
struct usb_process *up = arg;
struct usb_proc_msg *pm;
struct thread *td;
/* in case of attach error, check for suspended */
USB_THREAD_SUSPEND_CHECK();
/* adjust priority */
td = curthread;
thread_lock(td);
sched_prio(td, up->up_prio);
thread_unlock(td);
mtx_lock(up->up_mtx);
up->up_curtd = td;
while (1) {
if (up->up_gone)
break;
/*
* NOTE to reimplementors: dequeueing a command from the
* "used" queue and executing it must be atomic, with regard
* to the "up_mtx" mutex. That means any attempt to queue a
* command by another thread must be blocked until either:
*
* 1) the command sleeps
*
* 2) the command returns
*
* Here is a practical example that shows how this helps
* solving a problem:
*
* Assume that you want to set the baud rate on a USB serial
* device. During the programming of the device you don't
* want to receive nor transmit any data, because it will be
* garbage most likely anyway. The programming of our USB
* device takes 20 milliseconds and it needs to call
* functions that sleep.
*
* Non-working solution: Before we queue the programming
* command, we stop transmission and reception of data. Then
* we queue a programming command. At the end of the
* programming command we enable transmission and reception
* of data.
*
* Problem: If a second programming command is queued while the
* first one is sleeping, we end up enabling transmission
* and reception of data too early.
*
* Working solution: Before we queue the programming command,
* we stop transmission and reception of data. Then we queue
* a programming command. Then we queue a second command
* that only enables transmission and reception of data.
*
* Why it works: If a second programming command is queued
* while the first one is sleeping, then the queueing of a
* second command to enable the data transfers, will cause
* the previous one, which is still on the queue, to be
* removed from the queue, and re-inserted after the last
* baud rate programming command, which then gives the
* desired result.
*/
pm = TAILQ_FIRST(&up->up_qhead);
if (pm) {
DPRINTF("Message pm=%p, cb=%p (enter)\n",
pm, pm->pm_callback);
(pm->pm_callback) (pm);
if (pm == TAILQ_FIRST(&up->up_qhead)) {
/* nothing changed */
TAILQ_REMOVE(&up->up_qhead, pm, pm_qentry);
pm->pm_qentry.tqe_prev = NULL;
}
DPRINTF("Message pm=%p (leave)\n", pm);
continue;
}
/* end if messages - check if anyone is waiting for sync */
if (up->up_dsleep) {
up->up_dsleep = 0;
cv_broadcast(&up->up_drain);
}
up->up_msleep = 1;
cv_wait(&up->up_cv, up->up_mtx);
}
up->up_ptr = NULL;
cv_signal(&up->up_cv);
mtx_unlock(up->up_mtx);
#if (__FreeBSD_version >= 800000)
/* Clear the proc pointer if this is the last thread. */
if (--usb_pcount == 0)
usbproc = NULL;
#endif
USB_THREAD_EXIT(0);
}
/*ARGSUSED*/
void *
segkmem_alloc_lp(vmem_t *vmp, size_t *sizep, size_t align, int vmflag)
{
size_t size;
kthread_t *t = curthread;
segkmem_lpcb_t *lpcb = &segkmem_lpcb;
ASSERT(sizep != NULL);
size = *sizep;
if (lpcb->lp_uselp && !(t->t_flag & T_PANIC) &&
!(vmflag & SEGKMEM_SHARELOCKED)) {
size_t kmemlp_qnt = segkmem_kmemlp_quantum;
size_t asize = P2ROUNDUP(size, kmemlp_qnt);
void *addr = NULL;
ulong_t *lpthrtp = &lpcb->lp_throttle;
ulong_t lpthrt = *lpthrtp;
int dowakeup = 0;
int doalloc = 1;
ASSERT(kmem_lp_arena != NULL);
ASSERT(asize >= size);
if (lpthrt != 0) {
/* try to update the throttle value */
lpthrt = atomic_inc_ulong_nv(lpthrtp);
if (lpthrt >= segkmem_lpthrottle_max) {
lpthrt = atomic_cas_ulong(lpthrtp, lpthrt,
segkmem_lpthrottle_max / 4);
}
/*
* when we get above throttle start do an exponential
* backoff at trying large pages and reaping
*/
if (lpthrt > segkmem_lpthrottle_start &&
!ISP2(lpthrt)) {
lpcb->allocs_throttled++;
lpthrt--;
if (ISP2(lpthrt))
kmem_reap();
return (segkmem_alloc(vmp, size, vmflag));
}
}
if (!(vmflag & VM_NOSLEEP) &&
segkmem_heaplp_quantum >= (8 * kmemlp_qnt) &&
vmem_size(kmem_lp_arena, VMEM_FREE) <= kmemlp_qnt &&
asize < (segkmem_heaplp_quantum - kmemlp_qnt)) {
/*
* we are low on free memory in kmem_lp_arena
* we let only one guy to allocate heap_lp
* quantum size chunk that everybody is going to
* share
*/
mutex_enter(&lpcb->lp_lock);
if (lpcb->lp_wait) {
/* we are not the first one - wait */
cv_wait(&lpcb->lp_cv, &lpcb->lp_lock);
if (vmem_size(kmem_lp_arena, VMEM_FREE) <
kmemlp_qnt) {
doalloc = 0;
}
} else if (vmem_size(kmem_lp_arena, VMEM_FREE) <=
kmemlp_qnt) {
/*
* we are the first one, make sure we import
* a large page
*/
if (asize == kmemlp_qnt)
asize += kmemlp_qnt;
dowakeup = 1;
lpcb->lp_wait = 1;
}
mutex_exit(&lpcb->lp_lock);
}
/*
* VM_ABORT flag prevents sleeps in vmem_xalloc when
* large pages are not available. In that case this allocation
* attempt will fail and we will retry allocation with small
* pages. We also do not want to panic if this allocation fails
* because we are going to retry.
*/
if (doalloc) {
addr = vmem_alloc(kmem_lp_arena, asize,
(vmflag | VM_ABORT) & ~VM_PANIC);
if (dowakeup) {
mutex_enter(&lpcb->lp_lock);
ASSERT(lpcb->lp_wait != 0);
lpcb->lp_wait = 0;
cv_broadcast(&lpcb->lp_cv);
mutex_exit(&lpcb->lp_lock);
}
}
if (addr != NULL) {
*sizep = asize;
*lpthrtp = 0;
return (addr);
}
if (vmflag & VM_NOSLEEP)
lpcb->nosleep_allocs_failed++;
else
lpcb->sleep_allocs_failed++;
lpcb->alloc_bytes_failed += size;
/* if large page throttling is not started yet do it */
if (segkmem_use_lpthrottle && lpthrt == 0) {
lpthrt = atomic_cas_ulong(lpthrtp, lpthrt, 1);
}
}
return (segkmem_alloc(vmp, size, vmflag));
}
/*
* atabus_configthread: finish attach of atabus's childrens, in a separate
* kernel thread.
*/
static void
atabusconfig_thread(void *arg)
{
struct atabus_softc *atabus_sc = arg;
struct ata_channel *chp = atabus_sc->sc_chan;
struct atac_softc *atac = chp->ch_atac;
struct atabus_initq *atabus_initq = NULL;
int i, s;
/* XXX seems wrong */
mutex_enter(&atabus_qlock);
atabus_initq = TAILQ_FIRST(&atabus_initq_head);
KASSERT(atabus_initq->atabus_sc == atabus_sc);
mutex_exit(&atabus_qlock);
/*
* First look for a port multiplier
*/
if (chp->ch_ndrives == PMP_MAX_DRIVES &&
chp->ch_drive[PMP_PORT_CTL].drive_type == ATA_DRIVET_PM) {
#if NSATA_PMP > 0
satapmp_attach(chp);
#else
aprint_error_dev(atabus_sc->sc_dev,
"SATA port multiplier not supported\n");
/* no problems going on, all drives are ATA_DRIVET_NONE */
#endif
}
/*
* Attach an ATAPI bus, if needed.
*/
KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
for (i = 0; i < chp->ch_ndrives && chp->atapibus == NULL; i++) {
if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI) {
#if NATAPIBUS > 0
(*atac->atac_atapibus_attach)(atabus_sc);
#else
/*
* Fake the autoconfig "not configured" message
*/
aprint_normal("atapibus at %s not configured\n",
device_xname(atac->atac_dev));
chp->atapibus = NULL;
s = splbio();
for (i = 0; i < chp->ch_ndrives; i++) {
if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATAPI)
chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
}
splx(s);
#endif
break;
}
}
for (i = 0; i < chp->ch_ndrives; i++) {
struct ata_device adev;
if (chp->ch_drive[i].drive_type != ATA_DRIVET_ATA &&
chp->ch_drive[i].drive_type != ATA_DRIVET_OLD) {
continue;
}
if (chp->ch_drive[i].drv_softc != NULL)
continue;
memset(&adev, 0, sizeof(struct ata_device));
adev.adev_bustype = atac->atac_bustype_ata;
adev.adev_channel = chp->ch_channel;
adev.adev_openings = 1;
adev.adev_drv_data = &chp->ch_drive[i];
chp->ch_drive[i].drv_softc = config_found_ia(atabus_sc->sc_dev,
"ata_hl", &adev, ataprint);
if (chp->ch_drive[i].drv_softc != NULL) {
ata_probe_caps(&chp->ch_drive[i]);
} else {
s = splbio();
chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
splx(s);
}
}
/* now that we know the drives, the controller can set its modes */
if (atac->atac_set_modes) {
(*atac->atac_set_modes)(chp);
ata_print_modes(chp);
}
#if NATARAID > 0
if (atac->atac_cap & ATAC_CAP_RAID) {
for (i = 0; i < chp->ch_ndrives; i++) {
if (chp->ch_drive[i].drive_type == ATA_DRIVET_ATA) {
ata_raid_check_component(
chp->ch_drive[i].drv_softc);
}
}
}
#endif /* NATARAID > 0 */
/*
* reset drive_flags for unattached devices, reset state for attached
* ones
*/
s = splbio();
for (i = 0; i < chp->ch_ndrives; i++) {
if (chp->ch_drive[i].drive_type == ATA_DRIVET_PM)
continue;
if (chp->ch_drive[i].drv_softc == NULL) {
chp->ch_drive[i].drive_flags = 0;
chp->ch_drive[i].drive_type = ATA_DRIVET_NONE;
} else
chp->ch_drive[i].state = 0;
}
splx(s);
mutex_enter(&atabus_qlock);
TAILQ_REMOVE(&atabus_initq_head, atabus_initq, atabus_initq);
cv_broadcast(&atabus_qcv);
mutex_exit(&atabus_qlock);
free(atabus_initq, M_DEVBUF);
ata_delref(chp);
config_pending_decr(atac->atac_dev);
kthread_exit(0);
}
static void
atabusconfig(struct atabus_softc *atabus_sc)
{
struct ata_channel *chp = atabus_sc->sc_chan;
struct atac_softc *atac = chp->ch_atac;
struct atabus_initq *atabus_initq = NULL;
int i, s, error;
/* we are in the atabus's thread context */
s = splbio();
chp->ch_flags |= ATACH_TH_RUN;
splx(s);
/*
* Probe for the drives attached to controller, unless a PMP
* is already known
*/
/* XXX for SATA devices we will power up all drives at once */
if (chp->ch_satapmp_nports == 0)
(*atac->atac_probe)(chp);
if (chp->ch_ndrives >= 2) {
ATADEBUG_PRINT(("atabusattach: ch_drive_type 0x%x 0x%x\n",
chp->ch_drive[0].drive_type, chp->ch_drive[1].drive_type),
DEBUG_PROBE);
}
/* next operations will occurs in a separate thread */
s = splbio();
chp->ch_flags &= ~ATACH_TH_RUN;
splx(s);
/* Make sure the devices probe in atabus order to avoid jitter. */
mutex_enter(&atabus_qlock);
for (;;) {
atabus_initq = TAILQ_FIRST(&atabus_initq_head);
if (atabus_initq->atabus_sc == atabus_sc)
break;
cv_wait(&atabus_qcv, &atabus_qlock);
}
mutex_exit(&atabus_qlock);
/* If no drives, abort here */
if (chp->ch_drive == NULL)
goto out;
KASSERT(chp->ch_ndrives == 0 || chp->ch_drive != NULL);
for (i = 0; i < chp->ch_ndrives; i++)
if (chp->ch_drive[i].drive_type != ATA_DRIVET_NONE)
break;
if (i == chp->ch_ndrives)
goto out;
/* Shortcut in case we've been shutdown */
if (chp->ch_flags & ATACH_SHUTDOWN)
goto out;
if ((error = kthread_create(PRI_NONE, 0, NULL, atabusconfig_thread,
atabus_sc, &atabus_cfg_lwp,
"%scnf", device_xname(atac->atac_dev))) != 0)
aprint_error_dev(atac->atac_dev,
"unable to create config thread: error %d\n", error);
return;
out:
mutex_enter(&atabus_qlock);
TAILQ_REMOVE(&atabus_initq_head, atabus_initq, atabus_initq);
cv_broadcast(&atabus_qcv);
mutex_exit(&atabus_qlock);
free(atabus_initq, M_DEVBUF);
ata_delref(chp);
config_pending_decr(atac->atac_dev);
}
/**
* instructor - Piazza answer-editing thread.
*
* Each instructor thread should, for NCYCLES iterations, choose a random
* Piazza question and then update the answer. The answer should always
* consist of a lowercase alphabetic character repeated 10 times, e.g.,
*
* "aaaaaaaaaa"
*
* and each update should increment all ten characters (cycling back to a's
* from z's if a question is updated enough times).
*
* After each update, (including the first update, in which you should create
* the question and initialize the answer to all a's), the instructor should
* print the answer string using piazza_print().
*
* TODO: Implement this.
*/
static void
instructor(void *p, unsigned long which)
{
(void)p;
(void)which;
int i, n;
char letter, *pos;
for (i = 0; i < NCYCLES; ++i) {
// Choose a random Piazza question.
n = random() % NANSWERS;
// If first instructor to see the question, initalize answers
lock_acquire(creation_lock[n]);
if (questions[n] == NULL) {
questions[n] = kmalloc(sizeof(struct piazza_question));
questions[n]->mutex = lock_create("mutex");
questions[n]->readerQ = cv_create("readerQ");
questions[n]->writerQ = cv_create("writerQ");
questions[n]->readers = 0;
questions[n]->writers = 0;
questions[n]->active_writer = 0;
const char *answer = "aaaaaaaaaa"; //TODO: have const here?
questions[n]->pq_answer = kstrdup(answer);
lock_release(creation_lock[n]);
// Print submitted answer
piazza_print(n);
}
// Not the first instructor
else{
lock_release(creation_lock[n]);
// Set up writer lock
lock_acquire(questions[n]->mutex);
questions[n]->writers++;
while(!((questions[n]->readers == 0) && (questions[n]->active_writer == 0)))
cv_wait(questions[n]->writerQ, questions[n]->mutex);
questions[n]->active_writer++;
lock_release(questions[n]->mutex);
/* Start write */
pos = questions[n]->pq_answer;
letter = *pos;
// Update answer
if(letter != 'z'){
while (*(pos) == letter) {
(*pos)++;
pos++;
}
}
// Loop answer back to A's
else{
while (*(pos) == letter) {
*pos = 'a';
pos++;
}
}
// Print submitted answer
piazza_print(n);
/* End write */
// Clean up writer lock
lock_acquire(questions[n]->mutex);
questions[n]->active_writer--;
questions[n]->writers--;
if(questions[n]->writers==0)
cv_broadcast(questions[n]->readerQ, questions[n]->mutex);
else
cv_signal(questions[n]->writerQ, questions[n]->mutex);
lock_release(questions[n]->mutex);
}
}
// Exiting thread
lock_acquire(thread_count_lock);
if(++finished_thread_count == NSTUDENTS + NINSTRUCTORS){
lock_release(thread_count_lock);
V(driver_sem);
}
else
lock_release(thread_count_lock);
}
/*
* This routine is used to notify the framework that the state of
* a cryptographic provider has changed. Valid state codes are:
*
* CRYPTO_PROVIDER_READY
* The provider indicates that it can process more requests. A provider
* will notify with this event if it previously has notified us with a
* CRYPTO_PROVIDER_BUSY.
*
* CRYPTO_PROVIDER_BUSY
* The provider can not take more requests.
*
* CRYPTO_PROVIDER_FAILED
* The provider encountered an internal error. The framework will not
* be sending any more requests to the provider. The provider may notify
* with a CRYPTO_PROVIDER_READY, if it is able to recover from the error.
*
* This routine can be called from user or interrupt context.
*/
void
crypto_provider_notification(crypto_kcf_provider_handle_t handle, uint_t state)
{
kcf_provider_desc_t *pd;
/* lookup the provider from the given handle */
if ((pd = kcf_prov_tab_lookup((crypto_provider_id_t)handle)) == NULL)
return;
mutex_enter(&pd->pd_lock);
if (pd->pd_state <= KCF_PROV_VERIFICATION_FAILED)
goto out;
if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) {
cmn_err(CE_WARN, "crypto_provider_notification: "
"logical provider (%x) ignored\n", handle);
goto out;
}
switch (state) {
case CRYPTO_PROVIDER_READY:
switch (pd->pd_state) {
case KCF_PROV_BUSY:
pd->pd_state = KCF_PROV_READY;
/*
* Signal the per-provider taskq threads that they
* can start submitting requests.
*/
cv_broadcast(&pd->pd_resume_cv);
break;
case KCF_PROV_FAILED:
/*
* The provider recovered from the error. Let us
* use it now.
*/
pd->pd_state = KCF_PROV_READY;
break;
default:
break;
}
break;
case CRYPTO_PROVIDER_BUSY:
switch (pd->pd_state) {
case KCF_PROV_READY:
pd->pd_state = KCF_PROV_BUSY;
break;
default:
break;
}
break;
case CRYPTO_PROVIDER_FAILED:
/*
* We note the failure and return. The per-provider taskq
* threads check this flag and start failing the
* requests, if it is set. See process_req_hwp() for details.
*/
switch (pd->pd_state) {
case KCF_PROV_READY:
pd->pd_state = KCF_PROV_FAILED;
break;
case KCF_PROV_BUSY:
pd->pd_state = KCF_PROV_FAILED;
/*
* The per-provider taskq threads may be waiting. We
* signal them so that they can start failing requests.
*/
cv_broadcast(&pd->pd_resume_cv);
break;
default:
break;
}
break;
default:
break;
}
out:
mutex_exit(&pd->pd_lock);
KCF_PROV_REFRELE(pd);
}
/*
* The server uses this to check the callback path supplied by the
* client. The callback connection is marked "in progress" while this
* work is going on and then eventually marked either OK or FAILED.
* This work can be done as part of a separate thread and at the end
* of this the thread will exit or it may be done such that the caller
* will continue with other work.
*/
static void
rfs4_do_cb_null(rfs4_client_t *cp)
{
struct timeval tv;
CLIENT *ch;
rfs4_cbstate_t newstate;
rfs4_cbinfo_t *cbp = &cp->rc_cbinfo;
mutex_enter(cbp->cb_lock);
/* If another thread is doing CB_NULL RPC then return */
if (cbp->cb_nullcaller == TRUE) {
mutex_exit(cbp->cb_lock);
rfs4_client_rele(cp);
return;
}
/* Mark the cbinfo as having a thread in the NULL callback */
cbp->cb_nullcaller = TRUE;
/*
* Are there other threads still using the cbinfo client
* handles? If so, this thread must wait before going and
* mucking aroiund with the callback information
*/
while (cbp->cb_refcnt != 0)
cv_wait(cbp->cb_cv_nullcaller, cbp->cb_lock);
/*
* This thread itself may find that new callback info has
* arrived and is set up to handle this case and redrive the
* call to the client's callback server.
*/
retry:
if (cbp->cb_newer.cb_new == TRUE &&
cbp->cb_newer.cb_confirmed == TRUE) {
char *addr = cbp->cb_callback.cb_location.r_addr;
char *netid = cbp->cb_callback.cb_location.r_netid;
/*
* Free the old stuff if it exists; may be the first
* time through this path
*/
if (addr)
kmem_free(addr, strlen(addr) + 1);
if (netid)
kmem_free(netid, strlen(netid) + 1);
/* Move over the addr/netid */
cbp->cb_callback.cb_location.r_addr =
cbp->cb_newer.cb_callback.cb_location.r_addr;
cbp->cb_newer.cb_callback.cb_location.r_addr = NULL;
cbp->cb_callback.cb_location.r_netid =
cbp->cb_newer.cb_callback.cb_location.r_netid;
cbp->cb_newer.cb_callback.cb_location.r_netid = NULL;
/* Get the program number */
cbp->cb_callback.cb_program =
cbp->cb_newer.cb_callback.cb_program;
cbp->cb_newer.cb_callback.cb_program = 0;
/* Don't forget the protocol's "cb_ident" field */
cbp->cb_ident = cbp->cb_newer.cb_ident;
cbp->cb_newer.cb_ident = 0;
/* no longer new */
cbp->cb_newer.cb_new = FALSE;
cbp->cb_newer.cb_confirmed = FALSE;
/* get rid of the old client handles that may exist */
rfs4_cb_chflush(cbp);
cbp->cb_state = CB_NONE;
cbp->cb_timefailed = 0; /* reset the clock */
cbp->cb_notified_of_cb_path_down = TRUE;
}
if (cbp->cb_state != CB_NONE) {
cv_broadcast(cbp->cb_cv); /* let the others know */
cbp->cb_nullcaller = FALSE;
mutex_exit(cbp->cb_lock);
rfs4_client_rele(cp);
return;
}
/* mark rfs4_client_t as CALLBACK NULL in progress */
cbp->cb_state = CB_INPROG;
mutex_exit(cbp->cb_lock);
/* get/generate a client handle */
if ((ch = rfs4_cb_getch(cbp)) == NULL) {
mutex_enter(cbp->cb_lock);
cbp->cb_state = CB_BAD;
cbp->cb_timefailed = gethrestime_sec(); /* observability */
goto retry;
}
tv.tv_sec = 30;
tv.tv_usec = 0;
if (clnt_call(ch, CB_NULL, xdr_void, NULL, xdr_void, NULL, tv) != 0) {
newstate = CB_BAD;
} else {
newstate = CB_OK;
#ifdef DEBUG
rfs4_cb_null++;
#endif
}
/* Check to see if the client has specified new callback info */
mutex_enter(cbp->cb_lock);
rfs4_cb_freech(cbp, ch, TRUE);
if (cbp->cb_newer.cb_new == TRUE &&
cbp->cb_newer.cb_confirmed == TRUE) {
goto retry; /* give the CB_NULL another chance */
}
cbp->cb_state = newstate;
if (cbp->cb_state == CB_BAD)
cbp->cb_timefailed = gethrestime_sec(); /* observability */
cv_broadcast(cbp->cb_cv); /* start up the other threads */
cbp->cb_nullcaller = FALSE;
mutex_exit(cbp->cb_lock);
rfs4_client_rele(cp);
}
static void
txg_sync_thread(dsl_pool_t *dp)
{
spa_t *spa = dp->dp_spa;
tx_state_t *tx = &dp->dp_tx;
callb_cpr_t cpr;
uint64_t start, delta;
txg_thread_enter(tx, &cpr);
start = delta = 0;
for (;;) {
uint64_t timer, timeout = zfs_txg_timeout * hz;
uint64_t txg;
/*
* We sync when we're scanning, there's someone waiting
* on us, or the quiesce thread has handed off a txg to
* us, or we have reached our timeout.
*/
timer = (delta >= timeout ? 0 : timeout - delta);
while (!dsl_scan_active(dp->dp_scan) &&
!tx->tx_exiting && timer > 0 &&
tx->tx_synced_txg >= tx->tx_sync_txg_waiting &&
tx->tx_quiesced_txg == 0) {
dprintf("waiting; tx_synced=%llu waiting=%llu dp=%p\n",
tx->tx_synced_txg, tx->tx_sync_txg_waiting, dp);
txg_thread_wait(tx, &cpr, &tx->tx_sync_more_cv, timer);
delta = ddi_get_lbolt() - start;
timer = (delta > timeout ? 0 : timeout - delta);
}
/*
* Wait until the quiesce thread hands off a txg to us,
* prompting it to do so if necessary.
*/
while (!tx->tx_exiting && tx->tx_quiesced_txg == 0) {
if (tx->tx_quiesce_txg_waiting < tx->tx_open_txg+1)
tx->tx_quiesce_txg_waiting = tx->tx_open_txg+1;
cv_broadcast(&tx->tx_quiesce_more_cv);
txg_thread_wait(tx, &cpr, &tx->tx_quiesce_done_cv, 0);
}
if (tx->tx_exiting)
txg_thread_exit(tx, &cpr, &tx->tx_sync_thread);
/*
* Consume the quiesced txg which has been handed off to
* us. This may cause the quiescing thread to now be
* able to quiesce another txg, so we must signal it.
*/
txg = tx->tx_quiesced_txg;
tx->tx_quiesced_txg = 0;
tx->tx_syncing_txg = txg;
cv_broadcast(&tx->tx_quiesce_more_cv);
dprintf("txg=%llu quiesce_txg=%llu sync_txg=%llu\n",
txg, tx->tx_quiesce_txg_waiting, tx->tx_sync_txg_waiting);
mutex_exit(&tx->tx_sync_lock);
start = ddi_get_lbolt();
spa_sync(spa, txg);
delta = ddi_get_lbolt() - start;
mutex_enter(&tx->tx_sync_lock);
tx->tx_synced_txg = txg;
tx->tx_syncing_txg = 0;
cv_broadcast(&tx->tx_sync_done_cv);
/*
* Dispatch commit callbacks to worker threads.
*/
txg_dispatch_callbacks(dp, txg);
}
}
static void
txg_sync_thread(dsl_pool_t *dp)
{
spa_t *spa = dp->dp_spa;
tx_state_t *tx = &dp->dp_tx;
callb_cpr_t cpr;
vdev_stat_t *vs1, *vs2;
uint64_t start, delta;
#ifdef _KERNEL
/*
* Annotate this process with a flag that indicates that it is
* unsafe to use KM_SLEEP during memory allocations due to the
* potential for a deadlock. KM_PUSHPAGE should be used instead.
*/
current->flags |= PF_NOFS;
#endif /* _KERNEL */
txg_thread_enter(tx, &cpr);
vs1 = kmem_alloc(sizeof (vdev_stat_t), KM_PUSHPAGE);
vs2 = kmem_alloc(sizeof (vdev_stat_t), KM_PUSHPAGE);
start = delta = 0;
for (;;) {
uint64_t timer, timeout;
uint64_t txg;
timeout = zfs_txg_timeout * hz;
/*
* We sync when we're scanning, there's someone waiting
* on us, or the quiesce thread has handed off a txg to
* us, or we have reached our timeout.
*/
timer = (delta >= timeout ? 0 : timeout - delta);
while (!dsl_scan_active(dp->dp_scan) &&
!tx->tx_exiting && timer > 0 &&
tx->tx_synced_txg >= tx->tx_sync_txg_waiting &&
tx->tx_quiesced_txg == 0 &&
dp->dp_dirty_total < zfs_dirty_data_sync) {
dprintf("waiting; tx_synced=%llu waiting=%llu dp=%p\n",
tx->tx_synced_txg, tx->tx_sync_txg_waiting, dp);
txg_thread_wait(tx, &cpr, &tx->tx_sync_more_cv, timer);
delta = ddi_get_lbolt() - start;
timer = (delta > timeout ? 0 : timeout - delta);
}
/*
* Wait until the quiesce thread hands off a txg to us,
* prompting it to do so if necessary.
*/
while (!tx->tx_exiting && tx->tx_quiesced_txg == 0) {
if (tx->tx_quiesce_txg_waiting < tx->tx_open_txg+1)
tx->tx_quiesce_txg_waiting = tx->tx_open_txg+1;
cv_broadcast(&tx->tx_quiesce_more_cv);
txg_thread_wait(tx, &cpr, &tx->tx_quiesce_done_cv, 0);
}
if (tx->tx_exiting) {
kmem_free(vs2, sizeof (vdev_stat_t));
kmem_free(vs1, sizeof (vdev_stat_t));
txg_thread_exit(tx, &cpr, &tx->tx_sync_thread);
}
vdev_get_stats(spa->spa_root_vdev, vs1);
/*
* Consume the quiesced txg which has been handed off to
* us. This may cause the quiescing thread to now be
* able to quiesce another txg, so we must signal it.
*/
txg = tx->tx_quiesced_txg;
tx->tx_quiesced_txg = 0;
tx->tx_syncing_txg = txg;
DTRACE_PROBE2(txg__syncing, dsl_pool_t *, dp, uint64_t, txg);
cv_broadcast(&tx->tx_quiesce_more_cv);
dprintf("txg=%llu quiesce_txg=%llu sync_txg=%llu\n",
txg, tx->tx_quiesce_txg_waiting, tx->tx_sync_txg_waiting);
mutex_exit(&tx->tx_sync_lock);
start = ddi_get_lbolt();
spa_sync(spa, txg);
delta = ddi_get_lbolt() - start;
mutex_enter(&tx->tx_sync_lock);
tx->tx_synced_txg = txg;
tx->tx_syncing_txg = 0;
DTRACE_PROBE2(txg__synced, dsl_pool_t *, dp, uint64_t, txg);
cv_broadcast(&tx->tx_sync_done_cv);
/*
* Dispatch commit callbacks to worker threads.
*/
txg_dispatch_callbacks(dp, txg);
vdev_get_stats(spa->spa_root_vdev, vs2);
spa_txg_history_set_io(spa, txg,
vs2->vs_bytes[ZIO_TYPE_READ]-vs1->vs_bytes[ZIO_TYPE_READ],
vs2->vs_bytes[ZIO_TYPE_WRITE]-vs1->vs_bytes[ZIO_TYPE_WRITE],
vs2->vs_ops[ZIO_TYPE_READ]-vs1->vs_ops[ZIO_TYPE_READ],
vs2->vs_ops[ZIO_TYPE_WRITE]-vs1->vs_ops[ZIO_TYPE_WRITE],
dp->dp_dirty_pertxg[txg & TXG_MASK]);
spa_txg_history_set(spa, txg, TXG_STATE_SYNCED, gethrtime());
}
}
/**
* radeon_fence_process - process a fence
*
* @rdev: radeon_device pointer
* @ring: ring index the fence is associated with
*
* Checks the current fence value and wakes the fence queue
* if the sequence number has increased (all asics).
*/
void radeon_fence_process(struct radeon_device *rdev, int ring)
{
uint64_t seq, last_seq, last_emitted;
unsigned count_loop = 0;
bool wake = false;
/* Note there is a scenario here for an infinite loop but it's
* very unlikely to happen. For it to happen, the current polling
* process need to be interrupted by another process and another
* process needs to update the last_seq btw the atomic read and
* xchg of the current process.
*
* More over for this to go in infinite loop there need to be
* continuously new fence signaled ie radeon_fence_read needs
* to return a different value each time for both the currently
* polling process and the other process that xchg the last_seq
* btw atomic read and xchg of the current process. And the
* value the other process set as last seq must be higher than
* the seq value we just read. Which means that current process
* need to be interrupted after radeon_fence_read and before
* atomic xchg.
*
* To be even more safe we count the number of time we loop and
* we bail after 10 loop just accepting the fact that we might
* have temporarly set the last_seq not to the true real last
* seq but to an older one.
*/
last_seq = atomic_load_acq_64(&rdev->fence_drv[ring].last_seq);
do {
last_emitted = rdev->fence_drv[ring].sync_seq[ring];
seq = radeon_fence_read(rdev, ring);
seq |= last_seq & 0xffffffff00000000LL;
if (seq < last_seq) {
seq &= 0xffffffff;
seq |= last_emitted & 0xffffffff00000000LL;
}
if (seq <= last_seq || seq > last_emitted) {
break;
}
/* If we loop over we don't want to return without
* checking if a fence is signaled as it means that the
* seq we just read is different from the previous on.
*/
wake = true;
last_seq = seq;
if ((count_loop++) > 10) {
/* We looped over too many time leave with the
* fact that we might have set an older fence
* seq then the current real last seq as signaled
* by the hw.
*/
break;
}
} while (atomic64_xchg(&rdev->fence_drv[ring].last_seq, seq) > seq);
if (wake) {
rdev->fence_drv[ring].last_activity = jiffies;
cv_broadcast(&rdev->fence_queue);
}
}
/*
* Same as binval, except can force-invalidate delayed-write buffers
* (which are not be already flushed because of device errors). Also
* makes sure that the retry write flag is cleared.
*/
int
bfinval(dev_t dev, int force)
{
struct buf *dp;
struct buf *bp;
struct buf *binval_list = EMPTY_LIST;
int i, error = 0;
kmutex_t *hmp;
uint_t index;
struct buf **backp;
mutex_enter(&blist_lock);
/*
* Wait for any flushes ahead of us to finish, it's ok to
* do invalidates in parallel.
*/
while (bio_doingflush) {
bio_flinv_cv_wanted = 1;
cv_wait(&bio_flushinval_cv, &blist_lock);
}
bio_doinginval++;
/* Gather bp's */
for (i = 0; i < v.v_hbuf; i++) {
dp = (struct buf *)&hbuf[i];
hmp = &hbuf[i].b_lock;
mutex_enter(hmp);
for (bp = dp->b_forw; bp != dp; bp = bp->b_forw) {
if (bp->b_edev == dev) {
if (bp->b_list == NULL) {
bp->b_list = binval_list;
binval_list = bp;
}
}
}
mutex_exit(hmp);
}
mutex_exit(&blist_lock);
/* Invalidate all bp's found */
while (binval_list != EMPTY_LIST) {
bp = binval_list;
sema_p(&bp->b_sem);
if (bp->b_edev == dev) {
if (force && (bp->b_flags & B_DELWRI)) {
/* clear B_DELWRI, move to non-dw freelist */
index = bio_bhash(bp->b_edev, bp->b_blkno);
hmp = &hbuf[index].b_lock;
dp = (struct buf *)&hbuf[index];
mutex_enter(hmp);
/* remove from delayed write freelist */
notavail(bp);
/* add to B_AGE side of non-dw freelist */
backp = &dp->av_forw;
(*backp)->av_back = bp;
bp->av_forw = *backp;
*backp = bp;
bp->av_back = dp;
/*
* make sure write retries and busy are cleared
*/
bp->b_flags &=
~(B_BUSY | B_DELWRI | B_RETRYWRI);
mutex_exit(hmp);
}
if ((bp->b_flags & B_DELWRI) == 0)
bp->b_flags |= B_STALE|B_AGE;
else
error = EIO;
}
sema_v(&bp->b_sem);
binval_list = bp->b_list;
bp->b_list = NULL;
}
mutex_enter(&blist_lock);
bio_doinginval--;
if (bio_flinv_cv_wanted) {
cv_broadcast(&bio_flushinval_cv);
bio_flinv_cv_wanted = 0;
}
mutex_exit(&blist_lock);
return (error);
}
static void
midi_in(void *addr, int data)
{
struct midi_softc *sc;
struct midi_buffer *mb;
int i, count;
enum fst_ret got;
MIDI_BUF_DECLARE(idx);
MIDI_BUF_DECLARE(buf);
sc = addr;
mb = &sc->inbuf;
KASSERT(mutex_owned(sc->lock));
if (!sc->isopen)
return;
if ((sc->flags & FREAD) == 0)
return; /* discard data if not reading */
sxp_again:
do {
got = midi_fst(&sc->rcv, data, FST_CANON);
} while (got == FST_HUH);
switch (got) {
case FST_MORE:
case FST_ERR:
return;
case FST_CHN:
case FST_COM:
case FST_RT:
#if NSEQUENCER > 0
if (sc->seqopen) {
extern void midiseq_in(struct midi_dev *,u_char *,int);
count = sc->rcv.end - sc->rcv.pos;
midiseq_in(sc->seq_md, sc->rcv.pos, count);
return;
}
#endif
/*
* Pass Active Sense to the sequencer if it's open, but not to
* a raw reader. (Really should do something intelligent with
* it then, though....)
*/
if (got == FST_RT && MIDI_ACK == sc->rcv.pos[0]) {
if (!sc->rcv_expect_asense) {
sc->rcv_expect_asense = 1;
callout_schedule(&sc->rcv_asense_co,
MIDI_RCV_ASENSE_PERIOD);
}
sc->rcv_quiescent = 0;
sc->rcv_eof = 0;
return;
}
/* FALLTHROUGH */
/*
* Ultimately SysEx msgs should be offered to the sequencer also; the
* sequencer API addresses them - but maybe our sequencer can't handle
* them yet, so offer only to raw reader. (Which means, ultimately,
* discard them if the sequencer's open, as it's not doing reads!)
* -> When SysEx support is added to the sequencer, be sure to handle
* FST_SXP there too.
*/
case FST_SYX:
case FST_SXP:
count = sc->rcv.end - sc->rcv.pos;
sc->rcv_quiescent = 0;
sc->rcv_eof = 0;
if (0 == count)
break;
MIDI_BUF_PRODUCER_INIT(mb,idx);
MIDI_BUF_PRODUCER_INIT(mb,buf);
if (count > buf_lim - buf_cur
|| 1 > idx_lim - idx_cur) {
sc->rcv.bytesDiscarded.ev_count += count;
DPRINTF(("midi_in: buffer full, discard data=0x%02x\n",
sc->rcv.pos[0]));
return;
}
for (i = 0; i < count; i++) {
*buf_cur++ = sc->rcv.pos[i];
MIDI_BUF_WRAP(buf);
}
*idx_cur++ = PACK_MB_IDX(got,count);
MIDI_BUF_WRAP(idx);
MIDI_BUF_PRODUCER_WBACK(mb,buf);
MIDI_BUF_PRODUCER_WBACK(mb,idx);
cv_broadcast(&sc->rchan);
selnotify(&sc->rsel, 0, NOTE_SUBMIT);
if (sc->async != 0)
softint_schedule(sc->sih);
break;
default: /* don't #ifdef this away, gcc will say FST_HUH not handled */
printf("midi_in: midi_fst returned %d?!\n", got);
}
if (FST_SXP == got)
goto sxp_again;
}
/*
* Make sure all write-behind blocks on dev (or NODEV for all)
* are flushed out.
*/
void
bflush(dev_t dev)
{
struct buf *bp, *dp;
struct hbuf *hp;
struct buf *delwri_list = EMPTY_LIST;
int i, index;
kmutex_t *hmp;
mutex_enter(&blist_lock);
/*
* Wait for any invalidates or flushes ahead of us to finish.
* We really could split blist_lock up per device for better
* parallelism here.
*/
while (bio_doinginval || bio_doingflush) {
bio_flinv_cv_wanted = 1;
cv_wait(&bio_flushinval_cv, &blist_lock);
}
bio_doingflush++;
/*
* Gather all B_DELWRI buffer for device.
* Lock ordering is b_sem > hash lock (brelse).
* Since we are finding the buffer via the delayed write list,
* it may be busy and we would block trying to get the
* b_sem lock while holding hash lock. So transfer all the
* candidates on the delwri_list and then drop the hash locks.
*/
for (i = 0; i < v.v_hbuf; i++) {
vfs_syncprogress();
hmp = &hbuf[i].b_lock;
dp = (struct buf *)&dwbuf[i];
mutex_enter(hmp);
for (bp = dp->av_forw; bp != dp; bp = bp->av_forw) {
if (dev == NODEV || bp->b_edev == dev) {
if (bp->b_list == NULL) {
bp->b_list = delwri_list;
delwri_list = bp;
}
}
}
mutex_exit(hmp);
}
mutex_exit(&blist_lock);
/*
* Now that the hash locks have been dropped grab the semaphores
* and write back all the buffers that have B_DELWRI set.
*/
while (delwri_list != EMPTY_LIST) {
vfs_syncprogress();
bp = delwri_list;
sema_p(&bp->b_sem); /* may block */
if ((dev != bp->b_edev && dev != NODEV) ||
(panicstr && bp->b_flags & B_BUSY)) {
sema_v(&bp->b_sem);
delwri_list = bp->b_list;
bp->b_list = NULL;
continue; /* No longer a candidate */
}
if (bp->b_flags & B_DELWRI) {
index = bio_bhash(bp->b_edev, bp->b_blkno);
hp = &hbuf[index];
hmp = &hp->b_lock;
dp = (struct buf *)hp;
bp->b_flags |= B_ASYNC;
mutex_enter(hmp);
hp->b_length--;
notavail(bp);
mutex_exit(hmp);
if (bp->b_vp == NULL) { /* !ufs */
BWRITE(bp);
} else { /* ufs */
UFS_BWRITE(VTOI(bp->b_vp)->i_ufsvfs, bp);
}
} else {
sema_v(&bp->b_sem);
}
delwri_list = bp->b_list;
bp->b_list = NULL;
}
mutex_enter(&blist_lock);
bio_doingflush--;
if (bio_flinv_cv_wanted) {
bio_flinv_cv_wanted = 0;
cv_broadcast(&bio_flushinval_cv);
}
mutex_exit(&blist_lock);
}
/*------------------------------------------------------------------------*
* usb_pc_common_mem_cb - BUS-DMA callback function
*------------------------------------------------------------------------*/
static void
usb_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs,
int nseg, int error, uint8_t isload)
{
struct usb_dma_parent_tag *uptag;
struct usb_page_cache *pc;
struct usb_page *pg;
usb_size_t rem;
bus_size_t off;
uint8_t owned;
pc = arg;
uptag = pc->tag_parent;
/*
* XXX There is sometimes recursive locking here.
* XXX We should try to find a better solution.
* XXX Until further the "owned" variable does
* XXX the trick.
*/
if (error) {
goto done;
}
off = 0;
pg = pc->page_start;
pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1);
rem = segs->ds_addr & (USB_PAGE_SIZE - 1);
pc->page_offset_buf = rem;
pc->page_offset_end += rem;
#ifdef USB_DEBUG
if (rem != (USB_P2U(pc->buffer) & (USB_PAGE_SIZE - 1))) {
/*
* This check verifies that the physical address is correct:
*/
DPRINTFN(0, "Page offset was not preserved\n");
error = 1;
goto done;
}
#endif
while (pc->ismultiseg) {
off += USB_PAGE_SIZE;
if (off >= (segs->ds_len + rem)) {
/* page crossing */
nseg--;
segs++;
off = 0;
rem = 0;
if (nseg == 0)
break;
}
pg++;
pg->physaddr = (segs->ds_addr + off) & ~(USB_PAGE_SIZE - 1);
}
done:
owned = mtx_owned(uptag->mtx);
if (!owned)
mtx_lock(uptag->mtx);
uptag->dma_error = (error ? 1 : 0);
if (isload) {
(uptag->func) (uptag);
} else {
cv_broadcast(uptag->cv);
}
if (!owned)
mtx_unlock(uptag->mtx);
}
/*------------------------------------------------------------------------*
* usb_pc_common_mem_cb - BUS-DMA callback function
*------------------------------------------------------------------------*/
static void
usb_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs,
int nseg, int error, uint8_t isload)
{
struct usb_dma_parent_tag *uptag;
struct usb_page_cache *pc;
struct usb_page *pg;
usb_size_t rem;
bus_size_t off;
uint8_t owned;
pc = arg;
uptag = pc->tag_parent;
/*
* XXX There is sometimes recursive locking here.
* XXX We should try to find a better solution.
* XXX Until further the "owned" variable does
* XXX the trick.
*/
if (error) {
goto done;
}
off = 0;
pg = pc->page_start;
pg->physaddr = rounddown2(segs->ds_addr, USB_PAGE_SIZE);
rem = segs->ds_addr & (USB_PAGE_SIZE - 1);
pc->page_offset_buf = rem;
pc->page_offset_end += rem;
#ifdef USB_DEBUG
if (nseg > 1) {
int x;
for (x = 0; x != nseg - 1; x++) {
if (((segs[x].ds_addr + segs[x].ds_len) & (USB_PAGE_SIZE - 1)) ==
((segs[x + 1].ds_addr & (USB_PAGE_SIZE - 1))))
continue;
/*
* This check verifies there is no page offset
* hole between any of the segments. See the
* BUS_DMA_KEEP_PG_OFFSET flag.
*/
DPRINTFN(0, "Page offset was not preserved\n");
error = 1;
goto done;
}
}
#endif
while (pc->ismultiseg) {
off += USB_PAGE_SIZE;
if (off >= (segs->ds_len + rem)) {
/* page crossing */
nseg--;
segs++;
off = 0;
rem = 0;
if (nseg == 0)
break;
}
pg++;
pg->physaddr = rounddown2(segs->ds_addr + off, USB_PAGE_SIZE);
}
done:
owned = mtx_owned(uptag->mtx);
if (!owned)
mtx_lock(uptag->mtx);
uptag->dma_error = (error ? 1 : 0);
if (isload) {
(uptag->func) (uptag);
} else {
cv_broadcast(uptag->cv);
}
if (!owned)
mtx_unlock(uptag->mtx);
}
static u_int
adb_mouse_receive_packet(device_t dev, u_char status, u_char command,
u_char reg, int len, u_char *data)
{
struct adb_mouse_softc *sc;
int i = 0;
int xdelta, ydelta;
int buttons, tmp_buttons;
sc = device_get_softc(dev);
if (command != ADB_COMMAND_TALK || reg != 0 || len < 2)
return (0);
ydelta = data[0] & 0x7f;
xdelta = data[1] & 0x7f;
buttons = 0;
buttons |= !(data[0] & 0x80);
buttons |= !(data[1] & 0x80) << 1;
if (sc->flags & AMS_EXTENDED) {
for (i = 2; i < len && i < 5; i++) {
xdelta |= (data[i] & 0x07) << (3*i + 1);
ydelta |= (data[i] & 0x70) << (3*i - 3);
buttons |= !(data[i] & 0x08) << (2*i - 2);
buttons |= !(data[i] & 0x80) << (2*i - 1);
}
} else {
len = 2; /* Ignore extra data */
}
/* Do sign extension as necessary */
if (xdelta & (0x40 << 3*(len-2)))
xdelta |= 0xffffffc0 << 3*(len - 2);
if (ydelta & (0x40 << 3*(len-2)))
ydelta |= 0xffffffc0 << 3*(len - 2);
if ((sc->flags & AMS_TOUCHPAD) && (sc->sc_tapping == 1)) {
tmp_buttons = buttons;
if (buttons == 0x12) {
/* Map a double tap on button 3.
Keep the button state for the next sequence.
A double tap sequence is followed by a single tap
sequence.
*/
tmp_buttons = 0x3;
sc->button_buf = tmp_buttons;
} else if (buttons == 0x2) {
/* Map a single tap on button 2. But only if it is
not a successor from a double tap.
*/
if (sc->button_buf != 0x3)
tmp_buttons = 0x2;
else
tmp_buttons = 0;
sc->button_buf = 0;
}
buttons = tmp_buttons;
}
/*
* Some mice report high-numbered buttons on the wrong button number,
* so set the highest-numbered real button as pressed if there are
* mysterious high-numbered ones set.
*
* Don't do this for touchpads, because touchpads also trigger
* high button events when they are touched.
*/
if (buttons & ~((1 << sc->hw.buttons) - 1)
&& !(sc->flags & AMS_TOUCHPAD)) {
buttons |= 1 << (sc->hw.buttons - 1);
}
buttons &= (1 << sc->hw.buttons) - 1;
mtx_lock(&sc->sc_mtx);
/* Add in our new deltas, and take into account
Apple's opposite meaning for Y axis motion */
sc->xdelta += xdelta;
sc->ydelta -= ydelta;
sc->buttons = buttons;
mtx_unlock(&sc->sc_mtx);
cv_broadcast(&sc->sc_cv);
selwakeuppri(&sc->rsel, PZERO);
return (0);
}
int
ldap_pvt_thread_cond_broadcast( ldap_pvt_thread_cond_t *cv )
{
return( cv->lcv_created ? cv_broadcast( cv->lcv_cv ) : 0 );
}
static void
txg_sync_thread(dsl_pool_t *dp)
{
spa_t *spa = dp->dp_spa;
tx_state_t *tx = &dp->dp_tx;
callb_cpr_t cpr;
uint64_t start, delta;
#ifdef _KERNEL
/*
* Annotate this process with a flag that indicates that it is
* unsafe to use KM_SLEEP during memory allocations due to the
* potential for a deadlock. KM_PUSHPAGE should be used instead.
*/
//current->flags |= PF_NOFS;
#endif /* _KERNEL */
txg_thread_enter(tx, &cpr);
start = delta = 0;
for (;;) {
hrtime_t hrstart;
txg_history_t *th;
uint64_t timer, timeout;
uint64_t txg;
timeout = zfs_txg_timeout * hz;
/*
* We sync when we're scanning, there's someone waiting
* on us, or the quiesce thread has handed off a txg to
* us, or we have reached our timeout.
*/
timer = (delta >= timeout ? 0 : timeout - delta);
while (!dsl_scan_active(dp->dp_scan) &&
!tx->tx_exiting && timer > 0 &&
tx->tx_synced_txg >= tx->tx_sync_txg_waiting &&
tx->tx_quiesced_txg == 0) {
dprintf("waiting; tx_synced=%llu waiting=%llu dp=%p\n",
tx->tx_synced_txg, tx->tx_sync_txg_waiting, dp);
txg_thread_wait(tx, &cpr, &tx->tx_sync_more_cv, timer);
delta = ddi_get_lbolt() - start;
timer = (delta > timeout ? 0 : timeout - delta);
}
/*
* Wait until the quiesce thread hands off a txg to us,
* prompting it to do so if necessary.
*/
while (!tx->tx_exiting && tx->tx_quiesced_txg == 0) {
if (tx->tx_quiesce_txg_waiting < tx->tx_open_txg+1)
tx->tx_quiesce_txg_waiting = tx->tx_open_txg+1;
cv_broadcast(&tx->tx_quiesce_more_cv);
txg_thread_wait(tx, &cpr, &tx->tx_quiesce_done_cv, 0);
}
if (tx->tx_exiting)
txg_thread_exit(tx, &cpr, &tx->tx_sync_thread);
/*
* Consume the quiesced txg which has been handed off to
* us. This may cause the quiescing thread to now be
* able to quiesce another txg, so we must signal it.
*/
txg = tx->tx_quiesced_txg;
tx->tx_quiesced_txg = 0;
tx->tx_syncing_txg = txg;
cv_broadcast(&tx->tx_quiesce_more_cv);
th = dsl_pool_txg_history_get(dp, txg);
th->th_kstat.state = TXG_STATE_SYNCING;
vdev_get_stats(spa->spa_root_vdev, &th->th_vs1);
dsl_pool_txg_history_put(th);
dprintf("txg=%llu quiesce_txg=%llu sync_txg=%llu\n",
txg, tx->tx_quiesce_txg_waiting, tx->tx_sync_txg_waiting);
mutex_exit(&tx->tx_sync_lock);
start = ddi_get_lbolt();
hrstart = gethrtime();
spa_sync(spa, txg);
delta = ddi_get_lbolt() - start;
mutex_enter(&tx->tx_sync_lock);
tx->tx_synced_txg = txg;
tx->tx_syncing_txg = 0;
cv_broadcast(&tx->tx_sync_done_cv);
/*
* Dispatch commit callbacks to worker threads.
*/
txg_dispatch_callbacks(dp, txg);
/*
* Measure the txg sync time determine the amount of I/O done.
*/
th = dsl_pool_txg_history_get(dp, txg);
vdev_get_stats(spa->spa_root_vdev, &th->th_vs2);
th->th_kstat.sync_time = gethrtime() - hrstart;
th->th_kstat.nread = th->th_vs2.vs_bytes[ZIO_TYPE_READ] -
th->th_vs1.vs_bytes[ZIO_TYPE_READ];
th->th_kstat.nwritten = th->th_vs2.vs_bytes[ZIO_TYPE_WRITE] -
th->th_vs1.vs_bytes[ZIO_TYPE_WRITE];
th->th_kstat.reads = th->th_vs2.vs_ops[ZIO_TYPE_READ] -
th->th_vs1.vs_ops[ZIO_TYPE_READ];
th->th_kstat.writes = th->th_vs2.vs_ops[ZIO_TYPE_WRITE] -
th->th_vs1.vs_ops[ZIO_TYPE_WRITE];
th->th_kstat.state = TXG_STATE_COMMITTED;
dsl_pool_txg_history_put(th);
}
}
/*
* Process a vnode's page list for all pages whose offset is >= off.
* Pages are to either be free'd, invalidated, or written back to disk.
*
* An "exclusive" lock is acquired for each page if B_INVAL or B_FREE
* is specified, otherwise they are "shared" locked.
*
* Flags are {B_ASYNC, B_INVAL, B_FREE, B_DONTNEED, B_TRUNC}
*
* Special marker page_t's are inserted in the list in order
* to keep track of where we are in the list when locks are dropped.
*
* Note the list is circular and insertions can happen only at the
* head and tail of the list. The algorithm ensures visiting all pages
* on the list in the following way:
*
* Drop two marker pages at the end of the list.
*
* Move one marker page backwards towards the start of the list until
* it is at the list head, processing the pages passed along the way.
*
* Due to race conditions when the vphm mutex is dropped, additional pages
* can be added to either end of the list, so we'll continue to move
* the marker and process pages until it is up against the end marker.
*
* There is one special exit condition. If we are processing a VMODSORT
* vnode and only writing back modified pages, we can stop as soon as
* we run into an unmodified page. This makes fsync(3) operations fast.
*/
int
pvn_vplist_dirty(
vnode_t *vp,
u_offset_t off,
int (*putapage)(vnode_t *, page_t *, u_offset_t *,
size_t *, int, cred_t *),
int flags,
cred_t *cred)
{
page_t *pp;
page_t *mark; /* marker page that moves toward head */
page_t *end; /* marker page at end of list */
int err = 0;
int error;
kmutex_t *vphm;
se_t se;
page_t **where_to_move;
ASSERT(vp->v_type != VCHR);
if (vp->v_pages == NULL)
return (0);
/*
* Serialize vplist_dirty operations on this vnode by setting VVMLOCK.
*
* Don't block on VVMLOCK if B_ASYNC is set. This prevents sync()
* from getting blocked while flushing pages to a dead NFS server.
*/
mutex_enter(&vp->v_lock);
if ((vp->v_flag & VVMLOCK) && (flags & B_ASYNC)) {
mutex_exit(&vp->v_lock);
return (EAGAIN);
}
while (vp->v_flag & VVMLOCK)
cv_wait(&vp->v_cv, &vp->v_lock);
if (vp->v_pages == NULL) {
mutex_exit(&vp->v_lock);
return (0);
}
vp->v_flag |= VVMLOCK;
mutex_exit(&vp->v_lock);
/*
* Set up the marker pages used to walk the list
*/
end = kmem_cache_alloc(marker_cache, KM_SLEEP);
end->p_vnode = vp;
end->p_offset = (u_offset_t)-2;
mark = kmem_cache_alloc(marker_cache, KM_SLEEP);
mark->p_vnode = vp;
mark->p_offset = (u_offset_t)-1;
/*
* Grab the lock protecting the vnode's page list
* note that this lock is dropped at times in the loop.
*/
vphm = page_vnode_mutex(vp);
mutex_enter(vphm);
if (vp->v_pages == NULL)
goto leave;
/*
* insert the markers and loop through the list of pages
*/
page_vpadd(&vp->v_pages->p_vpprev->p_vpnext, mark);
page_vpadd(&mark->p_vpnext, end);
for (;;) {
/*
* If only doing an async write back, then we can
* stop as soon as we get to start of the list.
*/
if (flags == B_ASYNC && vp->v_pages == mark)
break;
/*
* otherwise stop when we've gone through all the pages
*/
if (mark->p_vpprev == end)
break;
pp = mark->p_vpprev;
if (vp->v_pages == pp)
where_to_move = &vp->v_pages;
else
where_to_move = &pp->p_vpprev->p_vpnext;
ASSERT(pp->p_vnode == vp);
/*
* If just flushing dirty pages to disk and this vnode
* is using a sorted list of pages, we can stop processing
* as soon as we find an unmodified page. Since all the
* modified pages are visited first.
*/
if (IS_VMODSORT(vp) &&
!(flags & (B_INVAL | B_FREE | B_TRUNC))) {
if (!hat_ismod(pp) && !page_io_locked(pp)) {
#ifdef DEBUG
/*
* For debug kernels examine what should be
* all the remaining clean pages, asserting
* that they are not modified.
*/
page_t *chk = pp;
int attr;
page_vpsub(&vp->v_pages, mark);
page_vpadd(where_to_move, mark);
do {
chk = chk->p_vpprev;
ASSERT(chk != end);
if (chk == mark)
continue;
attr = hat_page_getattr(chk, P_MOD |
P_REF);
if ((attr & P_MOD) == 0)
continue;
panic("v_pages list not all clean: "
"page_t*=%p vnode=%p off=%lx "
"attr=0x%x last clean page_t*=%p\n",
(void *)chk, (void *)chk->p_vnode,
(long)chk->p_offset, attr,
(void *)pp);
} while (chk != vp->v_pages);
#endif
break;
} else if (!(flags & B_ASYNC) && !hat_ismod(pp)) {
/*
* Couldn't get io lock, wait until IO is done.
* Block only for sync IO since we don't want
* to block async IO.
*/
mutex_exit(vphm);
page_io_wait(pp);
mutex_enter(vphm);
continue;
}
}
/*
* Skip this page if the offset is out of the desired range.
* Just move the marker and continue.
*/
if (pp->p_offset < off) {
page_vpsub(&vp->v_pages, mark);
page_vpadd(where_to_move, mark);
continue;
}
/*
* If we are supposed to invalidate or free this
* page, then we need an exclusive lock.
*/
se = (flags & (B_INVAL | B_FREE)) ? SE_EXCL : SE_SHARED;
/*
* We must acquire the page lock for all synchronous
* operations (invalidate, free and write).
*/
if ((flags & B_INVAL) != 0 || (flags & B_ASYNC) == 0) {
/*
* If the page_lock() drops the mutex
* we must retry the loop.
*/
if (!page_lock(pp, se, vphm, P_NO_RECLAIM))
continue;
/*
* It's ok to move the marker page now.
*/
page_vpsub(&vp->v_pages, mark);
page_vpadd(where_to_move, mark);
} else {
/*
* update the marker page for all remaining cases
*/
page_vpsub(&vp->v_pages, mark);
page_vpadd(where_to_move, mark);
/*
* For write backs, If we can't lock the page, it's
* invalid or in the process of being destroyed. Skip
* it, assuming someone else is writing it.
*/
if (!page_trylock(pp, se))
continue;
}
ASSERT(pp->p_vnode == vp);
/*
* Successfully locked the page, now figure out what to
* do with it. Free pages are easily dealt with, invalidate
* if desired or just go on to the next page.
*/
if (PP_ISFREE(pp)) {
if ((flags & B_INVAL) == 0) {
page_unlock(pp);
continue;
}
/*
* Invalidate (destroy) the page.
*/
mutex_exit(vphm);
page_destroy_free(pp);
mutex_enter(vphm);
continue;
}
/*
* pvn_getdirty() figures out what do do with a dirty page.
* If the page is dirty, the putapage() routine will write it
* and will kluster any other adjacent dirty pages it can.
*
* pvn_getdirty() and `(*putapage)' unlock the page.
*/
mutex_exit(vphm);
if (pvn_getdirty(pp, flags)) {
error = (*putapage)(vp, pp, NULL, NULL, flags, cred);
if (!err)
err = error;
}
mutex_enter(vphm);
}
page_vpsub(&vp->v_pages, mark);
page_vpsub(&vp->v_pages, end);
leave:
/*
* Release v_pages mutex, also VVMLOCK and wakeup blocked thrds
*/
mutex_exit(vphm);
kmem_cache_free(marker_cache, mark);
kmem_cache_free(marker_cache, end);
mutex_enter(&vp->v_lock);
vp->v_flag &= ~VVMLOCK;
cv_broadcast(&vp->v_cv);
mutex_exit(&vp->v_lock);
return (err);
}
/*
* taskq_destroy().
*
* Assumes: by the time taskq_destroy is called no one will use this task queue
* in any way and no one will try to dispatch entries in it.
*/
void
taskq_destroy(taskq_t *tq)
{
taskq_bucket_t *b = tq->tq_buckets;
int bid = 0;
ASSERT(! (tq->tq_flags & TASKQ_CPR_SAFE));
/*
* Wait for any pending entries to complete.
*/
taskq_wait(tq);
mutex_enter(&tq->tq_lock);
ASSERT((tq->tq_task.tqent_next == &tq->tq_task) &&
(tq->tq_active == 0));
if ((tq->tq_nthreads > 1) && (tq->tq_threadlist != NULL))
kmem_free(tq->tq_threadlist, sizeof (kthread_t *) *
tq->tq_nthreads);
tq->tq_flags &= ~TASKQ_ACTIVE;
cv_broadcast(&tq->tq_dispatch_cv);
while (tq->tq_nthreads != 0)
cv_wait(&tq->tq_wait_cv, &tq->tq_lock);
tq->tq_minalloc = 0;
while (tq->tq_nalloc != 0)
taskq_ent_free(tq, taskq_ent_alloc(tq, TQ_SLEEP));
mutex_exit(&tq->tq_lock);
/*
* Mark each bucket as closing and wakeup all sleeping threads.
*/
for (; (b != NULL) && (bid < tq->tq_nbuckets); b++, bid++) {
taskq_ent_t *tqe;
mutex_enter(&b->tqbucket_lock);
b->tqbucket_flags |= TQBUCKET_CLOSE;
/* Wakeup all sleeping threads */
for (tqe = b->tqbucket_freelist.tqent_next;
tqe != &b->tqbucket_freelist; tqe = tqe->tqent_next)
cv_signal(&tqe->tqent_cv);
ASSERT(b->tqbucket_nalloc == 0);
/*
* At this point we waited for all pending jobs to complete (in
* both the task queue and the bucket and no new jobs should
* arrive. Wait for all threads to die.
*/
while (b->tqbucket_nfree > 0)
cv_wait(&b->tqbucket_cv, &b->tqbucket_lock);
mutex_exit(&b->tqbucket_lock);
mutex_destroy(&b->tqbucket_lock);
cv_destroy(&b->tqbucket_cv);
}
if (tq->tq_buckets != NULL) {
ASSERT(tq->tq_flags & TASKQ_DYNAMIC);
kmem_free(tq->tq_buckets,
sizeof (taskq_bucket_t) * tq->tq_nbuckets);
/* Cleanup fields before returning tq to the cache */
tq->tq_buckets = NULL;
tq->tq_tcreates = 0;
tq->tq_tdeaths = 0;
} else {
ASSERT(!(tq->tq_flags & TASKQ_DYNAMIC));
}
tq->tq_totaltime = 0;
tq->tq_tasks = 0;
tq->tq_maxtasks = 0;
tq->tq_executed = 0;
kmem_cache_free(taskq_cache, tq);
}
