static void
ptstart(struct cam_periph *periph, union ccb *start_ccb)
{
struct pt_softc *softc;
struct buf *bp;
struct bio *bio;
softc = (struct pt_softc *)periph->softc;
/*
* See if there is a buf with work for us to do..
*/
bio = bioq_first(&softc->bio_queue);
if (periph->immediate_priority <= periph->pinfo.priority) {
CAM_DEBUG_PRINT(CAM_DEBUG_SUBTRACE,
("queuing for immediate ccb\n"));
start_ccb->ccb_h.ccb_state = PT_CCB_WAITING;
SLIST_INSERT_HEAD(&periph->ccb_list, &start_ccb->ccb_h,
periph_links.sle);
periph->immediate_priority = CAM_PRIORITY_NONE;
wakeup(&periph->ccb_list);
} else if (bio == NULL) {
xpt_release_ccb(start_ccb);
} else {
bioq_remove(&softc->bio_queue, bio);
bp = bio->bio_buf;
devstat_start_transaction(&softc->device_stats);
scsi_send_receive(&start_ccb->csio,
/*retries*/4,
ptdone,
MSG_SIMPLE_Q_TAG,
(bp->b_cmd == BUF_CMD_READ),
/*byte2*/0,
bp->b_bcount,
bp->b_data,
/*sense_len*/SSD_FULL_SIZE,
/*timeout*/softc->io_timeout);
start_ccb->ccb_h.ccb_state = PT_CCB_BUFFER_IO_UA;
/*
* Block out any asyncronous callbacks
* while we touch the pending ccb list.
*/
LIST_INSERT_HEAD(&softc->pending_ccbs, &start_ccb->ccb_h,
periph_links.le);
start_ccb->ccb_h.ccb_bio = bio;
bio = bioq_first(&softc->bio_queue);
xpt_action(start_ccb);
if (bio != NULL) {
/* Have more work to do, so ensure we stay scheduled */
xpt_schedule(periph, /* XXX priority */1);
}
}
}
static void
adaschedule(struct cam_periph *periph)
{
struct ada_softc *softc = (struct ada_softc *)periph->softc;
if (bioq_first(&softc->bio_queue) ||
(!softc->trim_running && bioq_first(&softc->trim_queue))) {
/* Have more work to do, so ensure we stay scheduled */
xpt_schedule(periph, CAM_PRIORITY_NORMAL);
}
}
static void
ptstart(struct cam_periph *periph, union ccb *start_ccb)
{
struct pt_softc *softc;
struct bio *bp;
softc = (struct pt_softc *)periph->softc;
CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("ptstart\n"));
/*
* See if there is a buf with work for us to do..
*/
bp = bioq_first(&softc->bio_queue);
if (bp == NULL) {
xpt_release_ccb(start_ccb);
} else {
bioq_remove(&softc->bio_queue, bp);
devstat_start_transaction_bio(softc->device_stats, bp);
scsi_send_receive(&start_ccb->csio,
/*retries*/4,
ptdone,
MSG_SIMPLE_Q_TAG,
bp->bio_cmd == BIO_READ,
/*byte2*/0,
bp->bio_bcount,
bp->bio_data,
/*sense_len*/SSD_FULL_SIZE,
/*timeout*/softc->io_timeout);
start_ccb->ccb_h.ccb_state = PT_CCB_BUFFER_IO_UA;
/*
* Block out any asynchronous callbacks
* while we touch the pending ccb list.
*/
LIST_INSERT_HEAD(&softc->pending_ccbs, &start_ccb->ccb_h,
periph_links.le);
start_ccb->ccb_h.ccb_bp = bp;
bp = bioq_first(&softc->bio_queue);
xpt_action(start_ccb);
if (bp != NULL) {
/* Have more work to do, so ensure we stay scheduled */
xpt_schedule(periph, CAM_PRIORITY_NORMAL);
}
}
}
static void
pst_start(struct pst_softc *psc)
{
struct pst_request *request;
struct bio *bp;
u_int32_t mfa;
if (psc->iop->outstanding < (I2O_IOP_OUTBOUND_FRAME_COUNT - 1) &&
(bp = bioq_first(&psc->queue))) {
if ((mfa = iop_get_mfa(psc->iop)) != 0xffffffff) {
bioq_remove(&psc->queue, bp);
if (!(request = malloc(sizeof(struct pst_request),
M_PSTRAID, M_NOWAIT | M_ZERO))) {
printf("pst: out of memory in start\n");
biofinish(request->bp, NULL, ENOMEM);
iop_free_mfa(psc->iop, mfa);
return;
}
psc->iop->outstanding++;
request->psc = psc;
request->mfa = mfa;
request->bp = bp;
if (pst_rw(request)) {
biofinish(request->bp, NULL, EIO);
iop_free_mfa(request->psc->iop, request->mfa);
psc->iop->outstanding--;
free(request, M_PSTRAID);
}
}
}
}
static void
mcd_start(struct mcd_softc *sc)
{
struct bio *bp;
int s = splbio();
if (sc->data.flags & MCDMBXBSY) {
splx(s);
return;
}
bp = bioq_first(&sc->data.head);
if (bp != 0) {
/* block found to process, dequeue */
/*MCD_TRACE("mcd_start: found block bp=0x%x\n",bp,0,0,0);*/
bioq_remove(&sc->data.head, bp);
sc->data.flags |= MCDMBXBSY;
splx(s);
} else {
/* nothing to do */
splx(s);
return;
}
sc->data.mbx.retry = MCD_RETRYS;
sc->data.mbx.bp = bp;
mcd_doread(sc, MCD_S_BEGIN,&(sc->data.mbx));
return;
}
static void
altera_sdcard_task_io(struct altera_sdcard_softc *sc)
{
uint16_t asr;
ALTERA_SDCARD_LOCK_ASSERT(sc);
KASSERT(sc->as_currentbio != NULL, ("%s: no current I/O", __func__));
#ifdef ALTERA_SDCARD_FAST_SIM
recheck:
#endif
asr = altera_sdcard_read_asr(sc);
/*
* Check for unexpected card removal during an I/O.
*/
if (!(asr & ALTERA_SDCARD_ASR_CARDPRESENT)) {
altera_sdcard_disk_remove(sc);
if (sc->as_flags & ALTERA_SDCARD_FLAG_DETACHREQ)
sc->as_state = ALTERA_SDCARD_STATE_DETACHED;
else
sc->as_state = ALTERA_SDCARD_STATE_NOCARD;
return;
}
/*
* If the I/O isn't complete, remain in the IO state without further
* action, even if DETACHREQ is in flight.
*/
if (asr & ALTERA_SDCARD_ASR_CMDINPROGRESS)
return;
/*
* Handle various forms of I/O completion, successful and otherwise.
* The I/O layer may restart the transaction if an error occurred, in
* which case remain in the IO state and reschedule.
*/
if (!altera_sdcard_io_complete(sc, asr))
return;
/*
* Now that I/O is complete, process detach requests in preference to
* starting new I/O.
*/
if (sc->as_flags & ALTERA_SDCARD_FLAG_DETACHREQ) {
sc->as_state = ALTERA_SDCARD_STATE_DETACHED;
return;
}
/*
* Finally, either start the next I/O or transition to the IDLE state.
*/
if (bioq_first(&sc->as_bioq) != NULL) {
altera_sdcard_nextio(sc);
#ifdef ALTERA_SDCARD_FAST_SIM
goto recheck;
#endif
} else
sc->as_state = ALTERA_SDCARD_STATE_IDLE;
}
static void
adaschedule(struct cam_periph *periph)
{
struct ada_softc *softc = (struct ada_softc *)periph->softc;
uint32_t prio;
/* Check if cam_periph_getccb() was called. */
prio = periph->immediate_priority;
/* Check if we have more work to do. */
if (bioq_first(&softc->bio_queue) ||
(!softc->trim_running && bioq_first(&softc->trim_queue))) {
prio = CAM_PRIORITY_NORMAL;
}
/* Schedule CCB if any of above is true. */
if (prio != CAM_PRIORITY_NONE)
xpt_schedule(periph, prio);
}
void ips_start_io_request(ips_softc_t *sc)
{
struct bio *iobuf;
ips_command_t *command;
iobuf = bioq_first(&sc->queue);
if(!iobuf)
return;
if (ips_get_free_cmd(sc, &command, 0))
return;
bioq_remove(&sc->queue, iobuf);
ips_send_io_request(command, iobuf);
return;
}
static void
ida_construct_qcb(struct ida_softc *ida)
{
struct ida_hardware_qcb *hwqcb;
struct ida_qcb *qcb;
bus_dmasync_op_t op;
struct buf *bp;
struct bio *bio;
bio = bioq_first(&ida->bio_queue);
if (bio == NULL)
return; /* no more buffers */
qcb = ida_get_qcb(ida);
if (qcb == NULL)
return; /* out of resources */
bioq_remove(&ida->bio_queue, bio);
qcb->bio = bio;
qcb->flags = 0;
hwqcb = qcb->hwqcb;
bzero(hwqcb, sizeof(struct ida_hdr) + sizeof(struct ida_req));
bp = bio->bio_buf;
bus_dmamap_load(ida->buffer_dmat, qcb->dmamap,
(void *)bp->b_data, bp->b_bcount, ida_setup_dmamap, hwqcb, 0);
op = qcb->flags & DMA_DATA_IN ?
BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE;
bus_dmamap_sync(ida->buffer_dmat, qcb->dmamap, op);
{
struct idad_softc *drv;
drv = (struct idad_softc *)bio->bio_driver_info;
hwqcb->hdr.drive = drv->drive;
}
hwqcb->req.blkno = bio->bio_offset >> DEV_BSHIFT;
hwqcb->req.bcount = howmany(bp->b_bcount, DEV_BSIZE);
hwqcb->req.command = (bp->b_cmd == BUF_CMD_READ) ? CMD_READ : CMD_WRITE;
KKASSERT(bio->bio_offset < 0x100000000ULL * DEV_BSIZE);
STAILQ_INSERT_TAIL(&ida->qcb_queue, qcb, link.stqe);
}
static void
ptoninvalidate(struct cam_periph *periph)
{
int s;
struct pt_softc *softc;
struct bio *q_bp;
struct ccb_setasync csa;
softc = (struct pt_softc *)periph->softc;
/*
* De-register any async callbacks.
*/
xpt_setup_ccb(&csa.ccb_h, periph->path,
/* priority */ 5);
csa.ccb_h.func_code = XPT_SASYNC_CB;
csa.event_enable = 0;
csa.callback = ptasync;
csa.callback_arg = periph;
xpt_action((union ccb *)&csa);
softc->flags |= PT_FLAG_DEVICE_INVALID;
/*
* Although the oninvalidate() routines are always called at
* splsoftcam, we need to be at splbio() here to keep the buffer
* queue from being modified while we traverse it.
*/
s = splbio();
/*
* Return all queued I/O with ENXIO.
* XXX Handle any transactions queued to the card
* with XPT_ABORT_CCB.
*/
while ((q_bp = bioq_first(&softc->bio_queue)) != NULL){
bioq_remove(&softc->bio_queue, q_bp);
q_bp->bio_resid = q_bp->bio_bcount;
biofinish(q_bp, NULL, ENXIO);
}
splx(s);
xpt_print_path(periph->path);
printf("lost device\n");
}
void
afddetach(struct ata_device *atadev)
{
struct afd_softc *fdp = atadev->driver;
struct bio *bp;
while ((bp = bioq_first(&fdp->queue))) {
bioq_remove(&fdp->queue, bp);
biofinish(bp, NULL, ENXIO);
}
disk_invalidate(&fdp->disk);
disk_destroy(fdp->dev);
devstat_remove_entry(&fdp->stats);
ata_free_name(atadev);
ata_free_lun(&afd_lun_map, fdp->lun);
free(fdp, M_AFD);
atadev->driver = NULL;
}
void
isf_detach(struct isf_softc *sc)
{
/*
* Simulate a disk removal if one is present to deal with any pending
* or queued I/O. This will occur as a result of a device driver
* detach -- the Intel StrataFlash has no notion of removal itself.
*
* XXXRW: Is the locking here right?
*/
ISF_LOCK(sc);
isf_disk_remove(sc);
bioq_flush(&sc->isf_bioq, NULL, ENXIO);
KASSERT(bioq_first(&sc->isf_bioq) == NULL,
("%s: non-empty bioq", __func__));
ISF_UNLOCK(sc);
ISF_LOCK_DESTROY(sc);
}
void
altera_sdcard_detach(struct altera_sdcard_softc *sc)
{
KASSERT(sc->as_taskqueue != NULL, ("%s: taskqueue not present",
__func__));
/*
* Winding down the driver on detach is a bit complex. Update the
* flags to indicate that a detach has been requested, and then wait
* for in-progress I/O to wind down before continuing.
*/
ALTERA_SDCARD_LOCK(sc);
sc->as_flags |= ALTERA_SDCARD_FLAG_DETACHREQ;
while (sc->as_state != ALTERA_SDCARD_STATE_DETACHED)
ALTERA_SDCARD_CONDVAR_WAIT(sc);
ALTERA_SDCARD_UNLOCK(sc);
/*
* Now wait for the possibly still executing taskqueue to drain. In
* principle no more events will be scheduled as we've transitioned to
* a detached state, but there might still be a request in execution.
*/
while (taskqueue_cancel_timeout(sc->as_taskqueue, &sc->as_task, NULL))
taskqueue_drain_timeout(sc->as_taskqueue, &sc->as_task);
/*
* Simulate a disk removal if one is present to deal with any pending
* or queued I/O.
*/
if (sc->as_disk != NULL)
altera_sdcard_disk_remove(sc);
KASSERT(bioq_first(&sc->as_bioq) == NULL,
("%s: non-empty bioq", __func__));
/*
* Free any remaining allocated resources.
*/
taskqueue_free(sc->as_taskqueue);
sc->as_taskqueue = NULL;
ALTERA_SDCARD_CONDVAR_DESTROY(sc);
ALTERA_SDCARD_LOCK_DESTROY(sc);
}
/* Main flash handling task. */
static void
opalflash_task(void *arg)
{
struct opalflash_softc *sc;
struct bio *bp;
device_t dev;
sc = arg;
for (;;) {
dev = sc->sc_dev;
OPALFLASH_LOCK(sc);
do {
bp = bioq_first(&sc->sc_bio_queue);
if (bp == NULL)
msleep(sc, &sc->sc_mtx, PRIBIO, "opalflash", 0);
} while (bp == NULL);
bioq_remove(&sc->sc_bio_queue, bp);
OPALFLASH_UNLOCK(sc);
switch (bp->bio_cmd) {
case BIO_DELETE:
bp->bio_error = opalflash_erase(sc, bp->bio_offset,
bp->bio_bcount);
break;
case BIO_READ:
bp->bio_error = opalflash_read(sc, bp->bio_offset,
bp->bio_data, bp->bio_bcount);
break;
case BIO_WRITE:
bp->bio_error = opalflash_write(sc, bp->bio_offset,
bp->bio_data, bp->bio_bcount);
break;
default:
bp->bio_error = EINVAL;
}
biodone(bp);
}
}
static void
ida_startio(struct ida_softc *ida)
{
struct ida_hardware_qcb *hwqcb;
struct ida_qcb *qcb;
struct idad_softc *drv;
struct bio *bp;
int error;
mtx_assert(&ida->lock, MA_OWNED);
for (;;) {
if (ida->flags & IDA_QFROZEN)
return;
bp = bioq_first(&ida->bio_queue);
if (bp == NULL)
return; /* no more buffers */
qcb = ida_get_qcb(ida);
if (qcb == NULL)
return; /* out of resources */
bioq_remove(&ida->bio_queue, bp);
qcb->buf = bp;
qcb->flags = bp->bio_cmd == BIO_READ ? DMA_DATA_IN : DMA_DATA_OUT;
hwqcb = qcb->hwqcb;
drv = bp->bio_driver1;
hwqcb->hdr.drive = drv->drive;
hwqcb->req.blkno = bp->bio_pblkno;
hwqcb->req.bcount = howmany(bp->bio_bcount, DEV_BSIZE);
hwqcb->req.command = bp->bio_cmd == BIO_READ ? CMD_READ : CMD_WRITE;
error = ida_map_qcb(ida, qcb, bp->bio_data, bp->bio_bcount);
if (error) {
qcb->error = error;
ida_done(ida, qcb);
}
}
}
static void
isf_task(void *arg)
{
struct isf_softc *sc = arg;
struct bio *bp;
int ss = sc->isf_disk->d_sectorsize;
int error, i;
for (;;) {
ISF_LOCK(sc);
do {
bp = bioq_first(&sc->isf_bioq);
if (bp == NULL) {
if (sc->isf_doomed)
kproc_exit(0);
else
ISF_SLEEP(sc, sc, 0);
}
} while (bp == NULL);
bioq_remove(&sc->isf_bioq, bp);
error = 0;
switch (bp->bio_cmd) {
case BIO_READ:
isf_read(sc, bp->bio_pblkno * ss, bp->bio_data,
bp->bio_bcount);
break;
case BIO_WRITE:
/*
* In principle one could suspend the in-progress
* erase, process any pending writes to other
* blocks and then proceed, but that seems
* overly complex for the likely usage modes.
*/
if (sc->isf_erasing) {
error = EBUSY;
break;
}
/*
* Read in the block we want to write and check that
* we're only setting bits to 0. If an erase would
* be required return an I/O error.
*/
isf_read(sc, bp->bio_pblkno * ss, sc->isf_rbuf,
bp->bio_bcount);
for (i = 0; i < bp->bio_bcount / 2; i++)
if ((sc->isf_rbuf[i] &
((uint16_t *)bp->bio_data)[i]) !=
((uint16_t *)bp->bio_data)[i]) {
device_printf(sc->isf_dev, "write"
" requires erase at 0x%08jx\n",
bp->bio_pblkno * ss);
error = EIO;
break;
}
if (error != 0)
break;
error = isf_write(sc, bp->bio_pblkno * ss,
bp->bio_data, bp->bio_bcount);
break;
default:
panic("%s: unsupported I/O operation %d", __func__,
bp->bio_cmd);
}
if (error == 0)
biodone(bp);
else
biofinish(bp, NULL, error);
ISF_UNLOCK(sc);
}
}
static void
mambodisk_task(void *arg)
{
struct mambodisk_softc *sc = (struct mambodisk_softc*)arg;
struct bio *bp;
size_t sz;
int result;
daddr_t block, end;
device_t dev;
u_long unit;
dev = sc->dev;
unit = device_get_unit(dev);
while (sc->running) {
MBODISK_LOCK(sc);
do {
bp = bioq_first(&sc->bio_queue);
if (bp == NULL)
msleep(sc, &sc->sc_mtx, PRIBIO, "jobqueue", 0);
} while (bp == NULL && sc->running);
if (bp)
bioq_remove(&sc->bio_queue, bp);
MBODISK_UNLOCK(sc);
if (!sc->running)
break;
sz = sc->disk->d_sectorsize;
end = bp->bio_pblkno + (bp->bio_bcount / sz);
for (block = bp->bio_pblkno; block < end;) {
u_long numblocks;
char *vaddr = bp->bio_data +
(block - bp->bio_pblkno) * sz;
numblocks = end - block;
if (numblocks > sc->maxblocks)
numblocks = sc->maxblocks;
if (bp->bio_cmd == BIO_READ) {
result = mambocall(MAMBO_DISK_READ, vaddr,
(u_long)block, (numblocks << 16) | unit);
} else if (bp->bio_cmd == BIO_WRITE) {
result = mambocall(MAMBO_DISK_WRITE, vaddr,
(u_long)block, (numblocks << 16) | unit);
} else {
result = 1;
}
if (result)
break;
block += numblocks;
}
if (block < end) {
bp->bio_error = EIO;
bp->bio_resid = (end - block) * sz;
bp->bio_flags |= BIO_ERROR;
}
biodone(bp);
}
/* tell parent we're done */
MBODISK_LOCK(sc);
sc->running = -1;
wakeup(sc);
MBODISK_UNLOCK(sc);
kproc_exit(0);
}
static void
ptdone(struct cam_periph *periph, union ccb *done_ccb)
{
struct pt_softc *softc;
struct ccb_scsiio *csio;
softc = (struct pt_softc *)periph->softc;
csio = &done_ccb->csio;
switch (csio->ccb_h.ccb_state) {
case PT_CCB_BUFFER_IO:
case PT_CCB_BUFFER_IO_UA:
{
struct bio *bp;
int oldspl;
bp = (struct bio *)done_ccb->ccb_h.ccb_bp;
if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
int error;
int s;
int sf;
if ((csio->ccb_h.ccb_state & PT_CCB_RETRY_UA) != 0)
sf = SF_RETRY_UA;
else
sf = 0;
error = pterror(done_ccb, CAM_RETRY_SELTO, sf);
if (error == ERESTART) {
/*
* A retry was scheuled, so
* just return.
*/
return;
}
if (error != 0) {
struct bio *q_bp;
s = splbio();
if (error == ENXIO) {
/*
* Catastrophic error. Mark our device
* as invalid.
*/
xpt_print_path(periph->path);
printf("Invalidating device\n");
softc->flags |= PT_FLAG_DEVICE_INVALID;
}
/*
* return all queued I/O with EIO, so that
* the client can retry these I/Os in the
* proper order should it attempt to recover.
*/
while ((q_bp = bioq_first(&softc->bio_queue))
!= NULL) {
bioq_remove(&softc->bio_queue, q_bp);
q_bp->bio_resid = q_bp->bio_bcount;
biofinish(q_bp, NULL, EIO);
}
splx(s);
bp->bio_error = error;
bp->bio_resid = bp->bio_bcount;
bp->bio_flags |= BIO_ERROR;
} else {
bp->bio_resid = csio->resid;
bp->bio_error = 0;
if (bp->bio_resid != 0) {
/* Short transfer ??? */
bp->bio_flags |= BIO_ERROR;
}
}
if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0)
cam_release_devq(done_ccb->ccb_h.path,
/*relsim_flags*/0,
/*reduction*/0,
/*timeout*/0,
/*getcount_only*/0);
} else {
bp->bio_resid = csio->resid;
if (bp->bio_resid != 0)
bp->bio_flags |= BIO_ERROR;
}
/*
* Block out any asyncronous callbacks
* while we touch the pending ccb list.
*/
oldspl = splcam();
LIST_REMOVE(&done_ccb->ccb_h, periph_links.le);
splx(oldspl);
biofinish(bp, &softc->device_stats, 0);
break;
}
case PT_CCB_WAITING:
/* Caller will release the CCB */
wakeup(&done_ccb->ccb_h.cbfcnp);
return;
}
xpt_release_ccb(done_ccb);
}