/*
* The function is called after data encryption.
*
* g_eli_start -> g_eli_crypto_run -> G_ELI_CRYPTO_WRITE_DONE -> g_io_request -> g_eli_write_done -> g_io_deliver
*/
static int
g_eli_crypto_write_done(struct cryptop *crp)
{
struct g_eli_softc *sc;
struct g_geom *gp;
struct g_consumer *cp;
struct bio *bp, *cbp;
if (crp->crp_etype == EAGAIN) {
if (g_eli_crypto_rerun(crp) == 0)
return (0);
}
bp = (struct bio *)crp->crp_opaque;
bp->bio_inbed++;
if (crp->crp_etype == 0) {
G_ELI_DEBUG(3, "Crypto WRITE request done (%d/%d).",
bp->bio_inbed, bp->bio_children);
} else {
G_ELI_DEBUG(1, "Crypto WRITE request failed (%d/%d) error=%d.",
bp->bio_inbed, bp->bio_children, crp->crp_etype);
if (bp->bio_error == 0)
bp->bio_error = crp->crp_etype;
}
gp = bp->bio_to->geom;
sc = gp->softc;
g_eli_key_drop(sc, crp->crp_desc->crd_key);
/*
* All sectors are already encrypted?
*/
if (bp->bio_inbed < bp->bio_children)
return (0);
bp->bio_inbed = 0;
bp->bio_children = 1;
cbp = bp->bio_driver1;
bp->bio_driver1 = NULL;
if (bp->bio_error != 0) {
G_ELI_LOGREQ(0, bp, "Crypto WRITE request failed (error=%d).",
bp->bio_error);
free(bp->bio_driver2, M_ELI);
bp->bio_driver2 = NULL;
g_destroy_bio(cbp);
g_io_deliver(bp, bp->bio_error);
atomic_subtract_int(&sc->sc_inflight, 1);
return (0);
}
cbp->bio_data = bp->bio_driver2;
cbp->bio_done = g_eli_write_done;
cp = LIST_FIRST(&gp->consumer);
cbp->bio_to = cp->provider;
G_ELI_LOGREQ(2, cbp, "Sending request.");
/*
* Send encrypted data to the provider.
*/
g_io_request(cbp, cp);
return (0);
}
static void
nvme_ns_bio_test(void *arg)
{
struct nvme_io_test_internal *io_test = arg;
struct cdevsw *csw;
struct mtx *mtx;
struct bio *bio;
struct cdev *dev;
void *buf;
struct timeval t;
uint64_t offset;
uint32_t idx, io_completed = 0;
#if __FreeBSD_version >= 900017
int ref;
#endif
buf = malloc(io_test->size, M_NVME, M_WAITOK);
idx = atomic_fetchadd_int(&io_test->td_idx, 1);
dev = io_test->ns->cdev;
offset = idx * 2048 * nvme_ns_get_sector_size(io_test->ns);
while (1) {
bio = g_alloc_bio();
memset(bio, 0, sizeof(*bio));
bio->bio_cmd = (io_test->opc == NVME_OPC_READ) ?
BIO_READ : BIO_WRITE;
bio->bio_done = nvme_ns_bio_test_cb;
bio->bio_dev = dev;
bio->bio_offset = offset;
bio->bio_data = buf;
bio->bio_bcount = io_test->size;
if (io_test->flags & NVME_TEST_FLAG_REFTHREAD) {
#if __FreeBSD_version >= 900017
csw = dev_refthread(dev, &ref);
#else
csw = dev_refthread(dev);
#endif
} else
csw = dev->si_devsw;
mtx = mtx_pool_find(mtxpool_sleep, bio);
mtx_lock(mtx);
(*csw->d_strategy)(bio);
msleep(bio, mtx, PRIBIO, "biotestwait", 0);
mtx_unlock(mtx);
if (io_test->flags & NVME_TEST_FLAG_REFTHREAD) {
#if __FreeBSD_version >= 900017
dev_relthread(dev, ref);
#else
dev_relthread(dev);
#endif
}
if ((bio->bio_flags & BIO_ERROR) || (bio->bio_resid > 0))
break;
g_destroy_bio(bio);
io_completed++;
getmicrouptime(&t);
timevalsub(&t, &io_test->start);
if (t.tv_sec >= io_test->time)
break;
offset += io_test->size;
if ((offset + io_test->size) > nvme_ns_get_size(io_test->ns))
offset = 0;
}
io_test->io_completed[idx] = io_completed;
wakeup_one(io_test);
free(buf, M_NVME);
atomic_subtract_int(&io_test->td_active, 1);
mb();
#if __FreeBSD_version >= 800000
kthread_exit();
#else
kthread_exit(0);
#endif
}
static void
g_uzip_do(struct g_uzip_softc *sc, struct bio *bp)
{
struct bio *bp2;
struct g_provider *pp;
struct g_consumer *cp;
struct g_geom *gp;
char *data, *data2;
off_t ofs;
size_t blk, blkofs, len, ulen, firstblk;
int err;
bp2 = bp->bio_parent;
gp = bp2->bio_to->geom;
cp = LIST_FIRST(&gp->consumer);
pp = cp->provider;
bp2->bio_error = bp->bio_error;
if (bp2->bio_error != 0)
goto done;
/* Make sure there's forward progress. */
if (bp->bio_completed == 0) {
bp2->bio_error = ECANCELED;
goto done;
}
ofs = bp2->bio_offset + bp2->bio_completed;
firstblk = blk = ofs / sc->blksz;
blkofs = ofs % sc->blksz;
data = bp->bio_data + sc->toc[blk].offset % pp->sectorsize;
data2 = bp2->bio_data + bp2->bio_completed;
while (bp->bio_completed && bp2->bio_resid) {
if (blk > firstblk && !BLK_IS_CONT(sc, blk)) {
DPRINTF_BLK(GUZ_DBG_IO, blk, ("%s/%s: %p: backref'ed "
"cluster #%u requested, looping around\n",
__func__, gp->name, bp2, (u_int)blk));
goto done;
}
ulen = MIN(sc->blksz - blkofs, bp2->bio_resid);
len = sc->toc[blk].blen;
DPRINTF(GUZ_DBG_IO, ("%s/%s: %p/%ju: data2=%p, ulen=%u, "
"data=%p, len=%u\n", __func__, gp->name, gp,
bp->bio_completed, data2, (u_int)ulen, data, (u_int)len));
if (len == 0) {
/* All zero block: no cache update */
bzero(data2, ulen);
} else if (len <= bp->bio_completed) {
mtx_lock(&sc->last_mtx);
err = sc->dcp->decompress(sc->dcp, gp->name, data,
len, sc->last_buf);
if (err != 0) {
sc->last_blk = -1;
mtx_unlock(&sc->last_mtx);
bp2->bio_error = EILSEQ;
DPRINTF(GUZ_DBG_ERR, ("%s/%s: decompress"
"(%p) failed\n", __func__, gp->name,
sc->dcp));
goto done;
}
sc->last_blk = blk;
memcpy(data2, sc->last_buf + blkofs, ulen);
mtx_unlock(&sc->last_mtx);
err = sc->dcp->rewind(sc->dcp, gp->name);
if (err != 0) {
bp2->bio_error = EILSEQ;
DPRINTF(GUZ_DBG_ERR, ("%s/%s: rewind(%p) "
"failed\n", __func__, gp->name, sc->dcp));
goto done;
}
data += len;
} else
break;
data2 += ulen;
bp2->bio_completed += ulen;
bp2->bio_resid -= ulen;
bp->bio_completed -= len;
blkofs = 0;
blk++;
}
done:
/* Finish processing the request. */
free(bp->bio_data, M_GEOM_UZIP);
g_destroy_bio(bp);
if (bp2->bio_error != 0 || bp2->bio_resid == 0)
g_io_deliver(bp2, bp2->bio_error);
else
g_uzip_request(gp, bp2);
}
static int
g_uzip_request(struct g_geom *gp, struct bio *bp)
{
struct g_uzip_softc *sc;
struct bio *bp2;
struct g_consumer *cp;
struct g_provider *pp;
off_t ofs, start_blk_ofs;
size_t i, start_blk, end_blk, zsize;
if (g_uzip_cached(gp, bp) != 0)
return (1);
sc = gp->softc;
cp = LIST_FIRST(&gp->consumer);
pp = cp->provider;
ofs = bp->bio_offset + bp->bio_completed;
start_blk = ofs / sc->blksz;
KASSERT(start_blk < sc->nblocks, ("start_blk out of range"));
end_blk = howmany(ofs + bp->bio_resid, sc->blksz);
KASSERT(end_blk <= sc->nblocks, ("end_blk out of range"));
for (; BLK_IS_NIL(sc, start_blk) && start_blk < end_blk; start_blk++) {
/* Fill in any leading Nil blocks */
start_blk_ofs = ofs % sc->blksz;
zsize = MIN(sc->blksz - start_blk_ofs, bp->bio_resid);
DPRINTF_BLK(GUZ_DBG_IO, start_blk, ("%s/%s: %p/%ju: "
"filling %ju zero bytes\n", __func__, gp->name, gp,
(uintmax_t)bp->bio_completed, (uintmax_t)zsize));
bzero(bp->bio_data + bp->bio_completed, zsize);
bp->bio_completed += zsize;
bp->bio_resid -= zsize;
ofs += zsize;
}
if (start_blk == end_blk) {
KASSERT(bp->bio_resid == 0, ("bp->bio_resid is invalid"));
/*
* No non-Nil data is left, complete request immediately.
*/
DPRINTF(GUZ_DBG_IO, ("%s/%s: %p: all done returning %ju "
"bytes\n", __func__, gp->name, gp,
(uintmax_t)bp->bio_completed));
g_io_deliver(bp, 0);
return (1);
}
for (i = start_blk + 1; i < end_blk; i++) {
/* Trim discontinuous areas if any */
if (!BLK_IS_CONT(sc, i)) {
end_blk = i;
break;
}
}
DPRINTF_BRNG(GUZ_DBG_IO, start_blk, end_blk, ("%s/%s: %p: "
"start=%u (%ju), end=%u (%ju)\n", __func__, gp->name, bp,
(u_int)start_blk, (uintmax_t)sc->toc[start_blk].offset,
(u_int)end_blk, (uintmax_t)BLK_ENDS(sc, end_blk - 1)));
bp2 = g_clone_bio(bp);
if (bp2 == NULL) {
g_io_deliver(bp, ENOMEM);
return (1);
}
bp2->bio_done = g_uzip_read_done;
bp2->bio_offset = TOFF_2_BOFF(sc, pp, start_blk);
while (1) {
bp2->bio_length = TLEN_2_BLEN(sc, pp, bp2, end_blk - 1);
if (bp2->bio_length <= MAXPHYS)
break;
if (end_blk == (start_blk + 1)) {
break;
}
end_blk--;
}
DPRINTF(GUZ_DBG_IO, ("%s/%s: bp2->bio_length = %jd\n",
__func__, gp->name, (intmax_t)bp2->bio_length));
bp2->bio_data = malloc(bp2->bio_length, M_GEOM_UZIP, M_NOWAIT);
if (bp2->bio_data == NULL) {
g_destroy_bio(bp2);
g_io_deliver(bp, ENOMEM);
return (1);
}
DPRINTF_BRNG(GUZ_DBG_IO, start_blk, end_blk, ("%s/%s: %p: "
"reading %jd bytes from offset %jd\n", __func__, gp->name, bp,
(intmax_t)bp2->bio_length, (intmax_t)bp2->bio_offset));
g_io_request(bp2, cp);
return (0);
}
int
physio(struct cdev *dev, struct uio *uio, int ioflag)
{
struct cdevsw *csw;
struct buf *pbuf;
struct bio *bp;
struct vm_page **pages;
caddr_t sa;
u_int iolen, poff;
int error, i, npages, maxpages;
vm_prot_t prot;
csw = dev->si_devsw;
npages = 0;
sa = NULL;
/* check if character device is being destroyed */
if (csw == NULL)
return (ENXIO);
/* XXX: sanity check */
if(dev->si_iosize_max < PAGE_SIZE) {
printf("WARNING: %s si_iosize_max=%d, using DFLTPHYS.\n",
devtoname(dev), dev->si_iosize_max);
dev->si_iosize_max = DFLTPHYS;
}
/*
* If the driver does not want I/O to be split, that means that we
* need to reject any requests that will not fit into one buffer.
*/
if (dev->si_flags & SI_NOSPLIT &&
(uio->uio_resid > dev->si_iosize_max || uio->uio_resid > MAXPHYS ||
uio->uio_iovcnt > 1)) {
/*
* Tell the user why his I/O was rejected.
*/
if (uio->uio_resid > dev->si_iosize_max)
uprintf("%s: request size=%zd > si_iosize_max=%d; "
"cannot split request\n", devtoname(dev),
uio->uio_resid, dev->si_iosize_max);
if (uio->uio_resid > MAXPHYS)
uprintf("%s: request size=%zd > MAXPHYS=%d; "
"cannot split request\n", devtoname(dev),
uio->uio_resid, MAXPHYS);
if (uio->uio_iovcnt > 1)
uprintf("%s: request vectors=%d > 1; "
"cannot split request\n", devtoname(dev),
uio->uio_iovcnt);
return (EFBIG);
}
/*
* Keep the process UPAGES from being swapped. Processes swapped
* out while holding pbufs, used by swapper, may lead to deadlock.
*/
PHOLD(curproc);
bp = g_alloc_bio();
if (uio->uio_segflg != UIO_USERSPACE) {
pbuf = NULL;
pages = NULL;
} else if ((dev->si_flags & SI_UNMAPPED) && unmapped_buf_allowed) {
pbuf = NULL;
maxpages = btoc(MIN(uio->uio_resid, MAXPHYS)) + 1;
pages = malloc(sizeof(*pages) * maxpages, M_DEVBUF, M_WAITOK);
} else {
pbuf = uma_zalloc(pbuf_zone, M_WAITOK);
sa = pbuf->b_data;
maxpages = btoc(MAXPHYS);
pages = pbuf->b_pages;
}
prot = VM_PROT_READ;
if (uio->uio_rw == UIO_READ)
prot |= VM_PROT_WRITE; /* Less backwards than it looks */
error = 0;
for (i = 0; i < uio->uio_iovcnt; i++) {
#ifdef RACCT
if (racct_enable) {
PROC_LOCK(curproc);
if (uio->uio_rw == UIO_READ) {
racct_add_force(curproc, RACCT_READBPS,
uio->uio_iov[i].iov_len);
racct_add_force(curproc, RACCT_READIOPS, 1);
} else {
racct_add_force(curproc, RACCT_WRITEBPS,
uio->uio_iov[i].iov_len);
racct_add_force(curproc, RACCT_WRITEIOPS, 1);
}
PROC_UNLOCK(curproc);
}
#endif /* RACCT */
while (uio->uio_iov[i].iov_len) {
g_reset_bio(bp);
if (uio->uio_rw == UIO_READ) {
bp->bio_cmd = BIO_READ;
curthread->td_ru.ru_inblock++;
} else {
bp->bio_cmd = BIO_WRITE;
curthread->td_ru.ru_oublock++;
}
bp->bio_offset = uio->uio_offset;
bp->bio_data = uio->uio_iov[i].iov_base;
bp->bio_length = uio->uio_iov[i].iov_len;
if (bp->bio_length > dev->si_iosize_max)
bp->bio_length = dev->si_iosize_max;
if (bp->bio_length > MAXPHYS)
bp->bio_length = MAXPHYS;
/*
* Make sure the pbuf can map the request.
* The pbuf has kvasize = MAXPHYS, so a request
* larger than MAXPHYS - PAGE_SIZE must be
* page aligned or it will be fragmented.
*/
poff = (vm_offset_t)bp->bio_data & PAGE_MASK;
if (pbuf && bp->bio_length + poff > pbuf->b_kvasize) {
if (dev->si_flags & SI_NOSPLIT) {
uprintf("%s: request ptr %p is not "
"on a page boundary; cannot split "
"request\n", devtoname(dev),
bp->bio_data);
error = EFBIG;
goto doerror;
}
bp->bio_length = pbuf->b_kvasize;
if (poff != 0)
bp->bio_length -= PAGE_SIZE;
}
bp->bio_bcount = bp->bio_length;
bp->bio_dev = dev;
if (pages) {
if ((npages = vm_fault_quick_hold_pages(
&curproc->p_vmspace->vm_map,
(vm_offset_t)bp->bio_data, bp->bio_length,
prot, pages, maxpages)) < 0) {
error = EFAULT;
goto doerror;
}
if (pbuf && sa) {
pmap_qenter((vm_offset_t)sa,
pages, npages);
bp->bio_data = sa + poff;
} else {
bp->bio_ma = pages;
bp->bio_ma_n = npages;
bp->bio_ma_offset = poff;
bp->bio_data = unmapped_buf;
bp->bio_flags |= BIO_UNMAPPED;
}
}
csw->d_strategy(bp);
if (uio->uio_rw == UIO_READ)
biowait(bp, "physrd");
else
biowait(bp, "physwr");
if (pages) {
if (pbuf)
pmap_qremove((vm_offset_t)sa, npages);
vm_page_unhold_pages(pages, npages);
}
iolen = bp->bio_length - bp->bio_resid;
if (iolen == 0 && !(bp->bio_flags & BIO_ERROR))
goto doerror; /* EOF */
uio->uio_iov[i].iov_len -= iolen;
uio->uio_iov[i].iov_base =
(char *)uio->uio_iov[i].iov_base + iolen;
uio->uio_resid -= iolen;
uio->uio_offset += iolen;
if (bp->bio_flags & BIO_ERROR) {
error = bp->bio_error;
goto doerror;
}
}
}
doerror:
if (pbuf)
uma_zfree(pbuf_zone, pbuf);
else if (pages)
free(pages, M_DEVBUF);
g_destroy_bio(bp);
PRELE(curproc);
return (error);
}
/*
* The function is called after data encryption.
*
* g_eli_start -> g_eli_auth_run -> G_ELI_AUTH_WRITE_DONE -> g_io_request -> g_eli_write_done -> g_io_deliver
*/
static int
g_eli_auth_write_done(struct cryptop *crp)
{
struct g_eli_softc *sc;
struct g_consumer *cp;
struct bio *bp, *cbp, *cbp2;
u_int nsec;
if (crp->crp_etype == EAGAIN) {
if (g_eli_crypto_rerun(crp) == 0)
return (0);
}
bp = (struct bio *)crp->crp_opaque;
bp->bio_inbed++;
if (crp->crp_etype == 0) {
G_ELI_DEBUG(3, "Crypto WRITE request done (%d/%d).",
bp->bio_inbed, bp->bio_children);
} else {
G_ELI_DEBUG(1, "Crypto WRITE request failed (%d/%d) error=%d.",
bp->bio_inbed, bp->bio_children, crp->crp_etype);
if (bp->bio_error == 0)
bp->bio_error = crp->crp_etype;
}
sc = bp->bio_to->geom->softc;
g_eli_key_drop(sc, crp->crp_desc->crd_key);
/*
* All sectors are already encrypted?
*/
if (bp->bio_inbed < bp->bio_children)
return (0);
if (bp->bio_error != 0) {
G_ELI_LOGREQ(0, bp, "Crypto WRITE request failed (error=%d).",
bp->bio_error);
free(bp->bio_driver2, M_ELI);
bp->bio_driver2 = NULL;
cbp = bp->bio_driver1;
bp->bio_driver1 = NULL;
g_destroy_bio(cbp);
g_io_deliver(bp, bp->bio_error);
atomic_subtract_int(&sc->sc_inflight, 1);
return (0);
}
cp = LIST_FIRST(&sc->sc_geom->consumer);
cbp = bp->bio_driver1;
bp->bio_driver1 = NULL;
cbp->bio_to = cp->provider;
cbp->bio_done = g_eli_write_done;
/* Number of sectors from decrypted provider, eg. 1. */
nsec = bp->bio_length / bp->bio_to->sectorsize;
/* Number of sectors from encrypted provider, eg. 9. */
nsec = (nsec * sc->sc_bytes_per_sector) / cp->provider->sectorsize;
cbp->bio_length = cp->provider->sectorsize * nsec;
cbp->bio_offset = (bp->bio_offset / bp->bio_to->sectorsize) * sc->sc_bytes_per_sector;
cbp->bio_data = bp->bio_driver2;
/*
* We write more than what is requested, so we have to be ready to write
* more than MAXPHYS.
*/
cbp2 = NULL;
if (cbp->bio_length > MAXPHYS) {
cbp2 = g_duplicate_bio(bp);
cbp2->bio_length = cbp->bio_length - MAXPHYS;
cbp2->bio_data = cbp->bio_data + MAXPHYS;
cbp2->bio_offset = cbp->bio_offset + MAXPHYS;
cbp2->bio_to = cp->provider;
cbp2->bio_done = g_eli_write_done;
cbp->bio_length = MAXPHYS;
}
/*
* Send encrypted data to the provider.
*/
G_ELI_LOGREQ(2, cbp, "Sending request.");
bp->bio_inbed = 0;
bp->bio_children = (cbp2 != NULL ? 2 : 1);
g_io_request(cbp, cp);
if (cbp2 != NULL) {
G_ELI_LOGREQ(2, cbp2, "Sending request.");
g_io_request(cbp2, cp);
}
return (0);
}
static void
g_uzip_done(struct bio *bp)
{
z_stream zs;
struct bio *bp2;
struct g_provider *pp;
struct g_consumer *cp;
struct g_geom *gp;
struct g_uzip_softc *sc;
char *data, *data2;
off_t ofs;
size_t blk, blkofs, len, ulen;
bp2 = bp->bio_parent;
gp = bp2->bio_to->geom;
sc = gp->softc;
cp = LIST_FIRST(&gp->consumer);
pp = cp->provider;
bp2->bio_error = bp->bio_error;
if (bp2->bio_error != 0)
goto done;
/* Make sure there's forward progress. */
if (bp->bio_completed == 0) {
bp2->bio_error = ECANCELED;
goto done;
}
zs.zalloc = z_alloc;
zs.zfree = z_free;
if (inflateInit(&zs) != Z_OK) {
bp2->bio_error = EILSEQ;
goto done;
}
ofs = bp2->bio_offset + bp2->bio_completed;
blk = ofs / sc->blksz;
blkofs = ofs % sc->blksz;
data = bp->bio_data + sc->offsets[blk] % pp->sectorsize;
data2 = bp2->bio_data + bp2->bio_completed;
while (bp->bio_completed && bp2->bio_resid) {
ulen = MIN(sc->blksz - blkofs, bp2->bio_resid);
len = sc->offsets[blk + 1] - sc->offsets[blk];
DPRINTF(("%s/%s: %p/%ju: data2=%p, ulen=%u, data=%p, len=%u\n",
__func__, gp->name, gp, bp->bio_completed,
data2, (u_int)ulen, data, (u_int)len));
if (len == 0) {
/* All zero block: no cache update */
bzero(data2, ulen);
} else if (len <= bp->bio_completed) {
zs.next_in = data;
zs.avail_in = len;
zs.next_out = sc->last_buf;
zs.avail_out = sc->blksz;
mtx_lock(&sc->last_mtx);
if (inflate(&zs, Z_FINISH) != Z_STREAM_END) {
sc->last_blk = -1;
mtx_unlock(&sc->last_mtx);
inflateEnd(&zs);
bp2->bio_error = EILSEQ;
goto done;
}
sc->last_blk = blk;
memcpy(data2, sc->last_buf + blkofs, ulen);
mtx_unlock(&sc->last_mtx);
if (inflateReset(&zs) != Z_OK) {
inflateEnd(&zs);
bp2->bio_error = EILSEQ;
goto done;
}
data += len;
} else
break;
data2 += ulen;
bp2->bio_completed += ulen;
bp2->bio_resid -= ulen;
bp->bio_completed -= len;
blkofs = 0;
blk++;
}
if (inflateEnd(&zs) != Z_OK)
bp2->bio_error = EILSEQ;
done:
/* Finish processing the request. */
free(bp->bio_data, M_GEOM_UZIP);
g_destroy_bio(bp);
if (bp2->bio_error != 0 || bp2->bio_resid == 0)
g_io_deliver(bp2, bp2->bio_error);
else
g_uzip_request(gp, bp2);
}
static int
g_uzip_request(struct g_geom *gp, struct bio *bp)
{
struct g_uzip_softc *sc;
struct bio *bp2;
struct g_consumer *cp;
struct g_provider *pp;
off_t ofs;
size_t start_blk, end_blk;
if (g_uzip_cached(gp, bp) != 0)
return (1);
sc = gp->softc;
bp2 = g_clone_bio(bp);
if (bp2 == NULL) {
g_io_deliver(bp, ENOMEM);
return (1);
}
bp2->bio_done = g_uzip_done;
cp = LIST_FIRST(&gp->consumer);
pp = cp->provider;
ofs = bp->bio_offset + bp->bio_completed;
start_blk = ofs / sc->blksz;
KASSERT(start_blk < sc->nblocks, ("start_blk out of range"));
end_blk = (ofs + bp->bio_resid + sc->blksz - 1) / sc->blksz;
KASSERT(end_blk <= sc->nblocks, ("end_blk out of range"));
DPRINTF(("%s/%s: %p: start=%u (%jd), end=%u (%jd)\n",
__func__, gp->name, bp,
(u_int)start_blk, (intmax_t)sc->offsets[start_blk],
(u_int)end_blk, (intmax_t)sc->offsets[end_blk]));
bp2->bio_offset = sc->offsets[start_blk] -
sc->offsets[start_blk] % pp->sectorsize;
while (1) {
bp2->bio_length = sc->offsets[end_blk] - bp2->bio_offset;
bp2->bio_length = (bp2->bio_length + pp->sectorsize - 1) /
pp->sectorsize * pp->sectorsize;
if (bp2->bio_length <= MAXPHYS)
break;
end_blk--;
}
bp2->bio_data = malloc(bp2->bio_length, M_GEOM_UZIP, M_NOWAIT);
if (bp2->bio_data == NULL) {
g_destroy_bio(bp2);
g_io_deliver(bp, ENOMEM);
return (1);
}
DPRINTF(("%s/%s: %p: reading %jd bytes from offset %jd\n",
__func__, gp->name, bp,
(intmax_t)bp2->bio_length, (intmax_t)bp2->bio_offset));
g_io_request(bp2, cp);
return (0);
}
static void
g_uzip_start(struct bio *bp)
{
struct bio *bp2;
struct g_provider *pp, *pp2;
struct g_geom *gp;
struct g_consumer *cp;
struct g_uzip_softc *sc;
uint32_t start_blk, end_blk;
size_t bsize;
pp = bp->bio_to;
gp = pp->geom;
DPRINTF(("%s: start (%d)\n", gp->name, bp->bio_cmd));
if (bp->bio_cmd != BIO_READ) {
g_io_deliver(bp, EOPNOTSUPP);
return;
}
cp = LIST_FIRST(&gp->consumer);
pp2 = cp->provider;
sc = gp->softc;
start_blk = bp->bio_offset / sc->blksz;
end_blk = (bp->bio_offset + bp->bio_length + sc->blksz - 1) / sc->blksz;
KASSERT(start_blk < sc->nblocks,
("start_blk out of range"));
KASSERT(end_blk <= sc->nblocks,
("end_blk out of range"));
sc->req_total++;
if (start_blk + 1 == end_blk) {
mtx_lock(&sc->last_mtx);
if (start_blk == sc->last_blk) {
off_t uoff;
uoff = bp->bio_offset % sc->blksz;
KASSERT(bp->bio_length <= sc->blksz - uoff,
("cached data error"));
memcpy(bp->bio_data, sc->last_buf + uoff,
bp->bio_length);
sc->req_cached++;
mtx_unlock(&sc->last_mtx);
DPRINTF(("%s: start: cached 0 + %lld, %lld + 0 + %lld\n",
gp->name, bp->bio_length, uoff, bp->bio_length));
bp->bio_completed = bp->bio_length;
g_io_deliver(bp, 0);
return;
}
mtx_unlock(&sc->last_mtx);
}
bp2 = g_clone_bio(bp);
if (bp2 == NULL) {
g_io_deliver(bp, ENOMEM);
return;
}
bp2->bio_done = g_uzip_done;
DPRINTF(("%s: start (%d..%d), %s: %d + %lld, %s: %d + %lld\n",
gp->name, start_blk, end_blk,
pp->name, pp->sectorsize, pp->mediasize,
pp2->name, pp2->sectorsize, pp2->mediasize));
bsize = pp2->sectorsize;
bp2->bio_offset = sc->offsets[start_blk] - sc->offsets[start_blk] % bsize;
bp2->bio_length = sc->offsets[end_blk] - bp2->bio_offset;
bp2->bio_length = (bp2->bio_length + bsize - 1) / bsize * bsize;
DPRINTF(("%s: start %lld + %lld -> %lld + %lld -> %lld + %lld\n",
gp->name,
bp->bio_offset, bp->bio_length,
sc->offsets[start_blk], sc->offsets[end_blk] - sc->offsets[start_blk],
bp2->bio_offset, bp2->bio_length));
bp2->bio_data = malloc(bp2->bio_length, M_GEOM_UZIP, M_NOWAIT);
if (bp2->bio_data == NULL) {
g_destroy_bio(bp2);
g_io_deliver(bp, ENOMEM);
return;
}
g_io_request(bp2, cp);
DPRINTF(("%s: start ok\n", gp->name));
}
static void
g_uzip_done(struct bio *bp)
{
int err;
struct bio *bp2;
z_stream zs;
struct g_provider *pp, *pp2;
struct g_consumer *cp;
struct g_geom *gp;
struct g_uzip_softc *sc;
off_t pos, upos;
uint32_t start_blk, i;
size_t bsize;
bp2 = bp->bio_parent;
pp = bp2->bio_to;
gp = pp->geom;
cp = LIST_FIRST(&gp->consumer);
pp2 = cp->provider;
sc = gp->softc;
DPRINTF(("%s: done\n", gp->name));
bp2->bio_error = bp->bio_error;
if (bp2->bio_error != 0)
goto done;
/*
* Uncompress data.
*/
zs.zalloc = z_alloc;
zs.zfree = z_free;
err = inflateInit(&zs);
if (err != Z_OK) {
bp2->bio_error = EIO;
goto done;
}
start_blk = bp2->bio_offset / sc->blksz;
bsize = pp2->sectorsize;
pos = sc->offsets[start_blk] % bsize;
upos = 0;
DPRINTF(("%s: done: start_blk %d, pos %lld, upos %lld (%lld, %d, %d)\n",
gp->name, start_blk, pos, upos,
bp2->bio_offset, sc->blksz, bsize));
for (i = start_blk; upos < bp2->bio_length; i++) {
off_t len, ulen, uoff;
uoff = i == start_blk ? bp2->bio_offset % sc->blksz : 0;
ulen = MIN(sc->blksz - uoff, bp2->bio_length - upos);
len = sc->offsets[i + 1] - sc->offsets[i];
if (len == 0) {
/* All zero block: no cache update */
bzero(bp2->bio_data + upos, ulen);
upos += ulen;
bp2->bio_completed += ulen;
continue;
}
zs.next_in = bp->bio_data + pos;
zs.avail_in = len;
zs.next_out = sc->last_buf;
zs.avail_out = sc->blksz;
mtx_lock(&sc->last_mtx);
err = inflate(&zs, Z_FINISH);
if (err != Z_STREAM_END) {
sc->last_blk = -1;
mtx_unlock(&sc->last_mtx);
DPRINTF(("%s: done: inflate failed (%lld + %lld -> %lld + %lld + %lld)\n",
gp->name, pos, len, uoff, upos, ulen));
inflateEnd(&zs);
bp2->bio_error = EIO;
goto done;
}
sc->last_blk = i;
DPRINTF(("%s: done: inflated %lld + %lld -> %lld + %lld + %lld\n",
gp->name,
pos, len,
uoff, upos, ulen));
memcpy(bp2->bio_data + upos, sc->last_buf + uoff, ulen);
mtx_unlock(&sc->last_mtx);
pos += len;
upos += ulen;
bp2->bio_completed += ulen;
err = inflateReset(&zs);
if (err != Z_OK) {
inflateEnd(&zs);
bp2->bio_error = EIO;
goto done;
}
}
err = inflateEnd(&zs);
if (err != Z_OK) {
bp2->bio_error = EIO;
goto done;
}
done:
/*
* Finish processing the request.
*/
DPRINTF(("%s: done: (%d, %lld, %ld)\n",
gp->name, bp2->bio_error, bp2->bio_completed, bp2->bio_resid));
free(bp->bio_data, M_GEOM_UZIP);
g_destroy_bio(bp);
g_io_deliver(bp2, bp2->bio_error);
}