ssize_t
solv_zchunk_read(struct solv_zchunk *zck, char *buf, size_t len)
{
size_t n = 0;
if (!zck || zck->eof == 2)
return -1;
while (n < len && !zck->eof)
{
unsigned int bite;
while (!zck->buf_avail)
{
if (!zck->nchunks)
{
/* verify data checksum if requested */
if (zck->streamid != 0 && zck->data_chk && memcmp(solv_chksum_get(zck->data_chk, 0), zck->data_chk_ptr, zck->hdr_chk_len) != 0) {
zck->eof = 2;
return -1;
}
zck->eof = 1;
return n;
}
if (!nextchunk(zck, zck->streamid))
{
zck->eof = 2;
return -1;
}
}
bite = len - n > zck->buf_avail ? zck->buf_avail : len - n;
memcpy(buf + n, zck->buf + zck->buf_used, bite);
n += bite;
zck->buf_used += bite;
zck->buf_avail -= bite;
}
return n;
}
static int
icachewritesect(Index *ix, ISect *is, u8int *buf)
{
int err, i, werr, h, bsize, t;
u32int lo, hi;
u64int addr, naddr;
uint nbuf, off;
DBlock *b;
IBucket ib;
IEntry *ie, *iedirty, **l, *chunk;
lo = is->start * ix->div;
if(TWID32/ix->div < is->stop)
hi = TWID32;
else
hi = is->stop * ix->div - 1;
trace(TraceProc, "icachewritesect enter %ud %ud %llud",
lo, hi, iwrite.as.aa);
iedirty = icachedirty(lo, hi, iwrite.as.aa);
iedirty = iesort(iedirty);
bsize = 1 << is->blocklog;
err = 0;
while(iedirty){
disksched();
while((t = icachesleeptime) == SleepForever){
sleep(1000);
disksched();
}
if(t < minicachesleeptime)
t = minicachesleeptime;
if(t > 0)
sleep(t);
trace(TraceProc, "icachewritesect nextchunk");
chunk = nextchunk(ix, is, &iedirty, &addr, &nbuf);
trace(TraceProc, "icachewritesect readpart 0x%llux+0x%ux",
addr, nbuf);
if(readpart(is->part, addr, buf, nbuf) < 0){
fprint(2, "%s: part %s addr 0x%llux: icachewritesect "
"readpart: %r\n", argv0, is->part->name, addr);
err = -1;
continue;
}
trace(TraceProc, "icachewritesect updatebuf");
addstat(StatIsectReadBytes, nbuf);
addstat(StatIsectRead, 1);
for(l=&chunk; (ie=*l)!=nil; l=&ie->nextdirty){
again:
naddr = ie2diskaddr(ix, is, ie);
off = naddr - addr;
if(off+bsize > nbuf){
fprint(2, "%s: whoops! addr=0x%llux nbuf=%ud "
"addr+nbuf=0x%llux naddr=0x%llux\n",
argv0, addr, nbuf, addr+nbuf, naddr);
assert(off+bsize <= nbuf);
}
unpackibucket(&ib, buf+off, is->bucketmagic);
if(okibucket(&ib, is) < 0){
fprint(2, "%s: bad bucket XXX\n", argv0);
goto skipit;
}
trace(TraceProc, "icachewritesect add %V at 0x%llux",
ie->score, naddr);
h = bucklook(ie->score, ie->ia.type, ib.data, ib.n);
if(h & 1){
h ^= 1;
packientry(ie, &ib.data[h]);
}else if(ib.n < is->buckmax){
memmove(&ib.data[h + IEntrySize], &ib.data[h],
ib.n*IEntrySize - h);
ib.n++;
packientry(ie, &ib.data[h]);
}else{
fprint(2, "%s: bucket overflow XXX\n", argv0);
skipit:
err = -1;
*l = ie->nextdirty;
ie = *l;
if(ie)
goto again;
else
break;
}
packibucket(&ib, buf+off, is->bucketmagic);
}
diskaccess(1);
trace(TraceProc, "icachewritesect writepart", addr, nbuf);
werr = 0;
if(writepart(is->part, addr, buf, nbuf) < 0 || flushpart(is->part) < 0)
werr = -1;
for(i=0; i<nbuf; i+=bsize){
if((b = _getdblock(is->part, addr+i, ORDWR, 0)) != nil){
memmove(b->data, buf+i, bsize);
putdblock(b);
}
}
if(werr < 0){
fprint(2, "%s: part %s addr 0x%llux: icachewritesect "
"writepart: %r\n", argv0, is->part->name, addr);
err = -1;
continue;
}
addstat(StatIsectWriteBytes, nbuf);
addstat(StatIsectWrite, 1);
icacheclean(chunk);
}
trace(TraceProc, "icachewritesect done");
return err;
}
struct solv_zchunk *
solv_zchunk_open(FILE *fp, unsigned int streamid)
{
struct solv_zchunk *zck;
unsigned char *p;
unsigned int hdr_size; /* preface + index + signatures */
unsigned int lead_size;
unsigned int preface_size;
unsigned int index_size;
zck = solv_calloc(1, sizeof(*zck));
/* read and parse the lead, read the complete header */
zck->hdr = solv_calloc(15, 1);
zck->hdr_end = zck->hdr + 15;
if (fread(zck->hdr, 15, 1, fp) != 1 || memcmp(zck->hdr, "\000ZCK1", 5) != 0)
return open_error(zck);
p = zck->hdr + 5;
if ((p = getchksum(p, zck->hdr_end, &zck->hdr_chk_type, &zck->hdr_chk_len, &zck->hdr_chk_id)) == 0)
return open_error(zck);
if ((p = getuint(p, zck->hdr_end, &hdr_size)) == 0 || hdr_size > MAX_HDR_SIZE)
return open_error(zck);
lead_size = p - zck->hdr + zck->hdr_chk_len;
zck->hdr = solv_realloc(zck->hdr, lead_size + hdr_size);
zck->hdr_end = zck->hdr + lead_size + hdr_size;
if (fread(zck->hdr + 15, lead_size + hdr_size - 15, 1, fp) != 1)
return open_error(zck);
/* verify header checksum to guard against corrupt files */
if (zck->hdr_chk_id)
{
Chksum *chk = solv_chksum_create(zck->hdr_chk_id);
if (!chk)
return open_error(zck);
solv_chksum_add(chk, zck->hdr, lead_size - zck->hdr_chk_len);
solv_chksum_add(chk, zck->hdr + lead_size, hdr_size);
if (memcmp(solv_chksum_get(chk, 0), zck->hdr + (lead_size - zck->hdr_chk_len), zck->hdr_chk_len) != 0)
{
solv_chksum_free(chk, 0);
return open_error(zck);
}
solv_chksum_free(chk, 0);
}
/* parse preface: data chksum, flags, compression */
p = zck->hdr + lead_size;
if (p + zck->hdr_chk_len > zck->hdr_end)
return open_error(zck);
zck->data_chk_ptr = p;
p += zck->hdr_chk_len;
#ifdef VERIFY_DATA_CHKSUM
if (zck->hdr_chk_id && (zck->data_chk = solv_chksum_create(zck->hdr_chk_id)) == 0)
return open_error(zck);
#endif
if ((p = getuint(p, zck->hdr_end, &zck->flags)) == 0)
return open_error(zck);
if ((zck->flags & ~(3)) != 0)
return open_error(zck);
if ((p = getuint(p, zck->hdr_end, &zck->comp)) == 0 || (zck->comp != 0 && zck->comp != 2))
return open_error(zck); /* only uncompressed + zstd supported */
/* skip all optional elements if present */
if ((zck->flags & 2) != 0)
{
unsigned int nopt, lopt;
if ((p = getuint(p, zck->hdr_end, &nopt)) == 0)
return open_error(zck);
for (; nopt != 0; nopt--)
{
if ((p = getuint(p, zck->hdr_end, &lopt)) == 0)
return open_error(zck);
if ((p = getuint(p, zck->hdr_end, &lopt)) == 0)
return open_error(zck);
if (p + lopt > zck->hdr_end)
return open_error(zck);
p += lopt;
}
}
preface_size = p - (zck->hdr + lead_size);
/* parse index: index size, index chksum type, num chunks, chunk data */
if ((p = getuint(p, zck->hdr_end, &index_size)) == 0)
return open_error(zck);
if (hdr_size < preface_size + index_size)
return open_error(zck);
if ((p = getchksum(p, zck->hdr_end, &zck->chunk_chk_type, &zck->chunk_chk_len, &zck->chunk_chk_id)) == 0)
return open_error(zck);
if ((p = getuint(p, zck->hdr_end, &zck->nchunks)) == 0 || zck->nchunks > MAX_CHUNK_CNT)
return open_error(zck);
/* setup decompressor */
if (zck->comp == 2)
{
if ((zck->dctx = ZSTD_createDCtx()) == 0)
return open_error(zck);
}
zck->fp = fp;
zck->chunks = p;
zck->streamid = streamid;
if (streamid == 0)
{
zck->nchunks = zck->nchunks ? 1 : 0; /* limit to dict chunk */
return zck;
}
/* setup dictionary */
if (!nextchunk(zck, 0))
{
zck->fp = 0;
return open_error(zck);
}
if (zck->comp == 2 && zck->buf_avail)
{
if ((zck->ddict = ZSTD_createDDict(zck->buf, zck->buf_avail)) == 0)
{
zck->fp = 0;
return open_error(zck);
}
}
zck->buf = solv_free(zck->buf);
zck->buf_used = 0;
zck->buf_avail = 0;
/* ready to read the rest of the chunks */
return zck;
}
