ISect*
initisect(Part *part)
{
ISect *is;
ZBlock *b;
int ok;
b = alloczblock(HeadSize, 0, 0);
if(b == nil || readpart(part, PartBlank, b->data, HeadSize) < 0){
seterr(EAdmin, "can't read index section header: %r");
return nil;
}
is = MKZ(ISect);
if(is == nil){
freezblock(b);
return nil;
}
is->part = part;
ok = unpackisect(is, b->data);
freezblock(b);
if(ok < 0){
seterr(ECorrupt, "corrupted index section header: %r");
freeisect(is);
return nil;
}
if(is->version != ISectVersion1 && is->version != ISectVersion2){
seterr(EAdmin, "unknown index section version %d", is->version);
freeisect(is);
return nil;
}
return initisect1(is);
}
int
wbisect(ISect *is)
{
ZBlock *b;
b = alloczblock(HeadSize, 1, 0);
if(b == nil){
/* ZZZ set error? */
return -1;
}
if(packisect(is, b->data) < 0){
seterr(ECorrupt, "can't make index section header: %r");
freezblock(b);
return -1;
}
if(writepart(is->part, PartBlank, b->data, HeadSize) < 0 || flushpart(is->part) < 0){
seterr(EAdmin, "can't write index section header: %r");
freezblock(b);
return -1;
}
freezblock(b);
return 0;
}
/*
* read the arena header and trailer blocks from disk
*/
static int
loadarena(Arena *arena)
{
ArenaHead head;
ZBlock *b;
b = alloczblock(arena->blocksize, 0, arena->part->blocksize);
if(b == nil)
return -1;
if(readpart(arena->part, arena->base + arena->size, b->data, arena->blocksize) < 0){
freezblock(b);
return -1;
}
if(unpackarena(arena, b->data) < 0){
freezblock(b);
return -1;
}
if(arena->version != ArenaVersion4 && arena->version != ArenaVersion5){
seterr(EAdmin, "unknown arena version %d", arena->version);
freezblock(b);
return -1;
}
scorecp(arena->score, &b->data[arena->blocksize - VtScoreSize]);
if(readpart(arena->part, arena->base - arena->blocksize, b->data, arena->blocksize) < 0){
logerr(EAdmin, "can't read arena header: %r");
freezblock(b);
return 0;
}
if(unpackarenahead(&head, b->data) < 0)
logerr(ECorrupt, "corrupted arena header: %r");
else if(namecmp(arena->name, head.name)!=0
|| arena->clumpmagic != head.clumpmagic
|| arena->version != head.version
|| arena->blocksize != head.blocksize
|| arena->size + 2 * arena->blocksize != head.size){
if(namecmp(arena->name, head.name)!=0)
logerr(ECorrupt, "arena tail name %s head %s",
arena->name, head.name);
else if(arena->clumpmagic != head.clumpmagic)
logerr(ECorrupt, "arena %d tail clumpmagic 0x%lux head 0x%lux",
debugarena, (uint32_t)arena->clumpmagic,
(uint32_t)head.clumpmagic);
else if(arena->version != head.version)
logerr(ECorrupt, "arena tail version %d head version %d",
arena->version, head.version);
else if(arena->blocksize != head.blocksize)
logerr(ECorrupt, "arena tail block size %d head %d",
arena->blocksize, head.blocksize);
else if(arena->size+2*arena->blocksize != head.size)
logerr(ECorrupt, "arena tail size %lud head %lud",
(uint32_t)arena->size+2*arena->blocksize,
head.size);
else
logerr(ECorrupt, "arena header inconsistent with arena data");
}
freezblock(b);
return 0;
}
static Packet*
readilump(Lump *u, IAddr *ia, u8int *score)
{
Arena *arena;
ZBlock *zb;
Packet *p, *pp;
Clump cl;
u64int aa;
u8int sc[VtScoreSize];
trace(TraceLump, "readilump enter");
arena = amapitoa(mainindex, ia->addr, &aa);
if(arena == nil){
trace(TraceLump, "readilump amapitoa failed");
return nil;
}
trace(TraceLump, "readilump loadclump");
zb = loadclump(arena, aa, ia->blocks, &cl, sc, paranoid);
if(zb == nil){
trace(TraceLump, "readilump loadclump failed");
return nil;
}
if(ia->size != cl.info.uncsize){
seterr(EInconsist, "index and clump size mismatch");
freezblock(zb);
return nil;
}
if(ia->type != cl.info.type){
seterr(EInconsist, "index and clump type mismatch");
freezblock(zb);
return nil;
}
if(scorecmp(score, sc) != 0){
seterr(ECrash, "score mismatch");
freezblock(zb);
return nil;
}
trace(TraceLump, "readilump success");
p = zblock2packet(zb, cl.info.uncsize);
freezblock(zb);
pp = packetdup(p, 0, packetsize(p));
trace(TraceLump, "readilump insertlump");
insertlump(u, pp);
trace(TraceLump, "readilump exit");
return p;
}
void
freeifile(IFile *f)
{
freezblock(f->b);
f->b = nil;
f->pos = 0;
}
int
wbarenahead(Arena *arena)
{
ZBlock *b;
ArenaHead head;
int bad;
namecp(head.name, arena->name);
head.version = arena->version;
head.size = arena->size + 2 * arena->blocksize;
head.blocksize = arena->blocksize;
head.clumpmagic = arena->clumpmagic;
b = alloczblock(arena->blocksize, 1, arena->part->blocksize);
if(b == nil){
logerr(EAdmin, "can't write arena header: %r");
/* ZZZ add error message? */
return -1;
}
/*
* this writepart is okay because it only happens
* during initialization.
*/
bad = packarenahead(&head, b->data)<0 ||
writepart(arena->part, arena->base - arena->blocksize, b->data, arena->blocksize)<0 ||
flushpart(arena->part)<0;
freezblock(b);
if(bad)
return -1;
return 0;
}
void
zeropart(Part *part, int blocksize)
{
ZBlock *b;
uint64_t addr;
int w;
fprint(2, "clearing %s\n", part->name);
b = alloczblock(MaxIoSize, 1, blocksize);
w = 0;
for(addr = PartBlank; addr + MaxIoSize <= part->size; addr += MaxIoSize){
if(writepart(part, addr, b->data, MaxIoSize) < 0)
sysfatal("can't initialize %s, writing block %d failed: %r", part->name, w);
w++;
}
for(; addr + blocksize <= part->size; addr += blocksize)
if(writepart(part, addr, b->data, blocksize) < 0)
sysfatal("can't initialize %s: %r", part->name);
if(flushpart(part) < 0)
sysfatal("can't flush writes to %s: %r", part->name);
freezblock(b);
}
void
vtsendthread(void *v)
{
ZClump zcl;
USED(v);
while(recv(c, &zcl) == 1){
if(zcl.lump == nil)
break;
if(vtwrite(z, zcl.cl.info.score, zcl.cl.info.type, zcl.lump->data, zcl.cl.info.uncsize) < 0)
sysfatal("failed writing clump %llud: %r", zcl.aa);
if(verbose)
print("%V\n", zcl.cl.info.score);
freezblock(zcl.lump);
}
/*
* All the send threads try to exit right when
* threadmain is calling threadexitsall.
* Either libthread or the Linux NPTL pthreads library
* can't handle this condition (I suspect NPTL but have
* not confirmed this) and we get a seg fault in exit.
* I spent a day tracking this down with no success,
* so we're going to work around it instead by just
* sitting here and waiting for the threadexitsall to
* take effect.
*/
qlock(&godot);
}
static void
rdarena(Arena *arena)
{
ZBlock *b;
uint64_t a, e;
uint32_t bs;
if (!quiet) {
fprint(2, "copying %s to standard output\n", arena->name);
printarena(2, arena);
}
bs = MaxIoSize;
if(bs < arena->blocksize)
bs = arena->blocksize;
b = alloczblock(bs, 0, arena->blocksize);
e = arena->base + arena->size + arena->blocksize;
for(a = arena->base - arena->blocksize; a + arena->blocksize <= e; a += bs){
if(a + bs > e)
bs = arena->blocksize;
if(readpart(arena->part, a, b->data, bs) < 0)
fprint(2, "can't copy %s, read at %lld failed: %r\n", arena->name, a);
if(write(1, b->data, bs) != bs)
sysfatal("can't copy %s, write at %lld failed: %r", arena->name, a);
}
freezblock(b);
}
int
wbindex(Index *ix)
{
Fmt f;
ZBlock *b;
int i;
if(ix->nsects == 0){
seterr(EOk, "no sections in index %s", ix->name);
return -1;
}
b = alloczblock(ix->tabsize, 1, ix->blocksize);
if(b == nil){
seterr(EOk, "can't write index configuration: out of memory");
return -1;
}
fmtzbinit(&f, b);
if(outputindex(&f, ix) < 0){
seterr(EOk, "can't make index configuration: table storage too small %d", ix->tabsize);
freezblock(b);
return -1;
}
for(i = 0; i < ix->nsects; i++){
if(writepart(ix->sects[i]->part, ix->sects[i]->tabbase, b->data, ix->tabsize) < 0
|| flushpart(ix->sects[i]->part) < 0){
seterr(EOk, "can't write index: %r");
freezblock(b);
return -1;
}
}
freezblock(b);
for(i = 0; i < ix->nsects; i++)
if(wbisect(ix->sects[i]) < 0)
return -1;
return 0;
}
ZBlock*
packet2zblock(Packet *p, uint32_t size)
{
ZBlock *b;
if(p == nil)
return nil;
b = alloczblock(size, 0, 0);
if(b == nil)
return nil;
if(packetcopy(p, b->data, 0, size) < 0){
freezblock(b);
return nil;
}
return b;
}
int
partifile(IFile *f, Part *part, u64int start, u32int size)
{
ZBlock *b;
b = alloczblock(size, 0, part->blocksize);
if(b == nil)
return -1;
if(readpart(part, start, b->data, size) < 0){
seterr(EAdmin, "can't read %s: %r", part->name);
freezblock(b);
return -1;
}
f->name = part->name;
f->b = b;
f->pos = 0;
return 0;
}
int
writeqlump(Lump *u, Packet *p, int creator, uint ms)
{
ZBlock *flat;
Packet *old;
IAddr ia;
int ok;
if(lookupscore(u->score, u->type, &ia) == 0){
if(verifywrites == 0){
/* assume the data is here! */
packetfree(p);
ms = msec() - ms;
addstat2(StatRpcWriteOld, 1, StatRpcWriteOldTime, ms);
return 0;
}
/*
* if the read fails,
* assume it was corrupted data and store the block again
*/
old = readilump(u, &ia, u->score);
if(old != nil){
ok = 0;
if(packetcmp(p, old) != 0){
uchar nscore[VtScoreSize];
packetsha1(old, nscore);
if(scorecmp(u->score, nscore) != 0)
seterr(EStrange, "readilump returned bad data %V not %V", nscore, u->score);
else
seterr(EStrange, "score collision %V", u->score);
ok = -1;
}
packetfree(p);
packetfree(old);
ms = msec() - ms;
addstat2(StatRpcWriteOld, 1, StatRpcWriteOldTime, ms);
return ok;
}
logerr(EAdmin, "writelump: read %V failed, rewriting: %r\n", u->score);
}
flat = packet2zblock(p, packetsize(p));
ok = storeclump(mainindex, flat, u->score, u->type, creator, &ia);
freezblock(flat);
if(ok == 0)
insertlump(u, p);
else
packetfree(p);
if(syncwrites){
flushdcache();
flushicache();
flushdcache();
}
ms = msec() - ms;
addstat2(StatRpcWriteNew, 1, StatRpcWriteNewTime, ms);
return ok;
}
void
sumarena(Arena *arena)
{
ZBlock *b;
DigestState s;
uint64_t a, e;
uint32_t bs;
int t;
uint8_t score[VtScoreSize];
bs = MaxIoSize;
if(bs < arena->blocksize)
bs = arena->blocksize;
/*
* read & sum all blocks except the last one
*/
flushdcache();
memset(&s, 0, sizeof s);
b = alloczblock(bs, 0, arena->part->blocksize);
e = arena->base + arena->size;
for(a = arena->base - arena->blocksize; a + arena->blocksize <= e; a += bs){
disksched();
while((t=arenasumsleeptime) == SleepForever){
sleep(1000);
disksched();
}
sleep(t);
if(a + bs > e)
bs = arena->blocksize;
if(readpart(arena->part, a, b->data, bs) < 0)
goto ReadErr;
addstat(StatSumRead, 1);
addstat(StatSumReadBytes, bs);
sha1(b->data, bs, nil, &s);
}
/*
* the last one is special, since it may already have the checksum included
*/
bs = arena->blocksize;
if(readpart(arena->part, e, b->data, bs) < 0){
ReadErr:
logerr(EOk, "sumarena can't sum %s, read at %lld failed: %r", arena->name, a);
freezblock(b);
return;
}
addstat(StatSumRead, 1);
addstat(StatSumReadBytes, bs);
sha1(b->data, bs-VtScoreSize, nil, &s);
sha1(zeroscore, VtScoreSize, nil, &s);
sha1(nil, 0, score, &s);
/*
* check for no checksum or the same
*/
if(scorecmp(score, &b->data[bs - VtScoreSize]) != 0
&& scorecmp(zeroscore, &b->data[bs - VtScoreSize]) != 0)
logerr(EOk, "overwriting mismatched checksums for arena=%s, found=%V calculated=%V",
arena->name, &b->data[bs - VtScoreSize], score);
freezblock(b);
qlock(&arena->lock);
scorecp(arena->score, score);
wbarena(arena);
qunlock(&arena->lock);
}
int
readifile(IFile *f, char *name)
{
Part *p;
ZBlock *b;
u8int *z;
p = initpart(name, OREAD);
if(p == nil)
return -1;
b = alloczblock(Maxconfig+1, 1, 0);
if(b == nil){
seterr(EOk, "can't alloc for %s: %R", name);
return -1;
}
if(p->size > PartBlank){
/*
* this is likely a real venti partition, in which case we're
* looking for the config file stored as 8k at end of PartBlank.
*/
if(readpart(p, PartBlank-Maxconfig, b->data, Maxconfig) < 0){
seterr(EOk, "can't read %s: %r", name);
freezblock(b);
freepart(p);
return -1;
}
b->data[Maxconfig] = '\0';
if(memcmp(b->data, vcmagic, Maglen) != 0){
seterr(EOk, "bad venti config magic in %s", name);
freezblock(b);
freepart(p);
return -1;
}
/*
* if we change b->data+b->_size, freezblock
* will blow an assertion, so don't.
*/
b->data += Maglen;
b->_size -= Maglen;
b->len -= Maglen;
z = memchr(b->data, '\0', b->len);
if(z)
b->len = z - b->data;
}else if(p->size > Maxconfig){
seterr(EOk, "config file is too large");
freepart(p);
freezblock(b);
return -1;
}else{
freezblock(b);
b = readfile(name);
if(b == nil){
freepart(p);
return -1;
}
}
freepart(p);
f->name = name;
f->b = b;
f->pos = 0;
return 0;
}
static void
rdarena(Arena *arena, u64int offset)
{
int i;
u64int a, aa, e;
uchar score[VtScoreSize];
Clump cl;
ClumpInfo ci;
ZBlock *lump;
ZClump zcl;
fprint(2, "wrarena: copying %s to venti\n", arena->name);
printarena(2, arena);
a = arena->base;
e = arena->base + arena->size;
if(offset != ~(u64int)0) {
if(offset >= e - a)
sysfatal("bad offset %#llx >= %#llx", offset, e - a);
aa = offset;
} else
aa = 0;
i = 0;
for(a = 0; maxwrites != 0 && i < arena->memstats.clumps;
a += ClumpSize + ci.size){
if(readclumpinfo(arena, i++, &ci) < 0)
break;
if(a < aa || ci.type == VtCorruptType){
if(ci.type == VtCorruptType)
fprint(2, "%s: corrupt clump read at %#llx: +%d\n",
argv0, a, ClumpSize+ci.size);
continue;
}
lump = loadclump(arena, a, 0, &cl, score, 0);
if(lump == nil) {
fprint(2, "clump %#llx failed to read: %r\n", a);
continue;
}
if(!fast && cl.info.type != VtCorruptType) {
scoremem(score, lump->data, cl.info.uncsize);
if(scorecmp(cl.info.score, score) != 0) {
fprint(2, "clump %#llx has mismatched score\n",
a);
break;
}
if(vttypevalid(cl.info.type) < 0) {
fprint(2, "clump %#llx has bad type %d\n",
a, cl.info.type);
break;
}
}
if(z && cl.info.type != VtCorruptType){
zcl.cl = cl;
zcl.lump = lump;
zcl.aa = a;
send(c, &zcl);
}else
freezblock(lump);
if(maxwrites > 0)
--maxwrites;
}
if(a > aa)
aa = a;
if(haveaoffset)
print("end offset %#llx\n", aa);
}
