ISect* initisect(Part *part) { ISect *is; ZBlock *b; int ok; b = alloczblock(HeadSize, 0, 0); if(b == nil || readpart(part, PartBlank, b->data, HeadSize) < 0){ seterr(EAdmin, "can't read index section header: %r"); return nil; } is = MKZ(ISect); if(is == nil){ freezblock(b); return nil; } is->part = part; ok = unpackisect(is, b->data); freezblock(b); if(ok < 0){ seterr(ECorrupt, "corrupted index section header: %r"); freeisect(is); return nil; } if(is->version != ISectVersion1 && is->version != ISectVersion2){ seterr(EAdmin, "unknown index section version %d", is->version); freeisect(is); return nil; } return initisect1(is); }
int wbisect(ISect *is) { ZBlock *b; b = alloczblock(HeadSize, 1, 0); if(b == nil){ /* ZZZ set error? */ return -1; } if(packisect(is, b->data) < 0){ seterr(ECorrupt, "can't make index section header: %r"); freezblock(b); return -1; } if(writepart(is->part, PartBlank, b->data, HeadSize) < 0 || flushpart(is->part) < 0){ seterr(EAdmin, "can't write index section header: %r"); freezblock(b); return -1; } freezblock(b); return 0; }
/* * read the arena header and trailer blocks from disk */ static int loadarena(Arena *arena) { ArenaHead head; ZBlock *b; b = alloczblock(arena->blocksize, 0, arena->part->blocksize); if(b == nil) return -1; if(readpart(arena->part, arena->base + arena->size, b->data, arena->blocksize) < 0){ freezblock(b); return -1; } if(unpackarena(arena, b->data) < 0){ freezblock(b); return -1; } if(arena->version != ArenaVersion4 && arena->version != ArenaVersion5){ seterr(EAdmin, "unknown arena version %d", arena->version); freezblock(b); return -1; } scorecp(arena->score, &b->data[arena->blocksize - VtScoreSize]); if(readpart(arena->part, arena->base - arena->blocksize, b->data, arena->blocksize) < 0){ logerr(EAdmin, "can't read arena header: %r"); freezblock(b); return 0; } if(unpackarenahead(&head, b->data) < 0) logerr(ECorrupt, "corrupted arena header: %r"); else if(namecmp(arena->name, head.name)!=0 || arena->clumpmagic != head.clumpmagic || arena->version != head.version || arena->blocksize != head.blocksize || arena->size + 2 * arena->blocksize != head.size){ if(namecmp(arena->name, head.name)!=0) logerr(ECorrupt, "arena tail name %s head %s", arena->name, head.name); else if(arena->clumpmagic != head.clumpmagic) logerr(ECorrupt, "arena %d tail clumpmagic 0x%lux head 0x%lux", debugarena, (uint32_t)arena->clumpmagic, (uint32_t)head.clumpmagic); else if(arena->version != head.version) logerr(ECorrupt, "arena tail version %d head version %d", arena->version, head.version); else if(arena->blocksize != head.blocksize) logerr(ECorrupt, "arena tail block size %d head %d", arena->blocksize, head.blocksize); else if(arena->size+2*arena->blocksize != head.size) logerr(ECorrupt, "arena tail size %lud head %lud", (uint32_t)arena->size+2*arena->blocksize, head.size); else logerr(ECorrupt, "arena header inconsistent with arena data"); } freezblock(b); return 0; }
static Packet* readilump(Lump *u, IAddr *ia, u8int *score) { Arena *arena; ZBlock *zb; Packet *p, *pp; Clump cl; u64int aa; u8int sc[VtScoreSize]; trace(TraceLump, "readilump enter"); arena = amapitoa(mainindex, ia->addr, &aa); if(arena == nil){ trace(TraceLump, "readilump amapitoa failed"); return nil; } trace(TraceLump, "readilump loadclump"); zb = loadclump(arena, aa, ia->blocks, &cl, sc, paranoid); if(zb == nil){ trace(TraceLump, "readilump loadclump failed"); return nil; } if(ia->size != cl.info.uncsize){ seterr(EInconsist, "index and clump size mismatch"); freezblock(zb); return nil; } if(ia->type != cl.info.type){ seterr(EInconsist, "index and clump type mismatch"); freezblock(zb); return nil; } if(scorecmp(score, sc) != 0){ seterr(ECrash, "score mismatch"); freezblock(zb); return nil; } trace(TraceLump, "readilump success"); p = zblock2packet(zb, cl.info.uncsize); freezblock(zb); pp = packetdup(p, 0, packetsize(p)); trace(TraceLump, "readilump insertlump"); insertlump(u, pp); trace(TraceLump, "readilump exit"); return p; }
void freeifile(IFile *f) { freezblock(f->b); f->b = nil; f->pos = 0; }
int wbarenahead(Arena *arena) { ZBlock *b; ArenaHead head; int bad; namecp(head.name, arena->name); head.version = arena->version; head.size = arena->size + 2 * arena->blocksize; head.blocksize = arena->blocksize; head.clumpmagic = arena->clumpmagic; b = alloczblock(arena->blocksize, 1, arena->part->blocksize); if(b == nil){ logerr(EAdmin, "can't write arena header: %r"); /* ZZZ add error message? */ return -1; } /* * this writepart is okay because it only happens * during initialization. */ bad = packarenahead(&head, b->data)<0 || writepart(arena->part, arena->base - arena->blocksize, b->data, arena->blocksize)<0 || flushpart(arena->part)<0; freezblock(b); if(bad) return -1; return 0; }
void zeropart(Part *part, int blocksize) { ZBlock *b; uint64_t addr; int w; fprint(2, "clearing %s\n", part->name); b = alloczblock(MaxIoSize, 1, blocksize); w = 0; for(addr = PartBlank; addr + MaxIoSize <= part->size; addr += MaxIoSize){ if(writepart(part, addr, b->data, MaxIoSize) < 0) sysfatal("can't initialize %s, writing block %d failed: %r", part->name, w); w++; } for(; addr + blocksize <= part->size; addr += blocksize) if(writepart(part, addr, b->data, blocksize) < 0) sysfatal("can't initialize %s: %r", part->name); if(flushpart(part) < 0) sysfatal("can't flush writes to %s: %r", part->name); freezblock(b); }
void vtsendthread(void *v) { ZClump zcl; USED(v); while(recv(c, &zcl) == 1){ if(zcl.lump == nil) break; if(vtwrite(z, zcl.cl.info.score, zcl.cl.info.type, zcl.lump->data, zcl.cl.info.uncsize) < 0) sysfatal("failed writing clump %llud: %r", zcl.aa); if(verbose) print("%V\n", zcl.cl.info.score); freezblock(zcl.lump); } /* * All the send threads try to exit right when * threadmain is calling threadexitsall. * Either libthread or the Linux NPTL pthreads library * can't handle this condition (I suspect NPTL but have * not confirmed this) and we get a seg fault in exit. * I spent a day tracking this down with no success, * so we're going to work around it instead by just * sitting here and waiting for the threadexitsall to * take effect. */ qlock(&godot); }
static void rdarena(Arena *arena) { ZBlock *b; uint64_t a, e; uint32_t bs; if (!quiet) { fprint(2, "copying %s to standard output\n", arena->name); printarena(2, arena); } bs = MaxIoSize; if(bs < arena->blocksize) bs = arena->blocksize; b = alloczblock(bs, 0, arena->blocksize); e = arena->base + arena->size + arena->blocksize; for(a = arena->base - arena->blocksize; a + arena->blocksize <= e; a += bs){ if(a + bs > e) bs = arena->blocksize; if(readpart(arena->part, a, b->data, bs) < 0) fprint(2, "can't copy %s, read at %lld failed: %r\n", arena->name, a); if(write(1, b->data, bs) != bs) sysfatal("can't copy %s, write at %lld failed: %r", arena->name, a); } freezblock(b); }
int wbindex(Index *ix) { Fmt f; ZBlock *b; int i; if(ix->nsects == 0){ seterr(EOk, "no sections in index %s", ix->name); return -1; } b = alloczblock(ix->tabsize, 1, ix->blocksize); if(b == nil){ seterr(EOk, "can't write index configuration: out of memory"); return -1; } fmtzbinit(&f, b); if(outputindex(&f, ix) < 0){ seterr(EOk, "can't make index configuration: table storage too small %d", ix->tabsize); freezblock(b); return -1; } for(i = 0; i < ix->nsects; i++){ if(writepart(ix->sects[i]->part, ix->sects[i]->tabbase, b->data, ix->tabsize) < 0 || flushpart(ix->sects[i]->part) < 0){ seterr(EOk, "can't write index: %r"); freezblock(b); return -1; } } freezblock(b); for(i = 0; i < ix->nsects; i++) if(wbisect(ix->sects[i]) < 0) return -1; return 0; }
ZBlock* packet2zblock(Packet *p, uint32_t size) { ZBlock *b; if(p == nil) return nil; b = alloczblock(size, 0, 0); if(b == nil) return nil; if(packetcopy(p, b->data, 0, size) < 0){ freezblock(b); return nil; } return b; }
int partifile(IFile *f, Part *part, u64int start, u32int size) { ZBlock *b; b = alloczblock(size, 0, part->blocksize); if(b == nil) return -1; if(readpart(part, start, b->data, size) < 0){ seterr(EAdmin, "can't read %s: %r", part->name); freezblock(b); return -1; } f->name = part->name; f->b = b; f->pos = 0; return 0; }
int writeqlump(Lump *u, Packet *p, int creator, uint ms) { ZBlock *flat; Packet *old; IAddr ia; int ok; if(lookupscore(u->score, u->type, &ia) == 0){ if(verifywrites == 0){ /* assume the data is here! */ packetfree(p); ms = msec() - ms; addstat2(StatRpcWriteOld, 1, StatRpcWriteOldTime, ms); return 0; } /* * if the read fails, * assume it was corrupted data and store the block again */ old = readilump(u, &ia, u->score); if(old != nil){ ok = 0; if(packetcmp(p, old) != 0){ uchar nscore[VtScoreSize]; packetsha1(old, nscore); if(scorecmp(u->score, nscore) != 0) seterr(EStrange, "readilump returned bad data %V not %V", nscore, u->score); else seterr(EStrange, "score collision %V", u->score); ok = -1; } packetfree(p); packetfree(old); ms = msec() - ms; addstat2(StatRpcWriteOld, 1, StatRpcWriteOldTime, ms); return ok; } logerr(EAdmin, "writelump: read %V failed, rewriting: %r\n", u->score); } flat = packet2zblock(p, packetsize(p)); ok = storeclump(mainindex, flat, u->score, u->type, creator, &ia); freezblock(flat); if(ok == 0) insertlump(u, p); else packetfree(p); if(syncwrites){ flushdcache(); flushicache(); flushdcache(); } ms = msec() - ms; addstat2(StatRpcWriteNew, 1, StatRpcWriteNewTime, ms); return ok; }
void sumarena(Arena *arena) { ZBlock *b; DigestState s; uint64_t a, e; uint32_t bs; int t; uint8_t score[VtScoreSize]; bs = MaxIoSize; if(bs < arena->blocksize) bs = arena->blocksize; /* * read & sum all blocks except the last one */ flushdcache(); memset(&s, 0, sizeof s); b = alloczblock(bs, 0, arena->part->blocksize); e = arena->base + arena->size; for(a = arena->base - arena->blocksize; a + arena->blocksize <= e; a += bs){ disksched(); while((t=arenasumsleeptime) == SleepForever){ sleep(1000); disksched(); } sleep(t); if(a + bs > e) bs = arena->blocksize; if(readpart(arena->part, a, b->data, bs) < 0) goto ReadErr; addstat(StatSumRead, 1); addstat(StatSumReadBytes, bs); sha1(b->data, bs, nil, &s); } /* * the last one is special, since it may already have the checksum included */ bs = arena->blocksize; if(readpart(arena->part, e, b->data, bs) < 0){ ReadErr: logerr(EOk, "sumarena can't sum %s, read at %lld failed: %r", arena->name, a); freezblock(b); return; } addstat(StatSumRead, 1); addstat(StatSumReadBytes, bs); sha1(b->data, bs-VtScoreSize, nil, &s); sha1(zeroscore, VtScoreSize, nil, &s); sha1(nil, 0, score, &s); /* * check for no checksum or the same */ if(scorecmp(score, &b->data[bs - VtScoreSize]) != 0 && scorecmp(zeroscore, &b->data[bs - VtScoreSize]) != 0) logerr(EOk, "overwriting mismatched checksums for arena=%s, found=%V calculated=%V", arena->name, &b->data[bs - VtScoreSize], score); freezblock(b); qlock(&arena->lock); scorecp(arena->score, score); wbarena(arena); qunlock(&arena->lock); }
int readifile(IFile *f, char *name) { Part *p; ZBlock *b; u8int *z; p = initpart(name, OREAD); if(p == nil) return -1; b = alloczblock(Maxconfig+1, 1, 0); if(b == nil){ seterr(EOk, "can't alloc for %s: %R", name); return -1; } if(p->size > PartBlank){ /* * this is likely a real venti partition, in which case we're * looking for the config file stored as 8k at end of PartBlank. */ if(readpart(p, PartBlank-Maxconfig, b->data, Maxconfig) < 0){ seterr(EOk, "can't read %s: %r", name); freezblock(b); freepart(p); return -1; } b->data[Maxconfig] = '\0'; if(memcmp(b->data, vcmagic, Maglen) != 0){ seterr(EOk, "bad venti config magic in %s", name); freezblock(b); freepart(p); return -1; } /* * if we change b->data+b->_size, freezblock * will blow an assertion, so don't. */ b->data += Maglen; b->_size -= Maglen; b->len -= Maglen; z = memchr(b->data, '\0', b->len); if(z) b->len = z - b->data; }else if(p->size > Maxconfig){ seterr(EOk, "config file is too large"); freepart(p); freezblock(b); return -1; }else{ freezblock(b); b = readfile(name); if(b == nil){ freepart(p); return -1; } } freepart(p); f->name = name; f->b = b; f->pos = 0; return 0; }
static void rdarena(Arena *arena, u64int offset) { int i; u64int a, aa, e; uchar score[VtScoreSize]; Clump cl; ClumpInfo ci; ZBlock *lump; ZClump zcl; fprint(2, "wrarena: copying %s to venti\n", arena->name); printarena(2, arena); a = arena->base; e = arena->base + arena->size; if(offset != ~(u64int)0) { if(offset >= e - a) sysfatal("bad offset %#llx >= %#llx", offset, e - a); aa = offset; } else aa = 0; i = 0; for(a = 0; maxwrites != 0 && i < arena->memstats.clumps; a += ClumpSize + ci.size){ if(readclumpinfo(arena, i++, &ci) < 0) break; if(a < aa || ci.type == VtCorruptType){ if(ci.type == VtCorruptType) fprint(2, "%s: corrupt clump read at %#llx: +%d\n", argv0, a, ClumpSize+ci.size); continue; } lump = loadclump(arena, a, 0, &cl, score, 0); if(lump == nil) { fprint(2, "clump %#llx failed to read: %r\n", a); continue; } if(!fast && cl.info.type != VtCorruptType) { scoremem(score, lump->data, cl.info.uncsize); if(scorecmp(cl.info.score, score) != 0) { fprint(2, "clump %#llx has mismatched score\n", a); break; } if(vttypevalid(cl.info.type) < 0) { fprint(2, "clump %#llx has bad type %d\n", a, cl.info.type); break; } } if(z && cl.info.type != VtCorruptType){ zcl.cl = cl; zcl.lump = lump; zcl.aa = a; send(c, &zcl); }else freezblock(lump); if(maxwrites > 0) --maxwrites; } if(a > aa) aa = a; if(haveaoffset) print("end offset %#llx\n", aa); }