static int flushblock(VtCache *c, VtBlock *bb, uint8_t score[VtScoreSize], int ppb, int epb, int type) { uint32_t addr; VtBlock *b; VtEntry e; int i; addr = vtglobaltolocal(score); if(addr == NilBlock) return 0; if(bb){ b = bb; if(memcmp(b->score, score, VtScoreSize) != 0) abort(); }else if((b = vtcachelocal(c, addr, type)) == nil) return -1; switch(type){ case VtDataType: break; case VtDirType: for(i=0; i<epb; i++){ if(vtentryunpack(&e, b->data, i) < 0) goto Err; if(!(e.flags&VtEntryActive)) continue; if(flushblock(c, nil, e.score, e.psize/VtScoreSize, e.dsize/VtEntrySize, e.type) < 0) goto Err; vtentrypack(&e, b->data, i); } break; default: /* VtPointerTypeX */ for(i=0; i<ppb; i++){ if(flushblock(c, nil, b->data+VtScoreSize*i, ppb, epb, type-1) < 0) goto Err; } break; } if(vtblockwrite(b) < 0) goto Err; memmove(score, b->score, VtScoreSize); if(b != bb) vtblockput(b); return 0; Err: if(b != bb) vtblockput(b); return -1; }
static VtBlock* fileload(VtFile *r, VtEntry *e) { VtBlock *b; assert(ISLOCKED(r)); b = r->b; if(vtentryunpack(e, b->data, r->offset % r->epb) < 0) return nil; vtblockduplock(b); return b; }
void walk(uint8_t score[VtScoreSize], uint type, int base) { int i, n; uint8_t *buf; uint8_t nscore[VtScoreSize]; VtEntry e; VtRoot root; if(memcmp(score, vtzeroscore, VtScoreSize) == 0 || memcmp(score, zeroscore, VtScoreSize) == 0) return; if(havevisited(score, type)){ nskip++; return; } buf = vtmallocz(VtMaxLumpSize); if(fast && vtread(zdst, score, type, buf, VtMaxLumpSize) >= 0){ if(verbose) fprint(2, "skip %V\n", score); free(buf); return; } n = vtread(zsrc, score, type, buf, VtMaxLumpSize); if(n < 0){ if(rewrite){ changes++; memmove(score, vtzeroscore, VtScoreSize); }else if(!ignoreerrors) sysfatal("reading block %V (type %d): %r", score, type); return; } switch(type){ case VtRootType: if(vtrootunpack(&root, buf) < 0){ fprint(2, "warning: could not unpack root in %V %d\n", score, type); break; } walk(root.prev, VtRootType, 0); walk(root.score, VtDirType, 0); if(rewrite) vtrootpack(&root, buf); /* walk might have changed score */ break; case VtDirType: for(i=0; i*VtEntrySize<n; i++){ if(vtentryunpack(&e, buf, i) < 0){ fprint(2, "warning: could not unpack entry #%d in %V %d\n", i, score, type); continue; } if(!(e.flags & VtEntryActive)) continue; walk(e.score, e.type, e.type&VtTypeBaseMask); /* * Don't repack unless we're rewriting -- some old * vac files have psize==0 and dsize==0, and these * get rewritten by vtentryunpack to have less strange * block sizes. So vtentryunpack; vtentrypack does not * guarantee to preserve the exact bytes in buf. */ if(rewrite) vtentrypack(&e, buf, i); } break; case VtDataType: break; default: /* pointers */ for(i=0; i<n; i+=VtScoreSize) if(memcmp(buf+i, vtzeroscore, VtScoreSize) != 0) walk(buf+i, type-1, base); break; } nwrite++; if(vtwrite(zdst, nscore, type, buf, n) < 0){ /* figure out score for better error message */ /* can't use input argument - might have changed contents */ n = vtzerotruncate(type, buf, n); sha1(buf, n, score, nil); sysfatal("writing block %V (type %d): %r", score, type); } if(!rewrite && memcmp(score, nscore, VtScoreSize) != 0){ fprint(2, "not rewriting: wrote %V got %V\n", score, nscore); abort(); sysfatal("not rewriting: wrote %V got %V", score, nscore); } markvisited(score, type); free(buf); }
void threadmain(int argc, char *argv[]) { int n; uchar score[VtScoreSize]; uchar *buf; char *host, *type; vlong off; VtEntry e; VtRoot root; VtCache *c; VtConn *z; VtFile *f; quotefmtinstall(); fmtinstall('F', vtfcallfmt); fmtinstall('V', vtscorefmt); host = nil; ARGBEGIN{ case 'V': chattyventi++; break; case 'h': host = EARGF(usage()); break; case 'v': chatty++; break; default: usage(); break; }ARGEND if(argc != 1) usage(); type = nil; if(vtparsescore(argv[0], &type, score) < 0) sysfatal("could not parse score '%s': %r", argv[0]); if(type == nil || strcmp(type, "file") != 0) sysfatal("bad score - not file:..."); buf = vtmallocz(VtMaxLumpSize); z = vtdial(host); if(z == nil) sysfatal("could not connect to server: %r"); if(vtconnect(z) < 0) sysfatal("vtconnect: %r"); // root block ... n = vtread(z, score, VtRootType, buf, VtMaxLumpSize); if(n < 0) sysfatal("could not read root %V: %r", score); if(n != VtRootSize) sysfatal("root block %V is wrong size %d != %d", score, n, VtRootSize); if(vtrootunpack(&root, buf) < 0) sysfatal("unpacking root block %V: %r", score); if(strcmp(root.type, "file") != 0) sysfatal("bad root type %q (not 'file')", root.type); if(chatty) fprint(2, "%V: %q %q %V %d %V\n", score, root.name, root.type, root.score, root.blocksize, root.prev); // ... points at entry block n = vtread(z, root.score, VtDirType, buf, VtMaxLumpSize); if(n < 0) sysfatal("could not read entry %V: %r", root.score); if(n != VtEntrySize) sysfatal("dir block %V is wrong size %d != %d", root.score, n, VtEntrySize); if(vtentryunpack(&e, buf, 0) < 0) sysfatal("unpacking dir block %V: %r", root.score); if((e.type&VtTypeBaseMask) != VtDataType) sysfatal("not a single file"); // open and read file c = vtcachealloc(z, root.blocksize*32); if(c == nil) sysfatal("vtcachealloc: %r"); f = vtfileopenroot(c, &e); if(f == nil) sysfatal("vtfileopenroot: %r"); off = 0; vtfilelock(f, VtOREAD); while((n = vtfileread(f, buf, VtMaxLumpSize, off)) > 0){ write(1, buf, n); off += n; } threadexitsall(0); }
void threadmain(int argc, char **argv) { char *pref, *mountname, *mountplace; uchar score[VtScoreSize], prev[VtScoreSize]; int i, fd, csize; vlong bsize; Tm tm; VtEntry e; VtBlock *b; VtCache *c; VtRoot root; char *tmp, *tmpnam; fmtinstall('F', vtfcallfmt); fmtinstall('H', encodefmt); fmtinstall('T', timefmt); fmtinstall('V', vtscorefmt); mountname = sysname(); mountplace = nil; ARGBEGIN{ default: usage(); break; case 'D': debug++; break; case 'V': chattyventi = 1; break; case 'f': fastwrites = 1; break; case 'i': incremental = 1; break; case 'm': mountname = EARGF(usage()); break; case 'M': mountplace = EARGF(usage()); i = strlen(mountplace); if(i > 0 && mountplace[i-1] == '/') mountplace[i-1] = 0; break; case 'n': nop = 1; break; case 's': statustime = atoi(EARGF(usage())); break; case 'v': verbose = 1; break; case 'w': nwritethread = atoi(EARGF(usage())); break; }ARGEND if(argc != 1 && argc != 2) usage(); if(statustime) print("# %T vbackup %s %s\n", argv[0], argc>=2 ? argv[1] : ""); /* * open fs */ if((disk = diskopenfile(argv[0])) == nil) sysfatal("diskopen: %r"); if((disk = diskcache(disk, 32768, 2*MAXQ+16)) == nil) sysfatal("diskcache: %r"); if((fsys = fsysopen(disk)) == nil) sysfatal("fsysopen: %r"); /* * connect to venti */ if((z = vtdial(nil)) == nil) sysfatal("vtdial: %r"); if(vtconnect(z) < 0) sysfatal("vtconnect: %r"); /* * set up venti block cache */ zero = vtmallocz(fsys->blocksize); bsize = fsys->blocksize; csize = 50; /* plenty; could probably do with 5 */ if(verbose) fprint(2, "cache %d blocks\n", csize); c = vtcachealloc(z, bsize*csize); zcache = c; /* * parse starting score */ memset(prev, 0, sizeof prev); if(argc == 1){ vfile = vtfilecreateroot(c, (fsys->blocksize/VtScoreSize)*VtScoreSize, fsys->blocksize, VtDataType); if(vfile == nil) sysfatal("vtfilecreateroot: %r"); vtfilelock(vfile, VtORDWR); if(vtfilewrite(vfile, zero, 1, bsize*fsys->nblock-1) != 1) sysfatal("vtfilewrite: %r"); if(vtfileflush(vfile) < 0) sysfatal("vtfileflush: %r"); }else{ if(vtparsescore(argv[1], &pref, score) < 0) sysfatal("bad score: %r"); if(pref!=nil && strcmp(pref, fsys->type) != 0) sysfatal("score is %s but fsys is %s", pref, fsys->type); b = vtcacheglobal(c, score, VtRootType, VtRootSize); if(b){ if(vtrootunpack(&root, b->data) < 0) sysfatal("bad root: %r"); if(strcmp(root.type, fsys->type) != 0) sysfatal("root is %s but fsys is %s", root.type, fsys->type); memmove(prev, score, VtScoreSize); memmove(score, root.score, VtScoreSize); vtblockput(b); } b = vtcacheglobal(c, score, VtDirType, VtEntrySize); if(b == nil) sysfatal("vtcacheglobal %V: %r", score); if(vtentryunpack(&e, b->data, 0) < 0) sysfatal("%V: vtentryunpack failed", score); if(verbose) fprint(2, "entry: size %llud psize %d dsize %d\n", e.size, e.psize, e.dsize); vtblockput(b); if((vfile = vtfileopenroot(c, &e)) == nil) sysfatal("vtfileopenroot: %r"); vtfilelock(vfile, VtORDWR); if(e.dsize != bsize) sysfatal("file system block sizes don't match %d %lld", e.dsize, bsize); if(e.size != fsys->nblock*bsize) sysfatal("file system block counts don't match %lld %lld", e.size, fsys->nblock*bsize); } tmpnam = nil; if(incremental){ if(vtfilegetentry(vfile, &e) < 0) sysfatal("vtfilegetentry: %r"); if((vscores = vtfileopenroot(c, &e)) == nil) sysfatal("vtfileopenroot: %r"); vtfileunlock(vfile); }else{ /* * write scores of blocks into temporary file */ if((tmp = getenv("TMP")) != nil){ /* okay, good */ }else if(access("/var/tmp", 0) >= 0) tmp = "/var/tmp"; else tmp = "/tmp"; tmpnam = smprint("%s/vbackup.XXXXXX", tmp); if(tmpnam == nil) sysfatal("smprint: %r"); if((fd = opentemp(tmpnam, ORDWR|ORCLOSE)) < 0) sysfatal("opentemp %s: %r", tmpnam); if(statustime) print("# %T reading scores into %s\n", tmpnam); if(verbose) fprint(2, "read scores into %s...\n", tmpnam); Binit(&bscores, fd, OWRITE); for(i=0; i<fsys->nblock; i++){ if(vtfileblockscore(vfile, i, score) < 0) sysfatal("vtfileblockhash %d: %r", i); if(Bwrite(&bscores, score, VtScoreSize) != VtScoreSize) sysfatal("Bwrite: %r"); } Bterm(&bscores); vtfileunlock(vfile); /* * prep scores for rereading */ seek(fd, 0, 0); Binit(&bscores, fd, OREAD); } /* * start the main processes */ if(statustime) print("# %T starting procs\n"); qcmp = qalloc(); qventi = qalloc(); rlock(&endlk); proccreate(fsysproc, nil, STACK); rlock(&endlk); proccreate(ventiproc, nil, STACK); rlock(&endlk); proccreate(cmpproc, nil, STACK); if(statustime){ rlock(&endlk); proccreate(statusproc, nil, STACK); } /* * wait for processes to finish */ wlock(&endlk); qfree(qcmp); qfree(qventi); if(statustime) print("# %T procs exited: %d of %d %d-byte blocks changed, " "%d read, %d written, %d skipped, %d copied\n", nchange, fsys->nblock, fsys->blocksize, vtcachenread, vtcachenwrite, nskip, vtcachencopy); /* * prepare root block */ if(incremental) vtfileclose(vscores); vtfilelock(vfile, -1); if(vtfileflush(vfile) < 0) sysfatal("vtfileflush: %r"); if(vtfilegetentry(vfile, &e) < 0) sysfatal("vtfilegetentry: %r"); vtfileunlock(vfile); vtfileclose(vfile); b = vtcacheallocblock(c, VtDirType, VtEntrySize); if(b == nil) sysfatal("vtcacheallocblock: %r"); vtentrypack(&e, b->data, 0); if(vtblockwrite(b) < 0) sysfatal("vtblockwrite: %r"); memset(&root, 0, sizeof root); strecpy(root.name, root.name+sizeof root.name, argv[0]); strecpy(root.type, root.type+sizeof root.type, fsys->type); memmove(root.score, b->score, VtScoreSize); root.blocksize = fsys->blocksize; memmove(root.prev, prev, VtScoreSize); vtblockput(b); b = vtcacheallocblock(c, VtRootType, VtRootSize); if(b == nil) sysfatal("vtcacheallocblock: %r"); vtrootpack(&root, b->data); if(vtblockwrite(b) < 0) sysfatal("vtblockwrite: %r"); tm = *localtime(time(0)); tm.year += 1900; tm.mon++; if(mountplace == nil) mountplace = guessmountplace(argv[0]); print("mount /%s/%d/%02d%02d%s %s:%V %d/%02d%02d/%02d%02d\n", mountname, tm.year, tm.mon, tm.mday, mountplace, root.type, b->score, tm.year, tm.mon, tm.mday, tm.hour, tm.min); print("# %T %s %s:%V\n", argv[0], root.type, b->score); if(statustime) print("# %T venti sync\n"); vtblockput(b); if(vtsync(z) < 0) sysfatal("vtsync: %r"); if(statustime) print("# %T synced\n"); fsysclose(fsys); diskclose(disk); vtcachefree(zcache); // Vtgoodbye hangs on Linux - not sure why. // Probably vtfcallrpc doesn't quite get the // remote hangup right. Also probably related // to the vtrecvproc problem below. // vtgoodbye(z); // Leak here, because I can't seem to make // the vtrecvproc exit. // vtfreeconn(z); free(tmpnam); z = nil; zcache = nil; fsys = nil; disk = nil; threadexitsall(nil); }
static VtFile * vtfilealloc(VtCache *c, VtBlock *b, VtFile *p, u32int offset, int mode) { int epb; u32int size; VtEntry e; VtFile *r; assert(p==nil || ISLOCKED(p)); if(p == nil){ assert(offset == 0); epb = 1; }else epb = p->dsize / VtEntrySize; if(b->type != VtDirType){ werrstr("bad block type %#uo", b->type); return nil; } /* * a non-active entry is the only thing that * can legitimately happen here. all the others * get prints. */ if(vtentryunpack(&e, b->data, offset % epb) < 0){ fprint(2, "vtentryunpack failed: %r (%.*H)\n", VtEntrySize, b->data+VtEntrySize*(offset%epb)); return nil; } if(!(e.flags & VtEntryActive)){ werrstr("entry not active"); return nil; } if(DEPTH(e.type) < sizetodepth(e.size, e.psize, e.dsize)){ fprint(2, "depth %ud size %llud psize %ud dsize %ud\n", DEPTH(e.type), e.size, e.psize, e.dsize); werrstr("bad depth"); return nil; } size = vtcacheblocksize(c); if(e.dsize > size || e.psize > size){ werrstr("block sizes %ud, %ud bigger than cache block size %ud", e.psize, e.dsize, size); return nil; } r = vtmallocz(sizeof(VtFile)); r->c = c; r->mode = mode; r->dsize = e.dsize; r->psize = e.psize; r->gen = e.gen; r->dir = (e.type & VtTypeBaseMask) == VtDirType; r->ref = 1; r->parent = p; if(p){ qlock(&p->lk); assert(mode == VtOREAD || p->mode == VtORDWR); p->ref++; qunlock(&p->lk); }else{ assert(b->addr != NilBlock); r->local = 1; } memmove(r->score, b->score, VtScoreSize); r->offset = offset; r->epb = epb; return r; }
VtFile* _vtfilecreate(VtFile *r, int o, int psize, int dsize, int type) { int i; VtBlock *b; u32int bn, size; VtEntry e; int epb; VtFile *rr; u32int offset; assert(ISLOCKED(r)); assert(psize <= VtMaxLumpSize); assert(dsize <= VtMaxLumpSize); assert(type == VtDirType || type == VtDataType); if(!r->dir){ werrstr(ENotDir); return nil; } epb = r->dsize/VtEntrySize; size = vtfilegetdirsize(r); /* * look at a random block to see if we can find an empty entry */ if(o == -1){ offset = lnrand(size+1); offset -= offset % epb; }else offset = o; /* try the given block and then try the last block */ for(;;){ bn = offset/epb; b = vtfileblock(r, bn, VtORDWR); if(b == nil) return nil; for(i=offset%r->epb; i<epb; i++){ if(vtentryunpack(&e, b->data, i) < 0) continue; if((e.flags&VtEntryActive) == 0 && e.gen != ~0) goto Found; } vtblockput(b); if(offset == size){ fprint(2, "vtfilecreate: cannot happen\n"); werrstr("vtfilecreate: cannot happen"); return nil; } offset = size; } Found: /* found an entry - gen already set */ e.psize = psize; e.dsize = dsize; e.flags = VtEntryActive; e.type = type; e.size = 0; memmove(e.score, vtzeroscore, VtScoreSize); vtentrypack(&e, b->data, i); offset = bn*epb + i; if(offset+1 > size){ if(vtfilesetdirsize(r, offset+1) < 0){ vtblockput(b); return nil; } } rr = vtfilealloc(r->c, b, r, offset, VtORDWR); vtblockput(b); return rr; }