int
vtfileflush(VtFile *f)
{
int ret;
VtBlock *b;
VtEntry e;
assert(ISLOCKED(f));
b = fileload(f, &e);
if(!(e.flags&VtEntryLocal)){
vtblockput(b);
return 0;
}
ret = flushblock(f->c, nil, e.score, e.psize/VtScoreSize, e.dsize/VtEntrySize,
e.type);
if(ret < 0){
vtblockput(b);
return -1;
}
vtentrypack(&e, b->data, f->offset % f->epb);
vtblockput(b);
return 0;
}
/*
* Change the depth of the VtFile r.
* The entry e for r is contained in block p.
*/
static int
growdepth(VtFile *r, VtBlock *p, VtEntry *e, int depth)
{
VtBlock *b, *bb;
assert(ISLOCKED(r));
assert(depth <= VtPointerDepth);
b = vtcacheglobal(r->c, e->score, e->type);
if(b == nil)
return -1;
/*
* Keep adding layers until we get to the right depth
* or an error occurs.
*/
while(DEPTH(e->type) < depth){
bb = vtcacheallocblock(r->c, e->type+1);
if(bb == nil)
break;
memmove(bb->data, b->score, VtScoreSize);
memmove(e->score, bb->score, VtScoreSize);
e->type++;
e->flags |= VtEntryLocal;
vtblockput(b);
b = bb;
}
vtentrypack(e, p->data, r->offset % r->epb);
vtblockput(b);
if(DEPTH(e->type) == depth)
return 0;
return -1;
}
static int
flushblock(VtCache *c, VtBlock *bb, uint8_t score[VtScoreSize], int ppb,
int epb,
int type)
{
uint32_t addr;
VtBlock *b;
VtEntry e;
int i;
addr = vtglobaltolocal(score);
if(addr == NilBlock)
return 0;
if(bb){
b = bb;
if(memcmp(b->score, score, VtScoreSize) != 0)
abort();
}else
if((b = vtcachelocal(c, addr, type)) == nil)
return -1;
switch(type){
case VtDataType:
break;
case VtDirType:
for(i=0; i<epb; i++){
if(vtentryunpack(&e, b->data, i) < 0)
goto Err;
if(!(e.flags&VtEntryActive))
continue;
if(flushblock(c, nil, e.score, e.psize/VtScoreSize, e.dsize/VtEntrySize,
e.type) < 0)
goto Err;
vtentrypack(&e, b->data, i);
}
break;
default: /* VtPointerTypeX */
for(i=0; i<ppb; i++){
if(flushblock(c, nil, b->data+VtScoreSize*i, ppb, epb, type-1) < 0)
goto Err;
}
break;
}
if(vtblockwrite(b) < 0)
goto Err;
memmove(score, b->score, VtScoreSize);
if(b != bb)
vtblockput(b);
return 0;
Err:
if(b != bb)
vtblockput(b);
return -1;
}
static VtBlock *
blockwalk(VtBlock *p, int index, VtCache *c, int mode, VtEntry *e)
{
VtBlock *b;
int type;
uchar *score;
VtEntry oe;
switch(p->type){
case VtDataType:
assert(0);
case VtDirType:
type = e->type;
score = e->score;
break;
default:
type = p->type - 1;
score = p->data+index*VtScoreSize;
break;
}
/*print("walk from %V/%d ty %d to %V ty %d\n", p->score, index, p->type, score, type); */
if(mode == VtOWRITE && vtglobaltolocal(score) == NilBlock){
print("VtOWRITE && NilBlock\n");
b = vtcacheallocblock(c, type);
print("vtcachealloblcok returns %p\n", b);
if(b)
goto HaveCopy;
}else
b = vtcacheglobal(c, score, type);
if(b == nil || mode == VtOREAD)
return b;
if(vtglobaltolocal(b->score) != NilBlock)
return b;
oe = *e;
/*
* Copy on write.
*/
e->flags |= VtEntryLocal;
b = vtblockcopy(b/*, e->tag, fs->ehi, fs->elo*/);
if(b == nil)
return nil;
HaveCopy:
if(p->type == VtDirType){
memmove(e->score, b->score, VtScoreSize);
vtentrypack(e, p->data, index);
}else{
memmove(p->data+index*VtScoreSize, b->score, VtScoreSize);
}
return b;
}
int
vtfilesetentry(VtFile *r, VtEntry *e)
{
VtBlock *b;
VtEntry ee;
assert(ISLOCKED(r));
b = fileload(r, &ee);
if(b == nil)
return -1;
vtentrypack(e, b->data, r->offset % r->epb);
vtblockput(b);
return 0;
}
int
vtfilesetsize(VtFile *r, u64int size)
{
int depth, edepth;
VtEntry e;
VtBlock *b;
assert(ISLOCKED(r));
if(size == 0)
return vtfiletruncate(r);
if(size > VtMaxFileSize || size > ((uvlong)MaxBlock)*r->dsize){
werrstr(ETooBig);
return -1;
}
b = fileload(r, &e);
if(b == nil)
return -1;
/* quick out */
if(e.size == size){
vtblockput(b);
return 0;
}
depth = sizetodepth(size, e.psize, e.dsize);
edepth = DEPTH(e.type);
if(depth < edepth){
if(shrinkdepth(r, b, &e, depth) < 0){
vtblockput(b);
return -1;
}
}else if(depth > edepth){
if(growdepth(r, b, &e, depth) < 0){
vtblockput(b);
return -1;
}
}
if(size < e.size)
shrinksize(r, &e, size);
e.size = size;
vtentrypack(&e, b->data, r->offset % r->epb);
vtblockput(b);
return 0;
}
VtFile*
vtfileopenroot(VtCache *c, VtEntry *e)
{
VtBlock *b;
VtFile *f;
b = vtcacheallocblock(c, VtDirType);
if(b == nil)
return nil;
vtentrypack(e, b->data, 0);
f = vtfilealloc(c, b, nil, 0, VtORDWR);
vtblockput(b);
return f;
}
static int
vtfilekill(VtFile *r, int doremove)
{
VtEntry e;
VtBlock *b;
assert(ISLOCKED(r));
b = fileload(r, &e);
if(b == nil)
return -1;
if(doremove==0 && e.size == 0){
/* already truncated */
vtblockput(b);
return 0;
}
if(doremove){
if(e.gen != ~0)
e.gen++;
e.dsize = 0;
e.psize = 0;
e.flags = 0;
}else
e.flags &= ~VtEntryLocal;
e.type = 0;
e.size = 0;
memmove(e.score, vtzeroscore, VtScoreSize);
vtentrypack(&e, b->data, r->offset % r->epb);
vtblockput(b);
if(doremove){
vtfileunlock(r);
vtfileclose(r);
}
return 0;
}
void
walk(uint8_t score[VtScoreSize], uint type, int base)
{
int i, n;
uint8_t *buf;
uint8_t nscore[VtScoreSize];
VtEntry e;
VtRoot root;
if(memcmp(score, vtzeroscore, VtScoreSize) == 0 || memcmp(score, zeroscore, VtScoreSize) == 0)
return;
if(havevisited(score, type)){
nskip++;
return;
}
buf = vtmallocz(VtMaxLumpSize);
if(fast && vtread(zdst, score, type, buf, VtMaxLumpSize) >= 0){
if(verbose)
fprint(2, "skip %V\n", score);
free(buf);
return;
}
n = vtread(zsrc, score, type, buf, VtMaxLumpSize);
if(n < 0){
if(rewrite){
changes++;
memmove(score, vtzeroscore, VtScoreSize);
}else if(!ignoreerrors)
sysfatal("reading block %V (type %d): %r", score, type);
return;
}
switch(type){
case VtRootType:
if(vtrootunpack(&root, buf) < 0){
fprint(2, "warning: could not unpack root in %V %d\n", score, type);
break;
}
walk(root.prev, VtRootType, 0);
walk(root.score, VtDirType, 0);
if(rewrite)
vtrootpack(&root, buf); /* walk might have changed score */
break;
case VtDirType:
for(i=0; i*VtEntrySize<n; i++){
if(vtentryunpack(&e, buf, i) < 0){
fprint(2, "warning: could not unpack entry #%d in %V %d\n", i, score, type);
continue;
}
if(!(e.flags & VtEntryActive))
continue;
walk(e.score, e.type, e.type&VtTypeBaseMask);
/*
* Don't repack unless we're rewriting -- some old
* vac files have psize==0 and dsize==0, and these
* get rewritten by vtentryunpack to have less strange
* block sizes. So vtentryunpack; vtentrypack does not
* guarantee to preserve the exact bytes in buf.
*/
if(rewrite)
vtentrypack(&e, buf, i);
}
break;
case VtDataType:
break;
default: /* pointers */
for(i=0; i<n; i+=VtScoreSize)
if(memcmp(buf+i, vtzeroscore, VtScoreSize) != 0)
walk(buf+i, type-1, base);
break;
}
nwrite++;
if(vtwrite(zdst, nscore, type, buf, n) < 0){
/* figure out score for better error message */
/* can't use input argument - might have changed contents */
n = vtzerotruncate(type, buf, n);
sha1(buf, n, score, nil);
sysfatal("writing block %V (type %d): %r", score, type);
}
if(!rewrite && memcmp(score, nscore, VtScoreSize) != 0){
fprint(2, "not rewriting: wrote %V got %V\n", score, nscore);
abort();
sysfatal("not rewriting: wrote %V got %V", score, nscore);
}
markvisited(score, type);
free(buf);
}
void
threadmain(int argc, char **argv)
{
char *pref, *mountname, *mountplace;
uchar score[VtScoreSize], prev[VtScoreSize];
int i, fd, csize;
vlong bsize;
Tm tm;
VtEntry e;
VtBlock *b;
VtCache *c;
VtRoot root;
char *tmp, *tmpnam;
fmtinstall('F', vtfcallfmt);
fmtinstall('H', encodefmt);
fmtinstall('T', timefmt);
fmtinstall('V', vtscorefmt);
mountname = sysname();
mountplace = nil;
ARGBEGIN{
default:
usage();
break;
case 'D':
debug++;
break;
case 'V':
chattyventi = 1;
break;
case 'f':
fastwrites = 1;
break;
case 'i':
incremental = 1;
break;
case 'm':
mountname = EARGF(usage());
break;
case 'M':
mountplace = EARGF(usage());
i = strlen(mountplace);
if(i > 0 && mountplace[i-1] == '/')
mountplace[i-1] = 0;
break;
case 'n':
nop = 1;
break;
case 's':
statustime = atoi(EARGF(usage()));
break;
case 'v':
verbose = 1;
break;
case 'w':
nwritethread = atoi(EARGF(usage()));
break;
}ARGEND
if(argc != 1 && argc != 2)
usage();
if(statustime)
print("# %T vbackup %s %s\n", argv[0], argc>=2 ? argv[1] : "");
/*
* open fs
*/
if((disk = diskopenfile(argv[0])) == nil)
sysfatal("diskopen: %r");
if((disk = diskcache(disk, 32768, 2*MAXQ+16)) == nil)
sysfatal("diskcache: %r");
if((fsys = fsysopen(disk)) == nil)
sysfatal("fsysopen: %r");
/*
* connect to venti
*/
if((z = vtdial(nil)) == nil)
sysfatal("vtdial: %r");
if(vtconnect(z) < 0)
sysfatal("vtconnect: %r");
/*
* set up venti block cache
*/
zero = vtmallocz(fsys->blocksize);
bsize = fsys->blocksize;
csize = 50; /* plenty; could probably do with 5 */
if(verbose)
fprint(2, "cache %d blocks\n", csize);
c = vtcachealloc(z, bsize*csize);
zcache = c;
/*
* parse starting score
*/
memset(prev, 0, sizeof prev);
if(argc == 1){
vfile = vtfilecreateroot(c, (fsys->blocksize/VtScoreSize)*VtScoreSize,
fsys->blocksize, VtDataType);
if(vfile == nil)
sysfatal("vtfilecreateroot: %r");
vtfilelock(vfile, VtORDWR);
if(vtfilewrite(vfile, zero, 1, bsize*fsys->nblock-1) != 1)
sysfatal("vtfilewrite: %r");
if(vtfileflush(vfile) < 0)
sysfatal("vtfileflush: %r");
}else{
if(vtparsescore(argv[1], &pref, score) < 0)
sysfatal("bad score: %r");
if(pref!=nil && strcmp(pref, fsys->type) != 0)
sysfatal("score is %s but fsys is %s", pref, fsys->type);
b = vtcacheglobal(c, score, VtRootType, VtRootSize);
if(b){
if(vtrootunpack(&root, b->data) < 0)
sysfatal("bad root: %r");
if(strcmp(root.type, fsys->type) != 0)
sysfatal("root is %s but fsys is %s", root.type, fsys->type);
memmove(prev, score, VtScoreSize);
memmove(score, root.score, VtScoreSize);
vtblockput(b);
}
b = vtcacheglobal(c, score, VtDirType, VtEntrySize);
if(b == nil)
sysfatal("vtcacheglobal %V: %r", score);
if(vtentryunpack(&e, b->data, 0) < 0)
sysfatal("%V: vtentryunpack failed", score);
if(verbose)
fprint(2, "entry: size %llud psize %d dsize %d\n",
e.size, e.psize, e.dsize);
vtblockput(b);
if((vfile = vtfileopenroot(c, &e)) == nil)
sysfatal("vtfileopenroot: %r");
vtfilelock(vfile, VtORDWR);
if(e.dsize != bsize)
sysfatal("file system block sizes don't match %d %lld", e.dsize, bsize);
if(e.size != fsys->nblock*bsize)
sysfatal("file system block counts don't match %lld %lld", e.size, fsys->nblock*bsize);
}
tmpnam = nil;
if(incremental){
if(vtfilegetentry(vfile, &e) < 0)
sysfatal("vtfilegetentry: %r");
if((vscores = vtfileopenroot(c, &e)) == nil)
sysfatal("vtfileopenroot: %r");
vtfileunlock(vfile);
}else{
/*
* write scores of blocks into temporary file
*/
if((tmp = getenv("TMP")) != nil){
/* okay, good */
}else if(access("/var/tmp", 0) >= 0)
tmp = "/var/tmp";
else
tmp = "/tmp";
tmpnam = smprint("%s/vbackup.XXXXXX", tmp);
if(tmpnam == nil)
sysfatal("smprint: %r");
if((fd = opentemp(tmpnam, ORDWR|ORCLOSE)) < 0)
sysfatal("opentemp %s: %r", tmpnam);
if(statustime)
print("# %T reading scores into %s\n", tmpnam);
if(verbose)
fprint(2, "read scores into %s...\n", tmpnam);
Binit(&bscores, fd, OWRITE);
for(i=0; i<fsys->nblock; i++){
if(vtfileblockscore(vfile, i, score) < 0)
sysfatal("vtfileblockhash %d: %r", i);
if(Bwrite(&bscores, score, VtScoreSize) != VtScoreSize)
sysfatal("Bwrite: %r");
}
Bterm(&bscores);
vtfileunlock(vfile);
/*
* prep scores for rereading
*/
seek(fd, 0, 0);
Binit(&bscores, fd, OREAD);
}
/*
* start the main processes
*/
if(statustime)
print("# %T starting procs\n");
qcmp = qalloc();
qventi = qalloc();
rlock(&endlk);
proccreate(fsysproc, nil, STACK);
rlock(&endlk);
proccreate(ventiproc, nil, STACK);
rlock(&endlk);
proccreate(cmpproc, nil, STACK);
if(statustime){
rlock(&endlk);
proccreate(statusproc, nil, STACK);
}
/*
* wait for processes to finish
*/
wlock(&endlk);
qfree(qcmp);
qfree(qventi);
if(statustime)
print("# %T procs exited: %d of %d %d-byte blocks changed, "
"%d read, %d written, %d skipped, %d copied\n",
nchange, fsys->nblock, fsys->blocksize,
vtcachenread, vtcachenwrite, nskip, vtcachencopy);
/*
* prepare root block
*/
if(incremental)
vtfileclose(vscores);
vtfilelock(vfile, -1);
if(vtfileflush(vfile) < 0)
sysfatal("vtfileflush: %r");
if(vtfilegetentry(vfile, &e) < 0)
sysfatal("vtfilegetentry: %r");
vtfileunlock(vfile);
vtfileclose(vfile);
b = vtcacheallocblock(c, VtDirType, VtEntrySize);
if(b == nil)
sysfatal("vtcacheallocblock: %r");
vtentrypack(&e, b->data, 0);
if(vtblockwrite(b) < 0)
sysfatal("vtblockwrite: %r");
memset(&root, 0, sizeof root);
strecpy(root.name, root.name+sizeof root.name, argv[0]);
strecpy(root.type, root.type+sizeof root.type, fsys->type);
memmove(root.score, b->score, VtScoreSize);
root.blocksize = fsys->blocksize;
memmove(root.prev, prev, VtScoreSize);
vtblockput(b);
b = vtcacheallocblock(c, VtRootType, VtRootSize);
if(b == nil)
sysfatal("vtcacheallocblock: %r");
vtrootpack(&root, b->data);
if(vtblockwrite(b) < 0)
sysfatal("vtblockwrite: %r");
tm = *localtime(time(0));
tm.year += 1900;
tm.mon++;
if(mountplace == nil)
mountplace = guessmountplace(argv[0]);
print("mount /%s/%d/%02d%02d%s %s:%V %d/%02d%02d/%02d%02d\n",
mountname, tm.year, tm.mon, tm.mday,
mountplace,
root.type, b->score,
tm.year, tm.mon, tm.mday, tm.hour, tm.min);
print("# %T %s %s:%V\n", argv[0], root.type, b->score);
if(statustime)
print("# %T venti sync\n");
vtblockput(b);
if(vtsync(z) < 0)
sysfatal("vtsync: %r");
if(statustime)
print("# %T synced\n");
fsysclose(fsys);
diskclose(disk);
vtcachefree(zcache);
// Vtgoodbye hangs on Linux - not sure why.
// Probably vtfcallrpc doesn't quite get the
// remote hangup right. Also probably related
// to the vtrecvproc problem below.
// vtgoodbye(z);
// Leak here, because I can't seem to make
// the vtrecvproc exit.
// vtfreeconn(z);
free(tmpnam);
z = nil;
zcache = nil;
fsys = nil;
disk = nil;
threadexitsall(nil);
}
static int
shrinkdepth(VtFile *r, VtBlock *p, VtEntry *e, int depth)
{
VtBlock *b, *nb, *ob, *rb;
VtEntry oe;
assert(ISLOCKED(r));
assert(depth <= VtPointerDepth);
rb = vtcacheglobal(r->c, e->score, e->type);
if(rb == nil)
return -1;
/*
* Walk down to the new root block.
* We may stop early, but something is better than nothing.
*/
oe = *e;
ob = nil;
b = rb;
for(; DEPTH(e->type) > depth; e->type--){
nb = vtcacheglobal(r->c, b->data, e->type-1);
if(nb == nil)
break;
if(ob!=nil && ob!=rb)
vtblockput(ob);
ob = b;
b = nb;
}
if(b == rb){
vtblockput(rb);
return 0;
}
/*
* Right now, e points at the root block rb, b is the new root block,
* and ob points at b. To update:
*
* (i) change e to point at b
* (ii) zero the pointer ob -> b
* (iii) free the root block
*
* p (the block containing e) must be written before
* anything else.
*/
/* (i) */
memmove(e->score, b->score, VtScoreSize);
vtentrypack(e, p->data, r->offset % r->epb);
/* (ii) */
memmove(ob->data, vtzeroscore, VtScoreSize);
/* (iii) */
vtblockput(rb);
if(ob!=nil && ob!=rb)
vtblockput(ob);
vtblockput(b);
if(DEPTH(e->type) == depth)
return 0;
return -1;
}
VtFile*
_vtfilecreate(VtFile *r, int o, int psize, int dsize, int type)
{
int i;
VtBlock *b;
u32int bn, size;
VtEntry e;
int epb;
VtFile *rr;
u32int offset;
assert(ISLOCKED(r));
assert(psize <= VtMaxLumpSize);
assert(dsize <= VtMaxLumpSize);
assert(type == VtDirType || type == VtDataType);
if(!r->dir){
werrstr(ENotDir);
return nil;
}
epb = r->dsize/VtEntrySize;
size = vtfilegetdirsize(r);
/*
* look at a random block to see if we can find an empty entry
*/
if(o == -1){
offset = lnrand(size+1);
offset -= offset % epb;
}else
offset = o;
/* try the given block and then try the last block */
for(;;){
bn = offset/epb;
b = vtfileblock(r, bn, VtORDWR);
if(b == nil)
return nil;
for(i=offset%r->epb; i<epb; i++){
if(vtentryunpack(&e, b->data, i) < 0)
continue;
if((e.flags&VtEntryActive) == 0 && e.gen != ~0)
goto Found;
}
vtblockput(b);
if(offset == size){
fprint(2, "vtfilecreate: cannot happen\n");
werrstr("vtfilecreate: cannot happen");
return nil;
}
offset = size;
}
Found:
/* found an entry - gen already set */
e.psize = psize;
e.dsize = dsize;
e.flags = VtEntryActive;
e.type = type;
e.size = 0;
memmove(e.score, vtzeroscore, VtScoreSize);
vtentrypack(&e, b->data, i);
offset = bn*epb + i;
if(offset+1 > size){
if(vtfilesetdirsize(r, offset+1) < 0){
vtblockput(b);
return nil;
}
}
rr = vtfilealloc(r->c, b, r, offset, VtORDWR);
vtblockput(b);
return rr;
}
int
vtfileflushbefore(VtFile *r, u64int offset)
{
VtBlock *b, *bb;
VtEntry e;
int i, base, depth, ppb, epb, doflush;
int index[VtPointerDepth+1], j, ret;
VtBlock *bi[VtPointerDepth+2];
uchar *score;
assert(ISLOCKED(r));
if(offset == 0)
return 0;
b = fileload(r, &e);
if(b == nil)
return -1;
/*
* compute path through tree for the last written byte and the next one.
*/
ret = -1;
memset(bi, 0, sizeof bi);
depth = DEPTH(e.type);
bi[depth+1] = b;
i = mkindices(&e, (offset-1)/e.dsize, index);
if(i < 0)
goto Err;
if(i > depth)
goto Err;
ppb = e.psize / VtScoreSize;
epb = e.dsize / VtEntrySize;
/*
* load the blocks along the last written byte
*/
index[depth] = r->offset % r->epb;
for(i=depth; i>=0; i--){
bb = blockwalk(b, index[i], r->c, VtORDWR, &e);
if(bb == nil)
goto Err;
bi[i] = bb;
b = bb;
}
ret = 0;
/*
* walk up the path from leaf to root, flushing anything that
* has been finished.
*/
base = e.type&~VtTypeDepthMask;
for(i=0; i<=depth; i++){
doflush = 0;
if(i == 0){
/* leaf: data or dir block */
if(offset%e.dsize == 0)
doflush = 1;
}else{
/*
* interior node: pointer blocks.
* specifically, b = bi[i] is a block whose index[i-1]'th entry
* points at bi[i-1].
*/
b = bi[i];
/*
* the index entries up to but not including index[i-1] point at
* finished blocks, so flush them for sure.
*/
for(j=0; j<index[i-1]; j++)
if(flushblock(r->c, nil, b->data+j*VtScoreSize, ppb, epb, base+i-1) < 0)
goto Err;
/*
* if index[i-1] is the last entry in the block and is global
* (i.e. the kid is flushed), then we can flush this block.
*/
if(j==ppb-1 && vtglobaltolocal(b->data+j*VtScoreSize)==NilBlock)
doflush = 1;
}
if(doflush){
if(i == depth)
score = e.score;
else
score = bi[i+1]->data+index[i]*VtScoreSize;
if(flushblock(r->c, bi[i], score, ppb, epb, base+i) < 0)
goto Err;
}
}
Err:
/* top: entry. do this always so that the score is up-to-date */
vtentrypack(&e, bi[depth+1]->data, index[depth]);
for(i=0; i<nelem(bi); i++)
if(bi[i])
vtblockput(bi[i]);
return ret;
}