static int browse_manifest_start(struct asfd *srfd, struct cstat *cstat,
struct bu *bu, const char *browse, struct conf **confs)
{
int ret=-1;
char *manifest=NULL;
struct sbuf *sb=NULL;
struct manio *manio=NULL;
if(!(manifest=prepend_s(bu->path,
cstat->protocol==PROTO_1?"manifest.gz":"manifest"))
|| !(manio=manio_alloc())
|| manio_init_read(manio, manifest)
|| !(sb=sbuf_alloc_protocol(cstat->protocol)))
goto end;
manio_set_protocol(manio, cstat->protocol);
if(get_int(confs[OPT_MONITOR_BROWSE_CACHE]))
ret=cache_load(srfd, manio, sb, cstat, bu);
else
ret=do_browse_manifest(srfd, manio, sb, browse);
end:
free_w(&manifest);
manio_free(&manio);
sbuf_free(&sb);
return ret;
}
// This is basically backup_phase3_server() from protocol1. It used to merge the
// unchanged and changed data into a single file. Now it splits the manifests
// into several files.
int backup_phase3_server_protocol2(struct sdirs *sdirs, struct conf **confs)
{
int ret=1;
int pcmp=0;
char *hooksdir=NULL;
char *dindexdir=NULL;
char *manifesttmp=NULL;
struct sbuf *usb=NULL;
struct sbuf *csb=NULL;
struct blk *blk=NULL;
int finished_ch=0;
int finished_un=0;
struct manio *newmanio=NULL;
struct manio *chmanio=NULL;
struct manio *unmanio=NULL;
uint64_t fcount=0;
logp("Start phase3\n");
if(!(manifesttmp=get_tmp_filename(sdirs->rmanifest))
|| !(newmanio=manio_alloc())
|| !(chmanio=manio_alloc())
|| !(unmanio=manio_alloc())
|| !(hooksdir=prepend_s(manifesttmp, "hooks"))
|| !(dindexdir=prepend_s(manifesttmp, "dindex"))
|| manio_init_write(newmanio, manifesttmp)
|| manio_init_write_hooks(newmanio,
get_string(confs[OPT_DIRECTORY]), hooksdir, sdirs->rmanifest)
|| manio_init_write_dindex(newmanio, dindexdir)
|| manio_init_read(chmanio, sdirs->changed)
|| manio_init_read(unmanio, sdirs->unchanged)
|| !(usb=sbuf_alloc(confs))
|| !(csb=sbuf_alloc(confs)))
goto end;
while(!finished_ch || !finished_un)
{
if(!blk && !(blk=blk_alloc())) goto end;
if(!finished_un
&& usb
&& !usb->path.buf)
{
switch(manio_sbuf_fill(unmanio, NULL /* no async */,
usb, NULL, NULL, confs))
{
case -1: goto end;
case 1: finished_un++;
}
}
if(!finished_ch
&& csb
&& !csb->path.buf)
{
switch(manio_sbuf_fill(chmanio, NULL /* no async */,
csb, NULL, NULL, confs))
{
case -1: goto end;
case 1: finished_ch++;
}
}
if((usb && usb->path.buf) && (!csb || !csb->path.buf))
{
switch(manio_copy_entry(NULL /* no async */,
&usb, usb,
&blk, unmanio, newmanio, confs))
{
case -1: goto end;
case 1: finished_un++;
}
}
else if((!usb || !usb->path.buf) && (csb && csb->path.buf))
{
switch(manio_copy_entry(NULL /* no async */,
&csb, csb, &blk, chmanio, newmanio, confs))
{
case -1: goto end;
case 1: finished_ch++;
}
}
else if((!usb || !usb->path.buf) && (!csb || !(csb->path.buf)))
{
continue;
}
else if(!(pcmp=sbuf_pathcmp(usb, csb)))
{
// They were the same - write one.
switch(manio_copy_entry(NULL /* no async */,
&csb, csb, &blk, chmanio, newmanio, confs))
{
case -1: goto end;
case 1: finished_ch++;
}
}
else if(pcmp<0)
{
switch(manio_copy_entry(NULL /* no async */,
&usb, usb, &blk, unmanio, newmanio, confs))
{
case -1: goto end;
case 1: finished_un++;
}
}
else
{
switch(manio_copy_entry(NULL /* no async */,
&csb, csb, &blk, chmanio, newmanio, confs))
{
case -1: goto end;
case 1: finished_ch++;
}
}
}
fcount=newmanio->fcount;
// Flush to disk and set up for reading.
if(manio_free(&newmanio)
|| !(newmanio=manio_alloc())
|| manio_init_read(newmanio, sdirs->rmanifest))
goto end;
// Rename race condition should be of no consequence here, as the
// manifest should just get recreated automatically.
if(do_rename(manifesttmp, sdirs->rmanifest))
goto end;
else
{
recursive_delete(sdirs->changed, NULL, 1);
recursive_delete(sdirs->unchanged, NULL, 1);
}
if(sparse_generation(newmanio, fcount, sdirs, confs))
goto end;
ret=0;
logp("End phase3\n");
end:
manio_free(&newmanio);
manio_free(&chmanio);
manio_free(&unmanio);
sbuf_free(&csb);
sbuf_free(&usb);
blk_free(&blk);
free_w(&hooksdir);
free_w(&dindexdir);
free_w(&manifesttmp);
return ret;
}
int backup_phase2_server(struct async *as, struct sdirs *sdirs,
const char *manifest_dir, int resume, struct conf *conf)
{
int ret=-1;
int sigs_end=0;
int backup_end=0;
int requests_end=0;
int blk_requests_end=0;
struct slist *slist=NULL;
struct blist *blist=NULL;
struct iobuf *wbuf=NULL;
struct dpth *dpth=NULL;
struct manio *cmanio=NULL; // current manifest
struct manio *p1manio=NULL; // phase1 scan manifest
struct manio *chmanio=NULL; // changed manifest
struct manio *unmanio=NULL; // unchanged manifest
// This is used to tell the client that a number of consecutive blocks
// have been found and can be freed.
uint64_t wrap_up=0;
// Main fd is first in the list.
struct asfd *asfd=as->asfd;
// Champ chooser fd is second in the list.
struct asfd *chfd=asfd->next;
logp("Phase 2 begin (recv backup data)\n");
//if(champ_chooser_init(sdirs->data, conf)
if(!(cmanio=manio_alloc())
|| !(p1manio=manio_alloc())
|| !(chmanio=manio_alloc())
|| !(unmanio=manio_alloc())
|| manio_init_read(cmanio, sdirs->cmanifest)
|| manio_init_read(p1manio, sdirs->phase1data)
|| manio_init_write(chmanio, sdirs->changed)
|| manio_init_write(unmanio, sdirs->unchanged)
|| !(slist=slist_alloc())
|| !(blist=blist_alloc())
|| !(wbuf=iobuf_alloc())
|| !(dpth=dpth_alloc(sdirs->data))
|| dpth_init(dpth))
goto end;
// The phase1 manifest looks the same as a burp1 one.
manio_set_protocol(p1manio, PROTO_BURP1);
while(!backup_end)
{
if(maybe_add_from_scan(asfd,
p1manio, cmanio, unmanio, slist, conf))
goto end;
if(!wbuf->len)
{
if(get_wbuf_from_sigs(wbuf, slist, blist,
sigs_end, &blk_requests_end, dpth, conf))
goto end;
if(!wbuf->len)
{
get_wbuf_from_files(wbuf, slist,
p1manio, &requests_end);
}
}
if(wbuf->len)
asfd->append_all_to_write_buffer(asfd, wbuf);
append_for_champ_chooser(chfd, blist, sigs_end);
if(as->read_write(as))
{
logp("error in %s\n", __func__);
goto end;
}
while(asfd->rbuf->buf)
{
if(deal_with_read(asfd->rbuf, slist, blist,
conf, &sigs_end, &backup_end, dpth))
goto end;
// Get as much out of the
// readbuf as possible.
if(asfd->parse_readbuf(asfd)) goto end;
}
while(chfd->rbuf->buf)
{
if(deal_with_read_from_chfd(asfd, chfd,
blist, &wrap_up, dpth)) goto end;
// Get as much out of the
// readbuf as possible.
if(chfd->parse_readbuf(chfd)) goto end;
}
if(write_to_changed_file(asfd, chfd, chmanio,
slist, blist, dpth, backup_end, conf))
goto end;
}
// Hack: If there are some entries left after the last entry that
// contains block data, it will not be written to the changed file
// yet because the last entry of block data has not had
// sb->burp2->bend set.
if(slist->head && slist->head->next)
{
slist->head=slist->head->next;
if(write_to_changed_file(asfd, chfd, chmanio,
slist, blist, dpth, backup_end, conf))
goto end;
}
if(manio_close(unmanio)
|| manio_close(chmanio))
goto end;
if(blist->head)
{
logp("ERROR: finishing but still want block: %lu\n",
blist->head->index);
goto end;
}
// Need to release the last left. There should be one at most.
if(dpth->head && dpth->head->next)
{
logp("ERROR: More data locks remaining after: %s\n",
dpth->head->save_path);
goto end;
}
if(dpth_release_all(dpth)) goto end;
ret=0;
end:
logp("End backup\n");
slist_free(slist);
blist_free(blist);
iobuf_free_content(asfd->rbuf);
iobuf_free_content(chfd->rbuf);
// Write buffer did not allocate 'buf'.
if(wbuf) wbuf->buf=NULL;
iobuf_free(wbuf);
dpth_release_all(dpth);
dpth_free(&dpth);
manio_free(&cmanio);
manio_free(&p1manio);
manio_free(&chmanio);
manio_free(&unmanio);
return ret;
}
int backup_phase4_server_protocol2(struct sdirs *sdirs, struct conf **confs)
{
int ret=-1;
uint64_t i=0;
uint64_t pass=0;
char *sparse=NULL;
char *global_sparse=NULL;
char *h1dir=NULL;
char *h2dir=NULL;
char *hooksdir=NULL;
char *srca=NULL;
char *srcb=NULL;
char *dst=NULL;
char compa[32]="";
char compb[32]="";
char compd[32]="";
struct manio *newmanio=NULL;
char *logpath=NULL;
char *fmanifest=NULL; // FIX THIS: should be part of sdirs.
if(!(logpath=prepend_s(sdirs->finishing, "log")))
goto end;
if(set_logfp(logpath, confs))
goto end;
logp("Begin phase4 (sparse generation)\n");
if(!(newmanio=manio_alloc())
|| !(fmanifest=prepend_s(sdirs->finishing, "manifest"))
|| manio_init_read(newmanio, fmanifest)
|| manio_read_fcount(newmanio)
|| !(hooksdir=prepend_s(fmanifest, "hooks"))
|| !(h1dir=prepend_s(fmanifest, "h1"))
|| !(h2dir=prepend_s(fmanifest, "h2")))
goto end;
while(1)
{
char *srcdir=NULL;
char *dstdir=NULL;
if(!pass)
{
srcdir=hooksdir;
dstdir=h1dir;
}
else if(pass%2)
{
srcdir=h1dir;
dstdir=h2dir;
}
else
{
srcdir=h2dir;
dstdir=h1dir;
}
pass++;
for(i=0; i<newmanio->offset.fcount; i+=2)
{
free_w(&srca);
free_w(&srcb);
free_w(&dst);
snprintf(compa, sizeof(compa), "%08"PRIX64, i);
snprintf(compb, sizeof(compb), "%08"PRIX64, i+1);
snprintf(compd, sizeof(compd), "%08"PRIX64, i/2);
if(!(srca=prepend_s(srcdir, compa))
|| !(dst=prepend_s(dstdir, compd)))
goto end;
if(i+1<newmanio->offset.fcount
&& !(srcb=prepend_s(srcdir, compb)))
goto end;
if(merge_sparse_indexes(srca, srcb, dst, confs))
goto end;
}
if((newmanio->offset.fcount=i/2)<2) break;
}
if(!(sparse=prepend_s(fmanifest, "sparse"))
|| !(global_sparse=prepend_s(sdirs->data, "sparse")))
goto end;
// FIX THIS: nasty race condition here needs to be automatically
// recoverable.
if(do_rename(dst, sparse)) goto end;
if(merge_into_global_sparse(sparse, global_sparse, confs)) goto end;
logp("End phase4 (sparse generation)\n");
ret=0;
end:
manio_free(&newmanio);
free_w(&sparse);
free_w(&global_sparse);
free_w(&srca);
free_w(&srcb);
recursive_delete(h1dir, NULL, 1);
recursive_delete(h2dir, NULL, 1);
free_w(&h1dir);
free_w(&h2dir);
free_w(&logpath);
free_w(&fmanifest);
return ret;
}
static int list_manifest(struct asfd *asfd,
const char *fullpath, regex_t *regex,
const char *browsedir, struct conf *conf)
{
int ars=0;
int ret=0;
struct sbuf *sb=NULL;
struct manio *manio=NULL;
char *manifest_dir=NULL;
char *last_bd_match=NULL;
size_t bdlen=0;
if(!(manifest_dir=prepend_s(fullpath,
conf->protocol==PROTO_BURP1?"manifest.gz":"manifest"))
|| !(manio=manio_alloc())
|| manio_init_read(manio, manifest_dir)
|| !(sb=sbuf_alloc(conf)))
{
log_and_send_oom(asfd, __func__);
goto error;
}
manio_set_protocol(manio, conf->protocol);
if(browsedir) bdlen=strlen(browsedir);
while(1)
{
int show=0;
if((ars=manio_sbuf_fill(manio, asfd, sb, NULL, NULL, conf))<0)
goto error;
else if(ars>0)
goto end; // Finished OK.
if(write_status(STATUS_LISTING, sb->path.buf, conf))
goto error;
if(browsedir)
{
int r;
if((r=check_browsedir(browsedir,
&sb->path.buf, bdlen, &last_bd_match))<0)
goto error;
if(!r) continue;
show++;
}
else
{
if(check_regex(regex, sb->path.buf))
show++;
}
if(show)
{
if(write_wrapper(asfd, &sb->attr)
|| write_wrapper(asfd, &sb->path))
goto error;
if(sbuf_is_link(sb)
&& write_wrapper(asfd, &sb->link))
goto error;
}
sbuf_free_content(sb);
}
error:
ret=-1;
end:
sbuf_free(&sb);
free_w(&manifest_dir);
manio_free(&manio);
free_w(&last_bd_match);
return ret;
}
int backup_phase2_server_protocol1(struct async *as, struct sdirs *sdirs,
const char *incexc, int resume, struct conf **cconfs)
{
int ret=0;
struct manio *p1manio=NULL;
struct dpth *dpth=NULL;
char *deltmppath=NULL;
char *last_requested=NULL;
// Where to write changed data.
// Data is not getting written to a compressed file.
// This is important for recovery if the power goes.
struct fzp *chfp=NULL;
struct fzp *ucfp=NULL; // unchanged data
struct fzp *cmanfp=NULL; // previous (current) manifest.
struct sbuf *cb=NULL; // file list in current manifest
struct sbuf *p1b=NULL; // file list from client
struct sbuf *rb=NULL; // receiving file from client
struct asfd *asfd=as->asfd;
int breaking=0;
int breakcount=0;
if(get_int(cconfs[OPT_BREAKPOINT])>=2000
&& get_int(cconfs[OPT_BREAKPOINT])<3000)
{
breaking=get_int(cconfs[OPT_BREAKPOINT]);
breakcount=breaking-2000;
}
logp("Begin phase2 (receive file data)\n");
if(!(dpth=dpth_alloc())
|| dpth_protocol1_init(dpth, sdirs->currentdata,
get_int(cconfs[OPT_MAX_STORAGE_SUBDIRS])))
goto error;
if(open_previous_manifest(&cmanfp, sdirs, incexc, cconfs))
goto error;
if(get_int(cconfs[OPT_DIRECTORY_TREE]))
{
// Need to make sure we do not try to create a path that is
// too long.
if(build_path_w(sdirs->treepath)) goto error;
treepathlen=strlen(sdirs->treepath);
init_fs_max(sdirs->treepath);
}
if(!(p1manio=manio_alloc())
|| manio_init_read(p1manio, sdirs->phase1data)
|| !(cb=sbuf_alloc(cconfs))
|| !(p1b=sbuf_alloc(cconfs))
|| !(rb=sbuf_alloc(cconfs)))
goto error;
manio_set_protocol(p1manio, PROTO_1);
if(resume && do_resume(p1manio, sdirs, dpth, cconfs))
goto error;
// Unchanged and changed should now be truncated correctly, we just
// have to open them for appending.
if(!(ucfp=fzp_open(sdirs->unchanged, "a+b"))
|| !(chfp=fzp_open(sdirs->changed, "a+b")))
goto error;
if(manio_closed(p1manio)
&& manio_open_next_fpath(p1manio))
goto error;
while(1)
{
if(breaking)
{
if(breakcount--==0) return breakpoint(cconfs, __func__);
}
//printf("in loop, %s %s %c\n",
// cmanfp?"got cmanfp":"no cmanfp",
// rb->path.buf?:"no rb->path",
// rb->path.buf?'X':rb->path.cmd);
if(write_status(CNTR_STATUS_BACKUP,
rb->path.buf?rb->path.buf:p1b->path.buf, cconfs))
goto error;
if(last_requested
|| manio_closed(p1manio)
|| asfd->writebuflen)
{
switch(do_stuff_to_receive(asfd, sdirs,
cconfs, rb, chfp, dpth, &last_requested))
{
case 0: break;
case 1: goto end; // Finished ok.
case -1: goto error;
}
}
switch(do_stuff_to_send(asfd, p1b, &last_requested))
{
case 0: break;
case 1: continue;
case -1: goto error;
}
if(manio_closed(p1manio)) continue;
sbuf_free_content(p1b);
switch(manio_sbuf_fill_phase1(p1manio, asfd,
p1b, NULL, sdirs, cconfs))
{
case 0: break;
case 1: manio_close(p1manio);
if(asfd->write_str(asfd,
CMD_GEN, "backupphase2end")) goto error;
break;
case -1: goto error;
}
if(!cmanfp)
{
// No old manifest, need to ask for a new file.
if(process_new(sdirs, cconfs, p1b, ucfp))
goto error;
continue;
}
// Have an old manifest, look for it there.
// Might already have it, or be ahead in the old
// manifest.
if(cb->path.buf) switch(maybe_process_file(asfd,
sdirs, cb, p1b, ucfp, cconfs))
{
case 0: break;
case 1: continue;
case -1: goto error;
}
while(cmanfp)
{
sbuf_free_content(cb);
switch(sbufl_fill(cb, asfd, cmanfp, cconfs))
{
case 0: break;
case 1: fzp_close(&cmanfp);
if(process_new(sdirs, cconfs, p1b,
ucfp)) goto error;
continue;
case -1: goto error;
}
switch(maybe_process_file(asfd, sdirs,
cb, p1b, ucfp, cconfs))
{
case 0: continue;
case 1: break;
case -1: goto error;
}
break;
}
}
error:
ret=-1;
end:
if(fzp_close(&chfp))
{
logp("error closing %s in %s\n", sdirs->changed, __func__);
ret=-1;
}
if(fzp_close(&ucfp))
{
logp("error closing %s in %s\n", sdirs->unchanged, __func__);
ret=-1;
}
free_w(&deltmppath);
sbuf_free(&cb);
sbuf_free(&p1b);
sbuf_free(&rb);
manio_free(&p1manio);
fzp_close(&cmanfp);
dpth_free(&dpth);
if(!ret) unlink(sdirs->phase1data);
logp("End phase2 (receive file data)\n");
return ret;
}