// Return -1 for error, 0 for stuff read OK, 1 for end of files.
int manio_sbuf_fill(struct manio *manio, struct asfd *asfd,
struct sbuf *sb, struct blk *blk,
struct dpth *dpth, struct conf *conf)
{
int ars;
while(1)
{
if(!manio->zp)
{
if(open_next_fpath(manio)) goto error;
if(!manio->zp) return 1; // No more files to read.
manio->first_entry=1;
}
else
{
manio->first_entry=0;
}
if((ars=sbuf_fill_from_gzfile(sb, asfd, manio->zp, blk,
dpth?dpth->base_path:NULL, conf))<0) goto error;
else if(!ars)
return 0; // Got something.
// Reached the end of the current file.
// Maybe there is another file to continue with.
if(manio_close(manio)) goto error;
// If in burp1 mode, there is only one file, so end.
if(manio->protocol==PROTO_BURP1) return 1;
}
error:
manio_close(manio);
return -1;
}
END_TEST
static void test_manifest_tell_seek(enum protocol protocol, int phase)
{
struct slist *slist;
struct manio *manio;
struct sbuf *sb=NULL;
man_off_t *offset=NULL;
int entries=1000;
prng_init(0);
base64_init();
hexmap_init();
recursive_delete(path);
slist=build_manifest(path, protocol, entries, phase);
fail_unless(slist!=NULL);
sb=slist->head;
fail_unless((manio=do_manio_open(path, "rb", protocol, phase))!=NULL);
read_manifest(&sb, manio, 0, entries/2, protocol, phase);
fail_unless((offset=manio_tell(manio))!=NULL);
fail_unless(sb!=NULL);
fail_unless(!manio_close(&manio));
fail_unless((manio=do_manio_open(path, "rb", protocol, phase))!=NULL);
fail_unless(!manio_seek(manio, offset));
read_manifest(&sb, manio, entries/2, entries, protocol, phase);
fail_unless(sb==NULL);
fail_unless(!manio_close(&manio));
fail_unless(!manio);
slist_free(&slist);
man_off_t_free(&offset);
tear_down();
}
int manios_close(struct manios **manios)
{
int ret=0;
if(!manios || !*manios) return 0;
ret|=manio_close(&(*manios)->current);
ret|=manio_close(&(*manios)->phase1);
ret|=manio_close(&(*manios)->changed);
ret|=manio_close(&(*manios)->unchanged);
free_v((void **)manios);
return ret;
}
struct slist *build_manifest_with_data_files(const char *path,
const char *datapath, int entries, int data_files)
{
struct blk *b=NULL;
struct slist *slist=NULL;
struct manio *manio=NULL;
struct fzp *fzp=NULL;
char spath[256]="";
char cpath[256]="";
fail_unless((manio=manio_open_phase3(path, "wb", PROTO_2,
RMANIFEST_RELATIVE))!=NULL);
slist=do_build_manifest(manio, PROTO_2, entries, data_files);
fail_unless(!manio_close(&manio));
for(b=slist->blist->head; b; b=b->next)
{
snprintf(spath, sizeof(spath), "%s/%s", datapath,
uint64_to_savepathstr(b->savepath));
if(strcmp(spath, cpath))
{
snprintf(cpath, sizeof(cpath), "%s", spath);
fzp_close(&fzp);
}
if(!fzp)
{
fail_unless(!build_path_w(cpath));
fail_unless((fzp=fzp_open(cpath, "wb"))!=NULL);
}
fzp_printf(fzp, "%c%04X%s", CMD_DATA, strlen("data"), "data");
}
fzp_close(&fzp);
return slist;
}
static struct manio *do_manio_open(const char *manifest, const char *mode,
enum protocol protocol, int phase)
{
struct manio *manio=NULL;
if(!(manio=manio_alloc())
|| !(manio->manifest=strdup_w(manifest, __func__))
|| !(manio->mode=strdup_w(mode, __func__))
|| !(manio->offset=man_off_t_alloc()))
goto error;
manio->protocol=protocol;
manio->phase=phase;
if(!strcmp(manio->mode, MANIO_MODE_APPEND))
{
if(manio->phase!=2)
{
logp("manio append mode only works for phase 2.\n");
logp("%s has phase: %s\n",
manio->manifest, manio->phase);
goto error;
}
if(manio_open_last_fpath(manio))
goto error;
}
else
{
if(manio_open_next_fpath(manio))
goto error;
}
return manio;
error:
manio_close(&manio);
return NULL;
}
static int manio_set_mode(struct manio *manio, const char *mode)
{
if(manio_close(manio)) return -1;
free_w(&manio->mode);
if(!(manio->mode=strdup_w(mode, __func__)))
return -1;
manio->offset.fcount=0;
return 0;
}
// Return -1 for error, 0 for stuff read OK, 1 for end of files.
static int do_manio_sbuf_fill(struct manio *manio, struct asfd *asfd,
struct sbuf *sb, struct blk *blk,
struct sdirs *sdirs, struct conf **confs, int phase1)
{
int ars;
while(1)
{
if(!manio->fzp)
{
if(manio_open_next_fpath(manio)) goto error;
if(!manio->fzp) return 1; // No more files to read.
}
if(manio->protocol==PROTO_2 || phase1)
{
ars=sbuf_fill(sb, asfd, manio->fzp, blk,
sdirs?sdirs->data:NULL, confs);
}
else
{
ars=sbufl_fill(sb, asfd, manio->fzp, confs);
}
switch(ars)
{
case 0: return 0; // Got something.
case 1: break; // Keep going.
default: goto error; // Error.
}
// Reached the end of the current file.
// Maybe there is another file to continue with.
if(manio_close(manio)) goto error;
// If in protocol1 mode, there is only one file, so end.
if(manio->protocol==PROTO_1) return 1;
}
error:
manio_close(manio);
return -1;
}
man_off_t *do_resume(struct sdirs *sdirs,
struct dpth *dpth, struct conf **cconfs)
{
man_off_t *p1pos=NULL;
struct manio *cmanio=NULL;
struct manio *umanio=NULL;
struct manio *p1manio=NULL;
enum protocol protocol=get_protocol(cconfs);
logp("Begin phase1 (read previous file system scan)\n");
if(!(p1manio=manio_open_phase1(sdirs->phase1data, "rb", protocol))
|| read_phase1(p1manio, cconfs))
goto end;
manio_close(&p1manio);
// First, open them in append mode, so that they will be created if
// they do not exist.
if(!(cmanio=manio_open_phase2(sdirs->changed, "ab", protocol))
|| !(umanio=manio_open_phase2(sdirs->unchanged, "ab", protocol)))
goto end;
manio_close(&cmanio);
manio_close(&umanio);
if(!(p1pos=do_resume_work(sdirs, dpth, cconfs))) goto end;
if(dpth_incr(dpth)) goto end;
logp("End phase1 (read previous file system scan)\n");
end:
manio_close(&p1manio);
manio_close(&cmanio);
manio_close(&umanio);
return p1pos;
}
static struct slist *build_manifest_phase2(const char *path,
enum protocol protocol, int entries)
{
struct slist *slist=NULL;
struct manio *manio=NULL;
fail_unless((manio=manio_open_phase2(path, "wb", protocol))!=NULL);
slist=do_build_manifest(manio,
protocol, entries, 0 /*with_data_files*/);
fail_unless(!manio_close(&manio));
return slist;
}
void build_manifest_phase2_from_slist(const char *path,
struct slist *slist, enum protocol protocol, int short_write)
{
struct sbuf *sb;
struct manio *manio=NULL;
for(sb=slist->head; sb; sb=sb->next)
set_sbuf(slist, sb, 0 /* with_data_files */);
fail_unless((manio=manio_open_phase2(path, "wb", protocol))!=NULL);
for(sb=slist->head; sb; sb=sb->next)
{
fail_unless(!manio_write_sbuf(manio, sb));
if(protocol==PROTO_2)
{
struct blk *blk=NULL;
for(blk=sb->protocol2->bstart;
blk && blk!=sb->protocol2->bend; blk=blk->next)
{
fail_unless(!manio_write_sig_and_path(manio,
blk));
}
if(sbuf_is_filedata(sb) || sbuf_is_vssdata(sb))
{
struct iobuf endfile;
iobuf_from_str(&endfile,
CMD_END_FILE, (char *)"0:0");
fail_unless(!iobuf_send_msg_fzp(&endfile,
manio->fzp));
}
hack_protocol2_attr(&sb->attr);
}
}
if(short_write)
{
man_off_t *pos;
fail_unless((pos=manio_tell(manio))!=NULL);
if(pos->offset>=short_write) pos->offset-=short_write;
fail_unless(!manio_close_and_truncate(&manio,
pos, 0 /* compression */));
man_off_t_free(&pos);
}
fail_unless(!manio_close(&manio));
}
static int maybe_add_from_scan(struct manios *manios,
struct slist *slist, struct asfd *chfd, struct sbuf **csb)
{
int ret=-1;
struct sbuf *snew=NULL;
while(1)
{
sbuf_free(&snew);
if(!manios->phase1) return 0;
// Limit the amount loaded into memory at any one time.
if(slist && slist->head)
{
if(slist->head->protocol2->index
- slist->tail->protocol2->index>4096)
return 0;
}
if(!(snew=sbuf_alloc(PROTO_2))) goto end;
switch(manio_read(manios->phase1, snew))
{
case 0: break;
case 1: manio_close(&manios->phase1);
ret=0; // Finished.
default: goto end;
}
switch(entry_changed(snew, manios, chfd, csb))
{
case 0: continue; // No change.
case 1: break;
default: goto end; // Error.
}
if(data_needed(snew)) snew->flags|=SBUF_NEED_DATA;
slist_add_sbuf(slist, snew);
snew=NULL;
}
return 0;
end:
sbuf_free(&snew);
return ret;
}
int manio_close_and_truncate(struct manio **manio,
man_off_t *offset, int compression)
{
int ret=-1;
errno=0;
if(!is_single_file(*manio)
&& remove_trailing_files(*manio, offset))
goto end;
if(manio_close(manio)) goto end;
if(fzp_truncate(offset->fpath, FZP_FILE, offset->offset, compression))
{
logp("Could not fzp_truncate %s in %s(): %s\n",
offset->fpath, __func__, strerror(errno));
goto end;
}
ret=0;
end:
return ret;
}
static int manio_free_contents(struct manio *manio)
{
int ret=0;
if(!manio) return ret;
if(manio_close(manio)) ret=-1;
if(manio->base_dir) free(manio->base_dir);
if(manio->directory) free(manio->directory);
if(manio->fpath) free(manio->fpath);
if(manio->lpath) free(manio->lpath);
if(manio->mode) free(manio->mode);
if(manio->hook_dir) free(manio->hook_dir);
if(manio->hook_sort)
{
int i;
for(i=0; i<MANIFEST_SIG_MAX; i++)
if(manio->hook_sort[i]) free(manio->hook_sort[i]);
free(manio->hook_sort);
}
memset(manio, 0, sizeof(struct manio));
return ret;
}
static int manio_free_content(struct manio *manio)
{
int ret=0;
if(!manio) return ret;
if(manio_close(manio)) ret=-1;
man_off_free_content(&manio->offset);
free_w(&manio->base_dir);
free_w(&manio->directory);
free_w(&manio->mode);
free_w(&manio->hook_dir);
free_w(&manio->rdirectory);
if(manio->hook_sort)
{
int i;
for(i=0; i<MANIFEST_SIG_MAX; i++)
free_w(&(manio->hook_sort[i]));
free_v((void **)&manio->hook_sort);
}
memset(manio, 0, sizeof(struct manio));
return ret;
}
static int browse_manifest_start(struct asfd *srfd, struct cstat *cstat,
struct bu *bu, const char *browse, struct conf **confs)
{
int ret=-1;
char *manifest=NULL;
struct sbuf *sb=NULL;
struct manio *manio=NULL;
if(!(manifest=prepend_s(bu->path,
cstat->protocol==PROTO_1?"manifest.gz":"manifest"))
|| !(manio=manio_open(manifest, "rb", cstat->protocol))
|| !(sb=sbuf_alloc(cstat->protocol)))
goto end;
if(get_int(confs[OPT_MONITOR_BROWSE_CACHE]))
ret=cache_load(srfd, manio, sb, cstat, bu);
else
ret=do_browse_manifest(srfd, manio, sb, browse);
end:
free_w(&manifest);
manio_close(&manio);
sbuf_free(&sb);
return ret;
}
static struct slist *build_manifest_phase1(const char *path,
enum protocol protocol, int entries)
{
struct sbuf *sb;
struct slist *slist=NULL;
struct manio *manio=NULL;
slist=build_slist_phase1(NULL /*prefix*/, protocol, entries);
fail_unless((manio=manio_open_phase1(path, "wb", protocol))!=NULL);
for(sb=slist->head; sb; sb=sb->next)
{
fail_unless(!manio_write_sbuf(manio, sb));
if(protocol==PROTO_2) hack_protocol2_attr(&sb->attr);
}
fail_unless(!send_msg_fzp(manio->fzp,
CMD_GEN, "phase1end", strlen("phase1end")));
fail_unless(!manio_close(&manio));
return slist;
}
struct manio *manio_open_phase3(const char *manifest, const char *mode,
enum protocol protocol, const char *rmanifest)
{
struct manio *manio=NULL;
if(!(manio=do_manio_open(manifest, mode, protocol, 3)))
goto end;
if(protocol==PROTO_2 && rmanifest)
{
char *hooksdir=NULL;
char *dindexdir=NULL;
if(!(hooksdir=prepend_s(manifest, "hooks"))
|| !(dindexdir=prepend_s(manifest, "dindex"))
|| init_write_hooks(manio, hooksdir, rmanifest)
|| init_write_dindex(manio, dindexdir))
manio_close(&manio);
free_w(&hooksdir);
free_w(&dindexdir);
}
end:
return manio;
}
void build_manifest_phase1_from_slist(const char *path,
struct slist *slist, enum protocol protocol)
{
struct sbuf *sb;
struct manio *manio=NULL;
struct iobuf datapth;
struct iobuf endfile;
iobuf_init(&datapth);
iobuf_init(&endfile);
for(sb=slist->head; sb; sb=sb->next)
set_sbuf(slist, sb, 0 /* with_data_files */);
fail_unless((manio=manio_open_phase1(path, "wb", protocol))!=NULL);
for(sb=slist->head; sb; sb=sb->next)
{
// Might be given an slist that has datapth or endfile set,
// which should not go into a phase1 scan. Deal with it.
if(sb->protocol1
&& sb->protocol1->datapth.buf)
iobuf_move(&datapth, &sb->protocol1->datapth);
if(sb->endfile.buf)
iobuf_move(&endfile, &sb->endfile);
fail_unless(!manio_write_sbuf(manio, sb));
if(datapth.buf)
iobuf_move(&sb->protocol1->datapth, &datapth);
if(endfile.buf)
iobuf_move(&sb->endfile, &endfile);
}
fail_unless(!send_msg_fzp(manio->fzp,
CMD_GEN, "phase1end", strlen("phase1end")));
fail_unless(!manio_close(&manio));
}
// Combine the phase1 and phase2 files into a new manifest.
int backup_phase3_server_all(struct sdirs *sdirs, struct conf **confs)
{
int ret=-1;
int pcmp=0;
struct blk *blk=NULL;
struct sbuf *usb=NULL;
struct sbuf *csb=NULL;
char *manifesttmp=NULL;
struct manio *newmanio=NULL;
struct manio *chmanio=NULL;
struct manio *unmanio=NULL;
enum protocol protocol=get_protocol(confs);
struct cntr *cntr=get_cntr(confs);
const char *rmanifest_relative=NULL;
logp("Begin phase3 (merge manifests)\n");
if(protocol==PROTO_2)
rmanifest_relative=get_rmanifest_relative(sdirs, confs);
if(!(manifesttmp=get_tmp_filename(sdirs->manifest))
|| !(newmanio=manio_open_phase3(manifesttmp,
comp_level(get_int(confs[OPT_COMPRESSION])),
protocol, rmanifest_relative))
|| !(chmanio=manio_open_phase2(sdirs->changed, "rb", protocol))
|| !(unmanio=manio_open_phase2(sdirs->unchanged, "rb", protocol))
|| !(usb=sbuf_alloc(protocol))
|| !(csb=sbuf_alloc(protocol)))
goto end;
while(chmanio || unmanio)
{
if(!blk && !(blk=blk_alloc())) goto end;
if(unmanio
&& !usb->path.buf)
{
switch(manio_read(unmanio, usb))
{
case -1: goto end;
case 1: manio_close(&unmanio);
}
}
if(chmanio
&& !csb->path.buf)
{
switch(manio_read(chmanio, csb))
{
case -1: goto end;
case 1: manio_close(&chmanio);
}
}
if(usb->path.buf && !csb->path.buf)
{
if(write_status(CNTR_STATUS_MERGING,
usb->path.buf, cntr)) goto end;
switch(manio_copy_entry(
usb, usb, &blk, unmanio, newmanio))
{
case -1: goto end;
case 1: manio_close(&unmanio);
}
}
else if(!usb->path.buf && csb->path.buf)
{
if(write_status(CNTR_STATUS_MERGING,
csb->path.buf, cntr)) goto end;
switch(manio_copy_entry(
csb, csb, &blk, chmanio, newmanio))
{
case -1: goto end;
case 1: manio_close(&chmanio);
}
}
else if(!usb->path.buf && !csb->path.buf)
{
continue;
}
else if(!(pcmp=sbuf_pathcmp(usb, csb)))
{
// They were the same - write one.
if(write_status(CNTR_STATUS_MERGING,
csb->path.buf, cntr)) goto end;
switch(manio_copy_entry(
csb, csb, &blk, chmanio, newmanio))
{
case -1: goto end;
case 1: manio_close(&chmanio);
}
}
else if(pcmp<0)
{
if(write_status(CNTR_STATUS_MERGING,
usb->path.buf, cntr)) goto end;
switch(manio_copy_entry(
usb, usb, &blk, unmanio, newmanio))
{
case -1: goto end;
case 1: manio_close(&unmanio);
}
}
else
{
if(write_status(CNTR_STATUS_MERGING,
csb->path.buf, cntr)) goto end;
switch(manio_copy_entry(
csb, csb, &blk, chmanio, newmanio))
{
case -1: goto end;
case 1: manio_close(&chmanio);
}
}
}
// Flush to disk.
if(manio_close(&newmanio))
{
logp("error gzclosing %s in backup_phase3_server\n",
manifesttmp);
goto end;
}
// Rename race condition should be of no consequence here, as the
// manifest should just get recreated automatically.
if(do_rename(manifesttmp, sdirs->manifest))
goto end;
else
{
recursive_delete(sdirs->changed);
recursive_delete(sdirs->unchanged);
}
logp("End phase3 (merge manifests)\n");
ret=0;
end:
manio_close(&newmanio);
manio_close(&chmanio);
manio_close(&unmanio);
sbuf_free(&csb);
sbuf_free(&usb);
blk_free(&blk);
free_w(&manifesttmp);
return ret;
}
// Used when restoring a hard link that we have not restored the destination
// for. Read through the manifest from the beginning and substitute the path
// and data to the new location.
static int hard_link_substitution(struct asfd *asfd,
struct sbuf *sb, struct f_link *lp,
struct bu *bu, enum action act, struct sdirs *sdirs,
enum cntr_status cntr_status, struct conf **cconfs,
const char *manifest, struct slist *slist)
{
int ret=-1;
struct sbuf *need_data=NULL;
int last_ent_was_dir=0;
struct sbuf *hb=NULL;
struct manio *manio=NULL;
struct blk *blk=NULL;
int pcmp;
enum protocol protocol=get_protocol(cconfs);
struct cntr *cntr=get_cntr(cconfs);
if(!(manio=manio_open(manifest, "rb", protocol))
|| !(need_data=sbuf_alloc(protocol))
|| !(hb=sbuf_alloc(protocol)))
goto end;
if(protocol==PROTO_2)
{
if(!(blk=blk_alloc()))
goto end;
}
while(1)
{
switch(manio_read_with_blk(manio,
hb, need_data->path.buf?blk:NULL, sdirs))
{
case 0: break; // Keep going.
case 1: ret=0; goto end; // Finished OK.
default: goto end; // Error;
}
if(protocol==PROTO_2)
{
if(hb->endfile.buf)
{
sbuf_free_content(hb);
continue;
}
if(blk->data)
{
if(protocol2_extra_restore_stream_bits(asfd,
blk, slist, act, need_data,
last_ent_was_dir, cntr)) goto end;
continue;
}
sbuf_free_content(need_data);
}
pcmp=pathcmp(lp->name, hb->path.buf);
if(!pcmp && (sbuf_is_filedata(hb) || sbuf_is_vssdata(hb)))
{
// Copy the path from sb to hb.
free_w(&hb->path.buf);
if(!(hb->path.buf=strdup_w(sb->path.buf, __func__)))
goto end;
// Should now be able to restore the original data
// to the new location.
ret=restore_sbuf(asfd, hb, bu, act, sdirs,
cntr_status, cconfs, need_data, manifest, slist);
// May still need to get protocol2 data.
if(!ret && need_data->path.buf) continue;
break;
}
sbuf_free_content(hb);
// Break out once we have gone past the entry that we are
// interested in.
if(pcmp<0) break;
}
end:
blk_free(&blk);
sbuf_free(&hb);
manio_close(&manio);
return ret;
}
int backup_phase2_server(struct async *as, struct sdirs *sdirs,
const char *manifest_dir, int resume, struct conf *conf)
{
int ret=-1;
int sigs_end=0;
int backup_end=0;
int requests_end=0;
int blk_requests_end=0;
struct slist *slist=NULL;
struct blist *blist=NULL;
struct iobuf *wbuf=NULL;
struct dpth *dpth=NULL;
struct manio *cmanio=NULL; // current manifest
struct manio *p1manio=NULL; // phase1 scan manifest
struct manio *chmanio=NULL; // changed manifest
struct manio *unmanio=NULL; // unchanged manifest
// This is used to tell the client that a number of consecutive blocks
// have been found and can be freed.
uint64_t wrap_up=0;
// Main fd is first in the list.
struct asfd *asfd=as->asfd;
// Champ chooser fd is second in the list.
struct asfd *chfd=asfd->next;
logp("Phase 2 begin (recv backup data)\n");
//if(champ_chooser_init(sdirs->data, conf)
if(!(cmanio=manio_alloc())
|| !(p1manio=manio_alloc())
|| !(chmanio=manio_alloc())
|| !(unmanio=manio_alloc())
|| manio_init_read(cmanio, sdirs->cmanifest)
|| manio_init_read(p1manio, sdirs->phase1data)
|| manio_init_write(chmanio, sdirs->changed)
|| manio_init_write(unmanio, sdirs->unchanged)
|| !(slist=slist_alloc())
|| !(blist=blist_alloc())
|| !(wbuf=iobuf_alloc())
|| !(dpth=dpth_alloc(sdirs->data))
|| dpth_init(dpth))
goto end;
// The phase1 manifest looks the same as a burp1 one.
manio_set_protocol(p1manio, PROTO_BURP1);
while(!backup_end)
{
if(maybe_add_from_scan(asfd,
p1manio, cmanio, unmanio, slist, conf))
goto end;
if(!wbuf->len)
{
if(get_wbuf_from_sigs(wbuf, slist, blist,
sigs_end, &blk_requests_end, dpth, conf))
goto end;
if(!wbuf->len)
{
get_wbuf_from_files(wbuf, slist,
p1manio, &requests_end);
}
}
if(wbuf->len)
asfd->append_all_to_write_buffer(asfd, wbuf);
append_for_champ_chooser(chfd, blist, sigs_end);
if(as->read_write(as))
{
logp("error in %s\n", __func__);
goto end;
}
while(asfd->rbuf->buf)
{
if(deal_with_read(asfd->rbuf, slist, blist,
conf, &sigs_end, &backup_end, dpth))
goto end;
// Get as much out of the
// readbuf as possible.
if(asfd->parse_readbuf(asfd)) goto end;
}
while(chfd->rbuf->buf)
{
if(deal_with_read_from_chfd(asfd, chfd,
blist, &wrap_up, dpth)) goto end;
// Get as much out of the
// readbuf as possible.
if(chfd->parse_readbuf(chfd)) goto end;
}
if(write_to_changed_file(asfd, chfd, chmanio,
slist, blist, dpth, backup_end, conf))
goto end;
}
// Hack: If there are some entries left after the last entry that
// contains block data, it will not be written to the changed file
// yet because the last entry of block data has not had
// sb->burp2->bend set.
if(slist->head && slist->head->next)
{
slist->head=slist->head->next;
if(write_to_changed_file(asfd, chfd, chmanio,
slist, blist, dpth, backup_end, conf))
goto end;
}
if(manio_close(unmanio)
|| manio_close(chmanio))
goto end;
if(blist->head)
{
logp("ERROR: finishing but still want block: %lu\n",
blist->head->index);
goto end;
}
// Need to release the last left. There should be one at most.
if(dpth->head && dpth->head->next)
{
logp("ERROR: More data locks remaining after: %s\n",
dpth->head->save_path);
goto end;
}
if(dpth_release_all(dpth)) goto end;
ret=0;
end:
logp("End backup\n");
slist_free(slist);
blist_free(blist);
iobuf_free_content(asfd->rbuf);
iobuf_free_content(chfd->rbuf);
// Write buffer did not allocate 'buf'.
if(wbuf) wbuf->buf=NULL;
iobuf_free(wbuf);
dpth_release_all(dpth);
dpth_free(&dpth);
manio_free(&cmanio);
manio_free(&p1manio);
manio_free(&chmanio);
manio_free(&unmanio);
return ret;
}
static int restore_stream(struct asfd *asfd, struct sdirs *sdirs,
struct slist *slist, struct bu *bu, const char *manifest,
regex_t *regex, int srestore, struct conf **cconfs, enum action act,
enum cntr_status cntr_status)
{
int ret=-1;
int last_ent_was_dir=0;
struct sbuf *sb=NULL;
struct iobuf *rbuf=asfd->rbuf;
struct manio *manio=NULL;
struct blk *blk=NULL;
struct sbuf *need_data=NULL;
enum protocol protocol=get_protocol(cconfs);
struct cntr *cntr=get_cntr(cconfs);
if(protocol==PROTO_2)
{
if(asfd->write_str(asfd, CMD_GEN, "restore_stream")
|| asfd->read_expect(asfd, CMD_GEN, "restore_stream_ok")
|| !(blk=blk_alloc()))
goto end;
}
if(!(manio=manio_open(manifest, "rb", protocol))
|| !(need_data=sbuf_alloc(protocol))
|| !(sb=sbuf_alloc(protocol)))
goto end;
while(1)
{
iobuf_free_content(rbuf);
if(asfd->as->read_quick(asfd->as))
{
logp("read quick error\n");
goto end;
}
if(rbuf->buf) switch(rbuf->cmd)
{
case CMD_MESSAGE:
case CMD_WARNING:
{
log_recvd(rbuf, cntr, 0);
continue;
}
case CMD_INTERRUPT:
// Client wanted to interrupt the
// sending of a file. But if we are
// here, we have already moved on.
// Ignore.
continue;
default:
iobuf_log_unexpected(rbuf, __func__);
goto end;
}
switch(manio_read_with_blk(manio,
sb, need_data->path.buf?blk:NULL, sdirs))
{
case 0: break; // Keep going.
case 1: ret=0; goto end; // Finished OK.
default: goto end; // Error;
}
if(protocol==PROTO_2)
{
if(sb->endfile.buf)
{
sbuf_free_content(sb);
continue;
}
if(blk->data)
{
if(protocol2_extra_restore_stream_bits(asfd,
blk, slist, act, need_data,
last_ent_was_dir, cntr)) goto end;
continue;
}
sbuf_free_content(need_data);
}
if(want_to_restore(srestore, sb, regex, cconfs))
{
if(restore_ent(asfd, &sb, slist,
bu, act, sdirs, cntr_status, cconfs,
need_data, &last_ent_was_dir, manifest))
goto end;
}
else if(sbuf_is_filedata(sb) || sbuf_is_vssdata(sb))
{
// Add it to the list of filedata that was not
// restored.
struct f_link **bucket=NULL;
if(!linkhash_search(&sb->statp, &bucket)
&& linkhash_add(sb->path.buf, &sb->statp, bucket))
goto end;
}
sbuf_free_content(sb);
}
end:
blk_free(&blk);
sbuf_free(&sb);
sbuf_free(&need_data);
iobuf_free_content(rbuf);
manio_close(&manio);
return ret;
}
// Return p1manio position.
static man_off_t *do_resume_work(struct sdirs *sdirs,
struct dpth *dpth, struct conf **cconfs)
{
man_off_t *pos=NULL;
man_off_t *p1pos=NULL;
struct iobuf *chb=NULL;
struct manio *cmanio=NULL;
struct manio *umanio=NULL;
struct manio *p1manio=NULL;
enum protocol protocol=get_protocol(cconfs);
struct cntr *cntr=get_cntr(cconfs);
int compression=get_int(cconfs[OPT_COMPRESSION]);
if(!(p1manio=manio_open_phase1(sdirs->phase1data, "rb", protocol))
|| !(cmanio=manio_open_phase2(sdirs->changed, "rb", protocol))
|| !(umanio=manio_open_phase2(sdirs->unchanged, "rb", protocol)))
goto end;
if(!(chb=iobuf_alloc()))
return NULL;
logp("Setting up resume positions...\n");
if(get_last_good_entry(cmanio, chb, cntr, dpth, protocol, &pos))
goto error;
if(manio_close_and_truncate(&cmanio, pos, compression)) goto error;
man_off_t_free(&pos);
if(chb->buf)
{
logp(" last good entry: %s\n", chb->buf);
// Now need to go to the appropriate places in p1manio and
// unchanged.
if(forward_past_entry(p1manio, chb, protocol, &p1pos))
goto error;
// The unchanged file needs to be positioned just before the
// found entry, otherwise it ends up having a duplicated entry.
if(forward_before_entry(umanio,
chb, cntr, dpth, protocol, &pos))
goto error;
if(manio_close_and_truncate(&umanio, pos, compression))
goto error;
man_off_t_free(&pos);
}
else
{
logp(" nothing previously transferred\n");
if(!(p1pos=manio_tell(p1manio)))
goto error;
if(!(pos=manio_tell(umanio)))
goto error;
if(manio_close_and_truncate(&umanio, pos, compression))
goto error;
}
// Now should have all file pointers in the right places to resume.
goto end;
error:
man_off_t_free(&p1pos);
end:
iobuf_free(&chb);
man_off_t_free(&pos);
manio_close(&p1manio);
manio_close(&cmanio);
manio_close(&umanio);
return p1pos;
}
static int list_manifest(const char *fullpath)
{
int ret=0;
struct sbuf *sb=NULL;
struct manio *manio=NULL;
char *manifest_dir=NULL;
char *last_bd_match=NULL;
size_t bdlen=0;
if(!(manifest_dir=prepend_s(fullpath,
protocol==PROTO_1?"manifest.gz":"manifest"))
|| !(manio=manio_open(manifest_dir, "rb", protocol))
|| !(sb=sbuf_alloc(protocol)))
{
log_and_send_oom(asfd);
goto error;
}
if(browsedir) bdlen=strlen(browsedir);
while(1)
{
sbuf_free_content(sb);
switch(manio_read(manio, sb))
{
case 0:
break;
case 1:
if(browsedir && *browsedir && !last_bd_match)
asfd_write_wrapper_str(asfd,
CMD_ERROR,
"directory not found");
goto end; // Finished OK.
default:
goto error;
}
if(protocol==PROTO_2 && sb->endfile.buf)
continue;
if(sbuf_is_metadata(sb))
continue;
if(write_status(CNTR_STATUS_LISTING, sb->path.buf, cntr))
goto error;
if(browsedir)
{
int r;
if((r=check_browsedir(browsedir,
sb, bdlen, &last_bd_match))<0)
goto error;
if(!r) continue;
}
if(regex && !regex_check(regex, sb->path.buf))
continue;
if(asfd_write_wrapper(asfd, &sb->attr)
|| asfd_write_wrapper(asfd, &sb->path))
goto error;
if(sbuf_is_link(sb)
&& asfd_write_wrapper(asfd, &sb->link))
goto error;
}
error:
ret=-1;
end:
sbuf_free(&sb);
free_w(&manifest_dir);
manio_close(&manio);
free_w(&last_bd_match);
return ret;
}
static int reset_sig_count_and_close(struct manio *manio)
{
if(manio_close(manio)) return -1;
manio->sig_count=0;
return 0;
}
int backup_phase2_server_protocol1(struct async *as, struct sdirs *sdirs,
const char *incexc, int resume, struct conf **cconfs)
{
int ret=0;
struct manio *p1manio=NULL;
struct dpth *dpth=NULL;
char *deltmppath=NULL;
char *last_requested=NULL;
// Where to write changed data.
// Data is not getting written to a compressed file.
// This is important for recovery if the power goes.
struct fzp *chfp=NULL;
struct fzp *ucfp=NULL; // unchanged data
struct fzp *cmanfp=NULL; // previous (current) manifest.
struct sbuf *cb=NULL; // file list in current manifest
struct sbuf *p1b=NULL; // file list from client
struct sbuf *rb=NULL; // receiving file from client
struct asfd *asfd=as->asfd;
int breaking=0;
int breakcount=0;
if(get_int(cconfs[OPT_BREAKPOINT])>=2000
&& get_int(cconfs[OPT_BREAKPOINT])<3000)
{
breaking=get_int(cconfs[OPT_BREAKPOINT]);
breakcount=breaking-2000;
}
logp("Begin phase2 (receive file data)\n");
if(!(dpth=dpth_alloc())
|| dpth_protocol1_init(dpth, sdirs->currentdata,
get_int(cconfs[OPT_MAX_STORAGE_SUBDIRS])))
goto error;
if(open_previous_manifest(&cmanfp, sdirs, incexc, cconfs))
goto error;
if(get_int(cconfs[OPT_DIRECTORY_TREE]))
{
// Need to make sure we do not try to create a path that is
// too long.
if(build_path_w(sdirs->treepath)) goto error;
treepathlen=strlen(sdirs->treepath);
init_fs_max(sdirs->treepath);
}
if(!(p1manio=manio_alloc())
|| manio_init_read(p1manio, sdirs->phase1data)
|| !(cb=sbuf_alloc(cconfs))
|| !(p1b=sbuf_alloc(cconfs))
|| !(rb=sbuf_alloc(cconfs)))
goto error;
manio_set_protocol(p1manio, PROTO_1);
if(resume && do_resume(p1manio, sdirs, dpth, cconfs))
goto error;
// Unchanged and changed should now be truncated correctly, we just
// have to open them for appending.
if(!(ucfp=fzp_open(sdirs->unchanged, "a+b"))
|| !(chfp=fzp_open(sdirs->changed, "a+b")))
goto error;
if(manio_closed(p1manio)
&& manio_open_next_fpath(p1manio))
goto error;
while(1)
{
if(breaking)
{
if(breakcount--==0) return breakpoint(cconfs, __func__);
}
//printf("in loop, %s %s %c\n",
// cmanfp?"got cmanfp":"no cmanfp",
// rb->path.buf?:"no rb->path",
// rb->path.buf?'X':rb->path.cmd);
if(write_status(CNTR_STATUS_BACKUP,
rb->path.buf?rb->path.buf:p1b->path.buf, cconfs))
goto error;
if(last_requested
|| manio_closed(p1manio)
|| asfd->writebuflen)
{
switch(do_stuff_to_receive(asfd, sdirs,
cconfs, rb, chfp, dpth, &last_requested))
{
case 0: break;
case 1: goto end; // Finished ok.
case -1: goto error;
}
}
switch(do_stuff_to_send(asfd, p1b, &last_requested))
{
case 0: break;
case 1: continue;
case -1: goto error;
}
if(manio_closed(p1manio)) continue;
sbuf_free_content(p1b);
switch(manio_sbuf_fill_phase1(p1manio, asfd,
p1b, NULL, sdirs, cconfs))
{
case 0: break;
case 1: manio_close(p1manio);
if(asfd->write_str(asfd,
CMD_GEN, "backupphase2end")) goto error;
break;
case -1: goto error;
}
if(!cmanfp)
{
// No old manifest, need to ask for a new file.
if(process_new(sdirs, cconfs, p1b, ucfp))
goto error;
continue;
}
// Have an old manifest, look for it there.
// Might already have it, or be ahead in the old
// manifest.
if(cb->path.buf) switch(maybe_process_file(asfd,
sdirs, cb, p1b, ucfp, cconfs))
{
case 0: break;
case 1: continue;
case -1: goto error;
}
while(cmanfp)
{
sbuf_free_content(cb);
switch(sbufl_fill(cb, asfd, cmanfp, cconfs))
{
case 0: break;
case 1: fzp_close(&cmanfp);
if(process_new(sdirs, cconfs, p1b,
ucfp)) goto error;
continue;
case -1: goto error;
}
switch(maybe_process_file(asfd, sdirs,
cb, p1b, ucfp, cconfs))
{
case 0: continue;
case 1: break;
case -1: goto error;
}
break;
}
}
error:
ret=-1;
end:
if(fzp_close(&chfp))
{
logp("error closing %s in %s\n", sdirs->changed, __func__);
ret=-1;
}
if(fzp_close(&ucfp))
{
logp("error closing %s in %s\n", sdirs->unchanged, __func__);
ret=-1;
}
free_w(&deltmppath);
sbuf_free(&cb);
sbuf_free(&p1b);
sbuf_free(&rb);
manio_free(&p1manio);
fzp_close(&cmanfp);
dpth_free(&dpth);
if(!ret) unlink(sdirs->phase1data);
logp("End phase2 (receive file data)\n");
return ret;
}
int backup_phase1_server_all(struct async *as,
struct sdirs *sdirs, struct conf **confs)
{
int ret=-1;
struct sbuf *sb=NULL;
char *phase1tmp=NULL;
struct asfd *asfd=as->asfd;
struct manio *manio=NULL;
enum protocol protocol=get_protocol(confs);
struct cntr *cntr=get_cntr(confs);
logp("Begin phase1 (file system scan)\n");
if(!(phase1tmp=get_tmp_filename(sdirs->phase1data))
|| !(manio=manio_open_phase1(phase1tmp,
comp_level(get_int(confs[OPT_COMPRESSION])), protocol))
|| !(sb=sbuf_alloc(protocol)))
goto error;
while(1)
{
sbuf_free_content(sb);
switch(sbuf_fill_from_net(sb, asfd, NULL, NULL, cntr))
{
case 0: break;
case 1: // Last thing the client sends is
// 'backupphase2', and it wants an 'ok' reply.
if(asfd->write_str(asfd, CMD_GEN, "ok")
|| send_msg_fzp(manio->fzp, CMD_GEN,
"phase1end", strlen("phase1end")))
goto error;
goto end;
case -1:
default: goto error;
}
if(write_status(CNTR_STATUS_SCANNING, sb->path.buf, cntr)
|| manio_write_sbuf(manio, sb))
goto error;
cntr_add_phase1(cntr, sb->path.cmd, 0);
if(sbuf_is_filedata(sb))
{
cntr_add_val(cntr, CMD_BYTES_ESTIMATED,
(uint64_t)sb->statp.st_size, 0);
}
}
end:
if(manio_close(&manio))
{
logp("error closing %s in backup_phase1_server\n", phase1tmp);
goto error;
}
if(check_quota(as, cntr,
get_uint64_t(confs[OPT_HARD_QUOTA]),
get_uint64_t(confs[OPT_SOFT_QUOTA])))
goto error;
// Possible rename race condition is of no consequence here, because
// the working directory will always get deleted if phase1 is not
// complete.
if(do_rename(phase1tmp, sdirs->phase1data))
goto error;
//cntr_print(p1cntr, cntr, ACTION_BACKUP);
logp("End phase1 (file system scan)\n");
ret=0;
error:
free_w(&phase1tmp);
manio_close(&manio);
sbuf_free(&sb);
return ret;
}