man_off_t *do_resume(struct sdirs *sdirs,
struct dpth *dpth, struct conf **cconfs)
{
man_off_t *p1pos=NULL;
struct manio *cmanio=NULL;
struct manio *umanio=NULL;
struct manio *p1manio=NULL;
enum protocol protocol=get_protocol(cconfs);
logp("Begin phase1 (read previous file system scan)\n");
if(!(p1manio=manio_open_phase1(sdirs->phase1data, "rb", protocol))
|| read_phase1(p1manio, cconfs))
goto end;
manio_close(&p1manio);
// First, open them in append mode, so that they will be created if
// they do not exist.
if(!(cmanio=manio_open_phase2(sdirs->changed, "ab", protocol))
|| !(umanio=manio_open_phase2(sdirs->unchanged, "ab", protocol)))
goto end;
manio_close(&cmanio);
manio_close(&umanio);
if(!(p1pos=do_resume_work(sdirs, dpth, cconfs))) goto end;
if(dpth_incr(dpth)) goto end;
logp("End phase1 (read previous file system scan)\n");
end:
manio_close(&p1manio);
manio_close(&cmanio);
manio_close(&umanio);
return p1pos;
}
struct manios *manios_open_phase2(struct sdirs *sdirs, man_off_t *p1pos,
enum protocol p)
{
struct manios *m=NULL;
if(!(m=manios_alloc())
|| !(m->current=manio_open(sdirs->cmanifest, "rb", p))
|| !(m->phase1=manio_open_phase1(sdirs->phase1data, "rb", p))
|| !(m->changed=manio_open_phase2(sdirs->changed, "ab", p))
|| !(m->unchanged=manio_open_phase2(sdirs->unchanged, "ab", p))
|| (p1pos && manio_seek(m->phase1, p1pos)))
manios_close(&m);
return m;
}
static struct slist *build_manifest_phase2(const char *path,
enum protocol protocol, int entries)
{
struct slist *slist=NULL;
struct manio *manio=NULL;
fail_unless((manio=manio_open_phase2(path, "wb", protocol))!=NULL);
slist=do_build_manifest(manio,
protocol, entries, 0 /*with_data_files*/);
fail_unless(!manio_close(&manio));
return slist;
}
struct manio *do_manio_open(const char *path, const char *mode,
enum protocol protocol, int phase)
{
switch(phase)
{
case 0: return manio_open(path, mode, protocol);
case 1: return manio_open_phase1(path, mode, protocol);
case 2: return manio_open_phase2(path, mode, protocol);
default:
fprintf(stderr,
"Do not know how to manio_open phase %d\n",
phase);
fail_unless(0);
return NULL;
}
}
void build_manifest_phase2_from_slist(const char *path,
struct slist *slist, enum protocol protocol, int short_write)
{
struct sbuf *sb;
struct manio *manio=NULL;
for(sb=slist->head; sb; sb=sb->next)
set_sbuf(slist, sb, 0 /* with_data_files */);
fail_unless((manio=manio_open_phase2(path, "wb", protocol))!=NULL);
for(sb=slist->head; sb; sb=sb->next)
{
fail_unless(!manio_write_sbuf(manio, sb));
if(protocol==PROTO_2)
{
struct blk *blk=NULL;
for(blk=sb->protocol2->bstart;
blk && blk!=sb->protocol2->bend; blk=blk->next)
{
fail_unless(!manio_write_sig_and_path(manio,
blk));
}
if(sbuf_is_filedata(sb) || sbuf_is_vssdata(sb))
{
struct iobuf endfile;
iobuf_from_str(&endfile,
CMD_END_FILE, (char *)"0:0");
fail_unless(!iobuf_send_msg_fzp(&endfile,
manio->fzp));
}
hack_protocol2_attr(&sb->attr);
}
}
if(short_write)
{
man_off_t *pos;
fail_unless((pos=manio_tell(manio))!=NULL);
if(pos->offset>=short_write) pos->offset-=short_write;
fail_unless(!manio_close_and_truncate(&manio,
pos, 0 /* compression */));
man_off_t_free(&pos);
}
fail_unless(!manio_close(&manio));
}
// Combine the phase1 and phase2 files into a new manifest.
int backup_phase3_server_all(struct sdirs *sdirs, struct conf **confs)
{
int ret=-1;
int pcmp=0;
struct blk *blk=NULL;
struct sbuf *usb=NULL;
struct sbuf *csb=NULL;
char *manifesttmp=NULL;
struct manio *newmanio=NULL;
struct manio *chmanio=NULL;
struct manio *unmanio=NULL;
enum protocol protocol=get_protocol(confs);
struct cntr *cntr=get_cntr(confs);
const char *rmanifest_relative=NULL;
logp("Begin phase3 (merge manifests)\n");
if(protocol==PROTO_2)
rmanifest_relative=get_rmanifest_relative(sdirs, confs);
if(!(manifesttmp=get_tmp_filename(sdirs->manifest))
|| !(newmanio=manio_open_phase3(manifesttmp,
comp_level(get_int(confs[OPT_COMPRESSION])),
protocol, rmanifest_relative))
|| !(chmanio=manio_open_phase2(sdirs->changed, "rb", protocol))
|| !(unmanio=manio_open_phase2(sdirs->unchanged, "rb", protocol))
|| !(usb=sbuf_alloc(protocol))
|| !(csb=sbuf_alloc(protocol)))
goto end;
while(chmanio || unmanio)
{
if(!blk && !(blk=blk_alloc())) goto end;
if(unmanio
&& !usb->path.buf)
{
switch(manio_read(unmanio, usb))
{
case -1: goto end;
case 1: manio_close(&unmanio);
}
}
if(chmanio
&& !csb->path.buf)
{
switch(manio_read(chmanio, csb))
{
case -1: goto end;
case 1: manio_close(&chmanio);
}
}
if(usb->path.buf && !csb->path.buf)
{
if(write_status(CNTR_STATUS_MERGING,
usb->path.buf, cntr)) goto end;
switch(manio_copy_entry(
usb, usb, &blk, unmanio, newmanio))
{
case -1: goto end;
case 1: manio_close(&unmanio);
}
}
else if(!usb->path.buf && csb->path.buf)
{
if(write_status(CNTR_STATUS_MERGING,
csb->path.buf, cntr)) goto end;
switch(manio_copy_entry(
csb, csb, &blk, chmanio, newmanio))
{
case -1: goto end;
case 1: manio_close(&chmanio);
}
}
else if(!usb->path.buf && !csb->path.buf)
{
continue;
}
else if(!(pcmp=sbuf_pathcmp(usb, csb)))
{
// They were the same - write one.
if(write_status(CNTR_STATUS_MERGING,
csb->path.buf, cntr)) goto end;
switch(manio_copy_entry(
csb, csb, &blk, chmanio, newmanio))
{
case -1: goto end;
case 1: manio_close(&chmanio);
}
}
else if(pcmp<0)
{
if(write_status(CNTR_STATUS_MERGING,
usb->path.buf, cntr)) goto end;
switch(manio_copy_entry(
usb, usb, &blk, unmanio, newmanio))
{
case -1: goto end;
case 1: manio_close(&unmanio);
}
}
else
{
if(write_status(CNTR_STATUS_MERGING,
csb->path.buf, cntr)) goto end;
switch(manio_copy_entry(
csb, csb, &blk, chmanio, newmanio))
{
case -1: goto end;
case 1: manio_close(&chmanio);
}
}
}
// Flush to disk.
if(manio_close(&newmanio))
{
logp("error gzclosing %s in backup_phase3_server\n",
manifesttmp);
goto end;
}
// Rename race condition should be of no consequence here, as the
// manifest should just get recreated automatically.
if(do_rename(manifesttmp, sdirs->manifest))
goto end;
else
{
recursive_delete(sdirs->changed);
recursive_delete(sdirs->unchanged);
}
logp("End phase3 (merge manifests)\n");
ret=0;
end:
manio_close(&newmanio);
manio_close(&chmanio);
manio_close(&unmanio);
sbuf_free(&csb);
sbuf_free(&usb);
blk_free(&blk);
free_w(&manifesttmp);
return ret;
}
// Return p1manio position.
static man_off_t *do_resume_work(struct sdirs *sdirs,
struct dpth *dpth, struct conf **cconfs)
{
man_off_t *pos=NULL;
man_off_t *p1pos=NULL;
struct iobuf *chb=NULL;
struct manio *cmanio=NULL;
struct manio *umanio=NULL;
struct manio *p1manio=NULL;
enum protocol protocol=get_protocol(cconfs);
struct cntr *cntr=get_cntr(cconfs);
int compression=get_int(cconfs[OPT_COMPRESSION]);
if(!(p1manio=manio_open_phase1(sdirs->phase1data, "rb", protocol))
|| !(cmanio=manio_open_phase2(sdirs->changed, "rb", protocol))
|| !(umanio=manio_open_phase2(sdirs->unchanged, "rb", protocol)))
goto end;
if(!(chb=iobuf_alloc()))
return NULL;
logp("Setting up resume positions...\n");
if(get_last_good_entry(cmanio, chb, cntr, dpth, protocol, &pos))
goto error;
if(manio_close_and_truncate(&cmanio, pos, compression)) goto error;
man_off_t_free(&pos);
if(chb->buf)
{
logp(" last good entry: %s\n", chb->buf);
// Now need to go to the appropriate places in p1manio and
// unchanged.
if(forward_past_entry(p1manio, chb, protocol, &p1pos))
goto error;
// The unchanged file needs to be positioned just before the
// found entry, otherwise it ends up having a duplicated entry.
if(forward_before_entry(umanio,
chb, cntr, dpth, protocol, &pos))
goto error;
if(manio_close_and_truncate(&umanio, pos, compression))
goto error;
man_off_t_free(&pos);
}
else
{
logp(" nothing previously transferred\n");
if(!(p1pos=manio_tell(p1manio)))
goto error;
if(!(pos=manio_tell(umanio)))
goto error;
if(manio_close_and_truncate(&umanio, pos, compression))
goto error;
}
// Now should have all file pointers in the right places to resume.
goto end;
error:
man_off_t_free(&p1pos);
end:
iobuf_free(&chb);
man_off_t_free(&pos);
manio_close(&p1manio);
manio_close(&cmanio);
manio_close(&umanio);
return p1pos;
}
int backup_phase2_server_protocol1(struct async *as, struct sdirs *sdirs,
const char *incexc, int resume, struct conf **cconfs)
{
int ret=0;
man_off_t *p1pos=NULL;
struct manio *p1manio=NULL;
struct dpth *dpth=NULL;
char *deltmppath=NULL;
char *last_requested=NULL;
struct manio *chmanio=NULL; // changed data
struct manio *ucmanio=NULL; // unchanged data
struct manio *cmanio=NULL; // previous (current) manifest.
struct sbuf *cb=NULL; // file list in current manifest
struct sbuf *p1b=NULL; // file list from client
struct sbuf *rb=NULL; // receiving file from client
struct asfd *asfd=NULL;
int breaking=0;
int breakcount=0;
struct cntr *cntr=NULL;
if(!as)
{
logp("async not provided to %s()\n", __func__);
goto error;
}
if(!sdirs)
{
logp("sdirs not provided to %s()\n", __func__);
goto error;
}
if(!cconfs)
{
logp("cconfs not provided to %s()\n", __func__);
goto error;
}
asfd=as->asfd;
if(!asfd)
{
logp("asfd not provided to %s()\n", __func__);
goto error;
}
cntr=get_cntr(cconfs);
if(get_int(cconfs[OPT_BREAKPOINT])>=2000
&& get_int(cconfs[OPT_BREAKPOINT])<3000)
{
breaking=get_int(cconfs[OPT_BREAKPOINT]);
breakcount=breaking-2000;
}
logp("Begin phase2 (receive file data)\n");
if(!(dpth=dpth_alloc())
|| dpth_protocol1_init(dpth, sdirs->currentdata,
get_int(cconfs[OPT_MAX_STORAGE_SUBDIRS])))
goto error;
if(open_previous_manifest(&cmanio, sdirs, incexc, cconfs))
goto error;
if(get_int(cconfs[OPT_DIRECTORY_TREE]))
{
// Need to make sure we do not try to create a path that is
// too long.
if(build_path_w(sdirs->treepath)) goto error;
mkdir(sdirs->treepath, 0777);
treepathlen=strlen(sdirs->treepath);
if(init_fs_max(sdirs->treepath))
goto error;
}
if(resume && !(p1pos=do_resume(sdirs, dpth, cconfs)))
goto error;
if(!(p1manio=manio_open_phase1(sdirs->phase1data, "rb", PROTO_1))
|| (resume && manio_seek(p1manio, p1pos)))
goto error;
if(!(cb=sbuf_alloc(PROTO_1))
|| !(p1b=sbuf_alloc(PROTO_1))
|| !(rb=sbuf_alloc(PROTO_1)))
goto error;
// Unchanged and changed should now be truncated correctly, we just
// have to open them for appending.
// Data is not getting written to a compressed file.
// This is important for recovery if the power goes.
if(!(ucmanio=manio_open_phase2(sdirs->unchanged, "ab", PROTO_1))
|| !(chmanio=manio_open_phase2(sdirs->changed, "ab", PROTO_1)))
goto error;
while(1)
{
if(breaking && breakcount--==0)
return breakpoint(breaking, __func__);
if(write_status(CNTR_STATUS_BACKUP,
rb->path.buf?rb->path.buf:p1b->path.buf, cntr))
goto error;
if(last_requested
|| !p1manio
|| asfd->writebuflen)
{
switch(do_stuff_to_receive(asfd, sdirs,
cconfs, rb, chmanio, dpth, &last_requested))
{
case 0: break;
case 1: goto end; // Finished ok.
case -1: goto error;
}
}
switch(do_stuff_to_send(asfd, p1b, &last_requested))
{
case 0: break;
case 1: continue;
case -1: goto error;
}
if(!p1manio) continue;
sbuf_free_content(p1b);
switch(manio_read(p1manio, p1b))
{
case 0: break;
case 1: manio_close(&p1manio);
if(asfd->write_str(asfd,
CMD_GEN, "backupphase2end")) goto error;
break;
case -1: goto error;
}
if(!cmanio)
{
// No old manifest, need to ask for a new file.
if(process_new(sdirs, cconfs, p1b, ucmanio))
goto error;
continue;
}
// Have an old manifest, look for it there.
// Might already have it, or be ahead in the old
// manifest.
if(cb->path.buf) switch(maybe_process_file(asfd,
sdirs, cb, p1b, ucmanio, cconfs))
{
case 0: break;
case 1: continue;
case -1: goto error;
}
while(cmanio)
{
sbuf_free_content(cb);
switch(manio_read(cmanio, cb))
{
case 0: break;
case 1: manio_close(&cmanio);
if(process_new(sdirs, cconfs, p1b,
ucmanio)) goto error;
continue;
case -1: goto error;
}
switch(maybe_process_file(asfd, sdirs,
cb, p1b, ucmanio, cconfs))
{
case 0: continue;
case 1: break;
case -1: goto error;
}
break;
}
}
error:
ret=-1;
end:
if(manio_close(&chmanio))
{
logp("error closing %s in %s\n", sdirs->changed, __func__);
ret=-1;
}
if(manio_close(&ucmanio))
{
logp("error closing %s in %s\n", sdirs->unchanged, __func__);
ret=-1;
}
free_w(&deltmppath);
free_w(&last_requested);
sbuf_free(&cb);
sbuf_free(&p1b);
sbuf_free(&rb);
manio_close(&p1manio);
manio_close(&cmanio);
dpth_free(&dpth);
man_off_t_free(&p1pos);
if(!ret && sdirs)
unlink(sdirs->phase1data);
logp("End phase2 (receive file data)\n");
return ret;
}
