static
#endif
int forward_past_entry(struct manio *manio, struct iobuf *target,
enum protocol protocol, man_off_t **pos)
{
struct sbuf *sb=NULL;
if(!(sb=sbuf_alloc(protocol)))
goto error;
man_off_t_free(pos);
if(!(*pos=manio_tell(manio)))
{
logp("Could not manio_tell first pos in %s(): %s\n",
__func__, strerror(errno));
goto error;
}
while(1)
{
sbuf_free_content(sb);
switch(manio_read(manio, sb))
{
case 0:
break;
case 1:
logp("End of file in %s()\n", __func__);
goto error;
default:
logp("Error in %s()\n", __func__);
// Treat error in unchanged manio as not OK.
goto error;
}
if(target->cmd==sb->path.cmd
&& !pathcmp(target->buf, sb->path.buf))
{
man_off_t_free(pos);
if(!(*pos=manio_tell(manio)))
{
logp("Could not get pos in %s(): %s\n",
__func__, strerror(errno));
goto error;
}
sbuf_free(&sb);
return 0;
}
}
error:
sbuf_free(&sb);
man_off_t_free(pos);
return -1;
}
END_TEST
static void test_manifest_tell_seek(enum protocol protocol, int phase)
{
struct slist *slist;
struct manio *manio;
struct sbuf *sb=NULL;
man_off_t *offset=NULL;
int entries=1000;
prng_init(0);
base64_init();
hexmap_init();
recursive_delete(path);
slist=build_manifest(path, protocol, entries, phase);
fail_unless(slist!=NULL);
sb=slist->head;
fail_unless((manio=do_manio_open(path, "rb", protocol, phase))!=NULL);
read_manifest(&sb, manio, 0, entries/2, protocol, phase);
fail_unless((offset=manio_tell(manio))!=NULL);
fail_unless(sb!=NULL);
fail_unless(!manio_close(&manio));
fail_unless((manio=do_manio_open(path, "rb", protocol, phase))!=NULL);
fail_unless(!manio_seek(manio, offset));
read_manifest(&sb, manio, entries/2, entries, protocol, phase);
fail_unless(sb==NULL);
fail_unless(!manio_close(&manio));
fail_unless(!manio);
slist_free(&slist);
man_off_t_free(&offset);
tear_down();
}
static void manio_free_content(struct manio *manio)
{
if(!manio) return;
man_off_t_free(&manio->offset);
free_w(&manio->manifest);
free_w(&manio->mode);
free_w(&manio->rmanifest);
free_w(&manio->hook_dir);
free_v((void **)&manio->hook_sort);
free_w(&manio->dindex_dir);
free_v((void **)&manio->dindex_sort);
memset(manio, 0, sizeof(struct manio));
}
void build_manifest_phase2_from_slist(const char *path,
struct slist *slist, enum protocol protocol, int short_write)
{
struct sbuf *sb;
struct manio *manio=NULL;
for(sb=slist->head; sb; sb=sb->next)
set_sbuf(slist, sb, 0 /* with_data_files */);
fail_unless((manio=manio_open_phase2(path, "wb", protocol))!=NULL);
for(sb=slist->head; sb; sb=sb->next)
{
fail_unless(!manio_write_sbuf(manio, sb));
if(protocol==PROTO_2)
{
struct blk *blk=NULL;
for(blk=sb->protocol2->bstart;
blk && blk!=sb->protocol2->bend; blk=blk->next)
{
fail_unless(!manio_write_sig_and_path(manio,
blk));
}
if(sbuf_is_filedata(sb) || sbuf_is_vssdata(sb))
{
struct iobuf endfile;
iobuf_from_str(&endfile,
CMD_END_FILE, (char *)"0:0");
fail_unless(!iobuf_send_msg_fzp(&endfile,
manio->fzp));
}
hack_protocol2_attr(&sb->attr);
}
}
if(short_write)
{
man_off_t *pos;
fail_unless((pos=manio_tell(manio))!=NULL);
if(pos->offset>=short_write) pos->offset-=short_write;
fail_unless(!manio_close_and_truncate(&manio,
pos, 0 /* compression */));
man_off_t_free(&pos);
}
fail_unless(!manio_close(&manio));
}
man_off_t *manio_tell(struct manio *manio)
{
man_off_t *offset=NULL;
if(!manio->fzp)
{
logp("manio_tell called on null fzp\n");
goto error;
}
if(!(offset=man_off_t_alloc())
|| !(offset->fpath=strdup_w(manio->offset->fpath, __func__))
|| (offset->offset=fzp_tell(manio->fzp))<0)
goto error;
offset->fcount=manio->offset->fcount;
return offset;
error:
man_off_t_free(&offset);
return NULL;
}
static
#endif
int do_backup_phase2_server_protocol2(struct async *as, struct asfd *chfd,
struct sdirs *sdirs, int resume, struct conf **confs)
{
int ret=-1;
uint8_t end_flags=0;
struct slist *slist=NULL;
struct iobuf wbuf;
struct dpth *dpth=NULL;
man_off_t *p1pos=NULL;
struct manios *manios=NULL;
// This is used to tell the client that a number of consecutive blocks
// have been found and can be freed.
uint64_t wrap_up=0;
struct asfd *asfd=NULL;
struct cntr *cntr=NULL;
struct sbuf *csb=NULL;
uint64_t file_no=1;
if(!as)
{
logp("async not provided to %s()\n", __func__);
goto end;
}
if(!sdirs)
{
logp("sdirs not provided to %s()\n", __func__);
goto end;
}
if(!confs)
{
logp("confs not provided to %s()\n", __func__);
goto end;
}
asfd=as->asfd;
if(!asfd)
{
logp("asfd not provided to %s()\n", __func__);
goto end;
}
if(!chfd)
{
logp("chfd not provided to %s()\n", __func__);
goto end;
}
cntr=get_cntr(confs);
if(get_int(confs[OPT_BREAKPOINT])>=2000
&& get_int(confs[OPT_BREAKPOINT])<3000)
{
breaking=get_int(confs[OPT_BREAKPOINT]);
breakcount=breaking-2000;
}
logp("Phase 2 begin (recv backup data)\n");
if(!(dpth=dpth_alloc())
|| dpth_protocol2_init(dpth,
sdirs->data,
get_string(confs[OPT_CNAME]),
sdirs->cfiles,
get_int(confs[OPT_MAX_STORAGE_SUBDIRS])))
goto end;
if(resume && !(p1pos=do_resume(sdirs, dpth, confs)))
goto end;
if(!(manios=manios_open_phase2(sdirs, p1pos, PROTO_2))
|| !(slist=slist_alloc())
|| !(csb=sbuf_alloc(PROTO_2)))
goto end;
iobuf_free_content(asfd->rbuf);
memset(&wbuf, 0, sizeof(struct iobuf));
while(!(end_flags&END_BACKUP))
{
if(maybe_add_from_scan(manios, slist, chfd, &csb))
goto end;
if(!wbuf.len)
{
if(get_wbuf_from_sigs(&wbuf, slist, &end_flags))
goto end;
if(!wbuf.len)
{
get_wbuf_from_files(&wbuf, slist,
manios, &end_flags, &file_no);
}
}
if(wbuf.len
&& asfd->append_all_to_write_buffer(asfd, &wbuf)==APPEND_ERROR)
goto end;
if(append_for_champ_chooser(chfd, slist->blist, end_flags))
goto end;
if(as->read_write(as))
{
logp("error from as->read_write in %s\n", __func__);
goto end;
}
while(asfd->rbuf->buf)
{
if(deal_with_read(asfd->rbuf, slist, cntr,
&end_flags, dpth))
goto end;
// Get as much out of the readbuf as possible.
if(asfd->parse_readbuf(asfd))
goto end;
}
while(chfd->rbuf->buf)
{
if(deal_with_read_from_chfd(chfd,
slist->blist, &wrap_up, dpth, cntr))
goto end;
// Get as much out of the readbuf as possible.
if(chfd->parse_readbuf(chfd))
goto end;
}
if(write_to_changed_file(asfd, chfd, manios,
slist, end_flags))
goto end;
}
// Hack: If there are some entries left after the last entry that
// contains block data, it will not be written to the changed file
// yet because the last entry of block data has not had
// sb->protocol2->bend set.
if(slist->head && slist->head->next)
{
struct sbuf *sb=NULL;
sb=slist->head;
slist->head=sb->next;
sbuf_free(&sb);
if(write_to_changed_file(asfd, chfd, manios,
slist, end_flags))
goto end;
}
if(manios_close(&manios))
goto end;
if(check_for_missing_work_in_slist(slist))
goto end;
// Need to release the last left. There should be one at most.
if(dpth->head && dpth->head->next)
{
logp("ERROR: More data locks remaining after: %s\n",
dpth->head->save_path);
goto end;
}
if(dpth_release_all(dpth)) goto end;
ret=0;
end:
logp("End backup\n");
sbuf_free(&csb);
slist_free(&slist);
if(asfd) iobuf_free_content(asfd->rbuf);
if(chfd) iobuf_free_content(chfd->rbuf);
dpth_free(&dpth);
manios_close(&manios);
man_off_t_free(&p1pos);
return ret;
}
// Return p1manio position.
static man_off_t *do_resume_work(struct sdirs *sdirs,
struct dpth *dpth, struct conf **cconfs)
{
man_off_t *pos=NULL;
man_off_t *p1pos=NULL;
struct iobuf *chb=NULL;
struct manio *cmanio=NULL;
struct manio *umanio=NULL;
struct manio *p1manio=NULL;
enum protocol protocol=get_protocol(cconfs);
struct cntr *cntr=get_cntr(cconfs);
int compression=get_int(cconfs[OPT_COMPRESSION]);
if(!(p1manio=manio_open_phase1(sdirs->phase1data, "rb", protocol))
|| !(cmanio=manio_open_phase2(sdirs->changed, "rb", protocol))
|| !(umanio=manio_open_phase2(sdirs->unchanged, "rb", protocol)))
goto end;
if(!(chb=iobuf_alloc()))
return NULL;
logp("Setting up resume positions...\n");
if(get_last_good_entry(cmanio, chb, cntr, dpth, protocol, &pos))
goto error;
if(manio_close_and_truncate(&cmanio, pos, compression)) goto error;
man_off_t_free(&pos);
if(chb->buf)
{
logp(" last good entry: %s\n", chb->buf);
// Now need to go to the appropriate places in p1manio and
// unchanged.
if(forward_past_entry(p1manio, chb, protocol, &p1pos))
goto error;
// The unchanged file needs to be positioned just before the
// found entry, otherwise it ends up having a duplicated entry.
if(forward_before_entry(umanio,
chb, cntr, dpth, protocol, &pos))
goto error;
if(manio_close_and_truncate(&umanio, pos, compression))
goto error;
man_off_t_free(&pos);
}
else
{
logp(" nothing previously transferred\n");
if(!(p1pos=manio_tell(p1manio)))
goto error;
if(!(pos=manio_tell(umanio)))
goto error;
if(manio_close_and_truncate(&umanio, pos, compression))
goto error;
}
// Now should have all file pointers in the right places to resume.
goto end;
error:
man_off_t_free(&p1pos);
end:
iobuf_free(&chb);
man_off_t_free(&pos);
manio_close(&p1manio);
manio_close(&cmanio);
manio_close(&umanio);
return p1pos;
}
static
#endif
int get_last_good_entry(struct manio *manio, struct iobuf *result,
struct cntr *cntr, struct dpth *dpth, enum protocol protocol,
man_off_t **pos)
{
int ars=0;
int got_vss_start=0;
struct sbuf *sb=NULL;
struct iobuf lastpath;
if(!(sb=sbuf_alloc(protocol)))
goto error;
iobuf_init(&lastpath);
man_off_t_free(pos);
if(!(*pos=manio_tell(manio)))
{
logp("Could not manio_tell first pos in %s(): %s\n",
__func__, strerror(errno));
goto error;
}
while(1)
{
if(sb->path.buf && !got_vss_start)
{
iobuf_free_content(&lastpath);
iobuf_move(&lastpath, &sb->path);
if(!sbuf_is_filedata(sb)
&& !sbuf_is_vssdata(sb))
{
iobuf_free_content(result);
iobuf_move(result, &lastpath);
man_off_t_free(pos);
if(!(*pos=manio_tell(manio)))
{
logp("Could not manio_tell pos in %s(): %s\n",
__func__, strerror(errno));
goto error;
}
}
}
if(sb->endfile.buf && !got_vss_start)
{
iobuf_free_content(result);
iobuf_move(result, &lastpath);
man_off_t_free(pos);
if(!(*pos=manio_tell(manio)))
{
logp("Could not manio_tell pos in %s(): %s\n",
__func__, strerror(errno));
goto error;
}
}
sbuf_free_content(sb);
ars=manio_read(manio, sb);
if(dpth && set_higher_datapth(sb, dpth)) goto error;
switch(ars)
{
case 0:
break;
case 1:
iobuf_free_content(&lastpath);
sbuf_free(&sb);
return 0;
default:
if(result->buf)
logp("Error after %s in %s()\n",
result->buf, __func__);
// Treat error in changed manio as
// OK - could have been a short write.
iobuf_free_content(&lastpath);
sbuf_free(&sb);
return 0;
}
// Some hacks for split_vss.
switch(sb->path.cmd)
{
case CMD_VSS:
case CMD_ENC_VSS:
got_vss_start=1;
break;
case CMD_VSS_T:
case CMD_ENC_VSS_T:
got_vss_start=0;
break;
case CMD_FILE:
case CMD_ENC_FILE:
if(S_ISDIR(sb->statp.st_mode))
got_vss_start=0;
break;
default:
break;
}
if(cntr)
{
// FIX THIS: cannot distinguish between new and
// changed files.
cntr_add_changed(cntr, sb->path.cmd);
if(sb->endfile.buf)
{
uint64_t e=strtoull(sb->endfile.buf, NULL, 10);
cntr_add_bytes(cntr, e);
}
}
}
error:
iobuf_free_content(&lastpath);
sbuf_free(&sb);
man_off_t_free(pos);
return -1;
}
static
#endif
int forward_before_entry(struct manio *manio, struct iobuf *target,
struct cntr *cntr, struct dpth *dpth, enum protocol protocol,
man_off_t **pos)
{
int ars=0;
struct sbuf *sb=NULL;
if(!(sb=sbuf_alloc(protocol)))
goto error;
man_off_t_free(pos);
if(!(*pos=manio_tell(manio)))
{
logp("Could not manio_tell first pos in %s(): %s\n",
__func__, strerror(errno));
goto error;
}
while(1)
{
if(sb->endfile.buf
|| (sb->path.buf && !sbuf_is_filedata(sb)))
{
man_off_t_free(pos);
if(!(*pos=manio_tell(manio)))
{
logp("Could not manio_tell pos in %s(): "
"%s\n", __func__, strerror(errno));
goto error;
}
}
sbuf_free_content(sb);
ars=manio_read(manio, sb);
if(dpth && set_higher_datapth(sb, dpth)) goto error;
switch(ars)
{
case 0:
break;
case 1:
sbuf_free(&sb);
return 0;
default:
logp("Error in %s(), but continuing\n",
__func__);
// Treat error in unchanged manio as
// OK - could have been a short write.
sbuf_free(&sb);
return 0;
}
if(iobuf_pathcmp(target, &sb->path)<=0)
{
sbuf_free(&sb);
return 0;
}
if(cntr)
{
cntr_add_same(cntr, sb->path.cmd);
if(sb->endfile.buf)
{
uint64_t e=strtoull(sb->endfile.buf, NULL, 10);
cntr_add_bytes(cntr, e);
}
}
}
error:
sbuf_free(&sb);
man_off_t_free(pos);
return -1;
}
int backup_phase2_server_protocol1(struct async *as, struct sdirs *sdirs,
const char *incexc, int resume, struct conf **cconfs)
{
int ret=0;
man_off_t *p1pos=NULL;
struct manio *p1manio=NULL;
struct dpth *dpth=NULL;
char *deltmppath=NULL;
char *last_requested=NULL;
struct manio *chmanio=NULL; // changed data
struct manio *ucmanio=NULL; // unchanged data
struct manio *cmanio=NULL; // previous (current) manifest.
struct sbuf *cb=NULL; // file list in current manifest
struct sbuf *p1b=NULL; // file list from client
struct sbuf *rb=NULL; // receiving file from client
struct asfd *asfd=NULL;
int breaking=0;
int breakcount=0;
struct cntr *cntr=NULL;
if(!as)
{
logp("async not provided to %s()\n", __func__);
goto error;
}
if(!sdirs)
{
logp("sdirs not provided to %s()\n", __func__);
goto error;
}
if(!cconfs)
{
logp("cconfs not provided to %s()\n", __func__);
goto error;
}
asfd=as->asfd;
if(!asfd)
{
logp("asfd not provided to %s()\n", __func__);
goto error;
}
cntr=get_cntr(cconfs);
if(get_int(cconfs[OPT_BREAKPOINT])>=2000
&& get_int(cconfs[OPT_BREAKPOINT])<3000)
{
breaking=get_int(cconfs[OPT_BREAKPOINT]);
breakcount=breaking-2000;
}
logp("Begin phase2 (receive file data)\n");
if(!(dpth=dpth_alloc())
|| dpth_protocol1_init(dpth, sdirs->currentdata,
get_int(cconfs[OPT_MAX_STORAGE_SUBDIRS])))
goto error;
if(open_previous_manifest(&cmanio, sdirs, incexc, cconfs))
goto error;
if(get_int(cconfs[OPT_DIRECTORY_TREE]))
{
// Need to make sure we do not try to create a path that is
// too long.
if(build_path_w(sdirs->treepath)) goto error;
mkdir(sdirs->treepath, 0777);
treepathlen=strlen(sdirs->treepath);
if(init_fs_max(sdirs->treepath))
goto error;
}
if(resume && !(p1pos=do_resume(sdirs, dpth, cconfs)))
goto error;
if(!(p1manio=manio_open_phase1(sdirs->phase1data, "rb", PROTO_1))
|| (resume && manio_seek(p1manio, p1pos)))
goto error;
if(!(cb=sbuf_alloc(PROTO_1))
|| !(p1b=sbuf_alloc(PROTO_1))
|| !(rb=sbuf_alloc(PROTO_1)))
goto error;
// Unchanged and changed should now be truncated correctly, we just
// have to open them for appending.
// Data is not getting written to a compressed file.
// This is important for recovery if the power goes.
if(!(ucmanio=manio_open_phase2(sdirs->unchanged, "ab", PROTO_1))
|| !(chmanio=manio_open_phase2(sdirs->changed, "ab", PROTO_1)))
goto error;
while(1)
{
if(breaking && breakcount--==0)
return breakpoint(breaking, __func__);
if(write_status(CNTR_STATUS_BACKUP,
rb->path.buf?rb->path.buf:p1b->path.buf, cntr))
goto error;
if(last_requested
|| !p1manio
|| asfd->writebuflen)
{
switch(do_stuff_to_receive(asfd, sdirs,
cconfs, rb, chmanio, dpth, &last_requested))
{
case 0: break;
case 1: goto end; // Finished ok.
case -1: goto error;
}
}
switch(do_stuff_to_send(asfd, p1b, &last_requested))
{
case 0: break;
case 1: continue;
case -1: goto error;
}
if(!p1manio) continue;
sbuf_free_content(p1b);
switch(manio_read(p1manio, p1b))
{
case 0: break;
case 1: manio_close(&p1manio);
if(asfd->write_str(asfd,
CMD_GEN, "backupphase2end")) goto error;
break;
case -1: goto error;
}
if(!cmanio)
{
// No old manifest, need to ask for a new file.
if(process_new(sdirs, cconfs, p1b, ucmanio))
goto error;
continue;
}
// Have an old manifest, look for it there.
// Might already have it, or be ahead in the old
// manifest.
if(cb->path.buf) switch(maybe_process_file(asfd,
sdirs, cb, p1b, ucmanio, cconfs))
{
case 0: break;
case 1: continue;
case -1: goto error;
}
while(cmanio)
{
sbuf_free_content(cb);
switch(manio_read(cmanio, cb))
{
case 0: break;
case 1: manio_close(&cmanio);
if(process_new(sdirs, cconfs, p1b,
ucmanio)) goto error;
continue;
case -1: goto error;
}
switch(maybe_process_file(asfd, sdirs,
cb, p1b, ucmanio, cconfs))
{
case 0: continue;
case 1: break;
case -1: goto error;
}
break;
}
}
error:
ret=-1;
end:
if(manio_close(&chmanio))
{
logp("error closing %s in %s\n", sdirs->changed, __func__);
ret=-1;
}
if(manio_close(&ucmanio))
{
logp("error closing %s in %s\n", sdirs->unchanged, __func__);
ret=-1;
}
free_w(&deltmppath);
free_w(&last_requested);
sbuf_free(&cb);
sbuf_free(&p1b);
sbuf_free(&rb);
manio_close(&p1manio);
manio_close(&cmanio);
dpth_free(&dpth);
man_off_t_free(&p1pos);
if(!ret && sdirs)
unlink(sdirs->phase1data);
logp("End phase2 (receive file data)\n");
return ret;
}
