// Used on resume, this just reads the phase1 file and sets up cntr.
static int read_phase1(struct manio *p1manio, struct conf **cconfs)
{
int ret=-1;
struct sbuf *p1b;
enum protocol protocol=get_protocol(cconfs);
struct cntr *cntr=get_cntr(cconfs);
if(!(p1b=sbuf_alloc(protocol))) return -1;
while(1)
{
sbuf_free_content(p1b);
switch(manio_read(p1manio, p1b))
{
case 0:
break;
case 1:
ret=0;
default:
goto end;
}
cntr_add_phase1(cntr, p1b->path.cmd, 0);
if(sbuf_is_estimatable(p1b))
cntr_add_val(cntr, CMD_BYTES_ESTIMATED,
(uint64_t)p1b->statp.st_size);
}
end:
sbuf_free(&p1b);
return ret;
}
static
#endif
int forward_past_entry(struct manio *manio, struct iobuf *target,
enum protocol protocol, man_off_t **pos)
{
struct sbuf *sb=NULL;
if(!(sb=sbuf_alloc(protocol)))
goto error;
man_off_t_free(pos);
if(!(*pos=manio_tell(manio)))
{
logp("Could not manio_tell first pos in %s(): %s\n",
__func__, strerror(errno));
goto error;
}
while(1)
{
sbuf_free_content(sb);
switch(manio_read(manio, sb))
{
case 0:
break;
case 1:
logp("End of file in %s()\n", __func__);
goto error;
default:
logp("Error in %s()\n", __func__);
// Treat error in unchanged manio as not OK.
goto error;
}
if(target->cmd==sb->path.cmd
&& !pathcmp(target->buf, sb->path.buf))
{
man_off_t_free(pos);
if(!(*pos=manio_tell(manio)))
{
logp("Could not get pos in %s(): %s\n",
__func__, strerror(errno));
goto error;
}
sbuf_free(&sb);
return 0;
}
}
error:
sbuf_free(&sb);
man_off_t_free(pos);
return -1;
}
static int maybe_add_from_scan(struct manios *manios,
struct slist *slist, struct asfd *chfd, struct sbuf **csb)
{
int ret=-1;
struct sbuf *snew=NULL;
while(1)
{
sbuf_free(&snew);
if(!manios->phase1) return 0;
// Limit the amount loaded into memory at any one time.
if(slist && slist->head)
{
if(slist->head->protocol2->index
- slist->tail->protocol2->index>4096)
return 0;
}
if(!(snew=sbuf_alloc(PROTO_2))) goto end;
switch(manio_read(manios->phase1, snew))
{
case 0: break;
case 1: manio_close(&manios->phase1);
ret=0; // Finished.
default: goto end;
}
switch(entry_changed(snew, manios, chfd, csb))
{
case 0: continue; // No change.
case 1: break;
default: goto end; // Error.
}
if(data_needed(snew)) snew->flags|=SBUF_NEED_DATA;
slist_add_sbuf(slist, snew);
snew=NULL;
}
return 0;
end:
sbuf_free(&snew);
return ret;
}
static int do_browse_manifest(struct asfd *srfd,
struct manio *manio, struct sbuf *sb, const char *browse)
{
int ret=-1;
int ars=0;
//char ls[1024]="";
//struct cntr cntr;
size_t blen=0;
char *last_bd_match=NULL;
if(browse) blen=strlen(browse);
while(1)
{
int r;
sbuf_free_content(sb);
if((ars=manio_read(manio, sb)))
{
if(ars<0) goto end;
// ars==1 means it ended ok.
break;
}
if(manio->protocol==PROTO_2 && sb->endfile.buf)
continue;
if(sb->path.cmd!=CMD_DIRECTORY
&& sb->path.cmd!=CMD_FILE
&& sb->path.cmd!=CMD_ENC_FILE
&& sb->path.cmd!=CMD_EFS_FILE
&& sb->path.cmd!=CMD_SPECIAL
&& !cmd_is_link(sb->path.cmd))
continue;
if((r=check_browsedir(browse, sb, blen, &last_bd_match))<0)
goto end;
if(!r) continue;
if(json_from_statp(sb->path.buf, &sb->statp)) goto end;
}
ret=0;
end:
free_w(&last_bd_match);
return ret;
}
// Combine the phase1 and phase2 files into a new manifest.
int backup_phase3_server_all(struct sdirs *sdirs, struct conf **confs)
{
int ret=-1;
int pcmp=0;
struct blk *blk=NULL;
struct sbuf *usb=NULL;
struct sbuf *csb=NULL;
char *manifesttmp=NULL;
struct manio *newmanio=NULL;
struct manio *chmanio=NULL;
struct manio *unmanio=NULL;
enum protocol protocol=get_protocol(confs);
struct cntr *cntr=get_cntr(confs);
const char *rmanifest_relative=NULL;
logp("Begin phase3 (merge manifests)\n");
if(protocol==PROTO_2)
rmanifest_relative=get_rmanifest_relative(sdirs, confs);
if(!(manifesttmp=get_tmp_filename(sdirs->manifest))
|| !(newmanio=manio_open_phase3(manifesttmp,
comp_level(get_int(confs[OPT_COMPRESSION])),
protocol, rmanifest_relative))
|| !(chmanio=manio_open_phase2(sdirs->changed, "rb", protocol))
|| !(unmanio=manio_open_phase2(sdirs->unchanged, "rb", protocol))
|| !(usb=sbuf_alloc(protocol))
|| !(csb=sbuf_alloc(protocol)))
goto end;
while(chmanio || unmanio)
{
if(!blk && !(blk=blk_alloc())) goto end;
if(unmanio
&& !usb->path.buf)
{
switch(manio_read(unmanio, usb))
{
case -1: goto end;
case 1: manio_close(&unmanio);
}
}
if(chmanio
&& !csb->path.buf)
{
switch(manio_read(chmanio, csb))
{
case -1: goto end;
case 1: manio_close(&chmanio);
}
}
if(usb->path.buf && !csb->path.buf)
{
if(write_status(CNTR_STATUS_MERGING,
usb->path.buf, cntr)) goto end;
switch(manio_copy_entry(
usb, usb, &blk, unmanio, newmanio))
{
case -1: goto end;
case 1: manio_close(&unmanio);
}
}
else if(!usb->path.buf && csb->path.buf)
{
if(write_status(CNTR_STATUS_MERGING,
csb->path.buf, cntr)) goto end;
switch(manio_copy_entry(
csb, csb, &blk, chmanio, newmanio))
{
case -1: goto end;
case 1: manio_close(&chmanio);
}
}
else if(!usb->path.buf && !csb->path.buf)
{
continue;
}
else if(!(pcmp=sbuf_pathcmp(usb, csb)))
{
// They were the same - write one.
if(write_status(CNTR_STATUS_MERGING,
csb->path.buf, cntr)) goto end;
switch(manio_copy_entry(
csb, csb, &blk, chmanio, newmanio))
{
case -1: goto end;
case 1: manio_close(&chmanio);
}
}
else if(pcmp<0)
{
if(write_status(CNTR_STATUS_MERGING,
usb->path.buf, cntr)) goto end;
switch(manio_copy_entry(
usb, usb, &blk, unmanio, newmanio))
{
case -1: goto end;
case 1: manio_close(&unmanio);
}
}
else
{
if(write_status(CNTR_STATUS_MERGING,
csb->path.buf, cntr)) goto end;
switch(manio_copy_entry(
csb, csb, &blk, chmanio, newmanio))
{
case -1: goto end;
case 1: manio_close(&chmanio);
}
}
}
// Flush to disk.
if(manio_close(&newmanio))
{
logp("error gzclosing %s in backup_phase3_server\n",
manifesttmp);
goto end;
}
// Rename race condition should be of no consequence here, as the
// manifest should just get recreated automatically.
if(do_rename(manifesttmp, sdirs->manifest))
goto end;
else
{
recursive_delete(sdirs->changed);
recursive_delete(sdirs->unchanged);
}
logp("End phase3 (merge manifests)\n");
ret=0;
end:
manio_close(&newmanio);
manio_close(&chmanio);
manio_close(&unmanio);
sbuf_free(&csb);
sbuf_free(&usb);
blk_free(&blk);
free_w(&manifesttmp);
return ret;
}
static
#endif
int get_last_good_entry(struct manio *manio, struct iobuf *result,
struct cntr *cntr, struct dpth *dpth, enum protocol protocol,
man_off_t **pos)
{
int ars=0;
int got_vss_start=0;
struct sbuf *sb=NULL;
struct iobuf lastpath;
if(!(sb=sbuf_alloc(protocol)))
goto error;
iobuf_init(&lastpath);
man_off_t_free(pos);
if(!(*pos=manio_tell(manio)))
{
logp("Could not manio_tell first pos in %s(): %s\n",
__func__, strerror(errno));
goto error;
}
while(1)
{
if(sb->path.buf && !got_vss_start)
{
iobuf_free_content(&lastpath);
iobuf_move(&lastpath, &sb->path);
if(!sbuf_is_filedata(sb)
&& !sbuf_is_vssdata(sb))
{
iobuf_free_content(result);
iobuf_move(result, &lastpath);
man_off_t_free(pos);
if(!(*pos=manio_tell(manio)))
{
logp("Could not manio_tell pos in %s(): %s\n",
__func__, strerror(errno));
goto error;
}
}
}
if(sb->endfile.buf && !got_vss_start)
{
iobuf_free_content(result);
iobuf_move(result, &lastpath);
man_off_t_free(pos);
if(!(*pos=manio_tell(manio)))
{
logp("Could not manio_tell pos in %s(): %s\n",
__func__, strerror(errno));
goto error;
}
}
sbuf_free_content(sb);
ars=manio_read(manio, sb);
if(dpth && set_higher_datapth(sb, dpth)) goto error;
switch(ars)
{
case 0:
break;
case 1:
iobuf_free_content(&lastpath);
sbuf_free(&sb);
return 0;
default:
if(result->buf)
logp("Error after %s in %s()\n",
result->buf, __func__);
// Treat error in changed manio as
// OK - could have been a short write.
iobuf_free_content(&lastpath);
sbuf_free(&sb);
return 0;
}
// Some hacks for split_vss.
switch(sb->path.cmd)
{
case CMD_VSS:
case CMD_ENC_VSS:
got_vss_start=1;
break;
case CMD_VSS_T:
case CMD_ENC_VSS_T:
got_vss_start=0;
break;
case CMD_FILE:
case CMD_ENC_FILE:
if(S_ISDIR(sb->statp.st_mode))
got_vss_start=0;
break;
default:
break;
}
if(cntr)
{
// FIX THIS: cannot distinguish between new and
// changed files.
cntr_add_changed(cntr, sb->path.cmd);
if(sb->endfile.buf)
{
uint64_t e=strtoull(sb->endfile.buf, NULL, 10);
cntr_add_bytes(cntr, e);
}
}
}
error:
iobuf_free_content(&lastpath);
sbuf_free(&sb);
man_off_t_free(pos);
return -1;
}
static
#endif
int forward_before_entry(struct manio *manio, struct iobuf *target,
struct cntr *cntr, struct dpth *dpth, enum protocol protocol,
man_off_t **pos)
{
int ars=0;
struct sbuf *sb=NULL;
if(!(sb=sbuf_alloc(protocol)))
goto error;
man_off_t_free(pos);
if(!(*pos=manio_tell(manio)))
{
logp("Could not manio_tell first pos in %s(): %s\n",
__func__, strerror(errno));
goto error;
}
while(1)
{
if(sb->endfile.buf
|| (sb->path.buf && !sbuf_is_filedata(sb)))
{
man_off_t_free(pos);
if(!(*pos=manio_tell(manio)))
{
logp("Could not manio_tell pos in %s(): "
"%s\n", __func__, strerror(errno));
goto error;
}
}
sbuf_free_content(sb);
ars=manio_read(manio, sb);
if(dpth && set_higher_datapth(sb, dpth)) goto error;
switch(ars)
{
case 0:
break;
case 1:
sbuf_free(&sb);
return 0;
default:
logp("Error in %s(), but continuing\n",
__func__);
// Treat error in unchanged manio as
// OK - could have been a short write.
sbuf_free(&sb);
return 0;
}
if(iobuf_pathcmp(target, &sb->path)<=0)
{
sbuf_free(&sb);
return 0;
}
if(cntr)
{
cntr_add_same(cntr, sb->path.cmd);
if(sb->endfile.buf)
{
uint64_t e=strtoull(sb->endfile.buf, NULL, 10);
cntr_add_bytes(cntr, e);
}
}
}
error:
sbuf_free(&sb);
man_off_t_free(pos);
return -1;
}
// Return -1 for error, 0 for entry not changed, 1 for entry changed (or new).
static int entry_changed(struct sbuf *sb,
struct manios *manios, struct asfd *chfd, struct sbuf **csb,
struct cntr *cntr)
{
static int finished=0;
int pcmp;
if(finished)
{
cntr_add_new(cntr, sb->path.cmd);
return 1;
}
if(*csb && (*csb)->path.buf)
{
// Already have an entry.
}
else
{
// Need to read another.
switch(manio_read(manios->current, *csb))
{
case 1: // Reached the end.
sbuf_free(csb);
finished=1;
cntr_add_new(cntr, sb->path.cmd);
return 1;
case -1: return -1;
}
if(!(*csb)->path.buf)
{
logp("Should have a path at this point, but do not, in %s\n", __func__);
return -1;
}
// Got an entry.
}
while(1)
{
if(!(pcmp=sbuf_pathcmp(*csb, sb)))
return found_in_current_manifest(*csb, sb,
manios, chfd, cntr);
else if(pcmp>0)
{
cntr_add_new(cntr, sb->path.cmd);
return 1;
}
// cntr_add_deleted(cntr, (*csb)->path.cmd);
// Behind - need to read more data from the old manifest.
switch(manio_read(manios->current, *csb))
{
case 1: // Reached the end.
sbuf_free(csb);
cntr_add_new(cntr, sb->path.cmd);
return 1;
case -1: return -1;
}
// Got something, go back around the loop.
}
return 0;
}
static int list_manifest(const char *fullpath)
{
int ret=0;
struct sbuf *sb=NULL;
struct manio *manio=NULL;
char *manifest_dir=NULL;
char *last_bd_match=NULL;
size_t bdlen=0;
if(!(manifest_dir=prepend_s(fullpath,
protocol==PROTO_1?"manifest.gz":"manifest"))
|| !(manio=manio_open(manifest_dir, "rb", protocol))
|| !(sb=sbuf_alloc(protocol)))
{
log_and_send_oom(asfd);
goto error;
}
if(browsedir) bdlen=strlen(browsedir);
while(1)
{
sbuf_free_content(sb);
switch(manio_read(manio, sb))
{
case 0:
break;
case 1:
if(browsedir && *browsedir && !last_bd_match)
asfd_write_wrapper_str(asfd,
CMD_ERROR,
"directory not found");
goto end; // Finished OK.
default:
goto error;
}
if(protocol==PROTO_2 && sb->endfile.buf)
continue;
if(sbuf_is_metadata(sb))
continue;
if(write_status(CNTR_STATUS_LISTING, sb->path.buf, cntr))
goto error;
if(browsedir)
{
int r;
if((r=check_browsedir(browsedir,
sb, bdlen, &last_bd_match))<0)
goto error;
if(!r) continue;
}
if(regex && !regex_check(regex, sb->path.buf))
continue;
if(asfd_write_wrapper(asfd, &sb->attr)
|| asfd_write_wrapper(asfd, &sb->path))
goto error;
if(sbuf_is_link(sb)
&& asfd_write_wrapper(asfd, &sb->link))
goto error;
}
error:
ret=-1;
end:
sbuf_free(&sb);
free_w(&manifest_dir);
manio_close(&manio);
free_w(&last_bd_match);
return ret;
}
int backup_phase2_server_protocol1(struct async *as, struct sdirs *sdirs,
const char *incexc, int resume, struct conf **cconfs)
{
int ret=0;
man_off_t *p1pos=NULL;
struct manio *p1manio=NULL;
struct dpth *dpth=NULL;
char *deltmppath=NULL;
char *last_requested=NULL;
struct manio *chmanio=NULL; // changed data
struct manio *ucmanio=NULL; // unchanged data
struct manio *cmanio=NULL; // previous (current) manifest.
struct sbuf *cb=NULL; // file list in current manifest
struct sbuf *p1b=NULL; // file list from client
struct sbuf *rb=NULL; // receiving file from client
struct asfd *asfd=NULL;
int breaking=0;
int breakcount=0;
struct cntr *cntr=NULL;
if(!as)
{
logp("async not provided to %s()\n", __func__);
goto error;
}
if(!sdirs)
{
logp("sdirs not provided to %s()\n", __func__);
goto error;
}
if(!cconfs)
{
logp("cconfs not provided to %s()\n", __func__);
goto error;
}
asfd=as->asfd;
if(!asfd)
{
logp("asfd not provided to %s()\n", __func__);
goto error;
}
cntr=get_cntr(cconfs);
if(get_int(cconfs[OPT_BREAKPOINT])>=2000
&& get_int(cconfs[OPT_BREAKPOINT])<3000)
{
breaking=get_int(cconfs[OPT_BREAKPOINT]);
breakcount=breaking-2000;
}
logp("Begin phase2 (receive file data)\n");
if(!(dpth=dpth_alloc())
|| dpth_protocol1_init(dpth, sdirs->currentdata,
get_int(cconfs[OPT_MAX_STORAGE_SUBDIRS])))
goto error;
if(open_previous_manifest(&cmanio, sdirs, incexc, cconfs))
goto error;
if(get_int(cconfs[OPT_DIRECTORY_TREE]))
{
// Need to make sure we do not try to create a path that is
// too long.
if(build_path_w(sdirs->treepath)) goto error;
mkdir(sdirs->treepath, 0777);
treepathlen=strlen(sdirs->treepath);
if(init_fs_max(sdirs->treepath))
goto error;
}
if(resume && !(p1pos=do_resume(sdirs, dpth, cconfs)))
goto error;
if(!(p1manio=manio_open_phase1(sdirs->phase1data, "rb", PROTO_1))
|| (resume && manio_seek(p1manio, p1pos)))
goto error;
if(!(cb=sbuf_alloc(PROTO_1))
|| !(p1b=sbuf_alloc(PROTO_1))
|| !(rb=sbuf_alloc(PROTO_1)))
goto error;
// Unchanged and changed should now be truncated correctly, we just
// have to open them for appending.
// Data is not getting written to a compressed file.
// This is important for recovery if the power goes.
if(!(ucmanio=manio_open_phase2(sdirs->unchanged, "ab", PROTO_1))
|| !(chmanio=manio_open_phase2(sdirs->changed, "ab", PROTO_1)))
goto error;
while(1)
{
if(breaking && breakcount--==0)
return breakpoint(breaking, __func__);
if(write_status(CNTR_STATUS_BACKUP,
rb->path.buf?rb->path.buf:p1b->path.buf, cntr))
goto error;
if(last_requested
|| !p1manio
|| asfd->writebuflen)
{
switch(do_stuff_to_receive(asfd, sdirs,
cconfs, rb, chmanio, dpth, &last_requested))
{
case 0: break;
case 1: goto end; // Finished ok.
case -1: goto error;
}
}
switch(do_stuff_to_send(asfd, p1b, &last_requested))
{
case 0: break;
case 1: continue;
case -1: goto error;
}
if(!p1manio) continue;
sbuf_free_content(p1b);
switch(manio_read(p1manio, p1b))
{
case 0: break;
case 1: manio_close(&p1manio);
if(asfd->write_str(asfd,
CMD_GEN, "backupphase2end")) goto error;
break;
case -1: goto error;
}
if(!cmanio)
{
// No old manifest, need to ask for a new file.
if(process_new(sdirs, cconfs, p1b, ucmanio))
goto error;
continue;
}
// Have an old manifest, look for it there.
// Might already have it, or be ahead in the old
// manifest.
if(cb->path.buf) switch(maybe_process_file(asfd,
sdirs, cb, p1b, ucmanio, cconfs))
{
case 0: break;
case 1: continue;
case -1: goto error;
}
while(cmanio)
{
sbuf_free_content(cb);
switch(manio_read(cmanio, cb))
{
case 0: break;
case 1: manio_close(&cmanio);
if(process_new(sdirs, cconfs, p1b,
ucmanio)) goto error;
continue;
case -1: goto error;
}
switch(maybe_process_file(asfd, sdirs,
cb, p1b, ucmanio, cconfs))
{
case 0: continue;
case 1: break;
case -1: goto error;
}
break;
}
}
error:
ret=-1;
end:
if(manio_close(&chmanio))
{
logp("error closing %s in %s\n", sdirs->changed, __func__);
ret=-1;
}
if(manio_close(&ucmanio))
{
logp("error closing %s in %s\n", sdirs->unchanged, __func__);
ret=-1;
}
free_w(&deltmppath);
free_w(&last_requested);
sbuf_free(&cb);
sbuf_free(&p1b);
sbuf_free(&rb);
manio_close(&p1manio);
manio_close(&cmanio);
dpth_free(&dpth);
man_off_t_free(&p1pos);
if(!ret && sdirs)
unlink(sdirs->phase1data);
logp("End phase2 (receive file data)\n");
return ret;
}
