static int make_rev_sig(const char *dst, const char *sig, const char *endfile, int compression, struct cntr *cntr)
{
FILE *dstfp=NULL;
gzFile dstzp=NULL;
FILE *sigp=NULL;
rs_result result;
//logp("make rev sig: %s %s\n", dst, sig);
if(dpth_is_compressed(compression, dst))
dstzp=gzopen_file(dst, "rb");
else
dstfp=open_file(dst, "rb");
if((!dstzp && !dstfp)
|| !(sigp=open_file(sig, "wb")))
{
gzclose_fp(&dstzp);
close_fp(&dstfp);
return -1;
}
result=rs_sig_gzfile(dstfp, dstzp, sigp,
get_librsync_block_len(endfile),
RS_DEFAULT_STRONG_LEN, NULL, cntr);
gzclose_fp(&dstzp);
close_fp(&dstfp);
close_fp(&sigp);
//logp("end of make rev sig\n");
return result;
}
static int compress(const char *src, const char *dst, struct config *cconf)
{
int res;
int got;
FILE *mp=NULL;
gzFile zp=NULL;
char buf[ZCHUNK];
if(!(mp=open_file(src, "rb"))
|| !(zp=gzopen_file(dst, comp_level(cconf))))
{
close_fp(&mp);
gzclose_fp(&zp);
return -1;
}
while((got=fread(buf, 1, sizeof(buf), mp))>0)
{
res=gzwrite(zp, buf, got);
if(res!=got)
{
logp("compressing manifest - read %d but wrote %d\n",
got, res);
close_fp(&mp);
gzclose_fp(&zp);
return -1;
}
}
close_fp(&mp);
return gzclose_fp(&zp); // this can give an error when out of space
}
static int make_rev_delta(const char *src, const char *sig, const char *del,
int compression, struct conf *cconf)
{
int ret=-1;
FILE *srcfp=NULL;
FILE *delfp=NULL;
FILE *sigp=NULL;
gzFile srczp=NULL;
gzFile delzp=NULL;
rs_signature_t *sumset=NULL;
//logp("make rev delta: %s %s %s\n", src, sig, del);
if(!(sigp=open_file(sig, "rb"))) goto end;
if(rs_loadsig_file(sigp, &sumset, NULL)!=RS_DONE
|| rs_build_hash_table(sumset)!=RS_DONE)
goto end;
//logp("make rev deltb: %s %s %s\n", src, sig, del);
if(dpthl_is_compressed(compression, src))
srczp=gzopen_file(src, "rb");
else
srcfp=open_file(src, "rb");
if(!srczp && !srcfp) goto end;
if(cconf->compression)
delzp=gzopen_file(del, comp_level(cconf));
else
delfp=open_file(del, "wb");
if(!delzp && !delfp) goto end;
if(rs_delta_gzfile(NULL, sumset, srcfp, srczp,
delfp, delzp, NULL, cconf->cntr)!=RS_DONE)
goto end;
ret=0;
end:
if(sumset) rs_free_sumset(sumset);
gzclose_fp(&srczp);
close_fp(&srcfp);
close_fp(&sigp);
if(gzclose_fp(&delzp))
{
logp("error closing zp %s in %s\n", del, __func__);
ret=-1;
}
if(close_fp(&delfp))
{
logp("error closing fp %s in %s\n", del, __func__);
ret=-1;
}
return ret;
}
int manio_close(struct manio *manio)
{
if(manio_closed(manio)) return 0;
if(sort_and_write_hooks(manio)
|| sort_and_write_dindex(manio))
{
gzclose_fp(&(manio->zp));
return -1;
}
return gzclose_fp(&(manio->zp));
}
// Also used by backup_phase4_server.c
int do_patch(const char *dst, const char *del, const char *upd, bool gzupd, struct cntr *cntr, struct config *cconf)
{
FILE *dstp=NULL;
FILE *delfp=NULL;
gzFile delzp=NULL;
gzFile updp=NULL;
FILE *updfp=NULL;
rs_result result;
//logp("patching...\n");
if(!(dstp=fopen(dst, "rb")))
{
logp("could not open %s for reading\n", dst);
return -1;
}
if(dpth_is_compressed(del))
delzp=gzopen(del, "rb");
else
delfp=fopen(del, "rb");
if(!delzp && !delfp)
{
logp("could not open %s for reading\n", del);
close_fp(&dstp);
return -1;
}
if(gzupd)
updp=gzopen(upd, comp_level(cconf));
else
updfp=fopen(upd, "wb");
if(!updp && !updfp)
{
logp("could not open %s for writing\n", upd);
close_fp(&dstp);
gzclose_fp(&delzp);
close_fp(&delfp);
return -1;
}
result=rs_patch_gzfile(dstp, delfp, delzp, updfp, updp, NULL, cntr);
fclose(dstp);
gzclose_fp(&delzp);
close_fp(&delfp);
if(updp) gzclose_fp(&updp);
if(updfp) fclose(updfp);
return result;
}
static int make_rev_sig(const char *dst, const char *sig, const char *endfile,
int compression, struct conf *conf)
{
int ret=-1;
FILE *dstfp=NULL;
gzFile dstzp=NULL;
FILE *sigp=NULL;
//logp("make rev sig: %s %s\n", dst, sig);
if(dpthl_is_compressed(compression, dst))
dstzp=gzopen_file(dst, "rb");
else
dstfp=open_file(dst, "rb");
if((!dstzp && !dstfp)
|| !(sigp=open_file(sig, "wb"))
|| rs_sig_gzfile(NULL, dstfp, dstzp, sigp,
get_librsync_block_len(endfile),
RS_DEFAULT_STRONG_LEN, NULL, conf->cntr)!=RS_DONE)
goto end;
ret=0;
end:
//logp("end of make rev sig\n");
gzclose_fp(&dstzp);
close_fp(&dstfp);
if(close_fp(&sigp))
{
logp("error closing %s in %s\n", sig, __func__);
return -1;
}
return ret;
}
// Also used by restore.c.
// FIX THIS: This stuff is very similar to make_rev_delta, can maybe share
// some code.
int do_patch(struct asfd *asfd, const char *dst, const char *del,
const char *upd, bool gzupd, int compression, struct conf *cconf)
{
FILE *dstp=NULL;
FILE *delfp=NULL;
gzFile delzp=NULL;
gzFile updp=NULL;
FILE *updfp=NULL;
rs_result result=RS_IO_ERROR;
//logp("patching...\n");
if(!(dstp=open_file(dst, "rb"))) goto end;
if(dpthl_is_compressed(compression, del))
delzp=gzopen_file(del, "rb");
else
delfp=open_file(del, "rb");
if(!delzp && !delfp) goto end;
if(gzupd)
updp=gzopen(upd, comp_level(cconf));
else
updfp=fopen(upd, "wb");
if(!updp && !updfp) goto end;
result=rs_patch_gzfile(asfd,
dstp, delfp, delzp, updfp, updp, NULL, cconf->cntr);
end:
close_fp(&dstp);
gzclose_fp(&delzp);
close_fp(&delfp);
if(close_fp(&updfp))
{
logp("error closing %s in %s\n", upd, __func__);
result=RS_IO_ERROR;
}
if(gzclose_fp(&updp))
{
logp("error gzclosing %s in %s\n", upd, __func__);
result=RS_IO_ERROR;
}
return result;
}
// When a backup is ongoing, use this to add newly complete candidates.
int candidate_add_fresh(const char *path, struct conf *conf)
{
int ars;
int ret=-1;
gzFile zp=NULL;
const char *cp=NULL;
struct sbuf *sb=NULL;
struct candidate *candidate=NULL;
struct blk *blk=NULL;
if(!(candidate=candidates_add_new())) goto end;
cp=path+strlen(conf->directory);
while(cp && *cp=='/') cp++;
if(!(candidate->path=strdup(cp)))
{
log_out_of_memory(__func__);
goto end;
}
if(!(sb=sbuf_alloc(conf))
|| !(blk=blk_alloc())
|| !(zp=gzopen_file(path, "rb")))
goto end;
while(zp)
{
if((ars=sbuf_fill_from_gzfile(sb, NULL /* struct async */,
zp, blk, NULL, conf))<0)
goto end;
else if(ars>0)
{
// Reached the end.
break;
}
if(!*(blk->weak)) continue;
if(is_hook(blk->weak))
{
if(sparse_add_candidate(blk->weak, candidate))
goto end;
}
*blk->weak='\0';
}
if(scores_grow(scores, candidates_len)) goto end;
candidates_set_score_pointers(candidates, candidates_len, scores);
scores_reset(scores);
//printf("HERE: %d candidates\n", (int)candidates_len);
ret=0;
end:
gzclose_fp(&zp);
sbuf_free(sb);
blk_free(blk);
return ret;
}
static int sort_and_write_hooks(struct manio *manio)
{
int i;
int ret=-1;
gzFile zp=NULL;
char comp[32]="";
char *path=NULL;
int hook_count=manio->hook_count;
char **hook_sort=manio->hook_sort;
if(!hook_sort) return 0;
snprintf(comp, sizeof(comp), "%08lX", manio->fcount-1);
if(!(path=prepend_s(manio->hook_dir, comp))
|| build_path_w(path)
|| !(zp=gzopen_file(path, manio->mode)))
goto end;
qsort(hook_sort, hook_count, sizeof(char *), strsort);
if(write_hook_header(manio, zp, comp)) goto end;
for(i=0; i<hook_count; i++)
{
// Do not bother with duplicates.
if(i && !strcmp(hook_sort[i],
hook_sort[i-1])) continue;
gzprintf(zp, "%c%04X%s\n", CMD_FINGERPRINT,
(unsigned int)strlen(hook_sort[i]), hook_sort[i]);
}
if(gzclose_fp(&zp))
{
logp("Error closing %s in %s: %s\n",
path, __func__, strerror(errno));
goto end;
}
manio->hook_count=0;
ret=0;
end:
gzclose_fp(&zp);
if(path) free(path);
return ret;
}
static int deal_with_receive_end_file(struct asfd *asfd, struct sdirs *sdirs,
struct sbuf *rb, FILE *p2fp, struct conf *cconf, char **last_requested)
{
static char *cp=NULL;
static struct iobuf *rbuf;
rbuf=asfd->rbuf;
// Finished the file.
// Write it to the phase2 file, and free the buffers.
if(close_fp(&(rb->burp1->fp)))
{
logp("error closing delta for %s in receive\n", rb->path);
goto error;
}
if(gzclose_fp(&(rb->burp1->zp)))
{
logp("error gzclosing delta for %s in receive\n", rb->path);
goto error;
}
iobuf_copy(&rb->burp1->endfile, rbuf);
rbuf->buf=NULL;
if(rb->flags & SBUFL_RECV_DELTA && finish_delta(sdirs, rb))
goto error;
if(sbufl_to_manifest(rb, p2fp, NULL))
goto error;
if(rb->flags & SBUFL_RECV_DELTA)
cntr_add_changed(cconf->cntr, rb->path.cmd);
else
cntr_add(cconf->cntr, rb->path.cmd, 0);
if(*last_requested && !strcmp(rb->path.buf, *last_requested))
{
free(*last_requested);
*last_requested=NULL;
}
cp=strchr(rb->burp1->endfile.buf, ':');
if(rb->burp1->endfile.buf)
cntr_add_bytes(cconf->cntr,
strtoull(rb->burp1->endfile.buf, NULL, 10));
if(cp)
{
// checksum stuff goes here
}
sbuf_free_content(rb);
return 0;
error:
sbuf_free_content(rb);
return -1;
}
static int copy_gzFile_to_gzpath(gzFile sp, const char *dst)
{
size_t b=0;
size_t w=0;
gzFile dp=NULL;
unsigned char in[ZCHUNK];
if(!(dp=gzopen_file(dst, "wb")))
return -1;
if(copy_gzFile_to_gzFile(sp, dp))
{
gzclose_fp(&dp);
return -1;
}
if(gzclose_fp(&dp))
{
logp("failed gzclose when copying to %s\n", dst);
return -1;
}
return 0;
}
int zlib_inflate(struct asfd *asfd, const char *source,
const char *dest, struct conf *conf)
{
int ret=-1;
size_t b=0;
FILE *fp=NULL;
gzFile zp=NULL;
unsigned char in[ZCHUNK];
if(!(zp=gzopen_file(source, "rb")))
{
logw(asfd, conf, "could not open %s in %s\n", source, __func__);
goto end;
}
if(!(fp=open_file(dest, "wb")))
{
logw(asfd, conf, "could not open %s in %s: %s\n",
dest, __func__, strerror(errno));
goto end;
}
while((b=gzread(zp, in, ZCHUNK))>0)
{
if(fwrite(in, 1, b, fp)!=b)
{
logw(asfd, conf, "error when writing to %s\n", dest);
goto end;
}
}
if(!gzeof(zp))
{
logw(asfd, conf,
"error while gzreading %s in %s\n", source, __func__);
goto end;
}
if(close_fp(&fp))
{
logw(asfd, conf,
"error when closing %s in %s: %s\n",
dest, __func__, strerror(errno));
goto end;
}
ret=0;
end:
gzclose_fp(&zp);
close_fp(&fp);
return ret;
}
int backup_phase1_server_all(struct async *as,
struct sdirs *sdirs, struct conf *conf)
{
int ars=0;
int ret=-1;
struct sbuf *sb=NULL;
gzFile p1zp=NULL;
char *phase1tmp=NULL;
struct asfd *asfd=as->asfd;
logp("Begin phase1 (file system scan)\n");
if(!(phase1tmp=get_tmp_filename(sdirs->phase1data)))
goto end;
if(!(p1zp=gzopen_file(phase1tmp, comp_level(conf))))
goto end;
if(!(sb=sbuf_alloc(conf)))
goto end;
while(1)
{
sbuf_free_content(sb);
if(conf->protocol==PROTO_BURP1)
ars=sbufl_fill(sb, asfd, NULL, NULL, conf->cntr);
else
ars=sbuf_fill(sb, asfd, NULL, NULL, NULL, conf);
if(ars)
{
if(ars<0) goto end;
//ars==1 means it ended ok.
// Last thing the client sends is 'backupphase2', and
// it wants an 'ok' reply.
if(asfd->write_str(asfd, CMD_GEN, "ok")
|| send_msg_zp(p1zp, CMD_GEN,
"phase1end", strlen("phase1end")))
goto end;
break;
}
if(write_status(STATUS_SCANNING, sb->path.buf, conf)
|| sbufl_to_manifest_phase1(sb, NULL, p1zp))
goto end;
cntr_add_phase1(conf->cntr, sb->path.cmd, 0);
if(sb->path.cmd==CMD_FILE
|| sb->path.cmd==CMD_ENC_FILE
|| sb->path.cmd==CMD_METADATA
|| sb->path.cmd==CMD_ENC_METADATA
|| sb->path.cmd==CMD_EFS_FILE)
cntr_add_val(conf->cntr, CMD_BYTES_ESTIMATED,
(unsigned long long)sb->statp.st_size, 0);
}
if(gzclose_fp(&p1zp))
{
logp("error closing %s in backup_phase1_server\n", phase1tmp);
goto end;
}
// Possible rename race condition is of no consequence here, because
// the working directory will always get deleted if phase1 is not
// complete.
if(do_rename(phase1tmp, sdirs->phase1data))
goto end;
//cntr_print(p1cntr, cntr, ACTION_BACKUP);
logp("End phase1 (file system scan)\n");
ret=0;
end:
free(phase1tmp);
gzclose_fp(&p1zp);
sbuf_free(&sb);
return ret;
}
// Combine the phase1 and phase2 files into a new manifest.
int backup_phase3_server(const char *phase2data, const char *unchangeddata, const char *manifest, int recovery, int compress, const char *client, struct cntr *p1cntr, struct cntr *cntr, struct config *cconf)
{
int ars=0;
int ret=0;
int pcmp=0;
FILE *ucfp=NULL;
FILE *p2fp=NULL;
FILE *mp=NULL;
gzFile mzp=NULL;
struct sbuf ucb;
struct sbuf p2b;
char *manifesttmp=NULL;
logp("Begin phase3 (merge manifests)\n");
if(!(manifesttmp=get_tmp_filename(manifest))) return -1;
if(!(ucfp=open_file(unchangeddata, "rb"))
|| !(p2fp=open_file(phase2data, "rb"))
|| (compress && !(mzp=gzopen_file(manifesttmp, comp_level(cconf))))
|| (!compress && !(mp=open_file(manifesttmp, "wb"))))
{
close_fp(&ucfp);
gzclose_fp(&mzp);
close_fp(&p2fp);
close_fp(&mp);
free(manifesttmp);
return -1;
}
init_sbuf(&ucb);
init_sbuf(&p2b);
while(ucfp || p2fp)
{
if(ucfp && !ucb.path && (ars=sbuf_fill(ucfp, NULL, &ucb, cntr)))
{
if(ars<0) { ret=-1; break; }
// ars==1 means it ended ok.
close_fp(&ucfp);
}
if(p2fp && !p2b.path && (ars=sbuf_fill(p2fp, NULL, &p2b, cntr)))
{
if(ars<0) { ret=-1; break; }
// ars==1 means it ended ok.
close_fp(&p2fp);
// In recovery mode, only want to read to the last
// entry in the phase 2 file.
if(recovery) break;
}
if(ucb.path && !p2b.path)
{
write_status(client, STATUS_MERGING, ucb.path,
p1cntr, cntr);
if(sbuf_to_manifest(&ucb, mp, mzp)) { ret=-1; break; }
free_sbuf(&ucb);
}
else if(!ucb.path && p2b.path)
{
write_status(client, STATUS_MERGING, p2b.path,
p1cntr, cntr);
if(sbuf_to_manifest(&p2b, mp, mzp)) { ret=-1; break; }
free_sbuf(&p2b);
}
else if(!ucb.path && !p2b.path)
{
continue;
}
else if(!(pcmp=sbuf_pathcmp(&ucb, &p2b)))
{
// They were the same - write one and free both.
write_status(client, STATUS_MERGING, p2b.path,
p1cntr, cntr);
if(sbuf_to_manifest(&p2b, mp, mzp)) { ret=-1; break; }
free_sbuf(&p2b);
free_sbuf(&ucb);
}
else if(pcmp<0)
{
write_status(client, STATUS_MERGING, ucb.path,
p1cntr, cntr);
if(sbuf_to_manifest(&ucb, mp, mzp)) { ret=-1; break; }
free_sbuf(&ucb);
}
else
{
write_status(client, STATUS_MERGING, p2b.path,
p1cntr, cntr);
if(sbuf_to_manifest(&p2b, mp, mzp)) { ret=-1; break; }
free_sbuf(&p2b);
}
}
free_sbuf(&ucb);
free_sbuf(&p2b);
close_fp(&p2fp);
close_fp(&ucfp);
if(close_fp(&mp))
{
logp("error closing %s in backup_phase3_server\n",
manifesttmp);
ret=-1;
}
if(gzclose_fp(&mzp))
{
logp("error gzclosing %s in backup_phase3_server\n",
manifesttmp);
ret=-1;
}
if(!ret)
{
if(do_rename(manifesttmp, manifest))
ret=-1;
else
{
unlink(phase2data);
unlink(unchangeddata);
}
}
free(manifesttmp);
logp("End phase3 (merge manifests)\n");
return ret;
}
int do_backup_server_burp1(struct async *as,
struct sdirs *sdirs, struct conf *cconf,
const char *incexc, int resume)
{
int ret=0;
char msg[256]="";
gzFile mzp=NULL;
// Real path to the working directory
char *realworking=NULL;
char tstmp[64]="";
struct asfd *asfd=as->asfd;
struct dpthl dpthl;
gzFile cmanfp=NULL;
struct stat statp;
logp("in do_backup_server\n");
if(init_dpthl(&dpthl, asfd, sdirs, cconf))
{
log_and_send(asfd, "could not init_dpthl\n");
goto error;
}
if(resume)
{
ssize_t len=0;
char real[256]="";
if((len=readlink(sdirs->working, real, sizeof(real)-1))<0)
len=0;
real[len]='\0';
if(!(realworking=prepend_s(sdirs->client, real)))
{
log_and_send_oom(asfd, __func__);
goto error;
}
if(open_log(asfd, realworking, cconf)) goto error;
}
else
{
// Not resuming - need to set everything up fresh.
if(get_new_timestamp(asfd, sdirs, cconf, tstmp, sizeof(tstmp)))
goto error;
if(!(realworking=prepend_s(sdirs->client, tstmp)))
{
log_and_send_oom(asfd, __func__);
goto error;
}
// Add the working symlink before creating the directory.
// This is because bedup checks the working symlink before
// going into a directory. If the directory got created first,
// bedup might go into it in the moment before the symlink
// gets added.
if(symlink(tstmp, sdirs->working)) // relative link to the real work dir
{
snprintf(msg, sizeof(msg),
"could not point working symlink to: %s",
realworking);
log_and_send(asfd, msg);
goto error;
}
else if(mkdir(realworking, 0777))
{
snprintf(msg, sizeof(msg),
"could not mkdir for next backup: %s",
sdirs->working);
log_and_send(asfd, msg);
unlink(sdirs->working);
goto error;
}
else if(open_log(asfd, realworking, cconf))
{
goto error;
}
else if(mkdir(sdirs->datadirtmp, 0777))
{
snprintf(msg, sizeof(msg),
"could not mkdir for datadir: %s", sdirs->datadirtmp);
log_and_send(asfd, msg);
goto error;
}
else if(write_timestamp(sdirs->timestamp, tstmp))
{
snprintf(msg, sizeof(msg),
"unable to write timestamp %s", sdirs->timestamp);
log_and_send(asfd, msg);
goto error;
}
else if(incexc && *incexc && write_incexc(realworking, incexc))
{
snprintf(msg, sizeof(msg), "unable to write incexc");
log_and_send(asfd, msg);
goto error;
}
if(backup_phase1_server(asfd, sdirs, cconf))
{
logp("error in phase 1\n");
goto error;
}
}
// Open the previous (current) manifest.
// If the split_vss setting changed between the previous backup
// and the new backup, do not open the previous manifest.
// This will have the effect of making the client back up everything
// fresh. Need to do this, otherwise toggling split_vss on and off
// will result in backups that do not work.
if(!lstat(sdirs->cmanifest, &statp)
&& !vss_opts_changed(sdirs, cconf, incexc))
{
if(!(cmanfp=gzopen_file(sdirs->cmanifest, "rb")))
{
if(!lstat(sdirs->cmanifest, &statp))
{
logp("could not open old manifest %s\n",
sdirs->cmanifest);
goto error;
}
}
}
//if(cmanfp) logp("Current manifest: %s\n", sdirs->cmanifest);
if(backup_phase2_server(asfd, sdirs, cconf, &cmanfp, &dpthl, resume))
{
logp("error in backup phase 2\n");
goto error;
}
if(backup_phase3_server(sdirs, cconf,
0 /* not recovery mode */, 1 /* compress */))
{
logp("error in backup phase 3\n");
goto error;
}
// will not write anything more to
// the new manifest
// finish_backup will open it again
// for reading
if(gzclose_fp(&mzp))
{
logp("Error closing manifest after phase3\n");
goto error;
}
asfd->write_str(asfd, CMD_GEN, "okbackupend");
logp("Backup ending - disconnect from client.\n");
// Close the connection with the client, the rest of the job
// we can do by ourselves.
asfd_free(&as->asfd);
// Move the symlink to indicate that we are now in the end
// phase.
if(do_rename(sdirs->working, sdirs->finishing))
goto error;
set_logfp(NULL, cconf); // does an fclose on logfp.
// finish_backup will open logfp again
ret=backup_phase4_server(sdirs, cconf);
if(!ret && cconf->keep>0)
ret=remove_old_backups(asfd, sdirs, cconf);
goto end;
error:
ret=-1;
end:
gzclose_fp(&cmanfp);
gzclose_fp(&mzp);
set_logfp(NULL, cconf); // does an fclose on logfp.
return ret;
}
// a = length of struct bu array
// i = position to restore from
static int restore_manifest(struct bu *arr, int a, int i, const char *tmppath1, const char *tmppath2, regex_t *regex, int srestore, enum action act, const char *client, char **dir_for_notify, struct cntr *p1cntr, struct cntr *cntr, struct config *cconf)
{
int ret=0;
gzFile zp=NULL;
char *manifest=NULL;
char *datadir=NULL;
FILE *logfp=NULL;
char *logpath=NULL;
char *logpathz=NULL;
// For sending status information up to the server.
char status=STATUS_RESTORING;
if(act==ACTION_RESTORE) status=STATUS_RESTORING;
else if(act==ACTION_VERIFY) status=STATUS_VERIFYING;
if(
(act==ACTION_RESTORE && !(logpath=prepend_s(arr[i].path, "restorelog", strlen("restorelog"))))
|| (act==ACTION_RESTORE && !(logpathz=prepend_s(arr[i].path, "restorelog.gz", strlen("restorelog.gz"))))
|| (act==ACTION_VERIFY && !(logpath=prepend_s(arr[i].path, "verifylog", strlen("verifylog"))))
|| (act==ACTION_VERIFY && !(logpathz=prepend_s(arr[i].path, "verifylog.gz", strlen("verifylog.gz"))))
|| !(manifest=prepend_s(arr[i].path, "manifest.gz", strlen("manifest.gz"))))
{
log_and_send_oom(__FUNCTION__);
ret=-1;
}
else if(!(logfp=open_file(logpath, "ab")) || set_logfp(logfp, cconf))
{
char msg[256]="";
snprintf(msg, sizeof(msg),
"could not open log file: %s", logpath);
log_and_send(msg);
ret=-1;
}
*dir_for_notify=strdup(arr[i].path);
log_restore_settings(cconf, srestore);
// First, do a pass through the manifest to set up the counters.
// This is the equivalent of a phase1 scan during backup.
if(!ret && !(zp=gzopen_file(manifest, "rb")))
{
log_and_send("could not open manifest");
ret=-1;
}
else
{
int ars=0;
int quit=0;
struct sbuf sb;
init_sbuf(&sb);
while(!quit)
{
if((ars=sbuf_fill(NULL, zp, &sb, cntr)))
{
if(ars<0) ret=-1;
// ars==1 means end ok
quit++;
}
else
{
if((!srestore
|| check_srestore(cconf, sb.path))
&& check_regex(regex, sb.path))
{
do_filecounter(p1cntr, sb.cmd, 0);
if(sb.endfile)
do_filecounter_bytes(p1cntr,
strtoull(sb.endfile, NULL, 10));
/*
if(sb.cmd==CMD_FILE
|| sb.cmd==CMD_ENC_FILE
|| sb.cmd==CMD_METADATA
|| sb.cmd==CMD_ENC_METADATA
|| sb.cmd==CMD_VSS
|| sb.cmd==CMD_ENC_VSS
|| sb.cmd==CMD_VSS_T
|| sb.cmd==CMD_ENC_VSS_T
|| sb.cmd==CMD_EFS_FILE)
do_filecounter_bytes(p1cntr,
(unsigned long long)
sb.statp.st_size);
*/
}
}
free_sbuf(&sb);
}
free_sbuf(&sb);
gzclose_fp(&zp);
}
if(cconf->send_client_counters)
{
if(send_counters(client, p1cntr, cntr))
{
ret=-1;
}
}
// Now, do the actual restore.
if(!ret && !(zp=gzopen_file(manifest, "rb")))
{
log_and_send("could not open manifest");
ret=-1;
}
else
{
char cmd;
int s=0;
int quit=0;
size_t len=0;
struct sbuf sb;
// For out-of-sequence directory restoring so that the
// timestamps come out right:
int scount=0;
struct sbuf **sblist=NULL;
init_sbuf(&sb);
while(!quit)
{
int ars=0;
char *buf=NULL;
if(async_read_quick(&cmd, &buf, &len))
{
logp("read quick error\n");
ret=-1; quit++; break;
}
if(buf)
{
//logp("got read quick\n");
if(cmd==CMD_WARNING)
{
logp("WARNING: %s\n", buf);
do_filecounter(cntr, cmd, 0);
free(buf); buf=NULL;
continue;
}
else if(cmd==CMD_INTERRUPT)
{
// Client wanted to interrupt the
// sending of a file. But if we are
// here, we have already moved on.
// Ignore.
free(buf); buf=NULL;
continue;
}
else
{
logp("unexpected cmd from client: %c:%s\n", cmd, buf);
free(buf); buf=NULL;
ret=-1; quit++; break;
}
}
if((ars=sbuf_fill(NULL, zp, &sb, cntr)))
{
if(ars<0) ret=-1;
// ars==1 means end ok
quit++;
}
else
{
if((!srestore
|| check_srestore(cconf, sb.path))
&& check_regex(regex, sb.path)
&& restore_ent(client,
&sb, &sblist, &scount,
arr, a, i, tmppath1, tmppath2,
act, status, cconf,
cntr, p1cntr))
{
ret=-1;
quit++;
}
}
free_sbuf(&sb);
}
gzclose_fp(&zp);
// Restore any directories that are left in the list.
if(!ret) for(s=scount-1; s>=0; s--)
{
if(restore_sbuf(sblist[s], arr, a, i,
tmppath1, tmppath2, act, client, status,
p1cntr, cntr, cconf))
{
ret=-1;
break;
}
}
free_sbufs(sblist, scount);
if(!ret) ret=do_restore_end(act, cntr);
//print_endcounter(cntr);
print_filecounters(p1cntr, cntr, act);
reset_filecounter(p1cntr, time(NULL));
reset_filecounter(cntr, time(NULL));
}
set_logfp(NULL, cconf);
compress_file(logpath, logpathz, cconf);
if(manifest) free(manifest);
if(datadir) free(datadir);
if(logpath) free(logpath);
if(logpathz) free(logpathz);
return ret;
}
static int resume_partial_new_file(struct sbuf *p1b, struct cntr *cntr, const char *currentdata, const char *datadirtmp, const char *deltmppath, struct dpth *dpth, struct config *cconf)
{
int ret=0;
int junk=0;
struct sbuf cb;
char *rpath=NULL;
int istreedata=0;
struct stat statp;
char *partial=NULL;
char *partialdir=NULL;
char *zdeltmp=NULL;
// It does not matter what this checksum is set to.
// This is just to get an endfile string in the format that
// process_changed_file expects.
unsigned char checksum[18]="0123456789ABCDEF";
// Need to set up a fake current sbuf.
init_sbuf(&cb);
cb.cmd=p1b->cmd;
cb.compression=p1b->compression;
cb.path=strdup(p1b->path);
cb.statbuf=strdup(p1b->statbuf);
if(!(rpath=set_new_datapth(&cb, datadirtmp, dpth, &istreedata, cconf)))
{
ret=-1;
goto end;
}
if(!(partialdir=prepend_s(datadirtmp, "p", strlen("p")))
|| !(partial=prepend_s(partialdir,
cb.datapth, strlen(cb.datapth)))
|| build_path(partialdir, cb.datapth, strlen(cb.datapth),
&partial, partialdir))
{
ret=-1;
goto end;
}
if(!lstat(partial, &statp) && S_ISREG(statp.st_mode))
{
// A previous resume was going on.
// Need to concatenate the possible delta onto the partial
// file.
FILE *dfp=NULL;
gzFile dzp=NULL;
logp("Resume previously resumed partial new file: %s %s\n",
cb.path, rpath);
if(!(cb.endfile=strdup(
get_endfile_str(statp.st_size, checksum))))
{
ret=-1;
goto end;
}
if(cb.compression)
{
// Recreate partial, in case it was only partially
// written and consequently has gz errors.
if(!(zdeltmp=prepend(deltmppath, ".z", strlen(".z"),
0 /* no slash */))
|| !(dzp=gzopen_file(zdeltmp, "wb"))
|| copy_gzpath_to_gzFile(partial, dzp)
|| do_rename(zdeltmp, partial))
{
ret=-1;
goto end;
}
}
else
{
// Just append to the existing one.
if(!(dfp=open_file(partial, "ab")))
{
ret=-1;
goto end;
}
}
if(!lstat(deltmppath, &statp) && S_ISREG(statp.st_mode))
{
if(cb.compression)
{
if(copy_gzpath_to_gzFile(deltmppath, dzp))
{
ret=-1;
goto end;
}
}
else
{
if(copy_path_to_File(deltmppath, dfp))
{
ret=-1;
goto end;
}
}
}
if(dfp && close_fp(&dfp))
{
ret=-1;
goto end;
}
if(dzp && gzclose_fp(&dzp))
{
ret=-1;
goto end;
}
if(process_changed_file(&cb, p1b, partialdir, NULL, NULL,
&junk /* resume_partial=0 */,
cntr, cconf))
{
ret=-1;
goto end;
}
if(!istreedata) incr_dpth(dpth, cconf);
goto end;
}
logp("Resume partial new file: %s %s\n", cb.path, rpath);
if(!lstat(rpath, &statp) && S_ISREG(statp.st_mode))
{
if(!(cb.endfile=strdup(
get_endfile_str(statp.st_size, checksum))))
{
ret=-1;
goto end;
}
// If compression is on, be careful with gzip unexpected
// end of file errors.
// Otherwise, just rename the whole file.
unlink(partial);
if(cb.compression)
{
if(copy_gzpath_to_gzpath(rpath, partial))
{
logp("Error in copy_gzpath_to_gzpath\n");
ret=-1;
goto end;
}
// delete the original.
if(unlink(rpath))
{
logp("Failed to unlink %s: %s\n",
rpath, strerror(errno));
return -1;
}
}
else
{
if(do_rename(rpath, partial))
{
ret=-1;
goto end;
}
}
// So, we have created a new directory beginning with 'p',
// and moved the partial download to it.
// We can now use the partial file as the basis of a librsync
// transfer.
if(process_changed_file(&cb, p1b, partialdir, NULL, NULL,
&junk /* resume_partial=0 */,
cntr, cconf))
{
ret=-1;
goto end;
}
if(!istreedata) incr_dpth(dpth, cconf);
goto end;
}
logp("Actually, no - just treat it as completely new\n");
end:
if(rpath) free(rpath);
if(partialdir) free(partialdir);
if(partial) free(partial);
if(zdeltmp) free(zdeltmp);
free_sbuf(&cb);
return ret;
}
int backup_phase2_server(gzFile *cmanfp, const char *phase1data, const char *phase2data, const char *unchangeddata, const char *datadirtmp, struct dpth *dpth, const char *currentdata, const char *working, const char *client, struct cntr *p1cntr, int resume, struct cntr *cntr, struct config *cconf)
{
int ars=0;
int ret=0;
gzFile p1zp=NULL;
char *deltmppath=NULL;
char *last_requested=NULL;
// Where to write phase2data.
// Data is not getting written to a compressed file.
// This is important for recovery if the power goes.
FILE *p2fp=NULL;
// unchanged data
FILE *ucfp=NULL;
int resume_partial=resume;
struct sbuf cb; // file list in current manifest
struct sbuf p1b; // file list from client
struct sbuf rb; // receiving file from client
init_sbuf(&cb);
init_sbuf(&p1b);
init_sbuf(&rb);
if(!(p1zp=gzopen_file(phase1data, "rb")))
goto error;
// Open in read+write mode, so that they can be read through if
// we need to resume.
// First, open them in a+ mode, so that they will be created if they
// do not exist.
if(!(ucfp=open_file(unchangeddata, "a+b")))
goto error;
if(!(p2fp=open_file(phase2data, "a+b")))
goto error;
close_fp(&ucfp);
close_fp(&p2fp);
if(!(ucfp=open_file(unchangeddata, "r+b")))
goto error;
if(!(p2fp=open_file(phase2data, "r+b")))
goto error;
if(resume && do_resume(p1zp, p2fp, ucfp, dpth, cconf, client,
p1cntr, cntr)) goto error;
logp("Begin phase2 (receive file data)\n");
if(!(deltmppath=prepend_s(working, "delta.tmp", strlen("delta.tmp"))))
goto error;
while(1)
{
int sts=0;
// logp("in loop, %s %s %c\n",
// *cmanfp?"got cmanfp":"no cmanfp",
// rb.path?:"no rb.path", rb.path?'X':rb.cmd);
if(rb.path) write_status(client, STATUS_BACKUP,
rb.path, p1cntr, cntr);
else write_status(client, STATUS_BACKUP,
p1b.path, p1cntr, cntr);
if((last_requested || !p1zp || writebuflen)
&& (ars=do_stuff_to_receive(&rb, p2fp, datadirtmp, dpth,
working, &last_requested, deltmppath, cntr, cconf)))
{
if(ars<0) goto error;
// 1 means ok.
break;
}
if((sts=do_stuff_to_send(&p1b, &last_requested))<0)
goto error;
if(!sts && p1zp)
{
free_sbuf(&p1b);
if((ars=sbuf_fill_phase1(NULL, p1zp, &p1b, cntr)))
{
if(ars<0) goto error;
// ars==1 means it ended ok.
gzclose_fp(&p1zp);
//logp("ended OK - write phase2end");
if(async_write_str(CMD_GEN, "backupphase2end"))
goto error;
}
//logp("check: %s\n", p1b.path);
if(!*cmanfp)
{
// No old manifest, need to ask for a new file.
//logp("no cmanfp\n");
if(process_new(&p1b, ucfp, currentdata,
datadirtmp, deltmppath,
dpth, &resume_partial,
cntr, cconf)) goto error;
}
else
{
// Have an old manifest, look for it there.
// Might already have it, or be ahead in the old
// manifest.
if(cb.path)
{
if((ars=maybe_process_file(&cb, &p1b,
ucfp, currentdata, datadirtmp,
deltmppath, dpth, &resume_partial,
cntr, cconf)))
{
if(ars<0) goto error;
// Do not free it - need to send stuff.
continue;
}
//free_sbuf(&p1b);
}
while(*cmanfp)
{
free_sbuf(&cb);
if((ars=sbuf_fill(NULL, *cmanfp, &cb, cntr)))
{
// ars==1 means it ended ok.
if(ars<0) goto error;
gzclose_fp(cmanfp);
//logp("ran out of current manifest\n");
if(process_new(&p1b, ucfp,
currentdata, datadirtmp,
deltmppath, dpth,
&resume_partial, cntr, cconf))
goto error;
break;
}
//logp("against: %s\n", cb.path);
if((ars=maybe_process_file(&cb, &p1b,
ucfp, currentdata, datadirtmp,
deltmppath, dpth, &resume_partial,
cntr, cconf)))
{
if(ars<0) goto error;
// Do not free it - need to send stuff.
break;
}
}
}
}
}
goto end;
error:
ret=-1;
end:
if(close_fp(&p2fp))
{
logp("error closing %s in backup_phase2_server\n",
phase2data);
ret=-1;
}
if(close_fp(&ucfp))
{
logp("error closing %s in backup_phase2_server\n",
unchangeddata);
ret=-1;
}
free(deltmppath);
free_sbuf(&cb);
free_sbuf(&p1b);
free_sbuf(&rb);
gzclose_fp(&p1zp);
if(!ret) unlink(phase1data);
logp("End phase2 (receive file data)\n");
return ret;
}
static int maybe_delete_files_from_manifest(const char *manifesttmp,
struct fdirs *fdirs, struct conf *cconf)
{
int ars=0;
int ret=-1;
int pcmp=0;
FILE *dfp=NULL;
gzFile nmzp=NULL;
gzFile omzp=NULL;
struct sbuf *db=NULL;
struct sbuf *mb=NULL;
struct stat statp;
if(lstat(fdirs->deletionsfile, &statp)) // No deletions, no problem.
return 0;
logp("Performing deletions on manifest\n");
if(!(manifesttmp=get_tmp_filename(fdirs->manifest)))
goto end;
if(!(dfp=open_file(fdirs->deletionsfile, "rb"))
|| !(omzp=gzopen_file(fdirs->manifest, "rb"))
|| !(nmzp=gzopen_file(manifesttmp, comp_level(cconf)))
|| !(db=sbuf_alloc(cconf))
|| !(mb=sbuf_alloc(cconf)))
goto end;
while(omzp || dfp)
{
if(dfp && !db->path.buf
&& (ars=sbufl_fill(db, NULL, dfp, NULL, cconf->cntr)))
{
if(ars<0) goto end;
// ars==1 means it ended ok.
close_fp(&dfp);
}
if(omzp && !mb->path.buf
&& (ars=sbufl_fill(mb, NULL, NULL, omzp, cconf->cntr)))
{
if(ars<0) goto end;
// ars==1 means it ended ok.
gzclose_fp(&omzp);
}
if(mb->path.buf && !db->path.buf)
{
if(sbufl_to_manifest(mb, NULL, nmzp)) goto end;
sbuf_free_content(mb);
}
else if(!mb->path.buf && db->path.buf)
{
sbuf_free_content(db);
}
else if(!mb->path.buf && !db->path.buf)
{
continue;
}
else if(!(pcmp=sbuf_pathcmp(mb, db)))
{
// They were the same - do not write.
sbuf_free_content(mb);
sbuf_free_content(db);
}
else if(pcmp<0)
{
// Behind in manifest. Write.
if(sbufl_to_manifest(mb, NULL, nmzp)) goto end;
sbuf_free_content(mb);
}
else
{
// Behind in deletions file. Do not write.
sbuf_free_content(db);
}
}
ret=0;
end:
if(gzclose_fp(&nmzp))
{
logp("error closing %s in %s\n", manifesttmp, __func__);
ret=-1;
}
close_fp(&dfp);
gzclose_fp(&omzp);
sbuf_free(&db);
sbuf_free(&mb);
if(!ret)
{
unlink(fdirs->deletionsfile);
// The rename race condition is not a problem here, as long
// as manifesttmp is the same path as that generated in the
// atomic data jiggle.
if(do_rename(manifesttmp, fdirs->manifest))
return -1;
}
if(manifesttmp) unlink(manifesttmp);
return ret;
}
// a = length of struct bu array
// i = position to restore from
static int restore_manifest(struct bu *arr, int a, int i, const char *tmppath1, const char *tmppath2, regex_t *regex, enum action act, const char *client, struct cntr *p1cntr, struct cntr *cntr, struct config *cconf, bool all)
{
int ret=0;
gzFile zp=NULL;
char *manifest=NULL;
char *datadir=NULL;
FILE *logfp=NULL;
char *logpath=NULL;
char *logpathz=NULL;
// For sending status information up to the server.
char status=STATUS_RESTORING;
if(act==ACTION_RESTORE) status=STATUS_RESTORING;
else if(act==ACTION_VERIFY) status=STATUS_VERIFYING;
if(
(act==ACTION_RESTORE && !(logpath=prepend_s(arr[i].path, "restorelog", strlen("restorelog"))))
|| (act==ACTION_RESTORE && !(logpathz=prepend_s(arr[i].path, "restorelog.gz", strlen("restorelog.gz"))))
|| (act==ACTION_VERIFY && !(logpath=prepend_s(arr[i].path, "verifylog", strlen("verifylog"))))
|| (act==ACTION_VERIFY && !(logpathz=prepend_s(arr[i].path, "verifylog.gz", strlen("verifylog.gz"))))
|| !(manifest=prepend_s(arr[i].path, "manifest.gz", strlen("manifest.gz"))))
{
log_and_send("out of memory");
ret=-1;
}
else if(!(logfp=open_file(logpath, "ab")) || set_logfp(logfp))
{
char msg[256]="";
snprintf(msg, sizeof(msg),
"could not open log file: %s", logpath);
log_and_send(msg);
ret=-1;
}
else if(!(zp=gzopen_file(manifest, "rb")))
{
log_and_send("could not open manifest");
ret=-1;
}
else
{
char cmd;
int quit=0;
size_t len=0;
struct sbuf sb;
// For out-of-sequence directory restoring so that the
// timestamps come out right:
int s=0;
int scount=0;
struct sbuf **sblist=NULL;
init_sbuf(&sb);
while(!quit)
{
int ars=0;
char *buf=NULL;
if(async_read_quick(&cmd, &buf, &len))
{
logp("read quick error\n");
ret=-1; quit++; break;
}
if(buf)
{
//logp("got read quick\n");
if(cmd==CMD_WARNING)
{
logp("WARNING: %s\n", buf);
do_filecounter(cntr, cmd, 0);
free(buf); buf=NULL;
continue;
}
else if(cmd==CMD_INTERRUPT)
{
// Client wanted to interrupt the
// sending of a file. But if we are
// here, we have already moved on.
// Ignore.
free(buf); buf=NULL;
continue;
}
else
{
logp("unexpected cmd from client: %c:%s\n", cmd, buf);
free(buf); buf=NULL;
ret=-1; quit++; break;
}
}
if((ars=sbuf_fill(NULL, zp, &sb, cntr)))
{
if(ars<0) ret=-1;
// ars==1 means end ok
quit++;
}
else
{
if(check_regex(regex, sb.path))
{
// Check if we have any directories waiting
// to be restored.
for(s=scount-1; s>=0; s--)
{
if(is_subdir(sblist[s]->path, sb.path))
{
// We are still in a subdir.
//printf(" subdir (%s %s)\n", sblist[s]->path, sb.path);
break;
}
else
{
// Can now restore sblist[s]
// because nothing else is
// fiddling in a subdirectory.
if(restore_sbuf(sblist[s], arr,
a, i, tmppath1, tmppath2, act,
client, status,
p1cntr, cntr, cconf))
{
ret=-1; quit++;
break;
}
else if(del_from_sbuf_arr(
&sblist, &scount))
{
ret=-1; quit++;
break;
}
}
}
/* If it is a directory, need to remember it
and restore it later, so that the
permissions come out right. */
/* Meta data of directories will also have
the stat stuff set to be a directory,
so will also come out at the end. */
if(!ret && S_ISDIR(sb.statp.st_mode))
{
if(add_to_sbuf_arr(&sblist, &sb, &scount))
{
ret=-1; quit++;
}
// Wipe out sb, without freeing up
// all the strings inside it, which
// have been added to sblist.
init_sbuf(&sb);
}
else if(!ret && restore_sbuf(&sb, arr, a, i,
tmppath1, tmppath2, act, client, status,
p1cntr, cntr, cconf))
{
ret=-1; quit++;
}
}
}
free_sbuf(&sb);
}
gzclose_fp(&zp);
// Restore any directories that are left in the list.
if(!ret) for(s=scount-1; s>=0; s--)
{
if(restore_sbuf(sblist[s], arr, a, i,
tmppath1, tmppath2, act, client, status,
p1cntr, cntr, cconf))
{
ret=-1;
break;
}
}
free_sbufs(sblist, scount);
if(!ret && !all) ret=do_restore_end(act, cntr);
print_endcounter(cntr);
print_filecounters(p1cntr, cntr, act, 0);
reset_filecounter(p1cntr);
reset_filecounter(cntr);
}
set_logfp(NULL);
compress_file(logpath, logpathz, cconf);
if(manifest) free(manifest);
if(datadir) free(datadir);
if(logpath) free(logpath);
if(logpathz) free(logpathz);
return ret;
}
static int make_rev_delta(const char *src, const char *sig, const char *del, int compression, struct cntr *cntr, struct config *cconf)
{
gzFile srczp=NULL;
FILE *srcfp=NULL;
FILE *sigp=NULL;
rs_result result;
rs_signature_t *sumset=NULL;
//logp("make rev delta: %s %s %s\n", src, sig, del);
if(!(sigp=open_file(sig, "rb"))) return -1;
if((result=rs_loadsig_file(sigp, &sumset, NULL))
|| (result=rs_build_hash_table(sumset)))
{
fclose(sigp);
rs_free_sumset(sumset);
return result;
}
fclose(sigp);
//logp("make rev deltb: %s %s %s\n", src, sig, del);
if(dpth_is_compressed(compression, src))
srczp=gzopen_file(src, "rb");
else
srcfp=open_file(src, "rb");
if(!srczp && !srcfp)
{
rs_free_sumset(sumset);
return -1;
}
if(cconf->compression)
{
gzFile delzp=NULL;
if(!(delzp=gzopen_file(del, comp_level(cconf))))
{
gzclose_fp(&srczp);
close_fp(&srcfp);
rs_free_sumset(sumset);
return -1;
}
result=rs_delta_gzfile(sumset, srcfp, srczp, NULL, delzp, NULL, cntr);
gzclose_fp(&delzp);
}
else
{
FILE *delfp=NULL;
if(!(delfp=open_file(del, "wb")))
{
gzclose_fp(&srczp);
close_fp(&srcfp);
rs_free_sumset(sumset);
return -1;
}
result=rs_delta_gzfile(sumset, srcfp, srczp, delfp, NULL, NULL, cntr);
close_fp(&delfp);
}
rs_free_sumset(sumset);
gzclose_fp(&srczp);
close_fp(&srcfp);
return result;
}
/* Need to make all the stuff that this does atomic so that existing backups
never get broken, even if somebody turns the power off on the server. */
static int atomic_data_jiggle(const char *finishing, const char *working, const char *manifest, const char *current, const char *currentdata, const char *datadir, const char *datadirtmp, struct config *cconf, const char *client, int hardlinked, unsigned long bno, struct cntr *p1cntr, struct cntr *cntr)
{
int ret=0;
int ars=0;
char *datapth=NULL;
char *tmpman=NULL;
struct stat statp;
char *deltabdir=NULL;
char *deltafdir=NULL;
char *sigpath=NULL;
gzFile zp=NULL;
struct sbuf sb;
logp("Doing the atomic data jiggle...\n");
if(!(tmpman=get_tmp_filename(manifest))) return -1;
if(lstat(manifest, &statp))
{
// Manifest does not exist - maybe the server was killed before
// it could be renamed.
logp("%s did not exist - trying %s\n", manifest, tmpman);
do_rename(tmpman, manifest);
}
free(tmpman);
if(!(zp=gzopen_file(manifest, "rb"))) return -1;
if(!(deltabdir=prepend_s(current,
"deltas.reverse", strlen("deltas.reverse")))
|| !(deltafdir=prepend_s(finishing,
"deltas.forward", strlen("deltas.forward")))
|| !(sigpath=prepend_s(current,
"sig.tmp", strlen("sig.tmp"))))
{
logp("out of memory\n");
gzclose_fp(&zp);
return -1;
}
mkdir(datadir, 0777);
init_sbuf(&sb);
while(!(ars=sbuf_fill(NULL, zp, &sb, cntr)))
{
if(sb.datapth)
{
write_status(client, STATUS_SHUFFLING,
sb.datapth, p1cntr, cntr);
if((ret=jiggle(sb.datapth, currentdata, datadirtmp,
datadir, deltabdir, deltafdir,
sigpath, sb.endfile,
hardlinked, sb.compression, cntr, cconf)))
break;
}
free_sbuf(&sb);
}
if(!ret)
{
if(ars>0) ret=0;
else ret=-1;
}
gzclose_fp(&zp);
if(ret)
{
// Remove the temporary data directory, we have now removed
// everything useful from it.
sync(); // try to help CIFS
recursive_delete(deltafdir, NULL, FALSE /* do not del files */);
}
if(deltabdir) free(deltabdir);
if(deltafdir) free(deltafdir);
if(sigpath) free(sigpath);
if(datapth) free(datapth);
return ret;
}
static int maybe_delete_files_from_manifest(const char *manifest, const char *deletionsfile, struct config *cconf, struct cntr *cntr)
{
int ars=0;
int ret=0;
int pcmp=0;
FILE *dfp=NULL;
struct sbuf db;
struct sbuf mb;
gzFile nmzp=NULL;
gzFile omzp=NULL;
char *manifesttmp=NULL;
struct stat statp;
if(lstat(deletionsfile, &statp))
{
// No deletions, no problem.
return 0;
}
logp("Performing deletions on manifest\n");
if(!(manifesttmp=get_tmp_filename(manifest)))
{
ret=-1;
goto end;
}
if(!(dfp=open_file(deletionsfile, "rb"))
|| !(omzp=gzopen_file(manifest, "rb"))
|| !(nmzp=gzopen_file(manifesttmp, comp_level(cconf))))
{
ret=-1;
goto end;
}
init_sbuf(&db);
init_sbuf(&mb);
while(omzp || dfp)
{
if(dfp && !db.path && (ars=sbuf_fill(dfp, NULL, &db, cntr)))
{
if(ars<0) { ret=-1; break; }
// ars==1 means it ended ok.
close_fp(&dfp);
}
if(omzp && !mb.path && (ars=sbuf_fill(NULL, omzp, &mb, cntr)))
{
if(ars<0) { ret=-1; break; }
// ars==1 means it ended ok.
gzclose_fp(&omzp);
}
if(mb.path && !db.path)
{
if(sbuf_to_manifest(&mb, NULL, nmzp)) { ret=-1; break; }
free_sbuf(&mb);
}
else if(!mb.path && db.path)
{
free_sbuf(&db);
}
else if(!mb.path && !db.path)
{
continue;
}
else if(!(pcmp=sbuf_pathcmp(&mb, &db)))
{
// They were the same - do not write.
free_sbuf(&mb);
free_sbuf(&db);
}
else if(pcmp<0)
{
// Behind in manifest. Write.
if(sbuf_to_manifest(&mb, NULL, nmzp)) { ret=-1; break; }
free_sbuf(&mb);
}
else
{
// Behind in deletions file. Do not write.
free_sbuf(&db);
}
}
end:
if(gzclose_fp(&nmzp))
{
logp("error closing %s in maybe_delete_files_from_manifest\n",
manifesttmp);
ret=-1;
}
close_fp(&dfp);
gzclose_fp(&omzp);
free_sbuf(&db);
free_sbuf(&mb);
if(!ret)
{
unlink(deletionsfile);
if(do_rename(manifesttmp, manifest))
{
free(manifesttmp);
return -1;
}
}
if(manifesttmp)
{
unlink(manifesttmp);
free(manifesttmp);
}
return ret;
}
// returns 1 for finished ok.
static int do_stuff_to_receive(struct sbuf *rb, FILE *p2fp, const char *datadirtmp, struct dpth *dpth, const char *working, char **last_requested, const char *deltmppath, struct cntr *cntr, struct config *cconf)
{
int ret=0;
char rcmd;
size_t rlen=0;
size_t wlen=0;
char *rbuf=NULL;
// This also attempts to write anything in the write buffer.
if(async_rw(&rcmd, &rbuf, &rlen, '\0', NULL, &wlen))
{
logp("error in async_rw\n");
return -1;
}
if(rbuf)
{
if(rcmd==CMD_WARNING)
{
logp("WARNING: %s\n", rbuf);
do_filecounter(cntr, rcmd, 0);
}
else if(rb->fp || rb->zp)
{
// Currently writing a file (or meta data)
if(rcmd==CMD_APPEND)
{
int app;
//logp("rlen: %d\n", rlen);
if((rb->zp
&& (app=gzwrite(rb->zp, rbuf, rlen))<=0)
|| (rb->fp
&& (app=fwrite(rbuf, 1, rlen, rb->fp))<=0))
{
logp("error when appending: %d\n", app);
async_write_str(CMD_ERROR, "write failed");
ret=-1;
}
do_filecounter_recvbytes(cntr, rlen);
}
else if(rcmd==CMD_END_FILE)
{
// Finished the file.
// Write it to the phase2 file, and free the
// buffers.
if(close_fp(&(rb->fp)))
{
logp("error closing delta for %s in receive\n", rb->path);
ret=-1;
}
if(gzclose_fp(&(rb->zp)))
{
logp("error gzclosing delta for %s in receive\n", rb->path);
ret=-1;
}
rb->endfile=rbuf;
rb->elen=rlen;
rbuf=NULL;
if(!ret && rb->receivedelta
&& finish_delta(rb, working, deltmppath))
ret=-1;
else if(!ret)
{
if(sbuf_to_manifest(rb, p2fp, NULL))
ret=-1;
else
{
char cmd=rb->cmd;
if(rb->receivedelta)
do_filecounter_changed(cntr,
cmd);
else
do_filecounter(cntr, cmd, 0);
if(*last_requested
&& !strcmp(rb->path, *last_requested))
{
free(*last_requested);
*last_requested=NULL;
}
}
}
if(!ret)
{
char *cp=NULL;
cp=strchr(rb->endfile, ':');
if(rb->endfile)
do_filecounter_bytes(cntr,
strtoull(rb->endfile, NULL, 10));
if(cp)
{
// checksum stuff goes here
}
}
free_sbuf(rb);
}
else
{
logp("unexpected cmd from client while writing file: %c %s\n", rcmd, rbuf);
ret=-1;
}
}
// Otherwise, expecting to be told of a file to save.
else if(rcmd==CMD_DATAPTH)
{
rb->datapth=rbuf;
rbuf=NULL;
}
else if(rcmd==CMD_STAT)
{
rb->statbuf=rbuf;
rb->slen=rlen;
rbuf=NULL;
}
else if(filedata(rcmd))
{
rb->cmd=rcmd;
rb->plen=rlen;
rb->path=rbuf;
rbuf=NULL;
if(rb->datapth)
{
// Receiving a delta.
if(start_to_receive_delta(rb,
working, deltmppath, cconf))
{
logp("error in start_to_receive_delta\n");
ret=-1;
}
}
else
{
// Receiving a whole new file.
if(start_to_receive_new_file(rb,
datadirtmp, dpth, cntr, cconf))
{
logp("error in start_to_receive_new_file\n");
ret=-1;
}
}
}
else if(rcmd==CMD_GEN && !strcmp(rbuf, "okbackupphase2end"))
{
ret=1;
//logp("got okbackupphase2end\n");
}
else if(rcmd==CMD_INTERRUPT)
{
// Interrupt - forget about the last requested file
// if it matches. Otherwise, we can get stuck on the
// select in the async stuff, waiting for something
// that will never arrive.
if(*last_requested && !strcmp(rbuf, *last_requested))
{
free(*last_requested);
*last_requested=NULL;
}
}
else
{
logp("unexpected cmd from client while expecting a file: %c %s\n", rcmd, rbuf);
ret=-1;
}
if(rbuf) { free(rbuf); rbuf=NULL; }
}
//logp("returning: %d\n", ret);
return ret;
}
static int list_manifest(const char *fullpath, regex_t *regex, const char *browsedir, const char *client, struct cntr *p1cntr, struct cntr *cntr)
{
int ars=0;
int ret=0;
int quit=0;
gzFile zp=NULL;
struct sbuf mb;
char *manifest=NULL;
size_t bdlen=0;
init_sbuf(&mb);
if(!(manifest=prepend_s(fullpath,
"manifest.gz", strlen("manifest.gz"))))
{
log_and_send_oom(__FUNCTION__);
return -1;
}
if(!(zp=gzopen_file(manifest, "rb")))
{
log_and_send("could not open manifest");
free(manifest);
return -1;
}
free(manifest);
if(browsedir) bdlen=strlen(browsedir);
while(!quit)
{
int show=0;
//logp("list manifest loop\n");
// Need to parse while sending, to take note of the regex.
free_sbuf(&mb);
if((ars=sbuf_fill(NULL, zp, &mb, cntr)))
{
if(ars<0) ret=-1;
// ars==1 means it ended ok.
break;
}
if(mb.cmd!=CMD_DIRECTORY
&& mb.cmd!=CMD_FILE
&& mb.cmd!=CMD_ENC_FILE
&& mb.cmd!=CMD_EFS_FILE
&& mb.cmd!=CMD_SPECIAL
&& !cmd_is_link(mb.cmd))
continue;
//if(mb.path[mb.plen]=='\n') mb.path[mb.plen]='\0';
write_status(client, STATUS_LISTING, mb.path, p1cntr, cntr);
if(browsedir)
{
int r;
if((r=check_browsedir(browsedir, &(mb.path), bdlen))<0)
{
quit++;
ret=-1;
}
if(!r) continue;
show++;
}
else
{
if(check_regex(regex, mb.path))
show++;
}
if(show)
{
if(async_write(CMD_STAT, mb.statbuf, mb.slen)
|| async_write(mb.cmd, mb.path, mb.plen))
{ quit++; ret=-1; }
else if(sbuf_is_link(&mb)
&& async_write(mb.cmd, mb.linkto, mb.llen))
{ quit++; ret=-1; }
}
}
gzclose_fp(&zp);
free_sbuf(&mb);
return ret;
}
/* Need to make all the stuff that this does atomic so that existing backups
never get broken, even if somebody turns the power off on the server. */
static int atomic_data_jiggle(struct sdirs *sdirs, struct fdirs *fdirs,
int hardlinked_current, struct conf *cconf, unsigned long bno)
{
int ars=0;
int ret=-1;
char *datapth=NULL;
char *tmpman=NULL;
struct stat statp;
char *deltabdir=NULL;
char *deltafdir=NULL;
char *sigpath=NULL;
gzFile zp=NULL;
struct sbuf *sb=NULL;
FILE *delfp=NULL;
logp("Doing the atomic data jiggle...\n");
if(!(tmpman=get_tmp_filename(fdirs->manifest)))
goto end;
if(lstat(fdirs->manifest, &statp))
{
// Manifest does not exist - maybe the server was killed before
// it could be renamed.
logp("%s did not exist - trying %s\n", fdirs->manifest, tmpman);
// Rename race condition is of no consequence, because manifest
// already does not exist.
do_rename(tmpman, fdirs->manifest);
}
if(!(zp=gzopen_file(fdirs->manifest, "rb")))
goto end;
if(!(deltabdir=prepend_s(fdirs->currentdup, "deltas.reverse"))
|| !(deltafdir=prepend_s(sdirs->finishing, "deltas.forward"))
|| !(sigpath=prepend_s(fdirs->currentdup, "sig.tmp"))
|| !(sb=sbuf_alloc(cconf)))
{
log_out_of_memory(__func__);
goto end;
}
mkdir(fdirs->datadir, 0777);
while(!(ars=sbufl_fill(sb, NULL, NULL, zp, cconf->cntr)))
{
if(sb->burp1->datapth.buf)
{
if(write_status(STATUS_SHUFFLING,
sb->burp1->datapth.buf, cconf)) goto end;
if((ret=jiggle(sdirs, fdirs, sb, hardlinked_current,
deltabdir, deltafdir,
sigpath, &delfp, cconf))) goto end;
}
sbuf_free_content(sb);
}
if(ars<0) goto end;
if(close_fp(&delfp))
{
logp("error closing %s in atomic_data_jiggle\n",
fdirs->deletionsfile);
goto end;
}
if(maybe_delete_files_from_manifest(tmpman, fdirs, cconf))
goto end;
// Remove the temporary data directory, we have probably removed
// useful files from it.
sync(); // try to help CIFS
recursive_delete(deltafdir, NULL, 0 /* do not del files */);
end:
gzclose_fp(&zp);
close_fp(&delfp);
sbuf_free(&sb);
free_w(&deltabdir);
free_w(&deltafdir);
free_w(&sigpath);
free_w(&datapth);
free_w(&tmpman);
return ret;
}
static int verify_file(const char *fname, int patches, const char *best, const char *datapth, unsigned long long *bytes, const char *endfile, char cmd, int compression, struct cntr *cntr)
{
MD5_CTX md5;
size_t b=0;
const char *cp=NULL;
const char *newsum=NULL;
unsigned char in[ZCHUNK];
unsigned char checksum[MD5_DIGEST_LENGTH+1];
unsigned long long cbytes=0;
if(!(cp=strrchr(endfile, ':')))
{
logw(cntr, "%s has no md5sum!\n", datapth);
return 0;
}
cp++;
if(!MD5_Init(&md5))
{
logp("MD5_Init() failed\n");
return -1;
}
if(patches
|| cmd==CMD_ENC_FILE || cmd==CMD_ENC_METADATA || cmd==CMD_EFS_FILE
|| cmd==CMD_ENC_VSS
|| (!patches && !dpth_is_compressed(compression, best)))
{
// If we did some patches or encryption, or the compression
// was turned off, the resulting file is not gzipped.
FILE *fp=NULL;
if(!(fp=open_file(best, "rb")))
{
logw(cntr, "could not open %s\n", best);
return 0;
}
while((b=fread(in, 1, ZCHUNK, fp))>0)
{
cbytes+=b;
if(!MD5_Update(&md5, in, b))
{
logp("MD5_Update() failed\n");
close_fp(&fp);
return -1;
}
}
if(!feof(fp))
{
logw(cntr, "error while reading %s\n", best);
close_fp(&fp);
return 0;
}
close_fp(&fp);
}
else
{
gzFile zp=NULL;
if(!(zp=gzopen_file(best, "rb")))
{
logw(cntr, "could not gzopen %s\n", best);
return 0;
}
while((b=gzread(zp, in, ZCHUNK))>0)
{
cbytes+=b;
if(!MD5_Update(&md5, in, b))
{
logp("MD5_Update() failed\n");
gzclose_fp(&zp);
return -1;
}
}
if(!gzeof(zp))
{
logw(cntr, "error while gzreading %s\n", best);
gzclose_fp(&zp);
return 0;
}
gzclose_fp(&zp);
}
if(!MD5_Final(checksum, &md5))
{
logp("MD5_Final() failed\n");
return -1;
}
newsum=get_checksum_str(checksum);
if(strcmp(newsum, cp))
{
logp("%s %s\n", newsum, cp);
logw(cntr, "md5sum for '%s (%s)' did not match!\n", fname, datapth);
logp("md5sum for '%s (%s)' did not match!\n", fname, datapth);
return 0;
}
*bytes+=cbytes;
// Just send the file name to the client, so that it can show counters.
if(async_write(cmd, fname, strlen(fname))) return -1;
return 0;
}
int backup_phase2_server(struct asfd *asfd,
struct sdirs *sdirs, struct conf *cconf,
gzFile *cmanfp, struct dpthl *dpthl, int resume)
{
int ars=0;
int ret=0;
gzFile p1zp=NULL;
char *deltmppath=NULL;
char *last_requested=NULL;
// Where to write phase2data.
// Data is not getting written to a compressed file.
// This is important for recovery if the power goes.
FILE *p2fp=NULL;
// unchanged data
FILE *ucfp=NULL;
struct sbuf *cb=NULL; // file list in current manifest
struct sbuf *p1b=NULL; // file list from client
struct sbuf *rb=NULL; // receiving file from client
if(!(cb=sbuf_alloc(cconf))
|| !(p1b=sbuf_alloc(cconf))
|| !(rb=sbuf_alloc(cconf)))
goto error;
if(!(p1zp=gzopen_file(sdirs->phase1data, "rb")))
goto error;
// Open in read+write mode, so that they can be read through if
// we need to resume.
// First, open them in a+ mode, so that they will be created if they
// do not exist.
if(!(ucfp=open_file(sdirs->unchangeddata, "a+b"))
|| !(p2fp=open_file(sdirs->phase2data, "a+b")))
goto error;
close_fp(&ucfp);
close_fp(&p2fp);
if(!(ucfp=open_file(sdirs->unchangeddata, "r+b"))
|| !(p2fp=open_file(sdirs->phase2data, "r+b")))
goto error;
if(resume && do_resume(p1zp, p2fp, ucfp, dpthl, cconf))
goto error;
logp("Begin phase2 (receive file data)\n");
while(1)
{
int sts=0;
//printf("in loop, %s %s %c\n",
// *cmanfp?"got cmanfp":"no cmanfp",
// rb->path.buf?:"no rb->path",
// rb->path.buf?'X':rb->path.cmd);
if(write_status(STATUS_BACKUP,
rb->path.buf?rb->path.buf:p1b->path.buf, cconf))
goto error;
if((last_requested || !p1zp || asfd->writebuflen)
&& (ars=do_stuff_to_receive(asfd, sdirs,
cconf, rb, p2fp, dpthl, &last_requested)))
{
if(ars<0) goto error;
// 1 means ok.
break;
}
if((sts=do_stuff_to_send(asfd, p1b, &last_requested))<0)
goto error;
if(!sts && p1zp)
{
sbuf_free_content(p1b);
if((ars=sbufl_fill_phase1(p1b, NULL, p1zp, cconf->cntr)))
{
if(ars<0) goto error;
// ars==1 means it ended ok.
gzclose_fp(&p1zp);
//logp("ended OK - write phase2end");
if(asfd->write_str(asfd, CMD_GEN, "backupphase2end"))
goto error;
}
//logp("check: %s\n", p1b.path);
if(!*cmanfp)
{
// No old manifest, need to ask for a new file.
//logp("no cmanfp\n");
if(process_new(sdirs, cconf, p1b, ucfp, dpthl))
goto error;
}
else
{
// Have an old manifest, look for it there.
// Might already have it, or be ahead in the old
// manifest.
if(cb->path.buf)
{
if((ars=maybe_process_file(asfd, sdirs, cconf,
cb, p1b, ucfp, dpthl)))
{
if(ars<0) goto error;
// Do not free it - need to send stuff.
continue;
}
//free_sbufl(&p1b);
}
while(*cmanfp)
{
sbuf_free_content(cb);
if((ars=sbufl_fill(cb, asfd, NULL,
*cmanfp, cconf->cntr)))
{
// ars==1 means it ended ok.
if(ars<0) goto error;
gzclose_fp(cmanfp);
//logp("ran out of current manifest\n");
if(process_new(sdirs, cconf,
p1b, ucfp, dpthl))
goto error;
break;
}
//logp("against: %s\n", cb.path);
if((ars=maybe_process_file(asfd, sdirs, cconf,
cb, p1b, ucfp, dpthl)))
{
if(ars<0) goto error;
// Do not free it - need to send stuff.
break;
}
}
}
}
}
goto end;
error:
ret=-1;
end:
if(close_fp(&p2fp))
{
logp("error closing %s in backup_phase2_server\n",
sdirs->phase2data);
ret=-1;
}
if(close_fp(&ucfp))
{
logp("error closing %s in backup_phase2_server\n",
sdirs->unchangeddata);
ret=-1;
}
free(deltmppath);
sbuf_free(cb);
sbuf_free(p1b);
sbuf_free(rb);
gzclose_fp(&p1zp);
if(!ret) unlink(sdirs->phase1data);
logp("End phase2 (receive file data)\n");
return ret;
}
// TODO: Some of the repeated code in this can be factored out.
static int resume_partial_changed_file(struct sbuf *cb, struct sbuf *p1b, const char *currentdata, const char *curpath, const char *datadirtmp, const char *deltmppath, struct config *cconf, struct cntr *cntr)
{
int ret=0;
int istreedata=0;
struct stat dstatp;
struct stat cstatp;
char *partial=NULL;
char *partialdir=NULL;
char *zdeltmp=NULL;
struct sbuf xb;
init_sbuf(&xb);
xb.cmd=cb->cmd;
xb.compression=cb->compression;
xb.path=strdup(cb->path);
xb.statbuf=strdup(cb->statbuf);
xb.datapth=strdup(p1b->datapth);
xb.endfile=strdup(cb->endfile);
logp("Resume partial changed file: %s\n", xb.path);
if(!lstat(deltmppath, &dstatp) && S_ISREG(dstatp.st_mode)
&& !lstat(curpath, &cstatp) && S_ISREG(cstatp.st_mode))
{
int junk=0;
gzFile dzp=NULL;
FILE *dfp=NULL;
struct stat pstatp;
if(!(partialdir=prepend_s(datadirtmp, "p", strlen("p")))
|| !(partial=prepend_s(partialdir,
xb.datapth, strlen(xb.datapth)))
|| build_path(partialdir, xb.datapth, strlen(xb.datapth),
&partial, partialdir))
{
ret=-1;
goto end;
}
if(!lstat(partial, &pstatp))
{
if(!S_ISREG(pstatp.st_mode))
{
logp("%s is not a regular file\n", partial);
goto actuallyno;
}
if(pstatp.st_size>cstatp.st_size)
{
// Looks like a previously resumed file.
if(xb.compression)
{
// Need to read and recreate it, in
// case it was not fully created.
if(!(zdeltmp=prepend(deltmppath,
".z", strlen(".z"),
0 /* no slash */))
|| !(dzp=gzopen_file(zdeltmp, "wb"))
|| copy_gzpath_to_gzFile(partial,
dzp)
|| do_rename(zdeltmp, partial))
{
ret=-1;
goto end;
}
}
else
{
// Append to the existing one.
if(!(dfp=open_file(partial, "ab")))
{
ret=-1;
goto end;
}
}
}
else
{
unlink(partial);
// Copy the whole of p1b->sigfp/sigzp to
// partial.
if(xb.compression)
{
if(!(dzp=gzopen_file(partial, "wb"))
|| copy_gzFile_to_gzFile(p1b->sigzp,
dzp))
{
ret=-1;
goto end;
}
}
else
{
if(!(dfp=open_file(partial, "wb"))
|| copy_File_to_File(p1b->sigfp, dfp))
{
ret=-1;
goto end;
}
}
}
// Now, copy the whole of deltmppath onto partial.
// dzp or dfp will be open by this point.
if(xb.compression)
{
if(copy_gzpath_to_gzFile(deltmppath, dzp))
{
ret=-1;
goto end;
}
}
else
{
if(copy_path_to_File(deltmppath, dfp))
{
ret=-1;
goto end;
}
}
}
else
{
// Copy the whole of p1b->sigfp/sigzp onto partial.
// Copy the whole of deltmppath onto partial.
if(xb.compression)
{
// There is no partial, this creates it.
if(!(dzp=gzopen_file(partial, "wb"))
|| copy_gzFile_to_gzFile(p1b->sigzp, dzp))
{
ret=-1;
goto end;
}
}
else
{
// There is no partial, this creates it.
if(!(dfp=open_file(partial, "wb"))
|| copy_File_to_File(p1b->sigfp, dfp))
{
ret=-1;
goto end;
}
}
if(xb.compression)
{
if(copy_gzpath_to_gzFile(deltmppath, dzp))
{
ret=-1;
goto end;
}
}
else
{
if(copy_path_to_File(deltmppath, dfp))
{
ret=-1;
goto end;
}
}
}
if(dfp && close_fp(&dfp))
{
ret=-1;
goto end;
}
if(dzp && gzclose_fp(&dzp))
{
ret=-1;
goto end;
}
// Use partial as the basis for a librsync transfer.
// So, we have created a new directory beginning with 'p',
// and moved the partial download to it.
// We can now use the partial file as the basis of a librsync
// transfer.
if(process_changed_file(&xb, p1b, partialdir, NULL, NULL,
&junk /* resume_partial=0 */,
cntr, cconf))
{
ret=-1;
goto end;
}
goto end;
}
actuallyno:
logp("Actually, no - just forget the previous delta\n");
end:
if(partialdir) free(partialdir);
if(partial) free(partial);
if(zdeltmp) free(zdeltmp);
free_sbuf(&xb);
return ret;
}
