static int process_data_dir_file(struct asfd *asfd,
struct bu *bu, struct bu *b, const char *path,
struct sbuf *sb, enum action act, struct sdirs *sdirs,
struct conf **cconfs)
{
int ret=-1;
int patches=0;
char *dpath=NULL;
struct stat dstatp;
const char *tmp=NULL;
const char *best=NULL;
uint64_t bytes=0;
static char *tmppath1=NULL;
static char *tmppath2=NULL;
struct cntr *cntr=NULL;
if(cconfs) cntr=get_cntr(cconfs);
if((!tmppath1 && !(tmppath1=prepend_s(sdirs->client, "tmp1")))
|| (!tmppath2 && !(tmppath2=prepend_s(sdirs->client, "tmp2"))))
goto end;
best=path;
tmp=tmppath1;
// Now go down the list, applying any deltas.
for(b=b->prev; b && b->next!=bu; b=b->prev)
{
free_w(&dpath);
if(!(dpath=prepend_s(b->delta, sb->protocol1->datapth.buf)))
goto end;
if(lstat(dpath, &dstatp) || !S_ISREG(dstatp.st_mode))
continue;
if(!patches)
{
// Need to gunzip the first one.
if(inflate_or_link_oldfile(asfd, best, tmp,
cconfs, sb->compression))
{
char msg[256]="";
snprintf(msg, sizeof(msg),
"error when inflating %s\n", best);
log_and_send(asfd, msg);
goto end;
}
best=tmp;
if(tmp==tmppath1) tmp=tmppath2;
else tmp=tmppath1;
}
if(do_patch(asfd, best, dpath, tmp,
0 /* do not gzip the result */,
sb->compression /* from the manifest */, cconfs))
{
char msg[256]="";
snprintf(msg, sizeof(msg), "error when patching %s\n",
path);
log_and_send(asfd, msg);
goto end;
}
best=tmp;
if(tmp==tmppath1) tmp=tmppath2;
else tmp=tmppath1;
unlink(tmp);
patches++;
}
switch(act)
{
case ACTION_RESTORE:
if(send_file(asfd, sb, patches, best, &bytes, cntr))
goto end;
break;
case ACTION_VERIFY:
if(verify_file(asfd, sb, patches, best, &bytes, cntr))
goto end;
break;
default:
logp("Unknown action: %d\n", act);
goto end;
}
cntr_add(cntr, sb->path.cmd, 0);
cntr_add_bytes(cntr, strtoull(sb->endfile.buf, NULL, 10));
cntr_add_sentbytes(cntr, bytes);
ret=0;
end:
free_w(&dpath);
return ret;
}
static int jiggle(struct sdirs *sdirs, struct fdirs *fdirs, struct sbuf *sb,
int hardlinked_current, const char *deltabdir, const char *deltafdir,
const char *sigpath, FILE **delfp, struct conf *cconf)
{
int ret=-1;
struct stat statp;
char *oldpath=NULL;
char *newpath=NULL;
char *finpath=NULL;
char *deltafpath=NULL;
const char *datapth=sb->burp1->datapth.buf;
// If the previous backup was a hardlinked_archive, there will not be
// a currentdup directory - just directly use the file in the previous
// backup.
if(!(oldpath=prepend_s(hardlinked_current?
sdirs->currentdata:fdirs->currentdupdata, datapth))
|| !(newpath=prepend_s(fdirs->datadirtmp, datapth))
|| !(finpath=prepend_s(fdirs->datadir, datapth))
|| !(deltafpath=prepend_s(deltafdir, datapth)))
goto end;
if(!lstat(finpath, &statp) && S_ISREG(statp.st_mode))
{
// Looks like an interrupted jiggle
// did this file already.
static int donemsg=0;
if(!lstat(deltafpath, &statp) && S_ISREG(statp.st_mode))
{
logp("deleting unneeded forward delta: %s\n",
deltafpath);
unlink(deltafpath);
}
if(!donemsg)
{
logp("skipping already present file: %s\n", finpath);
logp("to save log space, skips of other already present files will not be logged\n");
donemsg++;
}
}
else if(mkpath(&finpath, fdirs->datadir))
{
logp("could not create path for: %s\n", finpath);
goto end;
}
else if(mkpath(&newpath, fdirs->datadirtmp))
{
logp("could not create path for: %s\n", newpath);
goto end;
}
else if(!lstat(deltafpath, &statp) && S_ISREG(statp.st_mode))
{
int lrs;
char *infpath=NULL;
// Got a forward patch to do.
// First, need to gunzip the old file,
// otherwise the librsync patch will take
// forever, because it will be doing seeks
// all over the place, and gzseeks are slow.
if(!(infpath=prepend_s(deltafdir, "inflate")))
{
log_out_of_memory(__func__);
goto end;
}
//logp("Fixing up: %s\n", datapth);
if(inflate_or_link_oldfile(oldpath, infpath,
sb->compression, cconf))
{
logp("error when inflating old file: %s\n", oldpath);
free(infpath);
goto end;
}
if((lrs=do_patch(NULL, infpath, deltafpath, newpath,
cconf->compression,
sb->compression /* from the manifest */, cconf)))
{
logp("WARNING: librsync error when patching %s: %d\n",
oldpath, lrs);
cntr_add(cconf->cntr, CMD_WARNING, 1);
// Try to carry on with the rest of the backup
// regardless.
//ret=-1;
// Remove anything that got written.
unlink(newpath);
unlink(infpath);
free(infpath);
// First, note that we want to remove this entry from
// the manifest.
if(!*delfp
&& !(*delfp=open_file(fdirs->deletionsfile, "ab")))
{
// Could not mark this file as deleted. Fatal.
goto end;
}
if(sbufl_to_manifest(sb, *delfp, NULL))
goto end;
if(fflush(*delfp))
{
logp("error fflushing deletions file in %s: %s\n", __func__, strerror(errno));
goto end;
}
ret=0;
goto end;
}
// Get rid of the inflated old file.
unlink(infpath);
free(infpath);
// Need to generate a reverse diff, unless we are keeping a
// hardlinked archive.
if(!hardlinked_current)
{
if(gen_rev_delta(sigpath, deltabdir,
oldpath, newpath, datapth, sb, cconf))
goto end;
}
// Power interruptions should be recoverable. If it happens
// before this point, the data jiggle for this file has to be
// done again.
// Once finpath is in place, no more jiggle is required.
// Use the fresh new file.
// Rename race condition is of no consequence, because finpath
// will just get recreated automatically.
if(do_rename(newpath, finpath))
goto end;
// Remove the forward delta, as it is no longer needed. There
// is a reverse diff and the finished finished file is in place.
//logp("Deleting delta.forward...\n");
unlink(deltafpath);
// Remove the old file. If a power cut happens just before
// this, the old file will hang around forever.
// FIX THIS: maybe put in something to detect this.
// ie, both a reverse delta and the old file exist.
if(!hardlinked_current)
{
//logp("Deleting oldpath...\n");
unlink(oldpath);
}
}
else if(!lstat(newpath, &statp) && S_ISREG(statp.st_mode))
{
// Use the fresh new file.
// This needs to happen after checking
// for the forward delta, because the
// patching stuff writes to newpath.
// Rename race condition is of no consequence, because finpath
// will just get recreated automatically.
//logp("Using newly received file\n");
if(do_rename(newpath, finpath))
goto end;
}
else if(!lstat(oldpath, &statp) && S_ISREG(statp.st_mode))
{
// Use the old unchanged file.
// Hard link it first.
//logp("Hard linking to old file: %s\n", datapth);
if(do_link(oldpath, finpath, &statp, cconf,
0 /* do not overwrite finpath (should never need to) */))
goto end;
else
{
// If we are not keeping a hardlinked
// archive, delete the old link.
if(!hardlinked_current)
{
//logp("Unlinking old file: %s\n", oldpath);
unlink(oldpath);
}
}
}
else
{
logp("could not find: %s\n", oldpath);
goto end;
}
ret=0;
end:
free_w(&oldpath);
free_w(&newpath);
free_w(&finpath);
free_w(&deltafpath);
return ret;
}
static int forward_patch_and_reverse_diff(
struct fdirs *fdirs,
struct fzp **delfp,
const char *deltabdir,
const char *deltafdir,
const char *deltafpath,
const char *sigpath,
const char *oldpath,
const char *newpath,
const char *datapth,
const char *finpath,
int hardlinked_current,
struct sbuf *sb,
struct conf **cconfs
)
{
int lrs;
int ret=-1;
char *infpath=NULL;
// Got a forward patch to do.
// First, need to gunzip the old file, otherwise the librsync patch
// will take forever, because it will be doing seeks all over the
// place, and gzseeks are slow.
if(!(infpath=prepend_s(deltafdir, "inflate")))
{
log_out_of_memory(__func__);
goto end;
}
//logp("Fixing up: %s\n", datapth);
if(inflate_or_link_oldfile(oldpath, infpath, sb->compression, cconfs))
{
logp("error when inflating old file: %s\n", oldpath);
goto end;
}
if((lrs=do_patch(NULL, infpath, deltafpath, newpath,
sb->compression, sb->compression /* from manifest */)))
{
logp("WARNING: librsync error when patching %s: %d\n",
oldpath, lrs);
cntr_add(get_cntr(cconfs), CMD_WARNING, 1);
// Try to carry on with the rest of the backup regardless.
// Remove anything that got written.
unlink(newpath);
// First, note that we want to remove this entry from
// the manifest.
if(!*delfp
&& !(*delfp=fzp_open(fdirs->deletionsfile, "ab")))
{
// Could not mark this file as deleted. Fatal.
goto end;
}
if(sbuf_to_manifest(sb, *delfp))
goto end;
if(fzp_flush(*delfp))
{
logp("error fflushing deletions file in %s: %s\n",
__func__, strerror(errno));
goto end;
}
ret=0;
goto end;
}
// Need to generate a reverse diff, unless we are keeping a hardlinked
// archive.
if(!hardlinked_current)
{
if(gen_rev_delta(sigpath, deltabdir,
oldpath, newpath, datapth, sb, cconfs))
goto end;
}
// Power interruptions should be recoverable. If it happens before this
// point, the data jiggle for this file has to be done again.
// Once finpath is in place, no more jiggle is required.
// Use the fresh new file.
// Rename race condition is of no consequence, because finpath will
// just get recreated automatically.
if(do_rename(newpath, finpath))
goto end;
// Remove the forward delta, as it is no longer needed. There is a
// reverse diff and the finished finished file is in place.
//logp("Deleting delta.forward...\n");
unlink(deltafpath);
// Remove the old file. If a power cut happens just before this, the
// old file will hang around forever.
// FIX THIS: maybe put in something to detect this.
// ie, both a reverse delta and the old file exist.
if(!hardlinked_current)
{
//logp("Deleting oldpath...\n");
unlink(oldpath);
}
ret=0;
end:
if(infpath)
{
unlink(infpath);
free_w(&infpath);
}
return ret;
}
// a = length of struct bu array
// i = position to restore from
static int restore_file(struct bu *arr, int a, int i, const char *datapth, const char *fname, const char *tmppath1, const char *tmppath2, int act, const char *endfile, char cmd, int64_t winattr, int compression, struct cntr *cntr, struct config *cconf)
{
int x=0;
char msg[256]="";
// Go up the array until we find the file in the data directory.
for(x=i; x<a; x++)
{
char *path=NULL;
struct stat statp;
if(!(path=prepend_s(arr[x].data, datapth, strlen(datapth))))
{
log_and_send_oom(__FUNCTION__);
return -1;
}
//logp("server file: %s\n", path);
if(lstat(path, &statp) || !S_ISREG(statp.st_mode))
{
free(path);
continue;
}
else
{
int patches=0;
struct stat dstatp;
const char *tmp=NULL;
const char *best=NULL;
unsigned long long bytes=0;
best=path;
tmp=tmppath1;
// Now go down the array, applying any deltas.
for(x-=1; x>=i; x--)
{
char *dpath=NULL;
if(!(dpath=prepend_s(arr[x].delta,
datapth, strlen(datapth))))
{
log_and_send_oom(__FUNCTION__);
free(path);
return -1;
}
if(lstat(dpath, &dstatp)
|| !S_ISREG(dstatp.st_mode))
{
free(dpath);
continue;
}
if(!patches)
{
// Need to gunzip the first one.
if(inflate_or_link_oldfile(best, tmp,
compression))
{
logp("error when inflating %s\n", best);
free(path);
free(dpath);
return -1;
}
best=tmp;
if(tmp==tmppath1) tmp=tmppath2;
else tmp=tmppath1;
}
if(do_patch(best, dpath, tmp,
FALSE /* do not gzip the result */,
compression /* from the manifest */,
cntr, cconf))
{
char msg[256]="";
snprintf(msg, sizeof(msg),
"error when patching %s\n",
path);
log_and_send(msg);
free(path);
free(dpath);
return -1;
}
best=tmp;
if(tmp==tmppath1) tmp=tmppath2;
else tmp=tmppath1;
unlink(tmp);
patches++;
}
if(act==ACTION_RESTORE)
{
if(send_file(fname, patches, best, datapth,
&bytes, cmd, winattr, compression,
cntr, cconf))
{
free(path);
return -1;
}
else
{
do_filecounter(cntr, cmd, 0);
do_filecounter_bytes(cntr,
strtoull(endfile, NULL, 10));
}
}
else if(act==ACTION_VERIFY)
{
if(verify_file(fname, patches, best, datapth,
&bytes, endfile, cmd, compression,
cntr))
{
free(path);
return -1;
}
else
{
do_filecounter(cntr, cmd, 0);
do_filecounter_bytes(cntr,
strtoull(endfile, NULL, 10));
}
}
do_filecounter_sentbytes(cntr, bytes);
free(path);
return 0;
}
}
logw(cntr, "restore could not find %s (%s)\n", fname, datapth);
//return -1;
return 0;
}
static int jiggle(const char *datapth, const char *currentdata, const char *datadirtmp, const char *datadir, const char *deltabdir, const char *deltafdir, const char *sigpath, const char *endfile, int hardlinked, int compression, struct cntr *cntr, struct config *cconf)
{
int ret=0;
struct stat statp;
char *oldpath=NULL;
char *newpath=NULL;
char *finpath=NULL;
char *deltafpath=NULL;
if(!(oldpath=prepend_s(currentdata, datapth, strlen(datapth)))
|| !(newpath=prepend_s(datadirtmp, datapth, strlen(datapth)))
|| !(finpath=prepend_s(datadir, datapth, strlen(datapth)))
|| !(deltafpath=prepend_s(deltafdir, datapth, strlen(datapth))))
{
logp("out of memory\n");
ret=-1;
goto cleanup;
}
else if(!lstat(finpath, &statp) && S_ISREG(statp.st_mode))
{
// Looks like an interrupted jiggle
// did this file already.
static int donemsg=0;
if(!lstat(deltafpath, &statp) && S_ISREG(statp.st_mode))
{
logp("deleting unneeded forward delta: %s\n",
deltafpath);
unlink(deltafpath);
}
if(!donemsg)
{
logp("skipping already present file: %s\n", finpath);
logp("to save log space, skips of other already present files will not be logged\n");
donemsg++;
}
}
else if(mkpath(&finpath, datadir))
{
logp("could not create path for: %s\n", finpath);
ret=-1;
goto cleanup;
}
else if(mkpath(&newpath, datadirtmp))
{
logp("could not create path for: %s\n", newpath);
ret=-1;
goto cleanup;
}
else if(!lstat(deltafpath, &statp) && S_ISREG(statp.st_mode))
{
char *infpath=NULL;
// Got a forward patch to do.
// First, need to gunzip the old file,
// otherwise the librsync patch will take
// forever, because it will be doing seeks
// all over the place, and gzseeks are slow.
if(!(infpath=prepend_s(deltafdir,
"inflate", strlen("inflate"))))
{
logp("out of memory\n");
ret=-1;
goto cleanup;
}
//logp("Fixing up: %s\n", datapth);
if(inflate_or_link_oldfile(oldpath, infpath, compression, cconf))
{
logp("error when inflating old file: %s\n", oldpath);
ret=-1;
goto cleanup;
}
if(do_patch(infpath, deltafpath, newpath, cconf->compression,
compression /* from the manifest */, cntr, cconf))
{
logp("error when patching\n");
ret=-1;
// Remove anything that got written.
unlink(newpath);
goto cleanup;
}
// Get rid of the inflated old file.
// This will also remove it if there was an
// error.
unlink(infpath);
free(infpath);
// Need to generate a reverse diff,
// unless we are keeping a hardlinked
// archive.
if(!hardlinked)
{
if(gen_rev_delta(sigpath, deltabdir,
oldpath, newpath, datapth, endfile,
compression, cntr, cconf))
{
ret=-1;
goto cleanup;
}
}
// Power interruptions should be
// recoverable. If it happens before
// this point, the data jiggle for
// this file has to be done again.
// Once finpath is in place, no more
// jiggle is required.
// Use the fresh new file.
if(do_rename(newpath, finpath))
{
ret=-1;
goto cleanup;
}
else
{
// Remove the forward delta, as it is
// no longer needed. There is a
// reverse diff and the finished
// finished file is in place.
//logp("Deleting delta.forward...\n");
unlink(deltafpath);
// Remove the old file. If a power
// cut happens just before this,
// the old file will hang around
// forever.
// TODO: Put in something to
// detect this.
// ie, both a reverse delta and the
// old file exist.
if(!hardlinked)
{
//logp("Deleting oldpath...\n");
unlink(oldpath);
}
}
}
else if(!lstat(newpath, &statp) && S_ISREG(statp.st_mode))
{
// Use the fresh new file.
// This needs to happen after checking
// for the forward delta, because the
// patching stuff writes to newpath.
//logp("Using newly received file\n");
if(do_rename(newpath, finpath))
{
ret=-1;
goto cleanup;
}
}
else if(!lstat(oldpath, &statp) && S_ISREG(statp.st_mode))
{
// Use the old unchanged file.
// Hard link it first.
//logp("Hard linking to old file: %s\n", datapth);
if(do_link(oldpath, finpath, &statp, cconf))
{
ret=-1;
goto cleanup;
}
else
{
// If we are not keeping a hardlinked
// archive, delete the old link.
if(!hardlinked)
{
//logp("Unlinking old file: %s\n", oldpath);
unlink(oldpath);
}
}
}
else
{
logp("could not find: %s\n", oldpath);
ret=-1;
goto cleanup;
}
cleanup:
if(oldpath) { free(oldpath); oldpath=NULL; }
if(newpath) { free(newpath); newpath=NULL; }
if(finpath) { free(finpath); finpath=NULL; }
if(deltafpath) { free(deltafpath); deltafpath=NULL; }
return ret;
}
