static int restore_ent(const char *client, struct sbuf *sb, struct sbuf ***sblist, int *scount, struct bu *arr, int a, int i, const char *tmppath1, const char *tmppath2, enum action act, char status, struct config *cconf, struct cntr *cntr, struct cntr *p1cntr)
{
int s=0;
int ret=0;
// Check if we have any directories waiting to be restored.
for(s=(*scount)-1; s>=0; s--)
{
if(is_subdir((*sblist)[s]->path, sb->path))
{
// We are still in a subdir.
//printf(" subdir (%s %s)\n",
// (*sblist)[s]->path, sb->path);
break;
}
else
{
// Can now restore sblist[s] because nothing else is
// fiddling in a subdirectory.
if(restore_sbuf((*sblist)[s], arr, a, i, tmppath1,
tmppath2, act, client, status,
p1cntr, cntr, cconf))
{
ret=-1;
break;
}
else if(del_from_sbuf_arr(sblist, scount))
{
ret=-1;
break;
}
}
}
/* If it is a directory, need to remember it and restore it later, so
that the permissions come out right. */
/* Meta data of directories will also have the stat stuff set to be a
directory, so will also come out at the end. */
if(!ret && S_ISDIR(sb->statp.st_mode))
{
if(add_to_sbuf_arr(sblist, sb, scount))
ret=-1;
// Wipe out sb, without freeing up all the strings inside it,
// which have been added to sblist.
init_sbuf(sb);
}
else if(!ret && restore_sbuf(sb, arr, a, i, tmppath1, tmppath2, act,
client, status, p1cntr, cntr, cconf))
ret=-1;
return ret;
}
static int do_backup_phase2_client(struct config *conf, int resume, struct cntr *p1cntr, struct cntr *cntr)
{
int ret=0;
int quit=0;
char cmd;
char *buf=NULL;
size_t len=0;
char attribs[MAXSTRING];
// For efficiency, open Windows files for the VSS data, and do not
// close them until another time around the loop, when the actual
// data is read.
BFILE bfd;
// Windows VSS headers tell us how much file
// data to expect.
size_t datalen=0;
#ifdef HAVE_WIN32
binit(&bfd, 0);
#endif
struct sbuf sb;
init_sbuf(&sb);
if(!resume)
{
// Only do this bit if the server did not tell us to resume.
if(async_write_str(CMD_GEN, "backupphase2")
|| async_read_expect(CMD_GEN, "ok"))
return -1;
}
else if(conf->send_client_counters)
{
// On resume, the server might update the client with the
// counters.
if(recv_counters(p1cntr, cntr))
return -1;
}
while(!quit)
{
if(async_read(&cmd, &buf, &len))
{
ret=-1;
quit++;
}
else if(buf)
{
//logp("now: %c:%s\n", cmd, buf);
if(cmd==CMD_DATAPTH)
{
sb.datapth=buf;
buf=NULL;
continue;
}
else if(cmd==CMD_STAT)
{
// Ignore the stat data - we will fill it
// in again. Some time may have passed by now,
// and it is best to make it as fresh as
// possible.
free(buf);
buf=NULL;
continue;
}
else if(cmd==CMD_FILE
|| cmd==CMD_ENC_FILE
|| cmd==CMD_METADATA
|| cmd==CMD_ENC_METADATA
|| cmd==CMD_VSS
|| cmd==CMD_ENC_VSS
|| cmd==CMD_VSS_T
|| cmd==CMD_ENC_VSS_T
|| cmd==CMD_EFS_FILE)
{
int forget=0;
int64_t winattr=0;
struct stat statbuf;
char *extrameta=NULL;
size_t elen=0;
unsigned long long bytes=0;
FILE *fp=NULL;
int compression=conf->compression;
sb.path=buf;
buf=NULL;
#ifdef HAVE_WIN32
if(win32_lstat(sb.path, &statbuf, &winattr))
#else
if(lstat(sb.path, &statbuf))
#endif
{
logw(cntr, "Path has vanished: %s", sb.path);
if(forget_file(&sb, cmd, cntr))
{
ret=-1;
quit++;
}
free_sbuf(&sb);
continue;
}
if(conf->min_file_size
&& statbuf.st_size<
(boffset_t)conf->min_file_size
&& (cmd==CMD_FILE
|| cmd==CMD_ENC_FILE
|| cmd==CMD_EFS_FILE))
{
logw(cntr, "File size decreased below min_file_size after initial scan: %c:%s", cmd, sb.path);
forget++;
}
else if(conf->max_file_size
&& statbuf.st_size>
(boffset_t)conf->max_file_size
&& (cmd==CMD_FILE
|| cmd==CMD_ENC_FILE
|| cmd==CMD_EFS_FILE))
{
logw(cntr, "File size increased above max_file_size after initial scan: %c:%s", cmd, sb.path);
forget++;
}
if(!forget)
{
compression=in_exclude_comp(conf->excom,
conf->excmcount, sb.path,
conf->compression);
encode_stat(attribs,
&statbuf, winattr, compression);
if(open_file_for_send(
#ifdef HAVE_WIN32
&bfd, NULL,
#else
NULL, &fp,
#endif
sb.path, winattr,
&datalen, cntr))
forget++;
}
if(forget)
{
if(forget_file(&sb, cmd, cntr))
{
ret=-1;
quit++;
}
free_sbuf(&sb);
continue;
}
if(cmd==CMD_METADATA
|| cmd==CMD_ENC_METADATA
|| cmd==CMD_VSS
|| cmd==CMD_ENC_VSS
#ifdef HAVE_WIN32
|| conf->strip_vss
#endif
)
{
if(get_extrameta(
#ifdef HAVE_WIN32
&bfd,
#else
NULL,
#endif
sb.path,
&statbuf, &extrameta, &elen,
winattr, cntr,
&datalen))
{
logw(cntr, "Meta data error for %s", sb.path);
free_sbuf(&sb);
close_file_for_send(&bfd, &fp);
continue;
}
if(extrameta)
{
#ifdef HAVE_WIN32
if(conf->strip_vss)
{
free(extrameta);
extrameta=NULL;
elen=0;
}
#endif
}
else
{
logw(cntr, "No meta data after all: %s", sb.path);
free_sbuf(&sb);
close_file_for_send(&bfd, &fp);
continue;
}
}
if(cmd==CMD_FILE && sb.datapth)
{
unsigned long long sentbytes=0;
// Need to do sig/delta stuff.
if(async_write_str(CMD_DATAPTH, sb.datapth)
|| async_write_str(CMD_STAT, attribs)
|| async_write_str(CMD_FILE, sb.path)
|| load_signature_and_send_delta(
&bfd, fp,
&bytes, &sentbytes, cntr,
datalen))
{
logp("error in sig/delta for %s (%s)\n", sb.path, sb.datapth);
ret=-1;
quit++;
}
else
{
do_filecounter(cntr, CMD_FILE_CHANGED, 1);
do_filecounter_bytes(cntr, bytes);
do_filecounter_sentbytes(cntr, sentbytes);
}
}
else
{
//logp("need to send whole file: %s\n",
// sb.path);
// send the whole file.
if((async_write_str(CMD_STAT, attribs)
|| async_write_str(cmd, sb.path))
|| send_whole_file_w(cmd, sb.path,
NULL, 0, &bytes,
conf->encryption_password,
cntr, compression,
&bfd, fp,
extrameta, elen,
datalen))
{
ret=-1;
quit++;
}
else
{
do_filecounter(cntr, cmd, 1);
do_filecounter_bytes(cntr, bytes);
do_filecounter_sentbytes(cntr, bytes);
}
}
#ifdef HAVE_WIN32
// If using Windows do not close bfd - it needs
// to stay open to read VSS/file data/VSS.
// It will get closed either when given a
// different file path, or when this function
// exits.
//if(cmd!=CMD_VSS
// && cmd!=CMD_ENC_VSS)
// close_file_for_send(&bfd, NULL);
#else
close_file_for_send(NULL, &fp);
#endif
free_sbuf(&sb);
if(extrameta) free(extrameta);
}
else if(cmd==CMD_WARNING)
{
do_filecounter(cntr, cmd, 0);
free(buf);
buf=NULL;
}
else if(cmd==CMD_GEN && !strcmp(buf, "backupphase2end"))
{
if(async_write_str(CMD_GEN, "okbackupphase2end"))
ret=-1;
quit++;
}
else
{
logp("unexpected cmd from server: %c %s\n",
cmd, buf);
ret=-1;
quit++;
free(buf);
buf=NULL;
}
}
}
#ifdef HAVE_WIN32
// It is possible for a bfd to still be open.
close_file_for_send(&bfd, NULL);
#endif
return ret;
}
static int do_backup_phase2_client(struct config *conf, int resume, struct cntr *cntr)
{
int ret=0;
int quit=0;
char cmd;
char *buf=NULL;
size_t len=0;
char attribs[MAXSTRING];
struct sbuf sb;
init_sbuf(&sb);
if(!resume)
{
// Only do this bit if the server did not tell us to resume.
if(async_write_str(CMD_GEN, "backupphase2")
|| async_read_expect(CMD_GEN, "ok"))
return -1;
}
while(!quit)
{
if(async_read(&cmd, &buf, &len))
{
ret=-1;
quit++;
}
else if(buf)
{
//logp("now: %c:%s\n", cmd, buf);
if(cmd==CMD_DATAPTH)
{
sb.datapth=buf;
buf=NULL;
continue;
}
else if(cmd==CMD_STAT)
{
// Ignore the stat data - we will fill it
// in again. Some time may have passed by now,
// and it is best to make it as fresh as
// possible.
free(buf);
buf=NULL;
continue;
}
else if(cmd==CMD_FILE
|| cmd==CMD_ENC_FILE
|| cmd==CMD_METADATA
|| cmd==CMD_ENC_METADATA
|| cmd==CMD_EFS_FILE)
{
int forget=0;
int64_t winattr=0;
struct stat statbuf;
char *extrameta=NULL;
size_t elen=0;
unsigned long long bytes=0;
BFILE bfd;
FILE *fp=NULL;
sb.path=buf;
buf=NULL;
#ifdef HAVE_WIN32
if(win32_lstat(sb.path, &statbuf, &winattr))
#else
if(lstat(sb.path, &statbuf))
#endif
{
logw(cntr, "Path has vanished: %s", sb.path);
if(forget_file(&sb, cmd, cntr))
{
ret=-1;
quit++;
}
free_sbuf(&sb);
continue;
}
if(conf->min_file_size
&& statbuf.st_size<(boffset_t)conf->min_file_size)
{
logw(cntr, "File size decreased below min_file_size after initial scan: %s", sb.path);
forget++;
}
else if(conf->max_file_size
&& statbuf.st_size>(boffset_t)conf->max_file_size)
{
logw(cntr, "File size increased above max_file_size after initial scan: %s", sb.path);
forget++;
}
if(!forget)
{
encode_stat(attribs, &statbuf, winattr);
if(open_file_for_send(&bfd, &fp,
sb.path, winattr, cntr))
forget++;
}
if(forget)
{
if(forget_file(&sb, cmd, cntr))
{
ret=-1;
quit++;
}
free_sbuf(&sb);
continue;
}
if(cmd==CMD_METADATA
|| cmd==CMD_ENC_METADATA)
{
if(get_extrameta(sb.path,
&statbuf, &extrameta, &elen,
cntr))
{
logw(cntr, "Meta data error for %s", sb.path);
free_sbuf(&sb);
close_file_for_send(&bfd, &fp);
continue;
}
if(!extrameta)
{
logw(cntr, "No meta data after all: %s", sb.path);
free_sbuf(&sb);
close_file_for_send(&bfd, &fp);
continue;
}
}
if(cmd==CMD_FILE && sb.datapth)
{
unsigned long long sentbytes=0;
// Need to do sig/delta stuff.
if(async_write_str(CMD_DATAPTH, sb.datapth)
|| async_write_str(CMD_STAT, attribs)
|| async_write_str(CMD_FILE, sb.path)
|| load_signature_and_send_delta(
&bfd, fp,
&bytes, &sentbytes, cntr))
{
logp("error in sig/delta for %s (%s)\n", sb.path, sb.datapth);
ret=-1;
quit++;
}
else
{
do_filecounter(cntr, CMD_FILE_CHANGED, 1);
do_filecounter_bytes(cntr, bytes);
do_filecounter_sentbytes(cntr, sentbytes);
}
}
else
{
//logp("need to send whole file: %s\n",
// sb.path);
// send the whole file.
if(async_write_str(CMD_STAT, attribs)
|| async_write_str(cmd, sb.path)
|| send_whole_file_w(cmd, sb.path,
NULL, 0, &bytes,
conf->encryption_password,
cntr, conf->compression,
&bfd, fp,
extrameta, elen))
{
ret=-1;
quit++;
}
else
{
do_filecounter(cntr, cmd, 1);
do_filecounter_bytes(cntr, bytes);
do_filecounter_sentbytes(cntr, bytes);
}
}
close_file_for_send(&bfd, &fp);
free_sbuf(&sb);
if(extrameta) free(extrameta);
}
else if(cmd==CMD_WARNING)
{
do_filecounter(cntr, cmd, 0);
free(buf);
buf=NULL;
}
else if(cmd==CMD_GEN && !strcmp(buf, "backupphase2end"))
{
if(async_write_str(CMD_GEN, "okbackupphase2end"))
ret=-1;
quit++;
}
else
{
logp("unexpected cmd from server: %c %s\n",
cmd, buf);
ret=-1;
quit++;
free(buf);
buf=NULL;
}
}
}
return ret;
}
// Combine the phase1 and phase2 files into a new manifest.
int backup_phase3_server(const char *phase2data, const char *unchangeddata, const char *manifest, int recovery, int compress, const char *client, struct cntr *p1cntr, struct cntr *cntr, struct config *cconf)
{
int ars=0;
int ret=0;
int pcmp=0;
FILE *ucfp=NULL;
FILE *p2fp=NULL;
FILE *mp=NULL;
gzFile mzp=NULL;
struct sbuf ucb;
struct sbuf p2b;
char *manifesttmp=NULL;
logp("Begin phase3 (merge manifests)\n");
if(!(manifesttmp=get_tmp_filename(manifest))) return -1;
if(!(ucfp=open_file(unchangeddata, "rb"))
|| !(p2fp=open_file(phase2data, "rb"))
|| (compress && !(mzp=gzopen_file(manifesttmp, comp_level(cconf))))
|| (!compress && !(mp=open_file(manifesttmp, "wb"))))
{
close_fp(&ucfp);
gzclose_fp(&mzp);
close_fp(&p2fp);
close_fp(&mp);
free(manifesttmp);
return -1;
}
init_sbuf(&ucb);
init_sbuf(&p2b);
while(ucfp || p2fp)
{
if(ucfp && !ucb.path && (ars=sbuf_fill(ucfp, NULL, &ucb, cntr)))
{
if(ars<0) { ret=-1; break; }
// ars==1 means it ended ok.
close_fp(&ucfp);
}
if(p2fp && !p2b.path && (ars=sbuf_fill(p2fp, NULL, &p2b, cntr)))
{
if(ars<0) { ret=-1; break; }
// ars==1 means it ended ok.
close_fp(&p2fp);
// In recovery mode, only want to read to the last
// entry in the phase 2 file.
if(recovery) break;
}
if(ucb.path && !p2b.path)
{
write_status(client, STATUS_MERGING, ucb.path,
p1cntr, cntr);
if(sbuf_to_manifest(&ucb, mp, mzp)) { ret=-1; break; }
free_sbuf(&ucb);
}
else if(!ucb.path && p2b.path)
{
write_status(client, STATUS_MERGING, p2b.path,
p1cntr, cntr);
if(sbuf_to_manifest(&p2b, mp, mzp)) { ret=-1; break; }
free_sbuf(&p2b);
}
else if(!ucb.path && !p2b.path)
{
continue;
}
else if(!(pcmp=sbuf_pathcmp(&ucb, &p2b)))
{
// They were the same - write one and free both.
write_status(client, STATUS_MERGING, p2b.path,
p1cntr, cntr);
if(sbuf_to_manifest(&p2b, mp, mzp)) { ret=-1; break; }
free_sbuf(&p2b);
free_sbuf(&ucb);
}
else if(pcmp<0)
{
write_status(client, STATUS_MERGING, ucb.path,
p1cntr, cntr);
if(sbuf_to_manifest(&ucb, mp, mzp)) { ret=-1; break; }
free_sbuf(&ucb);
}
else
{
write_status(client, STATUS_MERGING, p2b.path,
p1cntr, cntr);
if(sbuf_to_manifest(&p2b, mp, mzp)) { ret=-1; break; }
free_sbuf(&p2b);
}
}
free_sbuf(&ucb);
free_sbuf(&p2b);
close_fp(&p2fp);
close_fp(&ucfp);
if(close_fp(&mp))
{
logp("error closing %s in backup_phase3_server\n",
manifesttmp);
ret=-1;
}
if(gzclose_fp(&mzp))
{
logp("error gzclosing %s in backup_phase3_server\n",
manifesttmp);
ret=-1;
}
if(!ret)
{
if(do_rename(manifesttmp, manifest))
ret=-1;
else
{
unlink(phase2data);
unlink(unchangeddata);
}
}
free(manifesttmp);
logp("End phase3 (merge manifests)\n");
return ret;
}
// a = length of struct bu array
// i = position to restore from
static int restore_manifest(struct bu *arr, int a, int i, const char *tmppath1, const char *tmppath2, regex_t *regex, int srestore, enum action act, const char *client, char **dir_for_notify, struct cntr *p1cntr, struct cntr *cntr, struct config *cconf)
{
int ret=0;
gzFile zp=NULL;
char *manifest=NULL;
char *datadir=NULL;
FILE *logfp=NULL;
char *logpath=NULL;
char *logpathz=NULL;
// For sending status information up to the server.
char status=STATUS_RESTORING;
if(act==ACTION_RESTORE) status=STATUS_RESTORING;
else if(act==ACTION_VERIFY) status=STATUS_VERIFYING;
if(
(act==ACTION_RESTORE && !(logpath=prepend_s(arr[i].path, "restorelog", strlen("restorelog"))))
|| (act==ACTION_RESTORE && !(logpathz=prepend_s(arr[i].path, "restorelog.gz", strlen("restorelog.gz"))))
|| (act==ACTION_VERIFY && !(logpath=prepend_s(arr[i].path, "verifylog", strlen("verifylog"))))
|| (act==ACTION_VERIFY && !(logpathz=prepend_s(arr[i].path, "verifylog.gz", strlen("verifylog.gz"))))
|| !(manifest=prepend_s(arr[i].path, "manifest.gz", strlen("manifest.gz"))))
{
log_and_send_oom(__FUNCTION__);
ret=-1;
}
else if(!(logfp=open_file(logpath, "ab")) || set_logfp(logfp, cconf))
{
char msg[256]="";
snprintf(msg, sizeof(msg),
"could not open log file: %s", logpath);
log_and_send(msg);
ret=-1;
}
*dir_for_notify=strdup(arr[i].path);
log_restore_settings(cconf, srestore);
// First, do a pass through the manifest to set up the counters.
// This is the equivalent of a phase1 scan during backup.
if(!ret && !(zp=gzopen_file(manifest, "rb")))
{
log_and_send("could not open manifest");
ret=-1;
}
else
{
int ars=0;
int quit=0;
struct sbuf sb;
init_sbuf(&sb);
while(!quit)
{
if((ars=sbuf_fill(NULL, zp, &sb, cntr)))
{
if(ars<0) ret=-1;
// ars==1 means end ok
quit++;
}
else
{
if((!srestore
|| check_srestore(cconf, sb.path))
&& check_regex(regex, sb.path))
{
do_filecounter(p1cntr, sb.cmd, 0);
if(sb.endfile)
do_filecounter_bytes(p1cntr,
strtoull(sb.endfile, NULL, 10));
/*
if(sb.cmd==CMD_FILE
|| sb.cmd==CMD_ENC_FILE
|| sb.cmd==CMD_METADATA
|| sb.cmd==CMD_ENC_METADATA
|| sb.cmd==CMD_VSS
|| sb.cmd==CMD_ENC_VSS
|| sb.cmd==CMD_VSS_T
|| sb.cmd==CMD_ENC_VSS_T
|| sb.cmd==CMD_EFS_FILE)
do_filecounter_bytes(p1cntr,
(unsigned long long)
sb.statp.st_size);
*/
}
}
free_sbuf(&sb);
}
free_sbuf(&sb);
gzclose_fp(&zp);
}
if(cconf->send_client_counters)
{
if(send_counters(client, p1cntr, cntr))
{
ret=-1;
}
}
// Now, do the actual restore.
if(!ret && !(zp=gzopen_file(manifest, "rb")))
{
log_and_send("could not open manifest");
ret=-1;
}
else
{
char cmd;
int s=0;
int quit=0;
size_t len=0;
struct sbuf sb;
// For out-of-sequence directory restoring so that the
// timestamps come out right:
int scount=0;
struct sbuf **sblist=NULL;
init_sbuf(&sb);
while(!quit)
{
int ars=0;
char *buf=NULL;
if(async_read_quick(&cmd, &buf, &len))
{
logp("read quick error\n");
ret=-1; quit++; break;
}
if(buf)
{
//logp("got read quick\n");
if(cmd==CMD_WARNING)
{
logp("WARNING: %s\n", buf);
do_filecounter(cntr, cmd, 0);
free(buf); buf=NULL;
continue;
}
else if(cmd==CMD_INTERRUPT)
{
// Client wanted to interrupt the
// sending of a file. But if we are
// here, we have already moved on.
// Ignore.
free(buf); buf=NULL;
continue;
}
else
{
logp("unexpected cmd from client: %c:%s\n", cmd, buf);
free(buf); buf=NULL;
ret=-1; quit++; break;
}
}
if((ars=sbuf_fill(NULL, zp, &sb, cntr)))
{
if(ars<0) ret=-1;
// ars==1 means end ok
quit++;
}
else
{
if((!srestore
|| check_srestore(cconf, sb.path))
&& check_regex(regex, sb.path)
&& restore_ent(client,
&sb, &sblist, &scount,
arr, a, i, tmppath1, tmppath2,
act, status, cconf,
cntr, p1cntr))
{
ret=-1;
quit++;
}
}
free_sbuf(&sb);
}
gzclose_fp(&zp);
// Restore any directories that are left in the list.
if(!ret) for(s=scount-1; s>=0; s--)
{
if(restore_sbuf(sblist[s], arr, a, i,
tmppath1, tmppath2, act, client, status,
p1cntr, cntr, cconf))
{
ret=-1;
break;
}
}
free_sbufs(sblist, scount);
if(!ret) ret=do_restore_end(act, cntr);
//print_endcounter(cntr);
print_filecounters(p1cntr, cntr, act);
reset_filecounter(p1cntr, time(NULL));
reset_filecounter(cntr, time(NULL));
}
set_logfp(NULL, cconf);
compress_file(logpath, logpathz, cconf);
if(manifest) free(manifest);
if(datadir) free(datadir);
if(logpath) free(logpath);
if(logpathz) free(logpathz);
return ret;
}
// a = length of struct bu array
// i = position to restore from
static int restore_manifest(struct bu *arr, int a, int i, const char *tmppath1, const char *tmppath2, regex_t *regex, enum action act, const char *client, struct cntr *p1cntr, struct cntr *cntr, struct config *cconf, bool all)
{
int ret=0;
gzFile zp=NULL;
char *manifest=NULL;
char *datadir=NULL;
FILE *logfp=NULL;
char *logpath=NULL;
char *logpathz=NULL;
// For sending status information up to the server.
char status=STATUS_RESTORING;
if(act==ACTION_RESTORE) status=STATUS_RESTORING;
else if(act==ACTION_VERIFY) status=STATUS_VERIFYING;
if(
(act==ACTION_RESTORE && !(logpath=prepend_s(arr[i].path, "restorelog", strlen("restorelog"))))
|| (act==ACTION_RESTORE && !(logpathz=prepend_s(arr[i].path, "restorelog.gz", strlen("restorelog.gz"))))
|| (act==ACTION_VERIFY && !(logpath=prepend_s(arr[i].path, "verifylog", strlen("verifylog"))))
|| (act==ACTION_VERIFY && !(logpathz=prepend_s(arr[i].path, "verifylog.gz", strlen("verifylog.gz"))))
|| !(manifest=prepend_s(arr[i].path, "manifest.gz", strlen("manifest.gz"))))
{
log_and_send("out of memory");
ret=-1;
}
else if(!(logfp=open_file(logpath, "ab")) || set_logfp(logfp))
{
char msg[256]="";
snprintf(msg, sizeof(msg),
"could not open log file: %s", logpath);
log_and_send(msg);
ret=-1;
}
else if(!(zp=gzopen_file(manifest, "rb")))
{
log_and_send("could not open manifest");
ret=-1;
}
else
{
char cmd;
int quit=0;
size_t len=0;
struct sbuf sb;
// For out-of-sequence directory restoring so that the
// timestamps come out right:
int s=0;
int scount=0;
struct sbuf **sblist=NULL;
init_sbuf(&sb);
while(!quit)
{
int ars=0;
char *buf=NULL;
if(async_read_quick(&cmd, &buf, &len))
{
logp("read quick error\n");
ret=-1; quit++; break;
}
if(buf)
{
//logp("got read quick\n");
if(cmd==CMD_WARNING)
{
logp("WARNING: %s\n", buf);
do_filecounter(cntr, cmd, 0);
free(buf); buf=NULL;
continue;
}
else if(cmd==CMD_INTERRUPT)
{
// Client wanted to interrupt the
// sending of a file. But if we are
// here, we have already moved on.
// Ignore.
free(buf); buf=NULL;
continue;
}
else
{
logp("unexpected cmd from client: %c:%s\n", cmd, buf);
free(buf); buf=NULL;
ret=-1; quit++; break;
}
}
if((ars=sbuf_fill(NULL, zp, &sb, cntr)))
{
if(ars<0) ret=-1;
// ars==1 means end ok
quit++;
}
else
{
if(check_regex(regex, sb.path))
{
// Check if we have any directories waiting
// to be restored.
for(s=scount-1; s>=0; s--)
{
if(is_subdir(sblist[s]->path, sb.path))
{
// We are still in a subdir.
//printf(" subdir (%s %s)\n", sblist[s]->path, sb.path);
break;
}
else
{
// Can now restore sblist[s]
// because nothing else is
// fiddling in a subdirectory.
if(restore_sbuf(sblist[s], arr,
a, i, tmppath1, tmppath2, act,
client, status,
p1cntr, cntr, cconf))
{
ret=-1; quit++;
break;
}
else if(del_from_sbuf_arr(
&sblist, &scount))
{
ret=-1; quit++;
break;
}
}
}
/* If it is a directory, need to remember it
and restore it later, so that the
permissions come out right. */
/* Meta data of directories will also have
the stat stuff set to be a directory,
so will also come out at the end. */
if(!ret && S_ISDIR(sb.statp.st_mode))
{
if(add_to_sbuf_arr(&sblist, &sb, &scount))
{
ret=-1; quit++;
}
// Wipe out sb, without freeing up
// all the strings inside it, which
// have been added to sblist.
init_sbuf(&sb);
}
else if(!ret && restore_sbuf(&sb, arr, a, i,
tmppath1, tmppath2, act, client, status,
p1cntr, cntr, cconf))
{
ret=-1; quit++;
}
}
}
free_sbuf(&sb);
}
gzclose_fp(&zp);
// Restore any directories that are left in the list.
if(!ret) for(s=scount-1; s>=0; s--)
{
if(restore_sbuf(sblist[s], arr, a, i,
tmppath1, tmppath2, act, client, status,
p1cntr, cntr, cconf))
{
ret=-1;
break;
}
}
free_sbufs(sblist, scount);
if(!ret && !all) ret=do_restore_end(act, cntr);
print_endcounter(cntr);
print_filecounters(p1cntr, cntr, act, 0);
reset_filecounter(p1cntr);
reset_filecounter(cntr);
}
set_logfp(NULL);
compress_file(logpath, logpathz, cconf);
if(manifest) free(manifest);
if(datadir) free(datadir);
if(logpath) free(logpath);
if(logpathz) free(logpathz);
return ret;
}
static int list_manifest(const char *fullpath, regex_t *regex, const char *browsedir, const char *client, struct cntr *p1cntr, struct cntr *cntr)
{
int ars=0;
int ret=0;
int quit=0;
gzFile zp=NULL;
struct sbuf mb;
char *manifest=NULL;
size_t bdlen=0;
init_sbuf(&mb);
if(!(manifest=prepend_s(fullpath,
"manifest.gz", strlen("manifest.gz"))))
{
log_and_send_oom(__FUNCTION__);
return -1;
}
if(!(zp=gzopen_file(manifest, "rb")))
{
log_and_send("could not open manifest");
free(manifest);
return -1;
}
free(manifest);
if(browsedir) bdlen=strlen(browsedir);
while(!quit)
{
int show=0;
//logp("list manifest loop\n");
// Need to parse while sending, to take note of the regex.
free_sbuf(&mb);
if((ars=sbuf_fill(NULL, zp, &mb, cntr)))
{
if(ars<0) ret=-1;
// ars==1 means it ended ok.
break;
}
if(mb.cmd!=CMD_DIRECTORY
&& mb.cmd!=CMD_FILE
&& mb.cmd!=CMD_ENC_FILE
&& mb.cmd!=CMD_EFS_FILE
&& mb.cmd!=CMD_SPECIAL
&& !cmd_is_link(mb.cmd))
continue;
//if(mb.path[mb.plen]=='\n') mb.path[mb.plen]='\0';
write_status(client, STATUS_LISTING, mb.path, p1cntr, cntr);
if(browsedir)
{
int r;
if((r=check_browsedir(browsedir, &(mb.path), bdlen))<0)
{
quit++;
ret=-1;
}
if(!r) continue;
show++;
}
else
{
if(check_regex(regex, mb.path))
show++;
}
if(show)
{
if(async_write(CMD_STAT, mb.statbuf, mb.slen)
|| async_write(mb.cmd, mb.path, mb.plen))
{ quit++; ret=-1; }
else if(sbuf_is_link(&mb)
&& async_write(mb.cmd, mb.linkto, mb.llen))
{ quit++; ret=-1; }
}
}
gzclose_fp(&zp);
free_sbuf(&mb);
return ret;
}
/* Need to make all the stuff that this does atomic so that existing backups
never get broken, even if somebody turns the power off on the server. */
static int atomic_data_jiggle(const char *finishing, const char *working, const char *manifest, const char *current, const char *currentdata, const char *datadir, const char *datadirtmp, struct config *cconf, const char *client, int hardlinked, unsigned long bno, struct cntr *p1cntr, struct cntr *cntr)
{
int ret=0;
int ars=0;
char *datapth=NULL;
char *tmpman=NULL;
struct stat statp;
char *deltabdir=NULL;
char *deltafdir=NULL;
char *sigpath=NULL;
gzFile zp=NULL;
struct sbuf sb;
logp("Doing the atomic data jiggle...\n");
if(!(tmpman=get_tmp_filename(manifest))) return -1;
if(lstat(manifest, &statp))
{
// Manifest does not exist - maybe the server was killed before
// it could be renamed.
logp("%s did not exist - trying %s\n", manifest, tmpman);
do_rename(tmpman, manifest);
}
free(tmpman);
if(!(zp=gzopen_file(manifest, "rb"))) return -1;
if(!(deltabdir=prepend_s(current,
"deltas.reverse", strlen("deltas.reverse")))
|| !(deltafdir=prepend_s(finishing,
"deltas.forward", strlen("deltas.forward")))
|| !(sigpath=prepend_s(current,
"sig.tmp", strlen("sig.tmp"))))
{
logp("out of memory\n");
gzclose_fp(&zp);
return -1;
}
mkdir(datadir, 0777);
init_sbuf(&sb);
while(!(ars=sbuf_fill(NULL, zp, &sb, cntr)))
{
if(sb.datapth)
{
write_status(client, STATUS_SHUFFLING,
sb.datapth, p1cntr, cntr);
if((ret=jiggle(sb.datapth, currentdata, datadirtmp,
datadir, deltabdir, deltafdir,
sigpath, sb.endfile,
hardlinked, sb.compression, cntr, cconf)))
break;
}
free_sbuf(&sb);
}
if(!ret)
{
if(ars>0) ret=0;
else ret=-1;
}
gzclose_fp(&zp);
if(ret)
{
// Remove the temporary data directory, we have now removed
// everything useful from it.
sync(); // try to help CIFS
recursive_delete(deltafdir, NULL, FALSE /* do not del files */);
}
if(deltabdir) free(deltabdir);
if(deltafdir) free(deltafdir);
if(sigpath) free(sigpath);
if(datapth) free(datapth);
return ret;
}
static int maybe_delete_files_from_manifest(const char *manifest, const char *deletionsfile, struct config *cconf, struct cntr *cntr)
{
int ars=0;
int ret=0;
int pcmp=0;
FILE *dfp=NULL;
struct sbuf db;
struct sbuf mb;
gzFile nmzp=NULL;
gzFile omzp=NULL;
char *manifesttmp=NULL;
struct stat statp;
if(lstat(deletionsfile, &statp))
{
// No deletions, no problem.
return 0;
}
logp("Performing deletions on manifest\n");
if(!(manifesttmp=get_tmp_filename(manifest)))
{
ret=-1;
goto end;
}
if(!(dfp=open_file(deletionsfile, "rb"))
|| !(omzp=gzopen_file(manifest, "rb"))
|| !(nmzp=gzopen_file(manifesttmp, comp_level(cconf))))
{
ret=-1;
goto end;
}
init_sbuf(&db);
init_sbuf(&mb);
while(omzp || dfp)
{
if(dfp && !db.path && (ars=sbuf_fill(dfp, NULL, &db, cntr)))
{
if(ars<0) { ret=-1; break; }
// ars==1 means it ended ok.
close_fp(&dfp);
}
if(omzp && !mb.path && (ars=sbuf_fill(NULL, omzp, &mb, cntr)))
{
if(ars<0) { ret=-1; break; }
// ars==1 means it ended ok.
gzclose_fp(&omzp);
}
if(mb.path && !db.path)
{
if(sbuf_to_manifest(&mb, NULL, nmzp)) { ret=-1; break; }
free_sbuf(&mb);
}
else if(!mb.path && db.path)
{
free_sbuf(&db);
}
else if(!mb.path && !db.path)
{
continue;
}
else if(!(pcmp=sbuf_pathcmp(&mb, &db)))
{
// They were the same - do not write.
free_sbuf(&mb);
free_sbuf(&db);
}
else if(pcmp<0)
{
// Behind in manifest. Write.
if(sbuf_to_manifest(&mb, NULL, nmzp)) { ret=-1; break; }
free_sbuf(&mb);
}
else
{
// Behind in deletions file. Do not write.
free_sbuf(&db);
}
}
end:
if(gzclose_fp(&nmzp))
{
logp("error closing %s in maybe_delete_files_from_manifest\n",
manifesttmp);
ret=-1;
}
close_fp(&dfp);
gzclose_fp(&omzp);
free_sbuf(&db);
free_sbuf(&mb);
if(!ret)
{
unlink(deletionsfile);
if(do_rename(manifesttmp, manifest))
{
free(manifesttmp);
return -1;
}
}
if(manifesttmp)
{
unlink(manifesttmp);
free(manifesttmp);
}
return ret;
}
static int resume_partial_new_file(struct sbuf *p1b, struct cntr *cntr, const char *currentdata, const char *datadirtmp, const char *deltmppath, struct dpth *dpth, struct config *cconf)
{
int ret=0;
int junk=0;
struct sbuf cb;
char *rpath=NULL;
int istreedata=0;
struct stat statp;
char *partial=NULL;
char *partialdir=NULL;
char *zdeltmp=NULL;
// It does not matter what this checksum is set to.
// This is just to get an endfile string in the format that
// process_changed_file expects.
unsigned char checksum[18]="0123456789ABCDEF";
// Need to set up a fake current sbuf.
init_sbuf(&cb);
cb.cmd=p1b->cmd;
cb.compression=p1b->compression;
cb.path=strdup(p1b->path);
cb.statbuf=strdup(p1b->statbuf);
if(!(rpath=set_new_datapth(&cb, datadirtmp, dpth, &istreedata, cconf)))
{
ret=-1;
goto end;
}
if(!(partialdir=prepend_s(datadirtmp, "p", strlen("p")))
|| !(partial=prepend_s(partialdir,
cb.datapth, strlen(cb.datapth)))
|| build_path(partialdir, cb.datapth, strlen(cb.datapth),
&partial, partialdir))
{
ret=-1;
goto end;
}
if(!lstat(partial, &statp) && S_ISREG(statp.st_mode))
{
// A previous resume was going on.
// Need to concatenate the possible delta onto the partial
// file.
FILE *dfp=NULL;
gzFile dzp=NULL;
logp("Resume previously resumed partial new file: %s %s\n",
cb.path, rpath);
if(!(cb.endfile=strdup(
get_endfile_str(statp.st_size, checksum))))
{
ret=-1;
goto end;
}
if(cb.compression)
{
// Recreate partial, in case it was only partially
// written and consequently has gz errors.
if(!(zdeltmp=prepend(deltmppath, ".z", strlen(".z"),
0 /* no slash */))
|| !(dzp=gzopen_file(zdeltmp, "wb"))
|| copy_gzpath_to_gzFile(partial, dzp)
|| do_rename(zdeltmp, partial))
{
ret=-1;
goto end;
}
}
else
{
// Just append to the existing one.
if(!(dfp=open_file(partial, "ab")))
{
ret=-1;
goto end;
}
}
if(!lstat(deltmppath, &statp) && S_ISREG(statp.st_mode))
{
if(cb.compression)
{
if(copy_gzpath_to_gzFile(deltmppath, dzp))
{
ret=-1;
goto end;
}
}
else
{
if(copy_path_to_File(deltmppath, dfp))
{
ret=-1;
goto end;
}
}
}
if(dfp && close_fp(&dfp))
{
ret=-1;
goto end;
}
if(dzp && gzclose_fp(&dzp))
{
ret=-1;
goto end;
}
if(process_changed_file(&cb, p1b, partialdir, NULL, NULL,
&junk /* resume_partial=0 */,
cntr, cconf))
{
ret=-1;
goto end;
}
if(!istreedata) incr_dpth(dpth, cconf);
goto end;
}
logp("Resume partial new file: %s %s\n", cb.path, rpath);
if(!lstat(rpath, &statp) && S_ISREG(statp.st_mode))
{
if(!(cb.endfile=strdup(
get_endfile_str(statp.st_size, checksum))))
{
ret=-1;
goto end;
}
// If compression is on, be careful with gzip unexpected
// end of file errors.
// Otherwise, just rename the whole file.
unlink(partial);
if(cb.compression)
{
if(copy_gzpath_to_gzpath(rpath, partial))
{
logp("Error in copy_gzpath_to_gzpath\n");
ret=-1;
goto end;
}
// delete the original.
if(unlink(rpath))
{
logp("Failed to unlink %s: %s\n",
rpath, strerror(errno));
return -1;
}
}
else
{
if(do_rename(rpath, partial))
{
ret=-1;
goto end;
}
}
// So, we have created a new directory beginning with 'p',
// and moved the partial download to it.
// We can now use the partial file as the basis of a librsync
// transfer.
if(process_changed_file(&cb, p1b, partialdir, NULL, NULL,
&junk /* resume_partial=0 */,
cntr, cconf))
{
ret=-1;
goto end;
}
if(!istreedata) incr_dpth(dpth, cconf);
goto end;
}
logp("Actually, no - just treat it as completely new\n");
end:
if(rpath) free(rpath);
if(partialdir) free(partialdir);
if(partial) free(partial);
if(zdeltmp) free(zdeltmp);
free_sbuf(&cb);
return ret;
}
// TODO: Some of the repeated code in this can be factored out.
static int resume_partial_changed_file(struct sbuf *cb, struct sbuf *p1b, const char *currentdata, const char *curpath, const char *datadirtmp, const char *deltmppath, struct config *cconf, struct cntr *cntr)
{
int ret=0;
int istreedata=0;
struct stat dstatp;
struct stat cstatp;
char *partial=NULL;
char *partialdir=NULL;
char *zdeltmp=NULL;
struct sbuf xb;
init_sbuf(&xb);
xb.cmd=cb->cmd;
xb.compression=cb->compression;
xb.path=strdup(cb->path);
xb.statbuf=strdup(cb->statbuf);
xb.datapth=strdup(p1b->datapth);
xb.endfile=strdup(cb->endfile);
logp("Resume partial changed file: %s\n", xb.path);
if(!lstat(deltmppath, &dstatp) && S_ISREG(dstatp.st_mode)
&& !lstat(curpath, &cstatp) && S_ISREG(cstatp.st_mode))
{
int junk=0;
gzFile dzp=NULL;
FILE *dfp=NULL;
struct stat pstatp;
if(!(partialdir=prepend_s(datadirtmp, "p", strlen("p")))
|| !(partial=prepend_s(partialdir,
xb.datapth, strlen(xb.datapth)))
|| build_path(partialdir, xb.datapth, strlen(xb.datapth),
&partial, partialdir))
{
ret=-1;
goto end;
}
if(!lstat(partial, &pstatp))
{
if(!S_ISREG(pstatp.st_mode))
{
logp("%s is not a regular file\n", partial);
goto actuallyno;
}
if(pstatp.st_size>cstatp.st_size)
{
// Looks like a previously resumed file.
if(xb.compression)
{
// Need to read and recreate it, in
// case it was not fully created.
if(!(zdeltmp=prepend(deltmppath,
".z", strlen(".z"),
0 /* no slash */))
|| !(dzp=gzopen_file(zdeltmp, "wb"))
|| copy_gzpath_to_gzFile(partial,
dzp)
|| do_rename(zdeltmp, partial))
{
ret=-1;
goto end;
}
}
else
{
// Append to the existing one.
if(!(dfp=open_file(partial, "ab")))
{
ret=-1;
goto end;
}
}
}
else
{
unlink(partial);
// Copy the whole of p1b->sigfp/sigzp to
// partial.
if(xb.compression)
{
if(!(dzp=gzopen_file(partial, "wb"))
|| copy_gzFile_to_gzFile(p1b->sigzp,
dzp))
{
ret=-1;
goto end;
}
}
else
{
if(!(dfp=open_file(partial, "wb"))
|| copy_File_to_File(p1b->sigfp, dfp))
{
ret=-1;
goto end;
}
}
}
// Now, copy the whole of deltmppath onto partial.
// dzp or dfp will be open by this point.
if(xb.compression)
{
if(copy_gzpath_to_gzFile(deltmppath, dzp))
{
ret=-1;
goto end;
}
}
else
{
if(copy_path_to_File(deltmppath, dfp))
{
ret=-1;
goto end;
}
}
}
else
{
// Copy the whole of p1b->sigfp/sigzp onto partial.
// Copy the whole of deltmppath onto partial.
if(xb.compression)
{
// There is no partial, this creates it.
if(!(dzp=gzopen_file(partial, "wb"))
|| copy_gzFile_to_gzFile(p1b->sigzp, dzp))
{
ret=-1;
goto end;
}
}
else
{
// There is no partial, this creates it.
if(!(dfp=open_file(partial, "wb"))
|| copy_File_to_File(p1b->sigfp, dfp))
{
ret=-1;
goto end;
}
}
if(xb.compression)
{
if(copy_gzpath_to_gzFile(deltmppath, dzp))
{
ret=-1;
goto end;
}
}
else
{
if(copy_path_to_File(deltmppath, dfp))
{
ret=-1;
goto end;
}
}
}
if(dfp && close_fp(&dfp))
{
ret=-1;
goto end;
}
if(dzp && gzclose_fp(&dzp))
{
ret=-1;
goto end;
}
// Use partial as the basis for a librsync transfer.
// So, we have created a new directory beginning with 'p',
// and moved the partial download to it.
// We can now use the partial file as the basis of a librsync
// transfer.
if(process_changed_file(&xb, p1b, partialdir, NULL, NULL,
&junk /* resume_partial=0 */,
cntr, cconf))
{
ret=-1;
goto end;
}
goto end;
}
actuallyno:
logp("Actually, no - just forget the previous delta\n");
end:
if(partialdir) free(partialdir);
if(partial) free(partial);
if(zdeltmp) free(zdeltmp);
free_sbuf(&xb);
return ret;
}
int backup_phase2_server(gzFile *cmanfp, const char *phase1data, const char *phase2data, const char *unchangeddata, const char *datadirtmp, struct dpth *dpth, const char *currentdata, const char *working, const char *client, struct cntr *p1cntr, int resume, struct cntr *cntr, struct config *cconf)
{
int ars=0;
int ret=0;
gzFile p1zp=NULL;
char *deltmppath=NULL;
char *last_requested=NULL;
// Where to write phase2data.
// Data is not getting written to a compressed file.
// This is important for recovery if the power goes.
FILE *p2fp=NULL;
// unchanged data
FILE *ucfp=NULL;
int resume_partial=resume;
struct sbuf cb; // file list in current manifest
struct sbuf p1b; // file list from client
struct sbuf rb; // receiving file from client
init_sbuf(&cb);
init_sbuf(&p1b);
init_sbuf(&rb);
if(!(p1zp=gzopen_file(phase1data, "rb")))
goto error;
// Open in read+write mode, so that they can be read through if
// we need to resume.
// First, open them in a+ mode, so that they will be created if they
// do not exist.
if(!(ucfp=open_file(unchangeddata, "a+b")))
goto error;
if(!(p2fp=open_file(phase2data, "a+b")))
goto error;
close_fp(&ucfp);
close_fp(&p2fp);
if(!(ucfp=open_file(unchangeddata, "r+b")))
goto error;
if(!(p2fp=open_file(phase2data, "r+b")))
goto error;
if(resume && do_resume(p1zp, p2fp, ucfp, dpth, cconf, client,
p1cntr, cntr)) goto error;
logp("Begin phase2 (receive file data)\n");
if(!(deltmppath=prepend_s(working, "delta.tmp", strlen("delta.tmp"))))
goto error;
while(1)
{
int sts=0;
// logp("in loop, %s %s %c\n",
// *cmanfp?"got cmanfp":"no cmanfp",
// rb.path?:"no rb.path", rb.path?'X':rb.cmd);
if(rb.path) write_status(client, STATUS_BACKUP,
rb.path, p1cntr, cntr);
else write_status(client, STATUS_BACKUP,
p1b.path, p1cntr, cntr);
if((last_requested || !p1zp || writebuflen)
&& (ars=do_stuff_to_receive(&rb, p2fp, datadirtmp, dpth,
working, &last_requested, deltmppath, cntr, cconf)))
{
if(ars<0) goto error;
// 1 means ok.
break;
}
if((sts=do_stuff_to_send(&p1b, &last_requested))<0)
goto error;
if(!sts && p1zp)
{
free_sbuf(&p1b);
if((ars=sbuf_fill_phase1(NULL, p1zp, &p1b, cntr)))
{
if(ars<0) goto error;
// ars==1 means it ended ok.
gzclose_fp(&p1zp);
//logp("ended OK - write phase2end");
if(async_write_str(CMD_GEN, "backupphase2end"))
goto error;
}
//logp("check: %s\n", p1b.path);
if(!*cmanfp)
{
// No old manifest, need to ask for a new file.
//logp("no cmanfp\n");
if(process_new(&p1b, ucfp, currentdata,
datadirtmp, deltmppath,
dpth, &resume_partial,
cntr, cconf)) goto error;
}
else
{
// Have an old manifest, look for it there.
// Might already have it, or be ahead in the old
// manifest.
if(cb.path)
{
if((ars=maybe_process_file(&cb, &p1b,
ucfp, currentdata, datadirtmp,
deltmppath, dpth, &resume_partial,
cntr, cconf)))
{
if(ars<0) goto error;
// Do not free it - need to send stuff.
continue;
}
//free_sbuf(&p1b);
}
while(*cmanfp)
{
free_sbuf(&cb);
if((ars=sbuf_fill(NULL, *cmanfp, &cb, cntr)))
{
// ars==1 means it ended ok.
if(ars<0) goto error;
gzclose_fp(cmanfp);
//logp("ran out of current manifest\n");
if(process_new(&p1b, ucfp,
currentdata, datadirtmp,
deltmppath, dpth,
&resume_partial, cntr, cconf))
goto error;
break;
}
//logp("against: %s\n", cb.path);
if((ars=maybe_process_file(&cb, &p1b,
ucfp, currentdata, datadirtmp,
deltmppath, dpth, &resume_partial,
cntr, cconf)))
{
if(ars<0) goto error;
// Do not free it - need to send stuff.
break;
}
}
}
}
}
goto end;
error:
ret=-1;
end:
if(close_fp(&p2fp))
{
logp("error closing %s in backup_phase2_server\n",
phase2data);
ret=-1;
}
if(close_fp(&ucfp))
{
logp("error closing %s in backup_phase2_server\n",
unchangeddata);
ret=-1;
}
free(deltmppath);
free_sbuf(&cb);
free_sbuf(&p1b);
free_sbuf(&rb);
gzclose_fp(&p1zp);
if(!ret) unlink(phase1data);
logp("End phase2 (receive file data)\n");
return ret;
}