// Combine the phase1 and phase2 files into a new manifest. int backup_phase3_server_all(struct sdirs *sdirs, struct conf **confs) { int ret=-1; int pcmp=0; struct blk *blk=NULL; struct sbuf *usb=NULL; struct sbuf *csb=NULL; char *manifesttmp=NULL; struct manio *newmanio=NULL; struct manio *chmanio=NULL; struct manio *unmanio=NULL; enum protocol protocol=get_protocol(confs); struct cntr *cntr=get_cntr(confs); const char *rmanifest_relative=NULL; logp("Begin phase3 (merge manifests)\n"); if(protocol==PROTO_2) rmanifest_relative=get_rmanifest_relative(sdirs, confs); if(!(manifesttmp=get_tmp_filename(sdirs->manifest)) || !(newmanio=manio_open_phase3(manifesttmp, comp_level(get_int(confs[OPT_COMPRESSION])), protocol, rmanifest_relative)) || !(chmanio=manio_open_phase2(sdirs->changed, "rb", protocol)) || !(unmanio=manio_open_phase2(sdirs->unchanged, "rb", protocol)) || !(usb=sbuf_alloc(protocol)) || !(csb=sbuf_alloc(protocol))) goto end; while(chmanio || unmanio) { if(!blk && !(blk=blk_alloc())) goto end; if(unmanio && !usb->path.buf) { switch(manio_read(unmanio, usb)) { case -1: goto end; case 1: manio_close(&unmanio); } } if(chmanio && !csb->path.buf) { switch(manio_read(chmanio, csb)) { case -1: goto end; case 1: manio_close(&chmanio); } } if(usb->path.buf && !csb->path.buf) { if(write_status(CNTR_STATUS_MERGING, usb->path.buf, cntr)) goto end; switch(manio_copy_entry( usb, usb, &blk, unmanio, newmanio)) { case -1: goto end; case 1: manio_close(&unmanio); } } else if(!usb->path.buf && csb->path.buf) { if(write_status(CNTR_STATUS_MERGING, csb->path.buf, cntr)) goto end; switch(manio_copy_entry( csb, csb, &blk, chmanio, newmanio)) { case -1: goto end; case 1: manio_close(&chmanio); } } else if(!usb->path.buf && !csb->path.buf) { continue; } else if(!(pcmp=sbuf_pathcmp(usb, csb))) { // They were the same - write one. if(write_status(CNTR_STATUS_MERGING, csb->path.buf, cntr)) goto end; switch(manio_copy_entry( csb, csb, &blk, chmanio, newmanio)) { case -1: goto end; case 1: manio_close(&chmanio); } } else if(pcmp<0) { if(write_status(CNTR_STATUS_MERGING, usb->path.buf, cntr)) goto end; switch(manio_copy_entry( usb, usb, &blk, unmanio, newmanio)) { case -1: goto end; case 1: manio_close(&unmanio); } } else { if(write_status(CNTR_STATUS_MERGING, csb->path.buf, cntr)) goto end; switch(manio_copy_entry( csb, csb, &blk, chmanio, newmanio)) { case -1: goto end; case 1: manio_close(&chmanio); } } } // Flush to disk. if(manio_close(&newmanio)) { logp("error gzclosing %s in backup_phase3_server\n", manifesttmp); goto end; } // Rename race condition should be of no consequence here, as the // manifest should just get recreated automatically. if(do_rename(manifesttmp, sdirs->manifest)) goto end; else { recursive_delete(sdirs->changed); recursive_delete(sdirs->unchanged); } logp("End phase3 (merge manifests)\n"); ret=0; end: manio_close(&newmanio); manio_close(&chmanio); manio_close(&unmanio); sbuf_free(&csb); sbuf_free(&usb); blk_free(&blk); free_w(&manifesttmp); return ret; }
/* Need to make all the stuff that this does atomic so that existing backups never get broken, even if somebody turns the power off on the server. */ static int atomic_data_jiggle(struct sdirs *sdirs, struct fdirs *fdirs, int hardlinked_current, struct conf *cconf, unsigned long bno) { int ars=0; int ret=-1; char *datapth=NULL; char *tmpman=NULL; struct stat statp; char *deltabdir=NULL; char *deltafdir=NULL; char *sigpath=NULL; gzFile zp=NULL; struct sbuf *sb=NULL; FILE *delfp=NULL; logp("Doing the atomic data jiggle...\n"); if(!(tmpman=get_tmp_filename(fdirs->manifest))) goto end; if(lstat(fdirs->manifest, &statp)) { // Manifest does not exist - maybe the server was killed before // it could be renamed. logp("%s did not exist - trying %s\n", fdirs->manifest, tmpman); // Rename race condition is of no consequence, because manifest // already does not exist. do_rename(tmpman, fdirs->manifest); } if(!(zp=gzopen_file(fdirs->manifest, "rb"))) goto end; if(!(deltabdir=prepend_s(fdirs->currentdup, "deltas.reverse")) || !(deltafdir=prepend_s(sdirs->finishing, "deltas.forward")) || !(sigpath=prepend_s(fdirs->currentdup, "sig.tmp")) || !(sb=sbuf_alloc(cconf))) { log_out_of_memory(__func__); goto end; } mkdir(fdirs->datadir, 0777); while(!(ars=sbufl_fill(sb, NULL, NULL, zp, cconf->cntr))) { if(sb->burp1->datapth.buf) { if(write_status(STATUS_SHUFFLING, sb->burp1->datapth.buf, cconf)) goto end; if((ret=jiggle(sdirs, fdirs, sb, hardlinked_current, deltabdir, deltafdir, sigpath, &delfp, cconf))) goto end; } sbuf_free_content(sb); } if(ars<0) goto end; if(close_fp(&delfp)) { logp("error closing %s in atomic_data_jiggle\n", fdirs->deletionsfile); goto end; } if(maybe_delete_files_from_manifest(tmpman, fdirs, cconf)) goto end; // Remove the temporary data directory, we have probably removed // useful files from it. sync(); // try to help CIFS recursive_delete(deltafdir, NULL, 0 /* do not del files */); end: gzclose_fp(&zp); close_fp(&delfp); sbuf_free(&sb); free_w(&deltabdir); free_w(&deltafdir); free_w(&sigpath); free_w(&datapth); free_w(&tmpman); return ret; }
// This is basically backup_phase3_server() from protocol1. It used to merge the // unchanged and changed data into a single file. Now it splits the manifests // into several files. int backup_phase3_server_protocol2(struct sdirs *sdirs, struct conf **confs) { int ret=1; int pcmp=0; char *hooksdir=NULL; char *dindexdir=NULL; char *manifesttmp=NULL; struct sbuf *usb=NULL; struct sbuf *csb=NULL; struct blk *blk=NULL; int finished_ch=0; int finished_un=0; struct manio *newmanio=NULL; struct manio *chmanio=NULL; struct manio *unmanio=NULL; uint64_t fcount=0; logp("Start phase3\n"); if(!(manifesttmp=get_tmp_filename(sdirs->rmanifest)) || !(newmanio=manio_alloc()) || !(chmanio=manio_alloc()) || !(unmanio=manio_alloc()) || !(hooksdir=prepend_s(manifesttmp, "hooks")) || !(dindexdir=prepend_s(manifesttmp, "dindex")) || manio_init_write(newmanio, manifesttmp) || manio_init_write_hooks(newmanio, get_string(confs[OPT_DIRECTORY]), hooksdir, sdirs->rmanifest) || manio_init_write_dindex(newmanio, dindexdir) || manio_init_read(chmanio, sdirs->changed) || manio_init_read(unmanio, sdirs->unchanged) || !(usb=sbuf_alloc(confs)) || !(csb=sbuf_alloc(confs))) goto end; while(!finished_ch || !finished_un) { if(!blk && !(blk=blk_alloc())) goto end; if(!finished_un && usb && !usb->path.buf) { switch(manio_sbuf_fill(unmanio, NULL /* no async */, usb, NULL, NULL, confs)) { case -1: goto end; case 1: finished_un++; } } if(!finished_ch && csb && !csb->path.buf) { switch(manio_sbuf_fill(chmanio, NULL /* no async */, csb, NULL, NULL, confs)) { case -1: goto end; case 1: finished_ch++; } } if((usb && usb->path.buf) && (!csb || !csb->path.buf)) { switch(manio_copy_entry(NULL /* no async */, &usb, usb, &blk, unmanio, newmanio, confs)) { case -1: goto end; case 1: finished_un++; } } else if((!usb || !usb->path.buf) && (csb && csb->path.buf)) { switch(manio_copy_entry(NULL /* no async */, &csb, csb, &blk, chmanio, newmanio, confs)) { case -1: goto end; case 1: finished_ch++; } } else if((!usb || !usb->path.buf) && (!csb || !(csb->path.buf))) { continue; } else if(!(pcmp=sbuf_pathcmp(usb, csb))) { // They were the same - write one. switch(manio_copy_entry(NULL /* no async */, &csb, csb, &blk, chmanio, newmanio, confs)) { case -1: goto end; case 1: finished_ch++; } } else if(pcmp<0) { switch(manio_copy_entry(NULL /* no async */, &usb, usb, &blk, unmanio, newmanio, confs)) { case -1: goto end; case 1: finished_un++; } } else { switch(manio_copy_entry(NULL /* no async */, &csb, csb, &blk, chmanio, newmanio, confs)) { case -1: goto end; case 1: finished_ch++; } } } fcount=newmanio->fcount; // Flush to disk and set up for reading. if(manio_free(&newmanio) || !(newmanio=manio_alloc()) || manio_init_read(newmanio, sdirs->rmanifest)) goto end; // Rename race condition should be of no consequence here, as the // manifest should just get recreated automatically. if(do_rename(manifesttmp, sdirs->rmanifest)) goto end; else { recursive_delete(sdirs->changed, NULL, 1); recursive_delete(sdirs->unchanged, NULL, 1); } if(sparse_generation(newmanio, fcount, sdirs, confs)) goto end; ret=0; logp("End phase3\n"); end: manio_free(&newmanio); manio_free(&chmanio); manio_free(&unmanio); sbuf_free(&csb); sbuf_free(&usb); blk_free(&blk); free_w(&hooksdir); free_w(&dindexdir); free_w(&manifesttmp); return ret; }
int backup_phase1_server_all(struct async *as, struct sdirs *sdirs, struct conf *conf) { int ars=0; int ret=-1; struct sbuf *sb=NULL; gzFile p1zp=NULL; char *phase1tmp=NULL; struct asfd *asfd=as->asfd; logp("Begin phase1 (file system scan)\n"); if(!(phase1tmp=get_tmp_filename(sdirs->phase1data))) goto end; if(!(p1zp=gzopen_file(phase1tmp, comp_level(conf)))) goto end; if(!(sb=sbuf_alloc(conf))) goto end; while(1) { sbuf_free_content(sb); if(conf->protocol==PROTO_BURP1) ars=sbufl_fill(sb, asfd, NULL, NULL, conf->cntr); else ars=sbuf_fill(sb, asfd, NULL, NULL, NULL, conf); if(ars) { if(ars<0) goto end; //ars==1 means it ended ok. // Last thing the client sends is 'backupphase2', and // it wants an 'ok' reply. if(asfd->write_str(asfd, CMD_GEN, "ok") || send_msg_zp(p1zp, CMD_GEN, "phase1end", strlen("phase1end"))) goto end; break; } if(write_status(STATUS_SCANNING, sb->path.buf, conf) || sbufl_to_manifest_phase1(sb, NULL, p1zp)) goto end; cntr_add_phase1(conf->cntr, sb->path.cmd, 0); if(sb->path.cmd==CMD_FILE || sb->path.cmd==CMD_ENC_FILE || sb->path.cmd==CMD_METADATA || sb->path.cmd==CMD_ENC_METADATA || sb->path.cmd==CMD_EFS_FILE) cntr_add_val(conf->cntr, CMD_BYTES_ESTIMATED, (unsigned long long)sb->statp.st_size, 0); } if(gzclose_fp(&p1zp)) { logp("error closing %s in backup_phase1_server\n", phase1tmp); goto end; } // Possible rename race condition is of no consequence here, because // the working directory will always get deleted if phase1 is not // complete. if(do_rename(phase1tmp, sdirs->phase1data)) goto end; //cntr_print(p1cntr, cntr, ACTION_BACKUP); logp("End phase1 (file system scan)\n"); ret=0; end: free(phase1tmp); gzclose_fp(&p1zp); sbuf_free(&sb); return ret; }
static int maybe_delete_files_from_manifest(const char *manifesttmp, struct fdirs *fdirs, struct conf *cconf) { int ars=0; int ret=-1; int pcmp=0; FILE *dfp=NULL; gzFile nmzp=NULL; gzFile omzp=NULL; struct sbuf *db=NULL; struct sbuf *mb=NULL; struct stat statp; if(lstat(fdirs->deletionsfile, &statp)) // No deletions, no problem. return 0; logp("Performing deletions on manifest\n"); if(!(manifesttmp=get_tmp_filename(fdirs->manifest))) goto end; if(!(dfp=open_file(fdirs->deletionsfile, "rb")) || !(omzp=gzopen_file(fdirs->manifest, "rb")) || !(nmzp=gzopen_file(manifesttmp, comp_level(cconf))) || !(db=sbuf_alloc(cconf)) || !(mb=sbuf_alloc(cconf))) goto end; while(omzp || dfp) { if(dfp && !db->path.buf && (ars=sbufl_fill(db, NULL, dfp, NULL, cconf->cntr))) { if(ars<0) goto end; // ars==1 means it ended ok. close_fp(&dfp); } if(omzp && !mb->path.buf && (ars=sbufl_fill(mb, NULL, NULL, omzp, cconf->cntr))) { if(ars<0) goto end; // ars==1 means it ended ok. gzclose_fp(&omzp); } if(mb->path.buf && !db->path.buf) { if(sbufl_to_manifest(mb, NULL, nmzp)) goto end; sbuf_free_content(mb); } else if(!mb->path.buf && db->path.buf) { sbuf_free_content(db); } else if(!mb->path.buf && !db->path.buf) { continue; } else if(!(pcmp=sbuf_pathcmp(mb, db))) { // They were the same - do not write. sbuf_free_content(mb); sbuf_free_content(db); } else if(pcmp<0) { // Behind in manifest. Write. if(sbufl_to_manifest(mb, NULL, nmzp)) goto end; sbuf_free_content(mb); } else { // Behind in deletions file. Do not write. sbuf_free_content(db); } } ret=0; end: if(gzclose_fp(&nmzp)) { logp("error closing %s in %s\n", manifesttmp, __func__); ret=-1; } close_fp(&dfp); gzclose_fp(&omzp); sbuf_free(&db); sbuf_free(&mb); if(!ret) { unlink(fdirs->deletionsfile); // The rename race condition is not a problem here, as long // as manifesttmp is the same path as that generated in the // atomic data jiggle. if(do_rename(manifesttmp, fdirs->manifest)) return -1; } if(manifesttmp) unlink(manifesttmp); return ret; }
static int recover_working(struct async *as, struct sdirs *sdirs, const char *incexc, int *resume, struct conf **cconfs) { int ret=-1; char msg[256]=""; char *logpath=NULL; struct stat statp; char *phase1datatmp=NULL; enum recovery_method recovery_method=get_e_recovery_method( cconfs[OPT_WORKING_DIR_RECOVERY_METHOD]); // The working directory has not finished being populated. // Check what to do. if(get_fullrealwork(as->asfd, sdirs, cconfs)) goto end; if(!sdirs->rworking) goto end; log_recovery_method(sdirs, recovery_method); if(!(phase1datatmp=get_tmp_filename(sdirs->phase1data))) goto end; // If there is still a phase1 tmp file... if(!lstat(phase1datatmp, &statp) || // ...or phase1 has not even got underway yet... (lstat(phase1datatmp, &statp) && lstat(sdirs->phase1data, &statp) && lstat(sdirs->changed, &statp) && lstat(sdirs->unchanged, &statp))) { // ...phase 1 did not complete - delete everything. logp("Phase 1 has not completed.\n"); recovery_method=RECOVERY_METHOD_DELETE; } if(recovery_method==RECOVERY_METHOD_DELETE) { ret=working_delete(as, sdirs, cconfs); goto end; } // We are not deleting the old working directory - open the log inside // for appending. if(!(logpath=prepend_s(sdirs->rworking, "log")) || log_fzp_set(logpath, cconfs)) goto end; switch(recovery_method) { case RECOVERY_METHOD_DELETE: // Dealt with above. break; case RECOVERY_METHOD_RESUME: ret=working_resume(as, sdirs, incexc, resume, cconfs); break; case RECOVERY_METHOD_UNSET: default: snprintf(msg, sizeof(msg), "Unknown working_dir_recovery_method: %d\n", (int)recovery_method); log_and_send(as->asfd, msg); break; } end: free_w(&logpath); free_w(&phase1datatmp); log_fzp_set(NULL, cconfs); // fclose the logfzp return ret; }
// Combine the phase1 and phase2 files into a new manifest. int backup_phase3_server(const char *phase2data, const char *unchangeddata, const char *manifest, int recovery, int compress, const char *client, struct cntr *p1cntr, struct cntr *cntr, struct config *cconf) { int ars=0; int ret=0; int pcmp=0; FILE *ucfp=NULL; FILE *p2fp=NULL; FILE *mp=NULL; gzFile mzp=NULL; struct sbuf ucb; struct sbuf p2b; char *manifesttmp=NULL; logp("Begin phase3 (merge manifests)\n"); if(!(manifesttmp=get_tmp_filename(manifest))) return -1; if(!(ucfp=open_file(unchangeddata, "rb")) || !(p2fp=open_file(phase2data, "rb")) || (compress && !(mzp=gzopen_file(manifesttmp, comp_level(cconf)))) || (!compress && !(mp=open_file(manifesttmp, "wb")))) { close_fp(&ucfp); gzclose_fp(&mzp); close_fp(&p2fp); close_fp(&mp); free(manifesttmp); return -1; } init_sbuf(&ucb); init_sbuf(&p2b); while(ucfp || p2fp) { if(ucfp && !ucb.path && (ars=sbuf_fill(ucfp, NULL, &ucb, cntr))) { if(ars<0) { ret=-1; break; } // ars==1 means it ended ok. close_fp(&ucfp); } if(p2fp && !p2b.path && (ars=sbuf_fill(p2fp, NULL, &p2b, cntr))) { if(ars<0) { ret=-1; break; } // ars==1 means it ended ok. close_fp(&p2fp); // In recovery mode, only want to read to the last // entry in the phase 2 file. if(recovery) break; } if(ucb.path && !p2b.path) { write_status(client, STATUS_MERGING, ucb.path, p1cntr, cntr); if(sbuf_to_manifest(&ucb, mp, mzp)) { ret=-1; break; } free_sbuf(&ucb); } else if(!ucb.path && p2b.path) { write_status(client, STATUS_MERGING, p2b.path, p1cntr, cntr); if(sbuf_to_manifest(&p2b, mp, mzp)) { ret=-1; break; } free_sbuf(&p2b); } else if(!ucb.path && !p2b.path) { continue; } else if(!(pcmp=sbuf_pathcmp(&ucb, &p2b))) { // They were the same - write one and free both. write_status(client, STATUS_MERGING, p2b.path, p1cntr, cntr); if(sbuf_to_manifest(&p2b, mp, mzp)) { ret=-1; break; } free_sbuf(&p2b); free_sbuf(&ucb); } else if(pcmp<0) { write_status(client, STATUS_MERGING, ucb.path, p1cntr, cntr); if(sbuf_to_manifest(&ucb, mp, mzp)) { ret=-1; break; } free_sbuf(&ucb); } else { write_status(client, STATUS_MERGING, p2b.path, p1cntr, cntr); if(sbuf_to_manifest(&p2b, mp, mzp)) { ret=-1; break; } free_sbuf(&p2b); } } free_sbuf(&ucb); free_sbuf(&p2b); close_fp(&p2fp); close_fp(&ucfp); if(close_fp(&mp)) { logp("error closing %s in backup_phase3_server\n", manifesttmp); ret=-1; } if(gzclose_fp(&mzp)) { logp("error gzclosing %s in backup_phase3_server\n", manifesttmp); ret=-1; } if(!ret) { if(do_rename(manifesttmp, manifest)) ret=-1; else { unlink(phase2data); unlink(unchangeddata); } } free(manifesttmp); logp("End phase3 (merge manifests)\n"); return ret; }
int check_for_rubble_burp1(struct asfd *asfd, struct sdirs *sdirs, struct conf *cconf, const char *incexc, int *resume) { int ret=0; ssize_t len=0; char msg[256]=""; char realwork[256]=""; struct stat statp; char *logpath=NULL; char *fullrealwork=NULL; char *phase1datatmp=NULL; const char *wdrm=cconf->recovery_method; // If there is a 'finishing' symlink, we need to // run the finish_backup stuff. if(!lstat(sdirs->finishing, &statp)) { logp("Found finishing symlink - attempting to complete prior backup!\n"); ret=backup_phase4_server(sdirs, cconf); if(!ret) logp("Prior backup completed OK.\n"); else log_and_send(asfd, "Problem with prior backup. Please check the client log on the server."); goto end; } if(lstat(sdirs->working, &statp)) { // No working directory - that is good. goto end; } if(!S_ISLNK(statp.st_mode)) { log_and_send(asfd, "Working directory is not a symlink.\n"); ret=-1; goto end; } // The working directory has not finished being populated. // Check what to do. if((len=readlink(sdirs->working, realwork, sizeof(realwork)-1))<0) { snprintf(msg, sizeof(msg), "Could not readlink on old working directory: %s\n", strerror(errno)); log_and_send(asfd, msg); ret=-1; goto end; } realwork[len]='\0'; if(!(fullrealwork=prepend_s(sdirs->client, realwork))) { ret=-1; goto end; } if(lstat(fullrealwork, &statp)) { logp("removing dangling working symlink -> %s\n", realwork); unlink(sdirs->working); goto end; } if(!(phase1datatmp=get_tmp_filename(sdirs->phase1data))) goto end; // We have found an old working directory - open the log inside // for appending. if(!(logpath=prepend_s(fullrealwork, "log"))) { ret=-1; goto end; } if(set_logfp(logpath, cconf)) { ret=-1; goto end; } logp("found old working directory: %s\n", fullrealwork); logp("working_dir_recovery_method: %s\n", wdrm); if(!lstat(phase1datatmp, &statp)) { // Phase 1 did not complete - delete everything. logp("Phase 1 has not completed.\n"); wdrm="delete"; } if(!strcmp(wdrm, "delete")) { // Try to remove it and start again. logp("deleting old working directory\n"); if(recursive_delete(fullrealwork, NULL, 1 /* delete files */)) { log_and_send(asfd, "Old working directory is in the way.\n"); ret=-1; goto end; } unlink(sdirs->working); // get rid of the symlink. goto end; } if(!strcmp(wdrm, "resume")) { if(cconf->restore_client) { // This client is not the original client, resuming // might cause all sorts of trouble. log_and_send(asfd, "Found interrupted backup - not resuming because the connected client is not the original"); ret=-1; goto end; } logp("Found interrupted backup.\n"); // Check that the current incexc configuration is the same // as before. if((ret=incexc_matches(fullrealwork, incexc))<0) goto end; if(ret) { // Attempt to resume on the next backup. logp("Will resume on the next backup request.\n"); *resume=1; ret=0; goto end; } logp("Includes/excludes have changed since the last backup.\n"); logp("Will treat last backup as finished.\n"); wdrm="use"; } if(!strcmp(wdrm, "use")) { // Use it as it is. logp("converting old working directory into the latest backup\n"); free(fullrealwork); fullrealwork=NULL; // TODO: There might be a partial file written that is not // yet logged to the manifest. It does no harm other than // taking up some disk space. Detect this and remove it. // Get us a partial manifest from the files lying around. if(maybe_rebuild_manifest(sdirs, cconf, 1 /* compress */)) { ret=-1; goto end; } // Now just rename the working link to be a finishing link, // then run this function again. if(do_rename(sdirs->working, sdirs->finishing)) { ret=-1; goto end; } ret=check_for_rubble_burp1(asfd, sdirs, cconf, incexc, resume); goto end; } snprintf(msg, sizeof(msg), "Unknown working_dir_recovery_method: %s\n", wdrm); log_and_send(asfd, msg); ret=-1; end: if(fullrealwork) free(fullrealwork); if(logpath) free(logpath); if(phase1datatmp) free(phase1datatmp); set_logfp(NULL, cconf); // fclose the logfp return ret; }
int backup_phase1_server_all(struct async *as, struct sdirs *sdirs, struct conf **confs) { int ret=-1; struct sbuf *sb=NULL; char *phase1tmp=NULL; struct asfd *asfd=as->asfd; struct manio *manio=NULL; enum protocol protocol=get_protocol(confs); struct cntr *cntr=get_cntr(confs); logp("Begin phase1 (file system scan)\n"); if(!(phase1tmp=get_tmp_filename(sdirs->phase1data)) || !(manio=manio_open_phase1(phase1tmp, comp_level(get_int(confs[OPT_COMPRESSION])), protocol)) || !(sb=sbuf_alloc(protocol))) goto error; while(1) { sbuf_free_content(sb); switch(sbuf_fill_from_net(sb, asfd, NULL, NULL, cntr)) { case 0: break; case 1: // Last thing the client sends is // 'backupphase2', and it wants an 'ok' reply. if(asfd->write_str(asfd, CMD_GEN, "ok") || send_msg_fzp(manio->fzp, CMD_GEN, "phase1end", strlen("phase1end"))) goto error; goto end; case -1: default: goto error; } if(write_status(CNTR_STATUS_SCANNING, sb->path.buf, cntr) || manio_write_sbuf(manio, sb)) goto error; cntr_add_phase1(cntr, sb->path.cmd, 0); if(sbuf_is_filedata(sb)) { cntr_add_val(cntr, CMD_BYTES_ESTIMATED, (uint64_t)sb->statp.st_size, 0); } } end: if(manio_close(&manio)) { logp("error closing %s in backup_phase1_server\n", phase1tmp); goto error; } if(check_quota(as, cntr, get_uint64_t(confs[OPT_HARD_QUOTA]), get_uint64_t(confs[OPT_SOFT_QUOTA]))) goto error; // Possible rename race condition is of no consequence here, because // the working directory will always get deleted if phase1 is not // complete. if(do_rename(phase1tmp, sdirs->phase1data)) goto error; //cntr_print(p1cntr, cntr, ACTION_BACKUP); logp("End phase1 (file system scan)\n"); ret=0; error: free_w(&phase1tmp); manio_close(&manio); sbuf_free(&sb); return ret; }
/* Need to make all the stuff that this does atomic so that existing backups never get broken, even if somebody turns the power off on the server. */ static int atomic_data_jiggle(struct sdirs *sdirs, struct fdirs *fdirs, int hardlinked_current, struct conf **cconfs) { int ret=-1; char *datapth=NULL; char *tmpman=NULL; struct stat statp; char *deltabdir=NULL; char *deltafdir=NULL; char *sigpath=NULL; struct fzp *zp=NULL; struct sbuf *sb=NULL; struct fzp *delfp=NULL; logp("Doing the atomic data jiggle...\n"); if(!(tmpman=get_tmp_filename(fdirs->manifest))) goto error; if(lstat(fdirs->manifest, &statp)) { // Manifest does not exist - maybe the server was killed before // it could be renamed. logp("%s did not exist - trying %s\n", fdirs->manifest, tmpman); // Rename race condition is of no consequence, because manifest // already does not exist. do_rename(tmpman, fdirs->manifest); } if(!(zp=fzp_gzopen(fdirs->manifest, "rb"))) goto error; if(!(deltabdir=prepend_s(fdirs->currentdup, "deltas.reverse")) || !(deltafdir=prepend_s(sdirs->finishing, "deltas.forward")) || !(sigpath=prepend_s(fdirs->currentdup, "sig.tmp")) || !(sb=sbuf_alloc(PROTO_1))) { log_out_of_memory(__func__); goto error; } mkdir(fdirs->datadir, 0777); while(1) { switch(sbuf_fill_from_file(sb, zp, NULL, NULL)) { case 0: break; case 1: goto end; default: goto error; } if(sb->protocol1->datapth.buf) { if(write_status(CNTR_STATUS_SHUFFLING, sb->protocol1->datapth.buf, get_cntr(cconfs)) || jiggle(sdirs, fdirs, sb, hardlinked_current, deltabdir, deltafdir, sigpath, &delfp, cconfs)) goto error; } sbuf_free_content(sb); } end: if(fzp_close(&delfp)) { logp("error closing %s in atomic_data_jiggle\n", fdirs->deletionsfile); goto error; } if(maybe_delete_files_from_manifest(tmpman, fdirs, cconfs)) goto error; // Remove the temporary data directory, we have probably removed // useful files from it. recursive_delete_dirs_only(deltafdir); ret=0; error: fzp_close(&zp); fzp_close(&delfp); sbuf_free(&sb); free_w(&deltabdir); free_w(&deltafdir); free_w(&sigpath); free_w(&datapth); free_w(&tmpman); return ret; }
/* Need to make all the stuff that this does atomic so that existing backups never get broken, even if somebody turns the power off on the server. */ static int atomic_data_jiggle(const char *finishing, const char *working, const char *manifest, const char *current, const char *currentdata, const char *datadir, const char *datadirtmp, struct config *cconf, const char *client, int hardlinked, unsigned long bno, struct cntr *p1cntr, struct cntr *cntr) { int ret=0; int ars=0; char *datapth=NULL; char *tmpman=NULL; struct stat statp; char *deltabdir=NULL; char *deltafdir=NULL; char *sigpath=NULL; gzFile zp=NULL; struct sbuf sb; logp("Doing the atomic data jiggle...\n"); if(!(tmpman=get_tmp_filename(manifest))) return -1; if(lstat(manifest, &statp)) { // Manifest does not exist - maybe the server was killed before // it could be renamed. logp("%s did not exist - trying %s\n", manifest, tmpman); do_rename(tmpman, manifest); } free(tmpman); if(!(zp=gzopen_file(manifest, "rb"))) return -1; if(!(deltabdir=prepend_s(current, "deltas.reverse", strlen("deltas.reverse"))) || !(deltafdir=prepend_s(finishing, "deltas.forward", strlen("deltas.forward"))) || !(sigpath=prepend_s(current, "sig.tmp", strlen("sig.tmp")))) { logp("out of memory\n"); gzclose_fp(&zp); return -1; } mkdir(datadir, 0777); init_sbuf(&sb); while(!(ars=sbuf_fill(NULL, zp, &sb, cntr))) { if(sb.datapth) { write_status(client, STATUS_SHUFFLING, sb.datapth, p1cntr, cntr); if((ret=jiggle(sb.datapth, currentdata, datadirtmp, datadir, deltabdir, deltafdir, sigpath, sb.endfile, hardlinked, sb.compression, cntr, cconf))) break; } free_sbuf(&sb); } if(!ret) { if(ars>0) ret=0; else ret=-1; } gzclose_fp(&zp); if(ret) { // Remove the temporary data directory, we have now removed // everything useful from it. sync(); // try to help CIFS recursive_delete(deltafdir, NULL, FALSE /* do not del files */); } if(deltabdir) free(deltabdir); if(deltafdir) free(deltafdir); if(sigpath) free(sigpath); if(datapth) free(datapth); return ret; }
static int maybe_delete_files_from_manifest(const char *manifest, const char *deletionsfile, struct config *cconf, struct cntr *cntr) { int ars=0; int ret=0; int pcmp=0; FILE *dfp=NULL; struct sbuf db; struct sbuf mb; gzFile nmzp=NULL; gzFile omzp=NULL; char *manifesttmp=NULL; struct stat statp; if(lstat(deletionsfile, &statp)) { // No deletions, no problem. return 0; } logp("Performing deletions on manifest\n"); if(!(manifesttmp=get_tmp_filename(manifest))) { ret=-1; goto end; } if(!(dfp=open_file(deletionsfile, "rb")) || !(omzp=gzopen_file(manifest, "rb")) || !(nmzp=gzopen_file(manifesttmp, comp_level(cconf)))) { ret=-1; goto end; } init_sbuf(&db); init_sbuf(&mb); while(omzp || dfp) { if(dfp && !db.path && (ars=sbuf_fill(dfp, NULL, &db, cntr))) { if(ars<0) { ret=-1; break; } // ars==1 means it ended ok. close_fp(&dfp); } if(omzp && !mb.path && (ars=sbuf_fill(NULL, omzp, &mb, cntr))) { if(ars<0) { ret=-1; break; } // ars==1 means it ended ok. gzclose_fp(&omzp); } if(mb.path && !db.path) { if(sbuf_to_manifest(&mb, NULL, nmzp)) { ret=-1; break; } free_sbuf(&mb); } else if(!mb.path && db.path) { free_sbuf(&db); } else if(!mb.path && !db.path) { continue; } else if(!(pcmp=sbuf_pathcmp(&mb, &db))) { // They were the same - do not write. free_sbuf(&mb); free_sbuf(&db); } else if(pcmp<0) { // Behind in manifest. Write. if(sbuf_to_manifest(&mb, NULL, nmzp)) { ret=-1; break; } free_sbuf(&mb); } else { // Behind in deletions file. Do not write. free_sbuf(&db); } } end: if(gzclose_fp(&nmzp)) { logp("error closing %s in maybe_delete_files_from_manifest\n", manifesttmp); ret=-1; } close_fp(&dfp); gzclose_fp(&omzp); free_sbuf(&db); free_sbuf(&mb); if(!ret) { unlink(deletionsfile); if(do_rename(manifesttmp, manifest)) { free(manifesttmp); return -1; } } if(manifesttmp) { unlink(manifesttmp); free(manifesttmp); } return ret; }