// Used on resume, this just reads the phase1 file and sets up cntr. static int read_phase1(struct manio *p1manio, struct conf **cconfs) { int ret=-1; struct sbuf *p1b; enum protocol protocol=get_protocol(cconfs); struct cntr *cntr=get_cntr(cconfs); if(!(p1b=sbuf_alloc(protocol))) return -1; while(1) { sbuf_free_content(p1b); switch(manio_read(p1manio, p1b)) { case 0: break; case 1: ret=0; default: goto end; } cntr_add_phase1(cntr, p1b->path.cmd, 0); if(sbuf_is_estimatable(p1b)) cntr_add_val(cntr, CMD_BYTES_ESTIMATED, (uint64_t)p1b->statp.st_size); } end: sbuf_free(&p1b); return ret; }
static #endif int forward_past_entry(struct manio *manio, struct iobuf *target, enum protocol protocol, man_off_t **pos) { struct sbuf *sb=NULL; if(!(sb=sbuf_alloc(protocol))) goto error; man_off_t_free(pos); if(!(*pos=manio_tell(manio))) { logp("Could not manio_tell first pos in %s(): %s\n", __func__, strerror(errno)); goto error; } while(1) { sbuf_free_content(sb); switch(manio_read(manio, sb)) { case 0: break; case 1: logp("End of file in %s()\n", __func__); goto error; default: logp("Error in %s()\n", __func__); // Treat error in unchanged manio as not OK. goto error; } if(target->cmd==sb->path.cmd && !pathcmp(target->buf, sb->path.buf)) { man_off_t_free(pos); if(!(*pos=manio_tell(manio))) { logp("Could not get pos in %s(): %s\n", __func__, strerror(errno)); goto error; } sbuf_free(&sb); return 0; } } error: sbuf_free(&sb); man_off_t_free(pos); return -1; }
static int maybe_add_from_scan(struct manios *manios, struct slist *slist, struct asfd *chfd, struct sbuf **csb) { int ret=-1; struct sbuf *snew=NULL; while(1) { sbuf_free(&snew); if(!manios->phase1) return 0; // Limit the amount loaded into memory at any one time. if(slist && slist->head) { if(slist->head->protocol2->index - slist->tail->protocol2->index>4096) return 0; } if(!(snew=sbuf_alloc(PROTO_2))) goto end; switch(manio_read(manios->phase1, snew)) { case 0: break; case 1: manio_close(&manios->phase1); ret=0; // Finished. default: goto end; } switch(entry_changed(snew, manios, chfd, csb)) { case 0: continue; // No change. case 1: break; default: goto end; // Error. } if(data_needed(snew)) snew->flags|=SBUF_NEED_DATA; slist_add_sbuf(slist, snew); snew=NULL; } return 0; end: sbuf_free(&snew); return ret; }
static int do_browse_manifest(struct asfd *srfd, struct manio *manio, struct sbuf *sb, const char *browse) { int ret=-1; int ars=0; //char ls[1024]=""; //struct cntr cntr; size_t blen=0; char *last_bd_match=NULL; if(browse) blen=strlen(browse); while(1) { int r; sbuf_free_content(sb); if((ars=manio_read(manio, sb))) { if(ars<0) goto end; // ars==1 means it ended ok. break; } if(manio->protocol==PROTO_2 && sb->endfile.buf) continue; if(sb->path.cmd!=CMD_DIRECTORY && sb->path.cmd!=CMD_FILE && sb->path.cmd!=CMD_ENC_FILE && sb->path.cmd!=CMD_EFS_FILE && sb->path.cmd!=CMD_SPECIAL && !cmd_is_link(sb->path.cmd)) continue; if((r=check_browsedir(browse, sb, blen, &last_bd_match))<0) goto end; if(!r) continue; if(json_from_statp(sb->path.buf, &sb->statp)) goto end; } ret=0; end: free_w(&last_bd_match); return ret; }
// Combine the phase1 and phase2 files into a new manifest. int backup_phase3_server_all(struct sdirs *sdirs, struct conf **confs) { int ret=-1; int pcmp=0; struct blk *blk=NULL; struct sbuf *usb=NULL; struct sbuf *csb=NULL; char *manifesttmp=NULL; struct manio *newmanio=NULL; struct manio *chmanio=NULL; struct manio *unmanio=NULL; enum protocol protocol=get_protocol(confs); struct cntr *cntr=get_cntr(confs); const char *rmanifest_relative=NULL; logp("Begin phase3 (merge manifests)\n"); if(protocol==PROTO_2) rmanifest_relative=get_rmanifest_relative(sdirs, confs); if(!(manifesttmp=get_tmp_filename(sdirs->manifest)) || !(newmanio=manio_open_phase3(manifesttmp, comp_level(get_int(confs[OPT_COMPRESSION])), protocol, rmanifest_relative)) || !(chmanio=manio_open_phase2(sdirs->changed, "rb", protocol)) || !(unmanio=manio_open_phase2(sdirs->unchanged, "rb", protocol)) || !(usb=sbuf_alloc(protocol)) || !(csb=sbuf_alloc(protocol))) goto end; while(chmanio || unmanio) { if(!blk && !(blk=blk_alloc())) goto end; if(unmanio && !usb->path.buf) { switch(manio_read(unmanio, usb)) { case -1: goto end; case 1: manio_close(&unmanio); } } if(chmanio && !csb->path.buf) { switch(manio_read(chmanio, csb)) { case -1: goto end; case 1: manio_close(&chmanio); } } if(usb->path.buf && !csb->path.buf) { if(write_status(CNTR_STATUS_MERGING, usb->path.buf, cntr)) goto end; switch(manio_copy_entry( usb, usb, &blk, unmanio, newmanio)) { case -1: goto end; case 1: manio_close(&unmanio); } } else if(!usb->path.buf && csb->path.buf) { if(write_status(CNTR_STATUS_MERGING, csb->path.buf, cntr)) goto end; switch(manio_copy_entry( csb, csb, &blk, chmanio, newmanio)) { case -1: goto end; case 1: manio_close(&chmanio); } } else if(!usb->path.buf && !csb->path.buf) { continue; } else if(!(pcmp=sbuf_pathcmp(usb, csb))) { // They were the same - write one. if(write_status(CNTR_STATUS_MERGING, csb->path.buf, cntr)) goto end; switch(manio_copy_entry( csb, csb, &blk, chmanio, newmanio)) { case -1: goto end; case 1: manio_close(&chmanio); } } else if(pcmp<0) { if(write_status(CNTR_STATUS_MERGING, usb->path.buf, cntr)) goto end; switch(manio_copy_entry( usb, usb, &blk, unmanio, newmanio)) { case -1: goto end; case 1: manio_close(&unmanio); } } else { if(write_status(CNTR_STATUS_MERGING, csb->path.buf, cntr)) goto end; switch(manio_copy_entry( csb, csb, &blk, chmanio, newmanio)) { case -1: goto end; case 1: manio_close(&chmanio); } } } // Flush to disk. if(manio_close(&newmanio)) { logp("error gzclosing %s in backup_phase3_server\n", manifesttmp); goto end; } // Rename race condition should be of no consequence here, as the // manifest should just get recreated automatically. if(do_rename(manifesttmp, sdirs->manifest)) goto end; else { recursive_delete(sdirs->changed); recursive_delete(sdirs->unchanged); } logp("End phase3 (merge manifests)\n"); ret=0; end: manio_close(&newmanio); manio_close(&chmanio); manio_close(&unmanio); sbuf_free(&csb); sbuf_free(&usb); blk_free(&blk); free_w(&manifesttmp); return ret; }
static #endif int get_last_good_entry(struct manio *manio, struct iobuf *result, struct cntr *cntr, struct dpth *dpth, enum protocol protocol, man_off_t **pos) { int ars=0; int got_vss_start=0; struct sbuf *sb=NULL; struct iobuf lastpath; if(!(sb=sbuf_alloc(protocol))) goto error; iobuf_init(&lastpath); man_off_t_free(pos); if(!(*pos=manio_tell(manio))) { logp("Could not manio_tell first pos in %s(): %s\n", __func__, strerror(errno)); goto error; } while(1) { if(sb->path.buf && !got_vss_start) { iobuf_free_content(&lastpath); iobuf_move(&lastpath, &sb->path); if(!sbuf_is_filedata(sb) && !sbuf_is_vssdata(sb)) { iobuf_free_content(result); iobuf_move(result, &lastpath); man_off_t_free(pos); if(!(*pos=manio_tell(manio))) { logp("Could not manio_tell pos in %s(): %s\n", __func__, strerror(errno)); goto error; } } } if(sb->endfile.buf && !got_vss_start) { iobuf_free_content(result); iobuf_move(result, &lastpath); man_off_t_free(pos); if(!(*pos=manio_tell(manio))) { logp("Could not manio_tell pos in %s(): %s\n", __func__, strerror(errno)); goto error; } } sbuf_free_content(sb); ars=manio_read(manio, sb); if(dpth && set_higher_datapth(sb, dpth)) goto error; switch(ars) { case 0: break; case 1: iobuf_free_content(&lastpath); sbuf_free(&sb); return 0; default: if(result->buf) logp("Error after %s in %s()\n", result->buf, __func__); // Treat error in changed manio as // OK - could have been a short write. iobuf_free_content(&lastpath); sbuf_free(&sb); return 0; } // Some hacks for split_vss. switch(sb->path.cmd) { case CMD_VSS: case CMD_ENC_VSS: got_vss_start=1; break; case CMD_VSS_T: case CMD_ENC_VSS_T: got_vss_start=0; break; case CMD_FILE: case CMD_ENC_FILE: if(S_ISDIR(sb->statp.st_mode)) got_vss_start=0; break; default: break; } if(cntr) { // FIX THIS: cannot distinguish between new and // changed files. cntr_add_changed(cntr, sb->path.cmd); if(sb->endfile.buf) { uint64_t e=strtoull(sb->endfile.buf, NULL, 10); cntr_add_bytes(cntr, e); } } } error: iobuf_free_content(&lastpath); sbuf_free(&sb); man_off_t_free(pos); return -1; }
static #endif int forward_before_entry(struct manio *manio, struct iobuf *target, struct cntr *cntr, struct dpth *dpth, enum protocol protocol, man_off_t **pos) { int ars=0; struct sbuf *sb=NULL; if(!(sb=sbuf_alloc(protocol))) goto error; man_off_t_free(pos); if(!(*pos=manio_tell(manio))) { logp("Could not manio_tell first pos in %s(): %s\n", __func__, strerror(errno)); goto error; } while(1) { if(sb->endfile.buf || (sb->path.buf && !sbuf_is_filedata(sb))) { man_off_t_free(pos); if(!(*pos=manio_tell(manio))) { logp("Could not manio_tell pos in %s(): " "%s\n", __func__, strerror(errno)); goto error; } } sbuf_free_content(sb); ars=manio_read(manio, sb); if(dpth && set_higher_datapth(sb, dpth)) goto error; switch(ars) { case 0: break; case 1: sbuf_free(&sb); return 0; default: logp("Error in %s(), but continuing\n", __func__); // Treat error in unchanged manio as // OK - could have been a short write. sbuf_free(&sb); return 0; } if(iobuf_pathcmp(target, &sb->path)<=0) { sbuf_free(&sb); return 0; } if(cntr) { cntr_add_same(cntr, sb->path.cmd); if(sb->endfile.buf) { uint64_t e=strtoull(sb->endfile.buf, NULL, 10); cntr_add_bytes(cntr, e); } } } error: sbuf_free(&sb); man_off_t_free(pos); return -1; }
// Return -1 for error, 0 for entry not changed, 1 for entry changed (or new). static int entry_changed(struct sbuf *sb, struct manios *manios, struct asfd *chfd, struct sbuf **csb, struct cntr *cntr) { static int finished=0; int pcmp; if(finished) { cntr_add_new(cntr, sb->path.cmd); return 1; } if(*csb && (*csb)->path.buf) { // Already have an entry. } else { // Need to read another. switch(manio_read(manios->current, *csb)) { case 1: // Reached the end. sbuf_free(csb); finished=1; cntr_add_new(cntr, sb->path.cmd); return 1; case -1: return -1; } if(!(*csb)->path.buf) { logp("Should have a path at this point, but do not, in %s\n", __func__); return -1; } // Got an entry. } while(1) { if(!(pcmp=sbuf_pathcmp(*csb, sb))) return found_in_current_manifest(*csb, sb, manios, chfd, cntr); else if(pcmp>0) { cntr_add_new(cntr, sb->path.cmd); return 1; } // cntr_add_deleted(cntr, (*csb)->path.cmd); // Behind - need to read more data from the old manifest. switch(manio_read(manios->current, *csb)) { case 1: // Reached the end. sbuf_free(csb); cntr_add_new(cntr, sb->path.cmd); return 1; case -1: return -1; } // Got something, go back around the loop. } return 0; }
static int list_manifest(const char *fullpath) { int ret=0; struct sbuf *sb=NULL; struct manio *manio=NULL; char *manifest_dir=NULL; char *last_bd_match=NULL; size_t bdlen=0; if(!(manifest_dir=prepend_s(fullpath, protocol==PROTO_1?"manifest.gz":"manifest")) || !(manio=manio_open(manifest_dir, "rb", protocol)) || !(sb=sbuf_alloc(protocol))) { log_and_send_oom(asfd); goto error; } if(browsedir) bdlen=strlen(browsedir); while(1) { sbuf_free_content(sb); switch(manio_read(manio, sb)) { case 0: break; case 1: if(browsedir && *browsedir && !last_bd_match) asfd_write_wrapper_str(asfd, CMD_ERROR, "directory not found"); goto end; // Finished OK. default: goto error; } if(protocol==PROTO_2 && sb->endfile.buf) continue; if(sbuf_is_metadata(sb)) continue; if(write_status(CNTR_STATUS_LISTING, sb->path.buf, cntr)) goto error; if(browsedir) { int r; if((r=check_browsedir(browsedir, sb, bdlen, &last_bd_match))<0) goto error; if(!r) continue; } if(regex && !regex_check(regex, sb->path.buf)) continue; if(asfd_write_wrapper(asfd, &sb->attr) || asfd_write_wrapper(asfd, &sb->path)) goto error; if(sbuf_is_link(sb) && asfd_write_wrapper(asfd, &sb->link)) goto error; } error: ret=-1; end: sbuf_free(&sb); free_w(&manifest_dir); manio_close(&manio); free_w(&last_bd_match); return ret; }
int backup_phase2_server_protocol1(struct async *as, struct sdirs *sdirs, const char *incexc, int resume, struct conf **cconfs) { int ret=0; man_off_t *p1pos=NULL; struct manio *p1manio=NULL; struct dpth *dpth=NULL; char *deltmppath=NULL; char *last_requested=NULL; struct manio *chmanio=NULL; // changed data struct manio *ucmanio=NULL; // unchanged data struct manio *cmanio=NULL; // previous (current) manifest. struct sbuf *cb=NULL; // file list in current manifest struct sbuf *p1b=NULL; // file list from client struct sbuf *rb=NULL; // receiving file from client struct asfd *asfd=NULL; int breaking=0; int breakcount=0; struct cntr *cntr=NULL; if(!as) { logp("async not provided to %s()\n", __func__); goto error; } if(!sdirs) { logp("sdirs not provided to %s()\n", __func__); goto error; } if(!cconfs) { logp("cconfs not provided to %s()\n", __func__); goto error; } asfd=as->asfd; if(!asfd) { logp("asfd not provided to %s()\n", __func__); goto error; } cntr=get_cntr(cconfs); if(get_int(cconfs[OPT_BREAKPOINT])>=2000 && get_int(cconfs[OPT_BREAKPOINT])<3000) { breaking=get_int(cconfs[OPT_BREAKPOINT]); breakcount=breaking-2000; } logp("Begin phase2 (receive file data)\n"); if(!(dpth=dpth_alloc()) || dpth_protocol1_init(dpth, sdirs->currentdata, get_int(cconfs[OPT_MAX_STORAGE_SUBDIRS]))) goto error; if(open_previous_manifest(&cmanio, sdirs, incexc, cconfs)) goto error; if(get_int(cconfs[OPT_DIRECTORY_TREE])) { // Need to make sure we do not try to create a path that is // too long. if(build_path_w(sdirs->treepath)) goto error; mkdir(sdirs->treepath, 0777); treepathlen=strlen(sdirs->treepath); if(init_fs_max(sdirs->treepath)) goto error; } if(resume && !(p1pos=do_resume(sdirs, dpth, cconfs))) goto error; if(!(p1manio=manio_open_phase1(sdirs->phase1data, "rb", PROTO_1)) || (resume && manio_seek(p1manio, p1pos))) goto error; if(!(cb=sbuf_alloc(PROTO_1)) || !(p1b=sbuf_alloc(PROTO_1)) || !(rb=sbuf_alloc(PROTO_1))) goto error; // Unchanged and changed should now be truncated correctly, we just // have to open them for appending. // Data is not getting written to a compressed file. // This is important for recovery if the power goes. if(!(ucmanio=manio_open_phase2(sdirs->unchanged, "ab", PROTO_1)) || !(chmanio=manio_open_phase2(sdirs->changed, "ab", PROTO_1))) goto error; while(1) { if(breaking && breakcount--==0) return breakpoint(breaking, __func__); if(write_status(CNTR_STATUS_BACKUP, rb->path.buf?rb->path.buf:p1b->path.buf, cntr)) goto error; if(last_requested || !p1manio || asfd->writebuflen) { switch(do_stuff_to_receive(asfd, sdirs, cconfs, rb, chmanio, dpth, &last_requested)) { case 0: break; case 1: goto end; // Finished ok. case -1: goto error; } } switch(do_stuff_to_send(asfd, p1b, &last_requested)) { case 0: break; case 1: continue; case -1: goto error; } if(!p1manio) continue; sbuf_free_content(p1b); switch(manio_read(p1manio, p1b)) { case 0: break; case 1: manio_close(&p1manio); if(asfd->write_str(asfd, CMD_GEN, "backupphase2end")) goto error; break; case -1: goto error; } if(!cmanio) { // No old manifest, need to ask for a new file. if(process_new(sdirs, cconfs, p1b, ucmanio)) goto error; continue; } // Have an old manifest, look for it there. // Might already have it, or be ahead in the old // manifest. if(cb->path.buf) switch(maybe_process_file(asfd, sdirs, cb, p1b, ucmanio, cconfs)) { case 0: break; case 1: continue; case -1: goto error; } while(cmanio) { sbuf_free_content(cb); switch(manio_read(cmanio, cb)) { case 0: break; case 1: manio_close(&cmanio); if(process_new(sdirs, cconfs, p1b, ucmanio)) goto error; continue; case -1: goto error; } switch(maybe_process_file(asfd, sdirs, cb, p1b, ucmanio, cconfs)) { case 0: continue; case 1: break; case -1: goto error; } break; } } error: ret=-1; end: if(manio_close(&chmanio)) { logp("error closing %s in %s\n", sdirs->changed, __func__); ret=-1; } if(manio_close(&ucmanio)) { logp("error closing %s in %s\n", sdirs->unchanged, __func__); ret=-1; } free_w(&deltmppath); free_w(&last_requested); sbuf_free(&cb); sbuf_free(&p1b); sbuf_free(&rb); manio_close(&p1manio); manio_close(&cmanio); dpth_free(&dpth); man_off_t_free(&p1pos); if(!ret && sdirs) unlink(sdirs->phase1data); logp("End phase2 (receive file data)\n"); return ret; }