void build_manifest_phase1_from_slist(const char *path, struct slist *slist, enum protocol protocol) { struct sbuf *sb; struct manio *manio=NULL; struct iobuf datapth; struct iobuf endfile; iobuf_init(&datapth); iobuf_init(&endfile); for(sb=slist->head; sb; sb=sb->next) set_sbuf(slist, sb, 0 /* with_data_files */); fail_unless((manio=manio_open_phase1(path, "wb", protocol))!=NULL); for(sb=slist->head; sb; sb=sb->next) { // Might be given an slist that has datapth or endfile set, // which should not go into a phase1 scan. Deal with it. if(sb->protocol1 && sb->protocol1->datapth.buf) iobuf_move(&datapth, &sb->protocol1->datapth); if(sb->endfile.buf) iobuf_move(&endfile, &sb->endfile); fail_unless(!manio_write_sbuf(manio, sb)); if(datapth.buf) iobuf_move(&sb->protocol1->datapth, &datapth); if(endfile.buf) iobuf_move(&sb->endfile, &endfile); } fail_unless(!send_msg_fzp(manio->fzp, CMD_GEN, "phase1end", strlen("phase1end"))); fail_unless(!manio_close(&manio)); }
// Return p1manio position. static man_off_t *do_resume_work(struct sdirs *sdirs, struct dpth *dpth, struct conf **cconfs) { man_off_t *pos=NULL; man_off_t *p1pos=NULL; struct iobuf *chb=NULL; struct manio *cmanio=NULL; struct manio *umanio=NULL; struct manio *p1manio=NULL; enum protocol protocol=get_protocol(cconfs); struct cntr *cntr=get_cntr(cconfs); int compression=get_int(cconfs[OPT_COMPRESSION]); if(!(p1manio=manio_open_phase1(sdirs->phase1data, "rb", protocol)) || !(cmanio=manio_open_phase2(sdirs->changed, "rb", protocol)) || !(umanio=manio_open_phase2(sdirs->unchanged, "rb", protocol))) goto end; if(!(chb=iobuf_alloc())) return NULL; logp("Setting up resume positions...\n"); if(get_last_good_entry(cmanio, chb, cntr, dpth, protocol, &pos)) goto error; if(manio_close_and_truncate(&cmanio, pos, compression)) goto error; man_off_t_free(&pos); if(chb->buf) { logp(" last good entry: %s\n", chb->buf); // Now need to go to the appropriate places in p1manio and // unchanged. if(forward_past_entry(p1manio, chb, protocol, &p1pos)) goto error; // The unchanged file needs to be positioned just before the // found entry, otherwise it ends up having a duplicated entry. if(forward_before_entry(umanio, chb, cntr, dpth, protocol, &pos)) goto error; if(manio_close_and_truncate(&umanio, pos, compression)) goto error; man_off_t_free(&pos); } else { logp(" nothing previously transferred\n"); if(!(p1pos=manio_tell(p1manio))) goto error; if(!(pos=manio_tell(umanio))) goto error; if(manio_close_and_truncate(&umanio, pos, compression)) goto error; } // Now should have all file pointers in the right places to resume. goto end; error: man_off_t_free(&p1pos); end: iobuf_free(&chb); man_off_t_free(&pos); manio_close(&p1manio); manio_close(&cmanio); manio_close(&umanio); return p1pos; }
int backup_phase1_server_all(struct async *as, struct sdirs *sdirs, struct conf **confs) { int ret=-1; struct sbuf *sb=NULL; char *phase1tmp=NULL; struct asfd *asfd=as->asfd; struct manio *manio=NULL; enum protocol protocol=get_protocol(confs); struct cntr *cntr=get_cntr(confs); logp("Begin phase1 (file system scan)\n"); if(!(phase1tmp=get_tmp_filename(sdirs->phase1data)) || !(manio=manio_open_phase1(phase1tmp, comp_level(get_int(confs[OPT_COMPRESSION])), protocol)) || !(sb=sbuf_alloc(protocol))) goto error; while(1) { sbuf_free_content(sb); switch(sbuf_fill_from_net(sb, asfd, NULL, NULL, cntr)) { case 0: break; case 1: // Last thing the client sends is // 'backupphase2', and it wants an 'ok' reply. if(asfd->write_str(asfd, CMD_GEN, "ok") || send_msg_fzp(manio->fzp, CMD_GEN, "phase1end", strlen("phase1end"))) goto error; goto end; case -1: default: goto error; } if(write_status(CNTR_STATUS_SCANNING, sb->path.buf, cntr) || manio_write_sbuf(manio, sb)) goto error; cntr_add_phase1(cntr, sb->path.cmd, 0); if(sbuf_is_filedata(sb)) { cntr_add_val(cntr, CMD_BYTES_ESTIMATED, (uint64_t)sb->statp.st_size, 0); } } end: if(manio_close(&manio)) { logp("error closing %s in backup_phase1_server\n", phase1tmp); goto error; } if(check_quota(as, cntr, get_uint64_t(confs[OPT_HARD_QUOTA]), get_uint64_t(confs[OPT_SOFT_QUOTA]))) goto error; // Possible rename race condition is of no consequence here, because // the working directory will always get deleted if phase1 is not // complete. if(do_rename(phase1tmp, sdirs->phase1data)) goto error; //cntr_print(p1cntr, cntr, ACTION_BACKUP); logp("End phase1 (file system scan)\n"); ret=0; error: free_w(&phase1tmp); manio_close(&manio); sbuf_free(&sb); return ret; }
int backup_phase2_server_protocol1(struct async *as, struct sdirs *sdirs, const char *incexc, int resume, struct conf **cconfs) { int ret=0; man_off_t *p1pos=NULL; struct manio *p1manio=NULL; struct dpth *dpth=NULL; char *deltmppath=NULL; char *last_requested=NULL; struct manio *chmanio=NULL; // changed data struct manio *ucmanio=NULL; // unchanged data struct manio *cmanio=NULL; // previous (current) manifest. struct sbuf *cb=NULL; // file list in current manifest struct sbuf *p1b=NULL; // file list from client struct sbuf *rb=NULL; // receiving file from client struct asfd *asfd=NULL; int breaking=0; int breakcount=0; struct cntr *cntr=NULL; if(!as) { logp("async not provided to %s()\n", __func__); goto error; } if(!sdirs) { logp("sdirs not provided to %s()\n", __func__); goto error; } if(!cconfs) { logp("cconfs not provided to %s()\n", __func__); goto error; } asfd=as->asfd; if(!asfd) { logp("asfd not provided to %s()\n", __func__); goto error; } cntr=get_cntr(cconfs); if(get_int(cconfs[OPT_BREAKPOINT])>=2000 && get_int(cconfs[OPT_BREAKPOINT])<3000) { breaking=get_int(cconfs[OPT_BREAKPOINT]); breakcount=breaking-2000; } logp("Begin phase2 (receive file data)\n"); if(!(dpth=dpth_alloc()) || dpth_protocol1_init(dpth, sdirs->currentdata, get_int(cconfs[OPT_MAX_STORAGE_SUBDIRS]))) goto error; if(open_previous_manifest(&cmanio, sdirs, incexc, cconfs)) goto error; if(get_int(cconfs[OPT_DIRECTORY_TREE])) { // Need to make sure we do not try to create a path that is // too long. if(build_path_w(sdirs->treepath)) goto error; mkdir(sdirs->treepath, 0777); treepathlen=strlen(sdirs->treepath); if(init_fs_max(sdirs->treepath)) goto error; } if(resume && !(p1pos=do_resume(sdirs, dpth, cconfs))) goto error; if(!(p1manio=manio_open_phase1(sdirs->phase1data, "rb", PROTO_1)) || (resume && manio_seek(p1manio, p1pos))) goto error; if(!(cb=sbuf_alloc(PROTO_1)) || !(p1b=sbuf_alloc(PROTO_1)) || !(rb=sbuf_alloc(PROTO_1))) goto error; // Unchanged and changed should now be truncated correctly, we just // have to open them for appending. // Data is not getting written to a compressed file. // This is important for recovery if the power goes. if(!(ucmanio=manio_open_phase2(sdirs->unchanged, "ab", PROTO_1)) || !(chmanio=manio_open_phase2(sdirs->changed, "ab", PROTO_1))) goto error; while(1) { if(breaking && breakcount--==0) return breakpoint(breaking, __func__); if(write_status(CNTR_STATUS_BACKUP, rb->path.buf?rb->path.buf:p1b->path.buf, cntr)) goto error; if(last_requested || !p1manio || asfd->writebuflen) { switch(do_stuff_to_receive(asfd, sdirs, cconfs, rb, chmanio, dpth, &last_requested)) { case 0: break; case 1: goto end; // Finished ok. case -1: goto error; } } switch(do_stuff_to_send(asfd, p1b, &last_requested)) { case 0: break; case 1: continue; case -1: goto error; } if(!p1manio) continue; sbuf_free_content(p1b); switch(manio_read(p1manio, p1b)) { case 0: break; case 1: manio_close(&p1manio); if(asfd->write_str(asfd, CMD_GEN, "backupphase2end")) goto error; break; case -1: goto error; } if(!cmanio) { // No old manifest, need to ask for a new file. if(process_new(sdirs, cconfs, p1b, ucmanio)) goto error; continue; } // Have an old manifest, look for it there. // Might already have it, or be ahead in the old // manifest. if(cb->path.buf) switch(maybe_process_file(asfd, sdirs, cb, p1b, ucmanio, cconfs)) { case 0: break; case 1: continue; case -1: goto error; } while(cmanio) { sbuf_free_content(cb); switch(manio_read(cmanio, cb)) { case 0: break; case 1: manio_close(&cmanio); if(process_new(sdirs, cconfs, p1b, ucmanio)) goto error; continue; case -1: goto error; } switch(maybe_process_file(asfd, sdirs, cb, p1b, ucmanio, cconfs)) { case 0: continue; case 1: break; case -1: goto error; } break; } } error: ret=-1; end: if(manio_close(&chmanio)) { logp("error closing %s in %s\n", sdirs->changed, __func__); ret=-1; } if(manio_close(&ucmanio)) { logp("error closing %s in %s\n", sdirs->unchanged, __func__); ret=-1; } free_w(&deltmppath); free_w(&last_requested); sbuf_free(&cb); sbuf_free(&p1b); sbuf_free(&rb); manio_close(&p1manio); manio_close(&cmanio); dpth_free(&dpth); man_off_t_free(&p1pos); if(!ret && sdirs) unlink(sdirs->phase1data); logp("End phase2 (receive file data)\n"); return ret; }