static #endif int forward_past_entry(struct manio *manio, struct iobuf *target, enum protocol protocol, man_off_t **pos) { struct sbuf *sb=NULL; if(!(sb=sbuf_alloc(protocol))) goto error; man_off_t_free(pos); if(!(*pos=manio_tell(manio))) { logp("Could not manio_tell first pos in %s(): %s\n", __func__, strerror(errno)); goto error; } while(1) { sbuf_free_content(sb); switch(manio_read(manio, sb)) { case 0: break; case 1: logp("End of file in %s()\n", __func__); goto error; default: logp("Error in %s()\n", __func__); // Treat error in unchanged manio as not OK. goto error; } if(target->cmd==sb->path.cmd && !pathcmp(target->buf, sb->path.buf)) { man_off_t_free(pos); if(!(*pos=manio_tell(manio))) { logp("Could not get pos in %s(): %s\n", __func__, strerror(errno)); goto error; } sbuf_free(&sb); return 0; } } error: sbuf_free(&sb); man_off_t_free(pos); return -1; }
END_TEST static void test_manifest_tell_seek(enum protocol protocol, int phase) { struct slist *slist; struct manio *manio; struct sbuf *sb=NULL; man_off_t *offset=NULL; int entries=1000; prng_init(0); base64_init(); hexmap_init(); recursive_delete(path); slist=build_manifest(path, protocol, entries, phase); fail_unless(slist!=NULL); sb=slist->head; fail_unless((manio=do_manio_open(path, "rb", protocol, phase))!=NULL); read_manifest(&sb, manio, 0, entries/2, protocol, phase); fail_unless((offset=manio_tell(manio))!=NULL); fail_unless(sb!=NULL); fail_unless(!manio_close(&manio)); fail_unless((manio=do_manio_open(path, "rb", protocol, phase))!=NULL); fail_unless(!manio_seek(manio, offset)); read_manifest(&sb, manio, entries/2, entries, protocol, phase); fail_unless(sb==NULL); fail_unless(!manio_close(&manio)); fail_unless(!manio); slist_free(&slist); man_off_t_free(&offset); tear_down(); }
static void manio_free_content(struct manio *manio) { if(!manio) return; man_off_t_free(&manio->offset); free_w(&manio->manifest); free_w(&manio->mode); free_w(&manio->rmanifest); free_w(&manio->hook_dir); free_v((void **)&manio->hook_sort); free_w(&manio->dindex_dir); free_v((void **)&manio->dindex_sort); memset(manio, 0, sizeof(struct manio)); }
void build_manifest_phase2_from_slist(const char *path, struct slist *slist, enum protocol protocol, int short_write) { struct sbuf *sb; struct manio *manio=NULL; for(sb=slist->head; sb; sb=sb->next) set_sbuf(slist, sb, 0 /* with_data_files */); fail_unless((manio=manio_open_phase2(path, "wb", protocol))!=NULL); for(sb=slist->head; sb; sb=sb->next) { fail_unless(!manio_write_sbuf(manio, sb)); if(protocol==PROTO_2) { struct blk *blk=NULL; for(blk=sb->protocol2->bstart; blk && blk!=sb->protocol2->bend; blk=blk->next) { fail_unless(!manio_write_sig_and_path(manio, blk)); } if(sbuf_is_filedata(sb) || sbuf_is_vssdata(sb)) { struct iobuf endfile; iobuf_from_str(&endfile, CMD_END_FILE, (char *)"0:0"); fail_unless(!iobuf_send_msg_fzp(&endfile, manio->fzp)); } hack_protocol2_attr(&sb->attr); } } if(short_write) { man_off_t *pos; fail_unless((pos=manio_tell(manio))!=NULL); if(pos->offset>=short_write) pos->offset-=short_write; fail_unless(!manio_close_and_truncate(&manio, pos, 0 /* compression */)); man_off_t_free(&pos); } fail_unless(!manio_close(&manio)); }
man_off_t *manio_tell(struct manio *manio) { man_off_t *offset=NULL; if(!manio->fzp) { logp("manio_tell called on null fzp\n"); goto error; } if(!(offset=man_off_t_alloc()) || !(offset->fpath=strdup_w(manio->offset->fpath, __func__)) || (offset->offset=fzp_tell(manio->fzp))<0) goto error; offset->fcount=manio->offset->fcount; return offset; error: man_off_t_free(&offset); return NULL; }
static #endif int do_backup_phase2_server_protocol2(struct async *as, struct asfd *chfd, struct sdirs *sdirs, int resume, struct conf **confs) { int ret=-1; uint8_t end_flags=0; struct slist *slist=NULL; struct iobuf wbuf; struct dpth *dpth=NULL; man_off_t *p1pos=NULL; struct manios *manios=NULL; // This is used to tell the client that a number of consecutive blocks // have been found and can be freed. uint64_t wrap_up=0; struct asfd *asfd=NULL; struct cntr *cntr=NULL; struct sbuf *csb=NULL; uint64_t file_no=1; if(!as) { logp("async not provided to %s()\n", __func__); goto end; } if(!sdirs) { logp("sdirs not provided to %s()\n", __func__); goto end; } if(!confs) { logp("confs not provided to %s()\n", __func__); goto end; } asfd=as->asfd; if(!asfd) { logp("asfd not provided to %s()\n", __func__); goto end; } if(!chfd) { logp("chfd not provided to %s()\n", __func__); goto end; } cntr=get_cntr(confs); if(get_int(confs[OPT_BREAKPOINT])>=2000 && get_int(confs[OPT_BREAKPOINT])<3000) { breaking=get_int(confs[OPT_BREAKPOINT]); breakcount=breaking-2000; } logp("Phase 2 begin (recv backup data)\n"); if(!(dpth=dpth_alloc()) || dpth_protocol2_init(dpth, sdirs->data, get_string(confs[OPT_CNAME]), sdirs->cfiles, get_int(confs[OPT_MAX_STORAGE_SUBDIRS]))) goto end; if(resume && !(p1pos=do_resume(sdirs, dpth, confs))) goto end; if(!(manios=manios_open_phase2(sdirs, p1pos, PROTO_2)) || !(slist=slist_alloc()) || !(csb=sbuf_alloc(PROTO_2))) goto end; iobuf_free_content(asfd->rbuf); memset(&wbuf, 0, sizeof(struct iobuf)); while(!(end_flags&END_BACKUP)) { if(maybe_add_from_scan(manios, slist, chfd, &csb)) goto end; if(!wbuf.len) { if(get_wbuf_from_sigs(&wbuf, slist, &end_flags)) goto end; if(!wbuf.len) { get_wbuf_from_files(&wbuf, slist, manios, &end_flags, &file_no); } } if(wbuf.len && asfd->append_all_to_write_buffer(asfd, &wbuf)==APPEND_ERROR) goto end; if(append_for_champ_chooser(chfd, slist->blist, end_flags)) goto end; if(as->read_write(as)) { logp("error from as->read_write in %s\n", __func__); goto end; } while(asfd->rbuf->buf) { if(deal_with_read(asfd->rbuf, slist, cntr, &end_flags, dpth)) goto end; // Get as much out of the readbuf as possible. if(asfd->parse_readbuf(asfd)) goto end; } while(chfd->rbuf->buf) { if(deal_with_read_from_chfd(chfd, slist->blist, &wrap_up, dpth, cntr)) goto end; // Get as much out of the readbuf as possible. if(chfd->parse_readbuf(chfd)) goto end; } if(write_to_changed_file(asfd, chfd, manios, slist, end_flags)) goto end; } // Hack: If there are some entries left after the last entry that // contains block data, it will not be written to the changed file // yet because the last entry of block data has not had // sb->protocol2->bend set. if(slist->head && slist->head->next) { struct sbuf *sb=NULL; sb=slist->head; slist->head=sb->next; sbuf_free(&sb); if(write_to_changed_file(asfd, chfd, manios, slist, end_flags)) goto end; } if(manios_close(&manios)) goto end; if(check_for_missing_work_in_slist(slist)) goto end; // Need to release the last left. There should be one at most. if(dpth->head && dpth->head->next) { logp("ERROR: More data locks remaining after: %s\n", dpth->head->save_path); goto end; } if(dpth_release_all(dpth)) goto end; ret=0; end: logp("End backup\n"); sbuf_free(&csb); slist_free(&slist); if(asfd) iobuf_free_content(asfd->rbuf); if(chfd) iobuf_free_content(chfd->rbuf); dpth_free(&dpth); manios_close(&manios); man_off_t_free(&p1pos); return ret; }
// Return p1manio position. static man_off_t *do_resume_work(struct sdirs *sdirs, struct dpth *dpth, struct conf **cconfs) { man_off_t *pos=NULL; man_off_t *p1pos=NULL; struct iobuf *chb=NULL; struct manio *cmanio=NULL; struct manio *umanio=NULL; struct manio *p1manio=NULL; enum protocol protocol=get_protocol(cconfs); struct cntr *cntr=get_cntr(cconfs); int compression=get_int(cconfs[OPT_COMPRESSION]); if(!(p1manio=manio_open_phase1(sdirs->phase1data, "rb", protocol)) || !(cmanio=manio_open_phase2(sdirs->changed, "rb", protocol)) || !(umanio=manio_open_phase2(sdirs->unchanged, "rb", protocol))) goto end; if(!(chb=iobuf_alloc())) return NULL; logp("Setting up resume positions...\n"); if(get_last_good_entry(cmanio, chb, cntr, dpth, protocol, &pos)) goto error; if(manio_close_and_truncate(&cmanio, pos, compression)) goto error; man_off_t_free(&pos); if(chb->buf) { logp(" last good entry: %s\n", chb->buf); // Now need to go to the appropriate places in p1manio and // unchanged. if(forward_past_entry(p1manio, chb, protocol, &p1pos)) goto error; // The unchanged file needs to be positioned just before the // found entry, otherwise it ends up having a duplicated entry. if(forward_before_entry(umanio, chb, cntr, dpth, protocol, &pos)) goto error; if(manio_close_and_truncate(&umanio, pos, compression)) goto error; man_off_t_free(&pos); } else { logp(" nothing previously transferred\n"); if(!(p1pos=manio_tell(p1manio))) goto error; if(!(pos=manio_tell(umanio))) goto error; if(manio_close_and_truncate(&umanio, pos, compression)) goto error; } // Now should have all file pointers in the right places to resume. goto end; error: man_off_t_free(&p1pos); end: iobuf_free(&chb); man_off_t_free(&pos); manio_close(&p1manio); manio_close(&cmanio); manio_close(&umanio); return p1pos; }
static #endif int get_last_good_entry(struct manio *manio, struct iobuf *result, struct cntr *cntr, struct dpth *dpth, enum protocol protocol, man_off_t **pos) { int ars=0; int got_vss_start=0; struct sbuf *sb=NULL; struct iobuf lastpath; if(!(sb=sbuf_alloc(protocol))) goto error; iobuf_init(&lastpath); man_off_t_free(pos); if(!(*pos=manio_tell(manio))) { logp("Could not manio_tell first pos in %s(): %s\n", __func__, strerror(errno)); goto error; } while(1) { if(sb->path.buf && !got_vss_start) { iobuf_free_content(&lastpath); iobuf_move(&lastpath, &sb->path); if(!sbuf_is_filedata(sb) && !sbuf_is_vssdata(sb)) { iobuf_free_content(result); iobuf_move(result, &lastpath); man_off_t_free(pos); if(!(*pos=manio_tell(manio))) { logp("Could not manio_tell pos in %s(): %s\n", __func__, strerror(errno)); goto error; } } } if(sb->endfile.buf && !got_vss_start) { iobuf_free_content(result); iobuf_move(result, &lastpath); man_off_t_free(pos); if(!(*pos=manio_tell(manio))) { logp("Could not manio_tell pos in %s(): %s\n", __func__, strerror(errno)); goto error; } } sbuf_free_content(sb); ars=manio_read(manio, sb); if(dpth && set_higher_datapth(sb, dpth)) goto error; switch(ars) { case 0: break; case 1: iobuf_free_content(&lastpath); sbuf_free(&sb); return 0; default: if(result->buf) logp("Error after %s in %s()\n", result->buf, __func__); // Treat error in changed manio as // OK - could have been a short write. iobuf_free_content(&lastpath); sbuf_free(&sb); return 0; } // Some hacks for split_vss. switch(sb->path.cmd) { case CMD_VSS: case CMD_ENC_VSS: got_vss_start=1; break; case CMD_VSS_T: case CMD_ENC_VSS_T: got_vss_start=0; break; case CMD_FILE: case CMD_ENC_FILE: if(S_ISDIR(sb->statp.st_mode)) got_vss_start=0; break; default: break; } if(cntr) { // FIX THIS: cannot distinguish between new and // changed files. cntr_add_changed(cntr, sb->path.cmd); if(sb->endfile.buf) { uint64_t e=strtoull(sb->endfile.buf, NULL, 10); cntr_add_bytes(cntr, e); } } } error: iobuf_free_content(&lastpath); sbuf_free(&sb); man_off_t_free(pos); return -1; }
static #endif int forward_before_entry(struct manio *manio, struct iobuf *target, struct cntr *cntr, struct dpth *dpth, enum protocol protocol, man_off_t **pos) { int ars=0; struct sbuf *sb=NULL; if(!(sb=sbuf_alloc(protocol))) goto error; man_off_t_free(pos); if(!(*pos=manio_tell(manio))) { logp("Could not manio_tell first pos in %s(): %s\n", __func__, strerror(errno)); goto error; } while(1) { if(sb->endfile.buf || (sb->path.buf && !sbuf_is_filedata(sb))) { man_off_t_free(pos); if(!(*pos=manio_tell(manio))) { logp("Could not manio_tell pos in %s(): " "%s\n", __func__, strerror(errno)); goto error; } } sbuf_free_content(sb); ars=manio_read(manio, sb); if(dpth && set_higher_datapth(sb, dpth)) goto error; switch(ars) { case 0: break; case 1: sbuf_free(&sb); return 0; default: logp("Error in %s(), but continuing\n", __func__); // Treat error in unchanged manio as // OK - could have been a short write. sbuf_free(&sb); return 0; } if(iobuf_pathcmp(target, &sb->path)<=0) { sbuf_free(&sb); return 0; } if(cntr) { cntr_add_same(cntr, sb->path.cmd); if(sb->endfile.buf) { uint64_t e=strtoull(sb->endfile.buf, NULL, 10); cntr_add_bytes(cntr, e); } } } error: sbuf_free(&sb); man_off_t_free(pos); return -1; }
int backup_phase2_server_protocol1(struct async *as, struct sdirs *sdirs, const char *incexc, int resume, struct conf **cconfs) { int ret=0; man_off_t *p1pos=NULL; struct manio *p1manio=NULL; struct dpth *dpth=NULL; char *deltmppath=NULL; char *last_requested=NULL; struct manio *chmanio=NULL; // changed data struct manio *ucmanio=NULL; // unchanged data struct manio *cmanio=NULL; // previous (current) manifest. struct sbuf *cb=NULL; // file list in current manifest struct sbuf *p1b=NULL; // file list from client struct sbuf *rb=NULL; // receiving file from client struct asfd *asfd=NULL; int breaking=0; int breakcount=0; struct cntr *cntr=NULL; if(!as) { logp("async not provided to %s()\n", __func__); goto error; } if(!sdirs) { logp("sdirs not provided to %s()\n", __func__); goto error; } if(!cconfs) { logp("cconfs not provided to %s()\n", __func__); goto error; } asfd=as->asfd; if(!asfd) { logp("asfd not provided to %s()\n", __func__); goto error; } cntr=get_cntr(cconfs); if(get_int(cconfs[OPT_BREAKPOINT])>=2000 && get_int(cconfs[OPT_BREAKPOINT])<3000) { breaking=get_int(cconfs[OPT_BREAKPOINT]); breakcount=breaking-2000; } logp("Begin phase2 (receive file data)\n"); if(!(dpth=dpth_alloc()) || dpth_protocol1_init(dpth, sdirs->currentdata, get_int(cconfs[OPT_MAX_STORAGE_SUBDIRS]))) goto error; if(open_previous_manifest(&cmanio, sdirs, incexc, cconfs)) goto error; if(get_int(cconfs[OPT_DIRECTORY_TREE])) { // Need to make sure we do not try to create a path that is // too long. if(build_path_w(sdirs->treepath)) goto error; mkdir(sdirs->treepath, 0777); treepathlen=strlen(sdirs->treepath); if(init_fs_max(sdirs->treepath)) goto error; } if(resume && !(p1pos=do_resume(sdirs, dpth, cconfs))) goto error; if(!(p1manio=manio_open_phase1(sdirs->phase1data, "rb", PROTO_1)) || (resume && manio_seek(p1manio, p1pos))) goto error; if(!(cb=sbuf_alloc(PROTO_1)) || !(p1b=sbuf_alloc(PROTO_1)) || !(rb=sbuf_alloc(PROTO_1))) goto error; // Unchanged and changed should now be truncated correctly, we just // have to open them for appending. // Data is not getting written to a compressed file. // This is important for recovery if the power goes. if(!(ucmanio=manio_open_phase2(sdirs->unchanged, "ab", PROTO_1)) || !(chmanio=manio_open_phase2(sdirs->changed, "ab", PROTO_1))) goto error; while(1) { if(breaking && breakcount--==0) return breakpoint(breaking, __func__); if(write_status(CNTR_STATUS_BACKUP, rb->path.buf?rb->path.buf:p1b->path.buf, cntr)) goto error; if(last_requested || !p1manio || asfd->writebuflen) { switch(do_stuff_to_receive(asfd, sdirs, cconfs, rb, chmanio, dpth, &last_requested)) { case 0: break; case 1: goto end; // Finished ok. case -1: goto error; } } switch(do_stuff_to_send(asfd, p1b, &last_requested)) { case 0: break; case 1: continue; case -1: goto error; } if(!p1manio) continue; sbuf_free_content(p1b); switch(manio_read(p1manio, p1b)) { case 0: break; case 1: manio_close(&p1manio); if(asfd->write_str(asfd, CMD_GEN, "backupphase2end")) goto error; break; case -1: goto error; } if(!cmanio) { // No old manifest, need to ask for a new file. if(process_new(sdirs, cconfs, p1b, ucmanio)) goto error; continue; } // Have an old manifest, look for it there. // Might already have it, or be ahead in the old // manifest. if(cb->path.buf) switch(maybe_process_file(asfd, sdirs, cb, p1b, ucmanio, cconfs)) { case 0: break; case 1: continue; case -1: goto error; } while(cmanio) { sbuf_free_content(cb); switch(manio_read(cmanio, cb)) { case 0: break; case 1: manio_close(&cmanio); if(process_new(sdirs, cconfs, p1b, ucmanio)) goto error; continue; case -1: goto error; } switch(maybe_process_file(asfd, sdirs, cb, p1b, ucmanio, cconfs)) { case 0: continue; case 1: break; case -1: goto error; } break; } } error: ret=-1; end: if(manio_close(&chmanio)) { logp("error closing %s in %s\n", sdirs->changed, __func__); ret=-1; } if(manio_close(&ucmanio)) { logp("error closing %s in %s\n", sdirs->unchanged, __func__); ret=-1; } free_w(&deltmppath); free_w(&last_requested); sbuf_free(&cb); sbuf_free(&p1b); sbuf_free(&rb); manio_close(&p1manio); manio_close(&cmanio); dpth_free(&dpth); man_off_t_free(&p1pos); if(!ret && sdirs) unlink(sdirs->phase1data); logp("End phase2 (receive file data)\n"); return ret; }