static struct slist *do_build_manifest(struct manio *manio, enum protocol protocol, int entries, int with_data_files) { struct sbuf *sb; struct slist *slist=NULL; slist=build_slist(protocol, entries, with_data_files); for(sb=slist->head; sb; sb=sb->next) { fail_unless(!manio_write_sbuf(manio, sb)); if(protocol==PROTO_2) { struct blk *blk=NULL; for(blk=sb->protocol2->bstart; blk && blk!=sb->protocol2->bend; blk=blk->next) { fail_unless(!manio_write_sig_and_path(manio, blk)); } if(sbuf_is_filedata(sb) || sbuf_is_vssdata(sb)) { struct iobuf endfile; iobuf_from_str(&endfile, CMD_END_FILE, (char *)"0:0"); fail_unless(!iobuf_send_msg_fzp(&endfile, manio->fzp)); } hack_protocol2_attr(&sb->attr); } } return slist; }
static int write_to_changed_file(struct asfd *asfd, struct asfd *chfd, struct manios *manios, struct slist *slist, int end_flags) { struct sbuf *sb; if(!slist) return 0; while((sb=slist->head)) { if(sb->flags & SBUF_NEED_DATA) { switch(sbuf_needs_data(sb, asfd, chfd, manios, slist, end_flags)) { case 0: return 0; case 1: continue; default: return -1; } } else { // No change, can go straight in. if(manio_write_sbuf(manios->changed, sb)) return -1; if(write_endfile(sb, manios)) return -1; // Move along. slist_advance(slist); } } return 0; }
static int changed_non_file(struct sbuf *p1b, struct manio *ucmanio, enum cmd cmd, struct conf **cconfs) { // As new_non_file. if(manio_write_sbuf(ucmanio, p1b)) return -1; cntr_add_changed(get_cntr(cconfs), cmd); sbuf_free_content(p1b); return 0; }
static int new_non_file(struct sbuf *p1b, struct manio *ucmanio, struct conf **cconfs) { // Is something that does not need more data backed up. // Like a directory or a link or something like that. // Goes into the unchanged file, so that it does not end up out of // order with normal files, which has to wait around for their data // to turn up. if(manio_write_sbuf(ucmanio, p1b)) return -1; cntr_add(get_cntr(cconfs), p1b->path.cmd, 0); sbuf_free_content(p1b); return 0; }
void build_manifest_phase2_from_slist(const char *path, struct slist *slist, enum protocol protocol, int short_write) { struct sbuf *sb; struct manio *manio=NULL; for(sb=slist->head; sb; sb=sb->next) set_sbuf(slist, sb, 0 /* with_data_files */); fail_unless((manio=manio_open_phase2(path, "wb", protocol))!=NULL); for(sb=slist->head; sb; sb=sb->next) { fail_unless(!manio_write_sbuf(manio, sb)); if(protocol==PROTO_2) { struct blk *blk=NULL; for(blk=sb->protocol2->bstart; blk && blk!=sb->protocol2->bend; blk=blk->next) { fail_unless(!manio_write_sig_and_path(manio, blk)); } if(sbuf_is_filedata(sb) || sbuf_is_vssdata(sb)) { struct iobuf endfile; iobuf_from_str(&endfile, CMD_END_FILE, (char *)"0:0"); fail_unless(!iobuf_send_msg_fzp(&endfile, manio->fzp)); } hack_protocol2_attr(&sb->attr); } } if(short_write) { man_off_t *pos; fail_unless((pos=manio_tell(manio))!=NULL); if(pos->offset>=short_write) pos->offset-=short_write; fail_unless(!manio_close_and_truncate(&manio, pos, 0 /* compression */)); man_off_t_free(&pos); } fail_unless(!manio_close(&manio)); }
// Return -1 on error, 0 on OK, 1 for srcmanio finished. int manio_copy_entry(struct asfd *asfd, struct sbuf **csb, struct sbuf *sb, struct blk **blk, struct manio *srcmanio, struct manio *dstmanio, struct conf *conf) { static int ars; static char *copy=NULL; // Use the most recent stat for the new manifest. if(dstmanio && manio_write_sbuf(dstmanio, sb)) goto error; if(!(copy=strdup((*csb)->path.buf))) { log_out_of_memory(__func__); goto error; } while(1) { if((ars=manio_sbuf_fill(srcmanio, asfd, *csb, *blk, NULL, conf))<0) goto error; else if(ars>0) { // Finished. sbuf_free(*csb); *csb=NULL; blk_free(*blk); *blk=NULL; free(copy); return 1; } // Got something. if(strcmp((*csb)->path.buf, copy)) { // Found the next entry. free(copy); return 0; } // Should have the next signature. // Write it to the destination manifest. if(dstmanio && manio_write_sig_and_path(dstmanio, *blk)) goto error; } error: if(copy) free(copy); return -1; }
static int deal_with_receive_end_file(struct asfd *asfd, struct sdirs *sdirs, struct sbuf *rb, struct manio *chmanio, struct conf **cconfs, char **last_requested) { int ret=-1; static char *cp=NULL; static struct iobuf *rbuf; struct cntr *cntr=get_cntr(cconfs); rbuf=asfd->rbuf; // Finished the file. // Write it to the phase2 file, and free the buffers. if(fzp_close(&(rb->protocol1->fzp))) { logp("error closing delta for %s in receive\n", rb->path.buf); goto end; } iobuf_move(&rb->endfile, rbuf); if(rb->flags & SBUF_RECV_DELTA && finish_delta(sdirs, rb)) goto end; if(manio_write_sbuf(chmanio, rb)) goto end; if(rb->flags & SBUF_RECV_DELTA) cntr_add_changed(cntr, rb->path.cmd); else cntr_add(cntr, rb->path.cmd, 0); if(*last_requested && !strcmp(rb->path.buf, *last_requested)) free_w(last_requested); cp=strchr(rb->endfile.buf, ':'); if(rb->endfile.buf) cntr_add_bytes(cntr, strtoull(rb->endfile.buf, NULL, 10)); if(cp) { // checksum stuff goes here } ret=0; end: sbuf_free_content(rb); return ret; }
static int process_unchanged_file(struct sbuf *p1b, struct sbuf *cb, struct manio *ucmanio, struct conf **cconfs) { // Need to re-encode the p1b attribs to include compression and // other bits and pieces that are recorded on cb. iobuf_move(&p1b->protocol1->datapth, &cb->protocol1->datapth); iobuf_move(&p1b->endfile, &cb->endfile); p1b->compression=cb->compression; p1b->winattr=cb->winattr; if(attribs_encode(p1b)) return -1; if(manio_write_sbuf(ucmanio, p1b)) return -1; cntr_add_same(get_cntr(cconfs), p1b->path.cmd); if(p1b->endfile.buf) cntr_add_bytes(get_cntr(cconfs), strtoull(p1b->endfile.buf, NULL, 10)); sbuf_free_content(cb); return 1; }
static struct slist *build_manifest_phase1(const char *path, enum protocol protocol, int entries) { struct sbuf *sb; struct slist *slist=NULL; struct manio *manio=NULL; slist=build_slist_phase1(NULL /*prefix*/, protocol, entries); fail_unless((manio=manio_open_phase1(path, "wb", protocol))!=NULL); for(sb=slist->head; sb; sb=sb->next) { fail_unless(!manio_write_sbuf(manio, sb)); if(protocol==PROTO_2) hack_protocol2_attr(&sb->attr); } fail_unless(!send_msg_fzp(manio->fzp, CMD_GEN, "phase1end", strlen("phase1end"))); fail_unless(!manio_close(&manio)); return slist; }
void build_manifest_phase1_from_slist(const char *path, struct slist *slist, enum protocol protocol) { struct sbuf *sb; struct manio *manio=NULL; struct iobuf datapth; struct iobuf endfile; iobuf_init(&datapth); iobuf_init(&endfile); for(sb=slist->head; sb; sb=sb->next) set_sbuf(slist, sb, 0 /* with_data_files */); fail_unless((manio=manio_open_phase1(path, "wb", protocol))!=NULL); for(sb=slist->head; sb; sb=sb->next) { // Might be given an slist that has datapth or endfile set, // which should not go into a phase1 scan. Deal with it. if(sb->protocol1 && sb->protocol1->datapth.buf) iobuf_move(&datapth, &sb->protocol1->datapth); if(sb->endfile.buf) iobuf_move(&endfile, &sb->endfile); fail_unless(!manio_write_sbuf(manio, sb)); if(datapth.buf) iobuf_move(&sb->protocol1->datapth, &datapth); if(endfile.buf) iobuf_move(&sb->endfile, &endfile); } fail_unless(!send_msg_fzp(manio->fzp, CMD_GEN, "phase1end", strlen("phase1end"))); fail_unless(!manio_close(&manio)); }
static int sbuf_needs_data(struct sbuf *sb, struct asfd *asfd, struct asfd *chfd, struct manios *manios, struct slist *slist, int end_flags) { int ret=-1; struct blk *blk; static struct iobuf wbuf; struct blist *blist=slist->blist; if(!(sb->flags & SBUF_HEADER_WRITTEN_TO_MANIFEST)) { if(manio_write_sbuf(manios->changed, sb)) goto end; sb->flags |= SBUF_HEADER_WRITTEN_TO_MANIFEST; } while((blk=sb->protocol2->bstart) && blk->got==BLK_GOT && (blk->next || end_flags&END_BACKUP)) { if(blk->got_save_path && !blk_is_zero_length(blk)) { if(breaking && breakcount--==0) { breakpoint(breaking, __func__); goto end; } if(manio_write_sig_and_path(manios->changed, blk)) goto end; if(manios->changed->sig_count==0) { // Have finished a manifest file. Want to start // using it as a dedup candidate now. if(manio_component_to_chfd(chfd, manios->changed->offset->ppath)) goto end; if(!blk->requested) { // Also let the client know, so that it // can free memory if there was a long // consecutive number of unrequested // blocks. get_wbuf_from_index(&wbuf, blk->index); if(asfd->write(asfd, &wbuf)) goto end; } } } if(blk==sb->protocol2->bend) { blist_adjust_head(blist, sb); if(write_endfile(sb, manios)) return -1; slist_advance(slist); return 1; } if(sb->protocol2->bsighead==sb->protocol2->bstart) sb->protocol2->bsighead=blk->next; sb->protocol2->bstart=blk->next; if(blk==blist->blk_from_champ_chooser) blist->blk_from_champ_chooser=blk->next; } if(!blk && sb && !sb->protocol2->bend && (end_flags&END_BACKUP)) { // Write endfile for the very last file. if(write_endfile(sb, manios)) return -1; } ret=0; end: blist_adjust_head(blist, sb); return ret; }
// Return -1 on error, 0 on OK, 1 for srcmanio finished. int manio_copy_entry(struct sbuf *csb, struct sbuf *sb, struct blk **blk, struct manio *srcmanio, struct manio *dstmanio) { static int ars; static char *copy=NULL; // Use the most recent stat for the new manifest. if(dstmanio) { if(manio_write_sbuf(dstmanio, sb)) goto error; if(dstmanio->protocol==PROTO_1) { sbuf_free_content(csb); return 0; } } if(!(copy=strdup_w(csb->path.buf, __func__))) goto error; while(1) { if((ars=manio_read_with_blk(srcmanio, csb, *blk, NULL))<0) goto error; else if(ars>0) { // Finished. sbuf_free_content(csb); blk_free(blk); free_w(©); return 1; } // Got something. if(strcmp(csb->path.buf, copy)) { // Found the next entry. free_w(©); return 0; } if(dstmanio) { if(!dstmanio->fzp && manio_open_next_fpath(dstmanio)) return -1; if(csb->endfile.buf) { if(iobuf_send_msg_fzp(&csb->endfile, dstmanio->fzp)) goto error; } else { // Should have the next signature. // Write it to the destination manifest. if(manio_write_sig_and_path(dstmanio, *blk)) goto error; } } } error: free_w(©); return -1; }
static int write_to_changed_file(struct asfd *asfd, struct asfd *chfd, struct manio *chmanio, struct slist *slist, struct blist *blist, struct dpth *dpth, int backup_end, struct conf *conf) { struct sbuf *sb; static struct iobuf *wbuf=NULL; if(!slist) return 0; if(!wbuf && !(wbuf=iobuf_alloc())) return -1; while((sb=slist->head)) { if(sb->flags & SBUF_NEED_DATA) { int hack=0; // Need data... struct blk *blk; if(!(sb->flags & SBUF_HEADER_WRITTEN_TO_MANIFEST)) { if(manio_write_sbuf(chmanio, sb)) return -1; sb->flags |= SBUF_HEADER_WRITTEN_TO_MANIFEST; } while((blk=sb->burp2->bstart) && blk->got==BLK_GOT && (blk->next || backup_end)) { if(*(blk->save_path)) { if(manio_write_sig_and_path(chmanio, blk)) return -1; if(chmanio->sig_count==0) { // Have finished a manifest // file. Want to start using // it as a dedup candidate // now. iobuf_from_str(wbuf, CMD_MANIFEST, chmanio->fpath); printf("send manifest path\n"); if(chfd->write(chfd, wbuf)) return -1; if(!blk->requested) { // Also let the client know, // so that it can free memory // if there was a long // consecutive number of // unrequested blocks. get_wbuf_from_wrap_up(wbuf, blk->index); if(asfd->write(asfd, wbuf)) return -1; } } } /* else { // This gets hit if there is a zero // length file. printf("!!!!!!!!!!!!! no data; %s\n", sb->path); exit(1); } */ if(blk==sb->burp2->bend) { slist->head=sb->next; if(!(blist->head=sb->burp2->bstart)) blist->tail=NULL; sanity_before_sbuf_free(slist, sb); sbuf_free(&sb); hack=1; break; } if(sb->burp2->bsighead==sb->burp2->bstart) sb->burp2->bsighead=blk->next; sb->burp2->bstart=blk->next; if(blk==blist->blk_from_champ_chooser) blist->blk_from_champ_chooser=blk->next; //printf("freeing blk %d\n", blk->index); blk_free(&blk); } if(hack) continue; if(!(blist->head=sb->burp2->bstart)) blist->tail=NULL; break; } else { // No change, can go straight in. if(manio_write_sbuf(chmanio, sb)) return -1; // Move along. slist->head=sb->next; sanity_before_sbuf_free(slist, sb); sbuf_free(&sb); } } return 0; }
int backup_phase1_server_all(struct async *as, struct sdirs *sdirs, struct conf **confs) { int ret=-1; struct sbuf *sb=NULL; char *phase1tmp=NULL; struct asfd *asfd=as->asfd; struct manio *manio=NULL; enum protocol protocol=get_protocol(confs); struct cntr *cntr=get_cntr(confs); logp("Begin phase1 (file system scan)\n"); if(!(phase1tmp=get_tmp_filename(sdirs->phase1data)) || !(manio=manio_open_phase1(phase1tmp, comp_level(get_int(confs[OPT_COMPRESSION])), protocol)) || !(sb=sbuf_alloc(protocol))) goto error; while(1) { sbuf_free_content(sb); switch(sbuf_fill_from_net(sb, asfd, NULL, NULL, cntr)) { case 0: break; case 1: // Last thing the client sends is // 'backupphase2', and it wants an 'ok' reply. if(asfd->write_str(asfd, CMD_GEN, "ok") || send_msg_fzp(manio->fzp, CMD_GEN, "phase1end", strlen("phase1end"))) goto error; goto end; case -1: default: goto error; } if(write_status(CNTR_STATUS_SCANNING, sb->path.buf, cntr) || manio_write_sbuf(manio, sb)) goto error; cntr_add_phase1(cntr, sb->path.cmd, 0); if(sbuf_is_filedata(sb)) { cntr_add_val(cntr, CMD_BYTES_ESTIMATED, (uint64_t)sb->statp.st_size, 0); } } end: if(manio_close(&manio)) { logp("error closing %s in backup_phase1_server\n", phase1tmp); goto error; } if(check_quota(as, cntr, get_uint64_t(confs[OPT_HARD_QUOTA]), get_uint64_t(confs[OPT_SOFT_QUOTA]))) goto error; // Possible rename race condition is of no consequence here, because // the working directory will always get deleted if phase1 is not // complete. if(do_rename(phase1tmp, sdirs->phase1data)) goto error; //cntr_print(p1cntr, cntr, ACTION_BACKUP); logp("End phase1 (file system scan)\n"); ret=0; error: free_w(&phase1tmp); manio_close(&manio); sbuf_free(&sb); return ret; }
static int sbuf_needs_data(struct sbuf *sb, struct asfd *asfd, struct asfd *chfd, struct manios *manios, struct slist *slist, int end_flags) { int ret=-1; struct blk *blk; static struct iobuf wbuf; struct blist *blist=slist->blist; static int unrequested=0; if(!(sb->flags & SBUF_HEADER_WRITTEN_TO_MANIFEST)) { if(manio_write_sbuf(manios->changed, sb)) goto end; sb->flags |= SBUF_HEADER_WRITTEN_TO_MANIFEST; } while((blk=sb->protocol2->bstart) && blk->got==BLK_GOT && (blk->next || end_flags&END_BACKUP)) { if(blk->got_save_path && !blk_is_zero_length(blk)) { if(breaking && breakcount--==0) { breakpoint(breaking, __func__); goto end; } if(manio_write_sig_and_path(manios->changed, blk)) goto end; if(manios->changed->sig_count==0) { // Have finished a manifest file. Want to start // using it as a dedup candidate now. if(manio_component_to_chfd(chfd, manios->changed->offset->ppath)) goto end; // The champ chooser has the candidate. Now, // empty our local hash table. hash_delete_all(); // Add the most recent block, so identical // adjacent blocks are deduplicated well. if(hash_load_blk(blk)) goto end; } } if(!blk->requested) unrequested++; if(unrequested>BLKS_MAX_UNREQUESTED) { unrequested=0; // Let the client know that it can free memory if there // was a long consecutive number of unrequested blocks. get_wbuf_from_index(&wbuf, blk->index); if(asfd->write(asfd, &wbuf)) goto end; } if(blk==sb->protocol2->bend) { blist_adjust_head(blist, sb); if(write_endfile(sb, manios)) return -1; slist_advance(slist); return 1; } if(sb->protocol2->bsighead==sb->protocol2->bstart) sb->protocol2->bsighead=blk->next; sb->protocol2->bstart=blk->next; if(blk==blist->blk_from_champ_chooser) blist->blk_from_champ_chooser=blk->next; } if(!blk && sb && !sb->protocol2->bend && (end_flags&END_BACKUP)) { // Write endfile for the very last file. if(write_endfile(sb, manios)) return -1; } ret=0; end: blist_adjust_head(blist, sb); return ret; }