static struct slist *do_build_manifest(struct manio *manio, enum protocol protocol, int entries, int with_data_files) { struct sbuf *sb; struct slist *slist=NULL; slist=build_slist(protocol, entries, with_data_files); for(sb=slist->head; sb; sb=sb->next) { fail_unless(!manio_write_sbuf(manio, sb)); if(protocol==PROTO_2) { struct blk *blk=NULL; for(blk=sb->protocol2->bstart; blk && blk!=sb->protocol2->bend; blk=blk->next) { fail_unless(!manio_write_sig_and_path(manio, blk)); } if(sbuf_is_filedata(sb) || sbuf_is_vssdata(sb)) { struct iobuf endfile; iobuf_from_str(&endfile, CMD_END_FILE, (char *)"0:0"); fail_unless(!iobuf_send_msg_fzp(&endfile, manio->fzp)); } hack_protocol2_attr(&sb->attr); } } return slist; }
void build_manifest_phase2_from_slist(const char *path, struct slist *slist, enum protocol protocol, int short_write) { struct sbuf *sb; struct manio *manio=NULL; for(sb=slist->head; sb; sb=sb->next) set_sbuf(slist, sb, 0 /* with_data_files */); fail_unless((manio=manio_open_phase2(path, "wb", protocol))!=NULL); for(sb=slist->head; sb; sb=sb->next) { fail_unless(!manio_write_sbuf(manio, sb)); if(protocol==PROTO_2) { struct blk *blk=NULL; for(blk=sb->protocol2->bstart; blk && blk!=sb->protocol2->bend; blk=blk->next) { fail_unless(!manio_write_sig_and_path(manio, blk)); } if(sbuf_is_filedata(sb) || sbuf_is_vssdata(sb)) { struct iobuf endfile; iobuf_from_str(&endfile, CMD_END_FILE, (char *)"0:0"); fail_unless(!iobuf_send_msg_fzp(&endfile, manio->fzp)); } hack_protocol2_attr(&sb->attr); } } if(short_write) { man_off_t *pos; fail_unless((pos=manio_tell(manio))!=NULL); if(pos->offset>=short_write) pos->offset-=short_write; fail_unless(!manio_close_and_truncate(&manio, pos, 0 /* compression */)); man_off_t_free(&pos); } fail_unless(!manio_close(&manio)); }
// Return -1 on error, 0 on OK, 1 for srcmanio finished. int manio_copy_entry(struct asfd *asfd, struct sbuf **csb, struct sbuf *sb, struct blk **blk, struct manio *srcmanio, struct manio *dstmanio, struct conf *conf) { static int ars; static char *copy=NULL; // Use the most recent stat for the new manifest. if(dstmanio && manio_write_sbuf(dstmanio, sb)) goto error; if(!(copy=strdup((*csb)->path.buf))) { log_out_of_memory(__func__); goto error; } while(1) { if((ars=manio_sbuf_fill(srcmanio, asfd, *csb, *blk, NULL, conf))<0) goto error; else if(ars>0) { // Finished. sbuf_free(*csb); *csb=NULL; blk_free(*blk); *blk=NULL; free(copy); return 1; } // Got something. if(strcmp((*csb)->path.buf, copy)) { // Found the next entry. free(copy); return 0; } // Should have the next signature. // Write it to the destination manifest. if(dstmanio && manio_write_sig_and_path(dstmanio, *blk)) goto error; } error: if(copy) free(copy); return -1; }
static int sbuf_needs_data(struct sbuf *sb, struct asfd *asfd, struct asfd *chfd, struct manios *manios, struct slist *slist, int end_flags) { int ret=-1; struct blk *blk; static struct iobuf wbuf; struct blist *blist=slist->blist; if(!(sb->flags & SBUF_HEADER_WRITTEN_TO_MANIFEST)) { if(manio_write_sbuf(manios->changed, sb)) goto end; sb->flags |= SBUF_HEADER_WRITTEN_TO_MANIFEST; } while((blk=sb->protocol2->bstart) && blk->got==BLK_GOT && (blk->next || end_flags&END_BACKUP)) { if(blk->got_save_path && !blk_is_zero_length(blk)) { if(breaking && breakcount--==0) { breakpoint(breaking, __func__); goto end; } if(manio_write_sig_and_path(manios->changed, blk)) goto end; if(manios->changed->sig_count==0) { // Have finished a manifest file. Want to start // using it as a dedup candidate now. if(manio_component_to_chfd(chfd, manios->changed->offset->ppath)) goto end; if(!blk->requested) { // Also let the client know, so that it // can free memory if there was a long // consecutive number of unrequested // blocks. get_wbuf_from_index(&wbuf, blk->index); if(asfd->write(asfd, &wbuf)) goto end; } } } if(blk==sb->protocol2->bend) { blist_adjust_head(blist, sb); if(write_endfile(sb, manios)) return -1; slist_advance(slist); return 1; } if(sb->protocol2->bsighead==sb->protocol2->bstart) sb->protocol2->bsighead=blk->next; sb->protocol2->bstart=blk->next; if(blk==blist->blk_from_champ_chooser) blist->blk_from_champ_chooser=blk->next; } if(!blk && sb && !sb->protocol2->bend && (end_flags&END_BACKUP)) { // Write endfile for the very last file. if(write_endfile(sb, manios)) return -1; } ret=0; end: blist_adjust_head(blist, sb); return ret; }
// Return -1 on error, 0 on OK, 1 for srcmanio finished. int manio_copy_entry(struct sbuf *csb, struct sbuf *sb, struct blk **blk, struct manio *srcmanio, struct manio *dstmanio) { static int ars; static char *copy=NULL; // Use the most recent stat for the new manifest. if(dstmanio) { if(manio_write_sbuf(dstmanio, sb)) goto error; if(dstmanio->protocol==PROTO_1) { sbuf_free_content(csb); return 0; } } if(!(copy=strdup_w(csb->path.buf, __func__))) goto error; while(1) { if((ars=manio_read_with_blk(srcmanio, csb, *blk, NULL))<0) goto error; else if(ars>0) { // Finished. sbuf_free_content(csb); blk_free(blk); free_w(©); return 1; } // Got something. if(strcmp(csb->path.buf, copy)) { // Found the next entry. free_w(©); return 0; } if(dstmanio) { if(!dstmanio->fzp && manio_open_next_fpath(dstmanio)) return -1; if(csb->endfile.buf) { if(iobuf_send_msg_fzp(&csb->endfile, dstmanio->fzp)) goto error; } else { // Should have the next signature. // Write it to the destination manifest. if(manio_write_sig_and_path(dstmanio, *blk)) goto error; } } } error: free_w(©); return -1; }
static int write_to_changed_file(struct asfd *asfd, struct asfd *chfd, struct manio *chmanio, struct slist *slist, struct blist *blist, struct dpth *dpth, int backup_end, struct conf *conf) { struct sbuf *sb; static struct iobuf *wbuf=NULL; if(!slist) return 0; if(!wbuf && !(wbuf=iobuf_alloc())) return -1; while((sb=slist->head)) { if(sb->flags & SBUF_NEED_DATA) { int hack=0; // Need data... struct blk *blk; if(!(sb->flags & SBUF_HEADER_WRITTEN_TO_MANIFEST)) { if(manio_write_sbuf(chmanio, sb)) return -1; sb->flags |= SBUF_HEADER_WRITTEN_TO_MANIFEST; } while((blk=sb->burp2->bstart) && blk->got==BLK_GOT && (blk->next || backup_end)) { if(*(blk->save_path)) { if(manio_write_sig_and_path(chmanio, blk)) return -1; if(chmanio->sig_count==0) { // Have finished a manifest // file. Want to start using // it as a dedup candidate // now. iobuf_from_str(wbuf, CMD_MANIFEST, chmanio->fpath); printf("send manifest path\n"); if(chfd->write(chfd, wbuf)) return -1; if(!blk->requested) { // Also let the client know, // so that it can free memory // if there was a long // consecutive number of // unrequested blocks. get_wbuf_from_wrap_up(wbuf, blk->index); if(asfd->write(asfd, wbuf)) return -1; } } } /* else { // This gets hit if there is a zero // length file. printf("!!!!!!!!!!!!! no data; %s\n", sb->path); exit(1); } */ if(blk==sb->burp2->bend) { slist->head=sb->next; if(!(blist->head=sb->burp2->bstart)) blist->tail=NULL; sanity_before_sbuf_free(slist, sb); sbuf_free(&sb); hack=1; break; } if(sb->burp2->bsighead==sb->burp2->bstart) sb->burp2->bsighead=blk->next; sb->burp2->bstart=blk->next; if(blk==blist->blk_from_champ_chooser) blist->blk_from_champ_chooser=blk->next; //printf("freeing blk %d\n", blk->index); blk_free(&blk); } if(hack) continue; if(!(blist->head=sb->burp2->bstart)) blist->tail=NULL; break; } else { // No change, can go straight in. if(manio_write_sbuf(chmanio, sb)) return -1; // Move along. slist->head=sb->next; sanity_before_sbuf_free(slist, sb); sbuf_free(&sb); } } return 0; }
static int sbuf_needs_data(struct sbuf *sb, struct asfd *asfd, struct asfd *chfd, struct manios *manios, struct slist *slist, int end_flags) { int ret=-1; struct blk *blk; static struct iobuf wbuf; struct blist *blist=slist->blist; static int unrequested=0; if(!(sb->flags & SBUF_HEADER_WRITTEN_TO_MANIFEST)) { if(manio_write_sbuf(manios->changed, sb)) goto end; sb->flags |= SBUF_HEADER_WRITTEN_TO_MANIFEST; } while((blk=sb->protocol2->bstart) && blk->got==BLK_GOT && (blk->next || end_flags&END_BACKUP)) { if(blk->got_save_path && !blk_is_zero_length(blk)) { if(breaking && breakcount--==0) { breakpoint(breaking, __func__); goto end; } if(manio_write_sig_and_path(manios->changed, blk)) goto end; if(manios->changed->sig_count==0) { // Have finished a manifest file. Want to start // using it as a dedup candidate now. if(manio_component_to_chfd(chfd, manios->changed->offset->ppath)) goto end; // The champ chooser has the candidate. Now, // empty our local hash table. hash_delete_all(); // Add the most recent block, so identical // adjacent blocks are deduplicated well. if(hash_load_blk(blk)) goto end; } } if(!blk->requested) unrequested++; if(unrequested>BLKS_MAX_UNREQUESTED) { unrequested=0; // Let the client know that it can free memory if there // was a long consecutive number of unrequested blocks. get_wbuf_from_index(&wbuf, blk->index); if(asfd->write(asfd, &wbuf)) goto end; } if(blk==sb->protocol2->bend) { blist_adjust_head(blist, sb); if(write_endfile(sb, manios)) return -1; slist_advance(slist); return 1; } if(sb->protocol2->bsighead==sb->protocol2->bstart) sb->protocol2->bsighead=blk->next; sb->protocol2->bstart=blk->next; if(blk==blist->blk_from_champ_chooser) blist->blk_from_champ_chooser=blk->next; } if(!blk && sb && !sb->protocol2->bend && (end_flags&END_BACKUP)) { // Write endfile for the very last file. if(write_endfile(sb, manios)) return -1; } ret=0; end: blist_adjust_head(blist, sb); return ret; }