static struct slist *do_build_manifest(struct manio *manio,
enum protocol protocol, int entries, int with_data_files)
{
struct sbuf *sb;
struct slist *slist=NULL;
slist=build_slist(protocol, entries, with_data_files);
for(sb=slist->head; sb; sb=sb->next)
{
fail_unless(!manio_write_sbuf(manio, sb));
if(protocol==PROTO_2)
{
struct blk *blk=NULL;
for(blk=sb->protocol2->bstart;
blk && blk!=sb->protocol2->bend; blk=blk->next)
{
fail_unless(!manio_write_sig_and_path(manio,
blk));
}
if(sbuf_is_filedata(sb) || sbuf_is_vssdata(sb))
{
struct iobuf endfile;
iobuf_from_str(&endfile,
CMD_END_FILE, (char *)"0:0");
fail_unless(!iobuf_send_msg_fzp(&endfile,
manio->fzp));
}
hack_protocol2_attr(&sb->attr);
}
}
return slist;
}
static int write_to_changed_file(struct asfd *asfd,
struct asfd *chfd, struct manios *manios,
struct slist *slist, int end_flags)
{
struct sbuf *sb;
if(!slist) return 0;
while((sb=slist->head))
{
if(sb->flags & SBUF_NEED_DATA)
{
switch(sbuf_needs_data(sb, asfd, chfd, manios, slist,
end_flags))
{
case 0: return 0;
case 1: continue;
default: return -1;
}
}
else
{
// No change, can go straight in.
if(manio_write_sbuf(manios->changed, sb)) return -1;
if(write_endfile(sb, manios)) return -1;
// Move along.
slist_advance(slist);
}
}
return 0;
}
static int changed_non_file(struct sbuf *p1b,
struct manio *ucmanio, enum cmd cmd, struct conf **cconfs)
{
// As new_non_file.
if(manio_write_sbuf(ucmanio, p1b))
return -1;
cntr_add_changed(get_cntr(cconfs), cmd);
sbuf_free_content(p1b);
return 0;
}
static int new_non_file(struct sbuf *p1b,
struct manio *ucmanio, struct conf **cconfs)
{
// Is something that does not need more data backed up.
// Like a directory or a link or something like that.
// Goes into the unchanged file, so that it does not end up out of
// order with normal files, which has to wait around for their data
// to turn up.
if(manio_write_sbuf(ucmanio, p1b))
return -1;
cntr_add(get_cntr(cconfs), p1b->path.cmd, 0);
sbuf_free_content(p1b);
return 0;
}
void build_manifest_phase2_from_slist(const char *path,
struct slist *slist, enum protocol protocol, int short_write)
{
struct sbuf *sb;
struct manio *manio=NULL;
for(sb=slist->head; sb; sb=sb->next)
set_sbuf(slist, sb, 0 /* with_data_files */);
fail_unless((manio=manio_open_phase2(path, "wb", protocol))!=NULL);
for(sb=slist->head; sb; sb=sb->next)
{
fail_unless(!manio_write_sbuf(manio, sb));
if(protocol==PROTO_2)
{
struct blk *blk=NULL;
for(blk=sb->protocol2->bstart;
blk && blk!=sb->protocol2->bend; blk=blk->next)
{
fail_unless(!manio_write_sig_and_path(manio,
blk));
}
if(sbuf_is_filedata(sb) || sbuf_is_vssdata(sb))
{
struct iobuf endfile;
iobuf_from_str(&endfile,
CMD_END_FILE, (char *)"0:0");
fail_unless(!iobuf_send_msg_fzp(&endfile,
manio->fzp));
}
hack_protocol2_attr(&sb->attr);
}
}
if(short_write)
{
man_off_t *pos;
fail_unless((pos=manio_tell(manio))!=NULL);
if(pos->offset>=short_write) pos->offset-=short_write;
fail_unless(!manio_close_and_truncate(&manio,
pos, 0 /* compression */));
man_off_t_free(&pos);
}
fail_unless(!manio_close(&manio));
}
// Return -1 on error, 0 on OK, 1 for srcmanio finished.
int manio_copy_entry(struct asfd *asfd, struct sbuf **csb, struct sbuf *sb,
struct blk **blk, struct manio *srcmanio,
struct manio *dstmanio, struct conf *conf)
{
static int ars;
static char *copy=NULL;
// Use the most recent stat for the new manifest.
if(dstmanio && manio_write_sbuf(dstmanio, sb)) goto error;
if(!(copy=strdup((*csb)->path.buf)))
{
log_out_of_memory(__func__);
goto error;
}
while(1)
{
if((ars=manio_sbuf_fill(srcmanio, asfd, *csb,
*blk, NULL, conf))<0) goto error;
else if(ars>0)
{
// Finished.
sbuf_free(*csb); *csb=NULL;
blk_free(*blk); *blk=NULL;
free(copy);
return 1;
}
// Got something.
if(strcmp((*csb)->path.buf, copy))
{
// Found the next entry.
free(copy);
return 0;
}
// Should have the next signature.
// Write it to the destination manifest.
if(dstmanio && manio_write_sig_and_path(dstmanio, *blk))
goto error;
}
error:
if(copy) free(copy);
return -1;
}
static int deal_with_receive_end_file(struct asfd *asfd, struct sdirs *sdirs,
struct sbuf *rb, struct manio *chmanio, struct conf **cconfs,
char **last_requested)
{
int ret=-1;
static char *cp=NULL;
static struct iobuf *rbuf;
struct cntr *cntr=get_cntr(cconfs);
rbuf=asfd->rbuf;
// Finished the file.
// Write it to the phase2 file, and free the buffers.
if(fzp_close(&(rb->protocol1->fzp)))
{
logp("error closing delta for %s in receive\n", rb->path.buf);
goto end;
}
iobuf_move(&rb->endfile, rbuf);
if(rb->flags & SBUF_RECV_DELTA && finish_delta(sdirs, rb))
goto end;
if(manio_write_sbuf(chmanio, rb))
goto end;
if(rb->flags & SBUF_RECV_DELTA)
cntr_add_changed(cntr, rb->path.cmd);
else
cntr_add(cntr, rb->path.cmd, 0);
if(*last_requested && !strcmp(rb->path.buf, *last_requested))
free_w(last_requested);
cp=strchr(rb->endfile.buf, ':');
if(rb->endfile.buf)
cntr_add_bytes(cntr, strtoull(rb->endfile.buf, NULL, 10));
if(cp)
{
// checksum stuff goes here
}
ret=0;
end:
sbuf_free_content(rb);
return ret;
}
static int process_unchanged_file(struct sbuf *p1b, struct sbuf *cb,
struct manio *ucmanio, struct conf **cconfs)
{
// Need to re-encode the p1b attribs to include compression and
// other bits and pieces that are recorded on cb.
iobuf_move(&p1b->protocol1->datapth, &cb->protocol1->datapth);
iobuf_move(&p1b->endfile, &cb->endfile);
p1b->compression=cb->compression;
p1b->winattr=cb->winattr;
if(attribs_encode(p1b))
return -1;
if(manio_write_sbuf(ucmanio, p1b))
return -1;
cntr_add_same(get_cntr(cconfs), p1b->path.cmd);
if(p1b->endfile.buf) cntr_add_bytes(get_cntr(cconfs),
strtoull(p1b->endfile.buf, NULL, 10));
sbuf_free_content(cb);
return 1;
}
static struct slist *build_manifest_phase1(const char *path,
enum protocol protocol, int entries)
{
struct sbuf *sb;
struct slist *slist=NULL;
struct manio *manio=NULL;
slist=build_slist_phase1(NULL /*prefix*/, protocol, entries);
fail_unless((manio=manio_open_phase1(path, "wb", protocol))!=NULL);
for(sb=slist->head; sb; sb=sb->next)
{
fail_unless(!manio_write_sbuf(manio, sb));
if(protocol==PROTO_2) hack_protocol2_attr(&sb->attr);
}
fail_unless(!send_msg_fzp(manio->fzp,
CMD_GEN, "phase1end", strlen("phase1end")));
fail_unless(!manio_close(&manio));
return slist;
}
void build_manifest_phase1_from_slist(const char *path,
struct slist *slist, enum protocol protocol)
{
struct sbuf *sb;
struct manio *manio=NULL;
struct iobuf datapth;
struct iobuf endfile;
iobuf_init(&datapth);
iobuf_init(&endfile);
for(sb=slist->head; sb; sb=sb->next)
set_sbuf(slist, sb, 0 /* with_data_files */);
fail_unless((manio=manio_open_phase1(path, "wb", protocol))!=NULL);
for(sb=slist->head; sb; sb=sb->next)
{
// Might be given an slist that has datapth or endfile set,
// which should not go into a phase1 scan. Deal with it.
if(sb->protocol1
&& sb->protocol1->datapth.buf)
iobuf_move(&datapth, &sb->protocol1->datapth);
if(sb->endfile.buf)
iobuf_move(&endfile, &sb->endfile);
fail_unless(!manio_write_sbuf(manio, sb));
if(datapth.buf)
iobuf_move(&sb->protocol1->datapth, &datapth);
if(endfile.buf)
iobuf_move(&sb->endfile, &endfile);
}
fail_unless(!send_msg_fzp(manio->fzp,
CMD_GEN, "phase1end", strlen("phase1end")));
fail_unless(!manio_close(&manio));
}
static int sbuf_needs_data(struct sbuf *sb, struct asfd *asfd,
struct asfd *chfd, struct manios *manios,
struct slist *slist, int end_flags)
{
int ret=-1;
struct blk *blk;
static struct iobuf wbuf;
struct blist *blist=slist->blist;
if(!(sb->flags & SBUF_HEADER_WRITTEN_TO_MANIFEST))
{
if(manio_write_sbuf(manios->changed, sb)) goto end;
sb->flags |= SBUF_HEADER_WRITTEN_TO_MANIFEST;
}
while((blk=sb->protocol2->bstart)
&& blk->got==BLK_GOT
&& (blk->next || end_flags&END_BACKUP))
{
if(blk->got_save_path
&& !blk_is_zero_length(blk))
{
if(breaking && breakcount--==0)
{
breakpoint(breaking, __func__);
goto end;
}
if(manio_write_sig_and_path(manios->changed, blk))
goto end;
if(manios->changed->sig_count==0)
{
// Have finished a manifest file. Want to start
// using it as a dedup candidate now.
if(manio_component_to_chfd(chfd,
manios->changed->offset->ppath))
goto end;
if(!blk->requested)
{
// Also let the client know, so that it
// can free memory if there was a long
// consecutive number of unrequested
// blocks.
get_wbuf_from_index(&wbuf, blk->index);
if(asfd->write(asfd, &wbuf)) goto end;
}
}
}
if(blk==sb->protocol2->bend)
{
blist_adjust_head(blist, sb);
if(write_endfile(sb, manios)) return -1;
slist_advance(slist);
return 1;
}
if(sb->protocol2->bsighead==sb->protocol2->bstart)
sb->protocol2->bsighead=blk->next;
sb->protocol2->bstart=blk->next;
if(blk==blist->blk_from_champ_chooser)
blist->blk_from_champ_chooser=blk->next;
}
if(!blk && sb && !sb->protocol2->bend && (end_flags&END_BACKUP))
{
// Write endfile for the very last file.
if(write_endfile(sb, manios)) return -1;
}
ret=0;
end:
blist_adjust_head(blist, sb);
return ret;
}
// Return -1 on error, 0 on OK, 1 for srcmanio finished.
int manio_copy_entry(struct sbuf *csb, struct sbuf *sb,
struct blk **blk, struct manio *srcmanio,
struct manio *dstmanio)
{
static int ars;
static char *copy=NULL;
// Use the most recent stat for the new manifest.
if(dstmanio)
{
if(manio_write_sbuf(dstmanio, sb)) goto error;
if(dstmanio->protocol==PROTO_1)
{
sbuf_free_content(csb);
return 0;
}
}
if(!(copy=strdup_w(csb->path.buf, __func__)))
goto error;
while(1)
{
if((ars=manio_read_with_blk(srcmanio, csb,
*blk, NULL))<0) goto error;
else if(ars>0)
{
// Finished.
sbuf_free_content(csb);
blk_free(blk);
free_w(©);
return 1;
}
// Got something.
if(strcmp(csb->path.buf, copy))
{
// Found the next entry.
free_w(©);
return 0;
}
if(dstmanio)
{
if(!dstmanio->fzp
&& manio_open_next_fpath(dstmanio)) return -1;
if(csb->endfile.buf)
{
if(iobuf_send_msg_fzp(&csb->endfile,
dstmanio->fzp)) goto error;
}
else
{
// Should have the next signature.
// Write it to the destination manifest.
if(manio_write_sig_and_path(dstmanio, *blk))
goto error;
}
}
}
error:
free_w(©);
return -1;
}
static int write_to_changed_file(struct asfd *asfd,
struct asfd *chfd, struct manio *chmanio,
struct slist *slist, struct blist *blist,
struct dpth *dpth, int backup_end, struct conf *conf)
{
struct sbuf *sb;
static struct iobuf *wbuf=NULL;
if(!slist) return 0;
if(!wbuf && !(wbuf=iobuf_alloc())) return -1;
while((sb=slist->head))
{
if(sb->flags & SBUF_NEED_DATA)
{
int hack=0;
// Need data...
struct blk *blk;
if(!(sb->flags & SBUF_HEADER_WRITTEN_TO_MANIFEST))
{
if(manio_write_sbuf(chmanio, sb)) return -1;
sb->flags |= SBUF_HEADER_WRITTEN_TO_MANIFEST;
}
while((blk=sb->burp2->bstart)
&& blk->got==BLK_GOT
&& (blk->next || backup_end))
{
if(*(blk->save_path))
{
if(manio_write_sig_and_path(chmanio,
blk)) return -1;
if(chmanio->sig_count==0)
{
// Have finished a manifest
// file. Want to start using
// it as a dedup candidate
// now.
iobuf_from_str(wbuf,
CMD_MANIFEST,
chmanio->fpath);
printf("send manifest path\n");
if(chfd->write(chfd, wbuf))
return -1;
if(!blk->requested)
{
// Also let the client know,
// so that it can free memory
// if there was a long
// consecutive number of
// unrequested blocks.
get_wbuf_from_wrap_up(wbuf,
blk->index);
if(asfd->write(asfd, wbuf))
return -1;
}
}
}
/*
else
{
// This gets hit if there is a zero
// length file.
printf("!!!!!!!!!!!!! no data; %s\n",
sb->path);
exit(1);
}
*/
if(blk==sb->burp2->bend)
{
slist->head=sb->next;
if(!(blist->head=sb->burp2->bstart))
blist->tail=NULL;
sanity_before_sbuf_free(slist, sb);
sbuf_free(&sb);
hack=1;
break;
}
if(sb->burp2->bsighead==sb->burp2->bstart)
sb->burp2->bsighead=blk->next;
sb->burp2->bstart=blk->next;
if(blk==blist->blk_from_champ_chooser)
blist->blk_from_champ_chooser=blk->next;
//printf("freeing blk %d\n", blk->index);
blk_free(&blk);
}
if(hack) continue;
if(!(blist->head=sb->burp2->bstart))
blist->tail=NULL;
break;
}
else
{
// No change, can go straight in.
if(manio_write_sbuf(chmanio, sb)) return -1;
// Move along.
slist->head=sb->next;
sanity_before_sbuf_free(slist, sb);
sbuf_free(&sb);
}
}
return 0;
}
int backup_phase1_server_all(struct async *as,
struct sdirs *sdirs, struct conf **confs)
{
int ret=-1;
struct sbuf *sb=NULL;
char *phase1tmp=NULL;
struct asfd *asfd=as->asfd;
struct manio *manio=NULL;
enum protocol protocol=get_protocol(confs);
struct cntr *cntr=get_cntr(confs);
logp("Begin phase1 (file system scan)\n");
if(!(phase1tmp=get_tmp_filename(sdirs->phase1data))
|| !(manio=manio_open_phase1(phase1tmp,
comp_level(get_int(confs[OPT_COMPRESSION])), protocol))
|| !(sb=sbuf_alloc(protocol)))
goto error;
while(1)
{
sbuf_free_content(sb);
switch(sbuf_fill_from_net(sb, asfd, NULL, NULL, cntr))
{
case 0: break;
case 1: // Last thing the client sends is
// 'backupphase2', and it wants an 'ok' reply.
if(asfd->write_str(asfd, CMD_GEN, "ok")
|| send_msg_fzp(manio->fzp, CMD_GEN,
"phase1end", strlen("phase1end")))
goto error;
goto end;
case -1:
default: goto error;
}
if(write_status(CNTR_STATUS_SCANNING, sb->path.buf, cntr)
|| manio_write_sbuf(manio, sb))
goto error;
cntr_add_phase1(cntr, sb->path.cmd, 0);
if(sbuf_is_filedata(sb))
{
cntr_add_val(cntr, CMD_BYTES_ESTIMATED,
(uint64_t)sb->statp.st_size, 0);
}
}
end:
if(manio_close(&manio))
{
logp("error closing %s in backup_phase1_server\n", phase1tmp);
goto error;
}
if(check_quota(as, cntr,
get_uint64_t(confs[OPT_HARD_QUOTA]),
get_uint64_t(confs[OPT_SOFT_QUOTA])))
goto error;
// Possible rename race condition is of no consequence here, because
// the working directory will always get deleted if phase1 is not
// complete.
if(do_rename(phase1tmp, sdirs->phase1data))
goto error;
//cntr_print(p1cntr, cntr, ACTION_BACKUP);
logp("End phase1 (file system scan)\n");
ret=0;
error:
free_w(&phase1tmp);
manio_close(&manio);
sbuf_free(&sb);
return ret;
}
static int sbuf_needs_data(struct sbuf *sb, struct asfd *asfd,
struct asfd *chfd, struct manios *manios,
struct slist *slist, int end_flags)
{
int ret=-1;
struct blk *blk;
static struct iobuf wbuf;
struct blist *blist=slist->blist;
static int unrequested=0;
if(!(sb->flags & SBUF_HEADER_WRITTEN_TO_MANIFEST))
{
if(manio_write_sbuf(manios->changed, sb)) goto end;
sb->flags |= SBUF_HEADER_WRITTEN_TO_MANIFEST;
}
while((blk=sb->protocol2->bstart)
&& blk->got==BLK_GOT
&& (blk->next || end_flags&END_BACKUP))
{
if(blk->got_save_path
&& !blk_is_zero_length(blk))
{
if(breaking && breakcount--==0)
{
breakpoint(breaking, __func__);
goto end;
}
if(manio_write_sig_and_path(manios->changed, blk))
goto end;
if(manios->changed->sig_count==0)
{
// Have finished a manifest file. Want to start
// using it as a dedup candidate now.
if(manio_component_to_chfd(chfd,
manios->changed->offset->ppath))
goto end;
// The champ chooser has the candidate. Now,
// empty our local hash table.
hash_delete_all();
// Add the most recent block, so identical
// adjacent blocks are deduplicated well.
if(hash_load_blk(blk))
goto end;
}
}
if(!blk->requested)
unrequested++;
if(unrequested>BLKS_MAX_UNREQUESTED)
{
unrequested=0;
// Let the client know that it can free memory if there
// was a long consecutive number of unrequested blocks.
get_wbuf_from_index(&wbuf, blk->index);
if(asfd->write(asfd, &wbuf))
goto end;
}
if(blk==sb->protocol2->bend)
{
blist_adjust_head(blist, sb);
if(write_endfile(sb, manios)) return -1;
slist_advance(slist);
return 1;
}
if(sb->protocol2->bsighead==sb->protocol2->bstart)
sb->protocol2->bsighead=blk->next;
sb->protocol2->bstart=blk->next;
if(blk==blist->blk_from_champ_chooser)
blist->blk_from_champ_chooser=blk->next;
}
if(!blk && sb && !sb->protocol2->bend && (end_flags&END_BACKUP))
{
// Write endfile for the very last file.
if(write_endfile(sb, manios)) return -1;
}
ret=0;
end:
blist_adjust_head(blist, sb);
return ret;
}
