int deduplicate(struct asfd *asfd, struct conf *conf)
{
struct blk *blk;
struct incoming *in=asfd->in;
struct candidate *champ;
struct candidate *champ_last=NULL;
int count=0;
int blk_count=0;
if(!in) return 0;
incoming_found_reset(in);
count=0;
while((champ=candidates_choose_champ(in, champ_last)))
{
// printf("Got champ: %s %d\n", champ->path, *(champ->score));
if(hash_load(champ->path, conf)) return -1;
if(++count==CHAMPS_MAX) break;
champ_last=champ;
}
blk_count=0;
for(blk=asfd->blist->blk_to_dedup; blk; blk=blk->next)
{
//printf("try: %lu\n", blk->index);
blk_count++;
if(blk_is_zero_length(blk))
{
//printf("got: %s %s\n", blk->weak, blk->strong);
blk->got=BLK_GOT;
in->got++;
continue;
}
// If already got, this function will set blk->save_path
// to be the location of the already got block.
if(already_got_block(asfd, blk)) return -1;
//printf("after agb: %lu %d\n", blk->index, blk->got);
}
logp("%s: %04d/%04d - %04d/%04d\n",
asfd->desc, count, candidates_len, in->got, blk_count);
//cntr_add_same_val(conf->cntr, CMD_DATA, in->got);
// Start the incoming array again.
in->size=0;
// Destroy the deduplication hash table.
hash_delete_all();
asfd->blist->blk_to_dedup=NULL;
return 0;
}
static int sbuf_needs_data(struct sbuf *sb, struct asfd *asfd,
struct asfd *chfd, struct manios *manios,
struct slist *slist, int end_flags)
{
int ret=-1;
struct blk *blk;
static struct iobuf wbuf;
struct blist *blist=slist->blist;
if(!(sb->flags & SBUF_HEADER_WRITTEN_TO_MANIFEST))
{
if(manio_write_sbuf(manios->changed, sb)) goto end;
sb->flags |= SBUF_HEADER_WRITTEN_TO_MANIFEST;
}
while((blk=sb->protocol2->bstart)
&& blk->got==BLK_GOT
&& (blk->next || end_flags&END_BACKUP))
{
if(blk->got_save_path
&& !blk_is_zero_length(blk))
{
if(breaking && breakcount--==0)
{
breakpoint(breaking, __func__);
goto end;
}
if(manio_write_sig_and_path(manios->changed, blk))
goto end;
if(manios->changed->sig_count==0)
{
// Have finished a manifest file. Want to start
// using it as a dedup candidate now.
if(manio_component_to_chfd(chfd,
manios->changed->offset->ppath))
goto end;
if(!blk->requested)
{
// Also let the client know, so that it
// can free memory if there was a long
// consecutive number of unrequested
// blocks.
get_wbuf_from_index(&wbuf, blk->index);
if(asfd->write(asfd, &wbuf)) goto end;
}
}
}
if(blk==sb->protocol2->bend)
{
blist_adjust_head(blist, sb);
if(write_endfile(sb, manios)) return -1;
slist_advance(slist);
return 1;
}
if(sb->protocol2->bsighead==sb->protocol2->bstart)
sb->protocol2->bsighead=blk->next;
sb->protocol2->bstart=blk->next;
if(blk==blist->blk_from_champ_chooser)
blist->blk_from_champ_chooser=blk->next;
}
if(!blk && sb && !sb->protocol2->bend && (end_flags&END_BACKUP))
{
// Write endfile for the very last file.
if(write_endfile(sb, manios)) return -1;
}
ret=0;
end:
blist_adjust_head(blist, sb);
return ret;
}
static int sbuf_needs_data(struct sbuf *sb, struct asfd *asfd,
struct asfd *chfd, struct manios *manios,
struct slist *slist, int end_flags)
{
int ret=-1;
struct blk *blk;
static struct iobuf wbuf;
struct blist *blist=slist->blist;
static int unrequested=0;
if(!(sb->flags & SBUF_HEADER_WRITTEN_TO_MANIFEST))
{
if(manio_write_sbuf(manios->changed, sb)) goto end;
sb->flags |= SBUF_HEADER_WRITTEN_TO_MANIFEST;
}
while((blk=sb->protocol2->bstart)
&& blk->got==BLK_GOT
&& (blk->next || end_flags&END_BACKUP))
{
if(blk->got_save_path
&& !blk_is_zero_length(blk))
{
if(breaking && breakcount--==0)
{
breakpoint(breaking, __func__);
goto end;
}
if(manio_write_sig_and_path(manios->changed, blk))
goto end;
if(manios->changed->sig_count==0)
{
// Have finished a manifest file. Want to start
// using it as a dedup candidate now.
if(manio_component_to_chfd(chfd,
manios->changed->offset->ppath))
goto end;
// The champ chooser has the candidate. Now,
// empty our local hash table.
hash_delete_all();
// Add the most recent block, so identical
// adjacent blocks are deduplicated well.
if(hash_load_blk(blk))
goto end;
}
}
if(!blk->requested)
unrequested++;
if(unrequested>BLKS_MAX_UNREQUESTED)
{
unrequested=0;
// Let the client know that it can free memory if there
// was a long consecutive number of unrequested blocks.
get_wbuf_from_index(&wbuf, blk->index);
if(asfd->write(asfd, &wbuf))
goto end;
}
if(blk==sb->protocol2->bend)
{
blist_adjust_head(blist, sb);
if(write_endfile(sb, manios)) return -1;
slist_advance(slist);
return 1;
}
if(sb->protocol2->bsighead==sb->protocol2->bstart)
sb->protocol2->bsighead=blk->next;
sb->protocol2->bstart=blk->next;
if(blk==blist->blk_from_champ_chooser)
blist->blk_from_champ_chooser=blk->next;
}
if(!blk && sb && !sb->protocol2->bend && (end_flags&END_BACKUP))
{
// Write endfile for the very last file.
if(write_endfile(sb, manios)) return -1;
}
ret=0;
end:
blist_adjust_head(blist, sb);
return ret;
}
