// Return -1 on error, 0 on OK, 1 for srcmanio finished.
int manio_copy_entry(struct asfd *asfd, struct sbuf **csb, struct sbuf *sb,
struct blk **blk, struct manio *srcmanio,
struct manio *dstmanio, struct conf *conf)
{
static int ars;
static char *copy=NULL;
// Use the most recent stat for the new manifest.
if(dstmanio && manio_write_sbuf(dstmanio, sb)) goto error;
if(!(copy=strdup((*csb)->path.buf)))
{
log_out_of_memory(__func__);
goto error;
}
while(1)
{
if((ars=manio_sbuf_fill(srcmanio, asfd, *csb,
*blk, NULL, conf))<0) goto error;
else if(ars>0)
{
// Finished.
sbuf_free(*csb); *csb=NULL;
blk_free(*blk); *blk=NULL;
free(copy);
return 1;
}
// Got something.
if(strcmp((*csb)->path.buf, copy))
{
// Found the next entry.
free(copy);
return 0;
}
// Should have the next signature.
// Write it to the destination manifest.
if(dstmanio && manio_write_sig_and_path(dstmanio, *blk))
goto error;
}
error:
if(copy) free(copy);
return -1;
}
static int maybe_add_from_scan(struct asfd *asfd,
struct manio *p1manio, struct manio *cmanio,
struct manio *unmanio, struct slist *slist, struct conf *conf)
{
int ret=-1;
static int ars;
static int ec=0;
struct sbuf *snew=NULL;
while(1)
{
if(manio_closed(p1manio)) return 0;
// Limit the amount loaded into memory at any one time.
if(slist && slist->head)
{
if(slist->head->burp2->index
- slist->tail->burp2->index>4096)
return 0;
}
if(!(snew=sbuf_alloc(conf))) goto end;
if((ars=manio_sbuf_fill(p1manio,
asfd, snew, NULL, NULL, conf))<0) goto end;
else if(ars>0) return 0; // Finished.
if(!(ec=entry_changed(asfd, snew, cmanio, unmanio, conf)))
{
// No change, no need to add to slist.
continue;
}
else if(ec<0) goto end; // Error.
if(data_needed(snew)) snew->flags|=SBUF_NEED_DATA;
slist_add_sbuf(slist, snew);
}
return 0;
end:
sbuf_free(&snew);
return ret;
}
static int do_browse_manifest(struct asfd *srfd,
struct manio *manio, struct sbuf *sb, const char *browse)
{
int ret=-1;
int ars=0;
//char ls[1024]="";
//struct cntr cntr;
size_t blen=0;
char *last_bd_match=NULL;
if(browse) blen=strlen(browse);
while(1)
{
int r;
sbuf_free_content(sb);
if((ars=manio_sbuf_fill(manio, NULL, sb, NULL, NULL, NULL)))
{
if(ars<0) goto end;
// ars==1 means it ended ok.
break;
}
if(sb->path.cmd!=CMD_DIRECTORY
&& sb->path.cmd!=CMD_FILE
&& sb->path.cmd!=CMD_ENC_FILE
&& sb->path.cmd!=CMD_EFS_FILE
&& sb->path.cmd!=CMD_SPECIAL
&& !cmd_is_link(sb->path.cmd))
continue;
if((r=check_browsedir(browse, sb, blen, &last_bd_match))<0)
goto end;
if(!r) continue;
if(json_from_statp(sb->path.buf, &sb->statp)) goto end;
}
ret=0;
end:
free_w(&last_bd_match);
return ret;
}
// This is basically backup_phase3_server() from protocol1. It used to merge the
// unchanged and changed data into a single file. Now it splits the manifests
// into several files.
int backup_phase3_server_protocol2(struct sdirs *sdirs, struct conf **confs)
{
int ret=1;
int pcmp=0;
char *hooksdir=NULL;
char *dindexdir=NULL;
char *manifesttmp=NULL;
struct sbuf *usb=NULL;
struct sbuf *csb=NULL;
struct blk *blk=NULL;
int finished_ch=0;
int finished_un=0;
struct manio *newmanio=NULL;
struct manio *chmanio=NULL;
struct manio *unmanio=NULL;
uint64_t fcount=0;
logp("Start phase3\n");
if(!(manifesttmp=get_tmp_filename(sdirs->rmanifest))
|| !(newmanio=manio_alloc())
|| !(chmanio=manio_alloc())
|| !(unmanio=manio_alloc())
|| !(hooksdir=prepend_s(manifesttmp, "hooks"))
|| !(dindexdir=prepend_s(manifesttmp, "dindex"))
|| manio_init_write(newmanio, manifesttmp)
|| manio_init_write_hooks(newmanio,
get_string(confs[OPT_DIRECTORY]), hooksdir, sdirs->rmanifest)
|| manio_init_write_dindex(newmanio, dindexdir)
|| manio_init_read(chmanio, sdirs->changed)
|| manio_init_read(unmanio, sdirs->unchanged)
|| !(usb=sbuf_alloc(confs))
|| !(csb=sbuf_alloc(confs)))
goto end;
while(!finished_ch || !finished_un)
{
if(!blk && !(blk=blk_alloc())) goto end;
if(!finished_un
&& usb
&& !usb->path.buf)
{
switch(manio_sbuf_fill(unmanio, NULL /* no async */,
usb, NULL, NULL, confs))
{
case -1: goto end;
case 1: finished_un++;
}
}
if(!finished_ch
&& csb
&& !csb->path.buf)
{
switch(manio_sbuf_fill(chmanio, NULL /* no async */,
csb, NULL, NULL, confs))
{
case -1: goto end;
case 1: finished_ch++;
}
}
if((usb && usb->path.buf) && (!csb || !csb->path.buf))
{
switch(manio_copy_entry(NULL /* no async */,
&usb, usb,
&blk, unmanio, newmanio, confs))
{
case -1: goto end;
case 1: finished_un++;
}
}
else if((!usb || !usb->path.buf) && (csb && csb->path.buf))
{
switch(manio_copy_entry(NULL /* no async */,
&csb, csb, &blk, chmanio, newmanio, confs))
{
case -1: goto end;
case 1: finished_ch++;
}
}
else if((!usb || !usb->path.buf) && (!csb || !(csb->path.buf)))
{
continue;
}
else if(!(pcmp=sbuf_pathcmp(usb, csb)))
{
// They were the same - write one.
switch(manio_copy_entry(NULL /* no async */,
&csb, csb, &blk, chmanio, newmanio, confs))
{
case -1: goto end;
case 1: finished_ch++;
}
}
else if(pcmp<0)
{
switch(manio_copy_entry(NULL /* no async */,
&usb, usb, &blk, unmanio, newmanio, confs))
{
case -1: goto end;
case 1: finished_un++;
}
}
else
{
switch(manio_copy_entry(NULL /* no async */,
&csb, csb, &blk, chmanio, newmanio, confs))
{
case -1: goto end;
case 1: finished_ch++;
}
}
}
fcount=newmanio->fcount;
// Flush to disk and set up for reading.
if(manio_free(&newmanio)
|| !(newmanio=manio_alloc())
|| manio_init_read(newmanio, sdirs->rmanifest))
goto end;
// Rename race condition should be of no consequence here, as the
// manifest should just get recreated automatically.
if(do_rename(manifesttmp, sdirs->rmanifest))
goto end;
else
{
recursive_delete(sdirs->changed, NULL, 1);
recursive_delete(sdirs->unchanged, NULL, 1);
}
if(sparse_generation(newmanio, fcount, sdirs, confs))
goto end;
ret=0;
logp("End phase3\n");
end:
manio_free(&newmanio);
manio_free(&chmanio);
manio_free(&unmanio);
sbuf_free(&csb);
sbuf_free(&usb);
blk_free(&blk);
free_w(&hooksdir);
free_w(&dindexdir);
free_w(&manifesttmp);
return ret;
}
// Return -1 for error, 0 for entry not changed, 1 for entry changed (or new).
static int entry_changed(struct asfd *asfd, struct sbuf *sb,
struct manio *cmanio, struct manio *unmanio, struct conf *conf)
{
static int finished=0;
static struct sbuf *csb=NULL;
static struct blk *blk=NULL;
if(finished) return 1;
if(!csb && !(csb=sbuf_alloc(conf))) return -1;
if(csb->path.buf)
{
// Already have an entry.
}
else
{
// Need to read another.
if(!blk && !(blk=blk_alloc())) return -1;
switch(manio_sbuf_fill(cmanio, asfd, csb, blk, NULL, conf))
{
case 1: // Reached the end.
sbuf_free(&csb);
blk_free(&blk);
finished=1;
return 1;
case -1: return -1;
}
if(!csb->path.buf)
{
logp("Should have a path at this point, but do not, in %s\n", __func__);
return -1;
}
// Got an entry.
}
while(1)
{
switch(sbuf_pathcmp(csb, sb))
{
case 0: return found_in_current_manifest(asfd, csb, sb,
cmanio, unmanio, &blk, conf);
case 1: return 1;
case -1:
// Behind - need to read more data from the old
// manifest.
switch(manio_sbuf_fill(cmanio, asfd,
csb, blk, NULL, conf))
{
case -1: return -1;
case 1:
{
// Reached the end.
sbuf_free(&csb);
blk_free(&blk);
return 1;
}
}
// Got something, go back around the loop.
}
}
return 0;
}
static int list_manifest(struct asfd *asfd,
const char *fullpath, regex_t *regex,
const char *browsedir, struct conf *conf)
{
int ars=0;
int ret=0;
struct sbuf *sb=NULL;
struct manio *manio=NULL;
char *manifest_dir=NULL;
char *last_bd_match=NULL;
size_t bdlen=0;
if(!(manifest_dir=prepend_s(fullpath,
conf->protocol==PROTO_BURP1?"manifest.gz":"manifest"))
|| !(manio=manio_alloc())
|| manio_init_read(manio, manifest_dir)
|| !(sb=sbuf_alloc(conf)))
{
log_and_send_oom(asfd, __func__);
goto error;
}
manio_set_protocol(manio, conf->protocol);
if(browsedir) bdlen=strlen(browsedir);
while(1)
{
int show=0;
if((ars=manio_sbuf_fill(manio, asfd, sb, NULL, NULL, conf))<0)
goto error;
else if(ars>0)
goto end; // Finished OK.
if(write_status(STATUS_LISTING, sb->path.buf, conf))
goto error;
if(browsedir)
{
int r;
if((r=check_browsedir(browsedir,
&sb->path.buf, bdlen, &last_bd_match))<0)
goto error;
if(!r) continue;
show++;
}
else
{
if(check_regex(regex, sb->path.buf))
show++;
}
if(show)
{
if(write_wrapper(asfd, &sb->attr)
|| write_wrapper(asfd, &sb->path))
goto error;
if(sbuf_is_link(sb)
&& write_wrapper(asfd, &sb->link))
goto error;
}
sbuf_free_content(sb);
}
error:
ret=-1;
end:
sbuf_free(&sb);
free_w(&manifest_dir);
manio_free(&manio);
free_w(&last_bd_match);
return ret;
}
/* This function reads the manifest to determine whether it may be more
efficient to just copy the data files across and unpack them on the other
side. If it thinks it is, it will then do it.
Return -1 on error, 1 if it copied the data across, 0 if it did not. */
static int maybe_copy_data_files_across(struct asfd *asfd,
const char *manifest,
const char *datadir, int srestore, regex_t *regex, struct conf *conf,
struct slist *slist,
enum action act, enum cstat_status status)
{
int ars;
int ret=-1;
struct sbuf *sb=NULL;
struct blk *blk=NULL;
struct manio *manio=NULL;
uint64_t blkcount=0;
uint64_t datcount=0;
struct hash_weak *tmpw;
struct hash_weak *hash_weak;
uint64_t estimate_blks;
uint64_t estimate_dats;
uint64_t estimate_one_dat;
int need_data=0;
int last_ent_was_dir=0;
char sig[128]="";
// If the client has no restore_spool directory, we have to fall back
// to the stream style restore.
if(!conf->restore_spool) return 0;
if(!(manio=manio_alloc())
|| manio_init_read(manio, manifest)
|| !(sb=sbuf_alloc(conf))
|| !(blk=blk_alloc()))
goto end;
while(1)
{
if((ars=manio_sbuf_fill(manio, asfd, sb, blk, NULL, conf))<0)
{
logp("Error from manio_sbuf_fill() in %s\n",
__func__);
goto end; // Error;
}
else if(ars>0)
break; // Finished OK.
if(!blk->got_save_path)
{
sbuf_free_content(sb);
continue;
}
if((!srestore || check_srestore(conf, sb->path.buf))
&& check_regex(regex, sb->path.buf))
{
blkcount++;
if(!hash_weak_find((uint64_t)blk->savepath))
{
if(!hash_weak_add((uint64_t)blk->savepath))
goto end;
datcount++;
}
}
sbuf_free_content(sb);
}
estimate_blks=blkcount*RABIN_AVG;
estimate_one_dat=DATA_FILE_SIG_MAX*RABIN_AVG;
estimate_dats=datcount*estimate_one_dat;
printf("%lu blocks = %lu bytes in stream approx\n",
blkcount, estimate_blks);
printf("%lu data files = %lu bytes approx\n",
datcount, estimate_dats);
if(estimate_blks < estimate_one_dat)
{
printf("Stream is less than the size of a data file.\n");
printf("Use restore stream\n");
return 0;
}
else if(estimate_dats >= 90*(estimate_blks/100))
{
printf("Stream is more than 90%% size of data files.\n");
printf("Use restore stream\n");
return 0;
}
else
{
printf("Data files are less than 90%% size of stream.\n");
printf("Use data files\n");
}
printf("Client is using restore_spool: %s\n", conf->restore_spool);
if(asfd->write_str(asfd, CMD_GEN, "restore_spool")
|| asfd->read_expect(asfd, CMD_GEN, "restore_spool_ok"))
goto end;
// Send each of the data files that we found to the client.
HASH_ITER(hh, hash_table, hash_weak, tmpw)
{
char msg[32];
char path[32];
char *fdatpath=NULL;
snprintf(path, sizeof(path), "%014lX", hash_weak->weak);
path[4]='/';
path[9]='/';
snprintf(msg, sizeof(msg), "dat=%s", path);
printf("got: %s\n", msg);
if(asfd->write_str(asfd, CMD_GEN, msg)) goto end;
if(!(fdatpath=prepend_s(datadir, path)))
goto end;
if(send_a_file(asfd, fdatpath, conf))
{
free(fdatpath);
goto end;
}
free(fdatpath);
}
/* This function reads the manifest to determine whether it may be more
efficient to just copy the data files across and unpack them on the other
side. If it thinks it is, it will then do it.
Return -1 on error, 1 if it copied the data across, 0 if it did not. */
int maybe_restore_spool(struct asfd *asfd, const char *manifest,
struct sdirs *sdirs, struct bu *bu, int srestore, regex_t *regex,
struct conf **confs, struct slist *slist,
enum action act, enum cntr_status cntr_status)
{
int ars;
int ret=-1;
struct sbuf *sb=NULL;
struct blk *blk=NULL;
struct manio *manio=NULL;
uint64_t blkcount=0;
uint64_t datcount=0;
struct hash_weak *tmpw;
struct hash_weak *hash_weak;
uint64_t estimate_blks;
uint64_t estimate_dats;
uint64_t estimate_one_dat;
struct sbuf *need_data=NULL;
int last_ent_was_dir=0;
char sig[128]="";
const char *restore_spool=get_string(confs[OPT_RESTORE_SPOOL]);
// If the client has no restore_spool directory, we have to fall back
// to the stream style restore.
if(!restore_spool) return 0;
if(!(manio=manio_alloc())
|| manio_init_read(manio, manifest)
|| !(need_data=sbuf_alloc(confs))
|| !(sb=sbuf_alloc(confs))
|| !(blk=blk_alloc()))
goto end;
while(1)
{
if((ars=manio_sbuf_fill(manio, asfd, sb, blk, NULL, confs))<0)
{
logp("Error from manio_sbuf_fill() in %s\n",
__func__);
goto end; // Error;
}
else if(ars>0)
break; // Finished OK.
if(!blk->got_save_path)
{
sbuf_free_content(sb);
continue;
}
if(want_to_restore(srestore, sb, regex, confs))
{
blkcount++;
if(!hash_weak_find((uint64_t)blk->savepath))
{
if(!hash_weak_add((uint64_t)blk->savepath))
goto end;
datcount++;
}
}
sbuf_free_content(sb);
}
// FIX THIS: should read rabin_avg of a struct rconf.
estimate_blks=blkcount*RABIN_AVG;
estimate_one_dat=DATA_FILE_SIG_MAX*RABIN_AVG;
estimate_dats=datcount*estimate_one_dat;
printf("%"PRIu64 " blocks = %"PRIu64 " bytes in stream approx\n",
blkcount, estimate_blks);
printf("%"PRIu64 " data files = %"PRIu64 " bytes approx\n",
datcount, estimate_dats);
if(estimate_blks < estimate_one_dat)
{
printf("Stream is less than the size of a data file.\n");
printf("Use restore stream\n");
return 0;
}
else if(estimate_dats >= 90*(estimate_blks/100))
{
printf("Stream is more than 90%% size of data files.\n");
printf("Use restore stream\n");
return 0;
}
else
{
printf("Data files are less than 90%% size of stream.\n");
printf("Use data files\n");
}
printf("Client is using restore_spool: %s\n", restore_spool);
if(asfd->write_str(asfd, CMD_GEN, "restore_spool")
|| asfd->read_expect(asfd, CMD_GEN, "restore_spool_ok"))
goto end;
// Send each of the data files that we found to the client.
HASH_ITER(hh, hash_table, hash_weak, tmpw)
{
char msg[32];
char path[32];
char *fdatpath=NULL;
snprintf(path, sizeof(path), "%014"PRIX64, hash_weak->weak);
path[4]='/';
path[9]='/';
snprintf(msg, sizeof(msg), "dat=%s", path);
printf("got: %s\n", msg);
if(asfd->write_str(asfd, CMD_GEN, msg)) goto end;
if(!(fdatpath=prepend_s(sdirs->data, path)))
goto end;
if(send_a_file(asfd, fdatpath, confs))
{
free(fdatpath);
goto end;
}
free(fdatpath);
}