void
ct_extract(struct ct_global_state *state, struct ct_op *op)
{
struct ct_extract_args *cea = op->op_args;
const char *ctfile = cea->cea_local_ctfile;
char **filelist = cea->cea_filelist;
int match_mode = cea->cea_matchmode;
struct ct_extract_priv *ex_priv = op->op_priv;
int ret;
struct ct_trans *trans;
char shat[SHA_DIGEST_STRING_LENGTH];
/* if we were woken up due to fatal, just clean up local state */
if (state->ct_dying != 0)
goto dying;
CNDBG(CT_LOG_TRANS, "entry");
switch (ct_get_file_state(state)) {
case CT_S_STARTING:
if (ex_priv == NULL) {
ex_priv = e_calloc(1, sizeof(*ex_priv));
TAILQ_INIT(&ex_priv->extract_head);
if ((ret = ct_match_compile(&ex_priv->inc_match,
match_mode, filelist)) != 0) {
ct_fatal(state,
"failed to compile include pattern", ret);
goto dying;
}
if (cea->cea_excllist != NULL &&
(ret = ct_match_compile(&ex_priv->ex_match,
match_mode, cea->cea_excllist)) != 0) {
ct_fatal(state,
"failed to compile exclude pattern", ret);
goto dying;
}
op->op_priv = ex_priv;
RB_INIT(&ex_priv->pending_tree);
}
if ((ret = ct_file_extract_init(&state->extract_state,
cea->cea_tdir, cea->cea_attr, cea->cea_follow_symlinks,
ex_priv->allfiles, cea->cea_log_state,
cea->cea_log_chown_failed)) != 0) {
ct_fatal(state, "Can not initialize extract state",
ret);
goto dying;
}
if (ct_extract_calculate_total(state, cea, ex_priv->inc_match,
ex_priv->ex_match) != 0) {
CWARNX("failed to calculate stats");
goto dying;
}
if ((ret = ct_extract_setup(&ex_priv->extract_head,
&ex_priv->xdr_ctx, ctfile, cea->cea_ctfile_basedir,
&ex_priv->allfiles)) != 0) {
ct_fatal(state, "can't setup extract queue", ret);
goto dying;
}
state->ct_print_ctfile_info(state->ct_print_state,
ex_priv->xdr_ctx.xs_filename, &ex_priv->xdr_ctx.xs_gh);
/* XXX we should handle this better */
if (state->ct_max_block_size <
ex_priv->xdr_ctx.xs_gh.cmg_chunk_size)
CABORTX("block size negotiated with server %d is "
"smaller than file max block size %d",
state->ct_max_block_size,
ex_priv->xdr_ctx.xs_gh.cmg_chunk_size);
/* create rb tree head, prepare to start inserting */
if (ex_priv->allfiles) {
ex_priv->fillrb = 1;
}
break;
case CT_S_FINISHED:
return;
default:
break;
}
ct_set_file_state(state, CT_S_RUNNING);
while (1) {
trans = ct_trans_alloc(state);
if (trans == NULL) {
/* system busy, return */
CNDBG(CT_LOG_TRANS, "ran out of transactions, waiting");
ct_set_file_state(state, CT_S_WAITING_TRANS);
return;
}
trans->tr_statemachine = ct_state_extract;
switch ((ret = ctfile_parse(&ex_priv->xdr_ctx))) {
case XS_RET_FILE:
if (ex_priv->fillrb == 0 &&
ex_priv->xdr_ctx.xs_hdr.cmh_nr_shas == -1) {
if (ex_priv->allfiles == 0)
CINFO("file %s has negative shas "
"and backup is not allfiles",
ex_priv->xdr_ctx.xs_hdr.cmh_filename);
ex_priv->doextract = 0;
goto skip; /* skip ze file for now */
}
trans = ct_trans_realloc_local(state, trans);
trans->tr_fl_node = ex_priv->fl_ex_node =
ct_alloc_fnode();
ct_populate_fnode(state->extract_state,
&ex_priv->xdr_ctx, trans->tr_fl_node,
&trans->tr_state, ex_priv->allfiles,
cea->cea_strip_slash);
if (trans->tr_state == TR_S_EX_SPECIAL) {
trans->tr_complete =
ct_extract_complete_special;
} else {
trans->tr_complete =
ct_extract_complete_file_start;
}
trans->tr_cleanup = ct_extract_cleanup_fnode;
if (ex_priv->haverb) {
struct ct_pending_file *cpf;
if ((cpf = ct_extract_find_entry(
&ex_priv->pending_tree,
trans->tr_fl_node->fn_fullname)) != NULL) {
struct fnode *hardlink;
/* copy permissions over */
trans->tr_fl_node->fn_uid =
cpf->cpf_uid;
trans->tr_fl_node->fn_gid =
cpf->cpf_gid;
trans->tr_fl_node->fn_mode =
cpf->cpf_mode;
trans->tr_fl_node->fn_mtime =
cpf->cpf_mtime;
trans->tr_fl_node->fn_atime =
cpf->cpf_atime;
/* copy list of pending links over */
while ((hardlink =
TAILQ_FIRST(&cpf->cpf_links))) {
TAILQ_REMOVE(&cpf->cpf_links,
hardlink, fn_list);
TAILQ_INSERT_TAIL(
&trans->tr_fl_node->fn_hardlinks,
hardlink, fn_list);
}
ex_priv->doextract = 1;
ct_extract_free_entry(
&ex_priv->pending_tree, cpf);
} else {
ex_priv->doextract = 0;
}
} else {
ex_priv->doextract =
!ct_match(ex_priv->inc_match,
trans->tr_fl_node->fn_fullname);
if (ex_priv->doextract &&
ex_priv->ex_match != NULL &&
!ct_match(ex_priv->ex_match,
trans->tr_fl_node->fn_fullname)) {
ex_priv->doextract = 0;
}
}
if (ex_priv->doextract &&
trans->tr_fl_node->fn_hardlink) {
struct ct_pending_file *file;
if ((file = ct_extract_find_entry(
&ex_priv->pending_tree,
trans->tr_fl_node->fn_hlname)) != NULL) {
CNDBG(CT_LOG_FILE,
"adding pending link for %s to %s",
file->cpf_name,
trans->tr_fl_node->fn_fullname);
/* our reference to node passed */
ct_pending_file_add_link(file,
trans->tr_fl_node);
ex_priv->doextract = 0;
goto skip;
}
}
/*
* If we're on the first ctfile in an allfiles backup
* put the matches with -1 on the rb tree so we'll
* remember to extract it from older files.
*/
if (ex_priv->doextract == 1 && ex_priv->fillrb &&
ex_priv->xdr_ctx.xs_hdr.cmh_nr_shas == -1) {
ct_extract_insert_entry(&ex_priv->pending_tree,
trans->tr_fl_node);
ex_priv->doextract = 0;
/* XXX reconsider the freeing */
}
if (ex_priv->doextract == 0) {
ct_free_fnode(trans->tr_fl_node);
skip:
ex_priv->fl_ex_node = NULL;
ct_trans_free(state, trans);
continue;
}
CNDBG(CT_LOG_CTFILE,
"file %s numshas %" PRId64,
trans->tr_fl_node->fn_fullname,
ex_priv->xdr_ctx.xs_hdr.cmh_nr_shas);
/*
* special files we give our refcount up
* regular files we need a new one since we need to
* keep ours.
*/
if (trans->tr_state != TR_S_EX_SPECIAL) {
ct_ref_fnode(trans->tr_fl_node);
} else {
ex_priv->fl_ex_node = NULL;
}
ct_queue_first(state, trans);
break;
case XS_RET_SHA:
if (ex_priv->doextract == 0 ||
ex_priv->fl_ex_node->fn_skip_file != 0) {
if (ctfile_parse_seek(&ex_priv->xdr_ctx)) {
ct_fatal(state, "Can't seek past shas",
ex_priv->xdr_ctx.xs_errno);
goto dying;
}
ct_trans_free(state, trans);
continue;
}
/* use saved fnode */
trans->tr_fl_node = ex_priv->fl_ex_node;
if (memcmp(zerosha, ex_priv->xdr_ctx.xs_sha,
SHA_DIGEST_LENGTH) == 0) {
CWARNX("\"%s\" truncated during backup",
trans->tr_fl_node->fn_fullname);
if (ctfile_parse_seek(&ex_priv->xdr_ctx)) {
ct_fatal(state, "Can't seek past "
"truncation shas",
ex_priv->xdr_ctx.xs_errno);
goto dying;
}
ct_trans_free(state, trans);
continue;
}
if (ex_priv->xdr_ctx.xs_gh.cmg_flags & CT_MD_CRYPTO) {
/*
* yes csha and sha are reversed, we want
* to download csha, but putting it in sha
* simplifies the code
*/
bcopy(ex_priv->xdr_ctx.xs_sha, trans->tr_csha,
sizeof(trans->tr_csha));
bcopy(ex_priv->xdr_ctx.xs_csha, trans->tr_sha,
sizeof(trans->tr_sha));
bcopy(ex_priv->xdr_ctx.xs_iv, trans->tr_iv,
sizeof(trans->tr_iv));
} else {
bcopy(ex_priv->xdr_ctx.xs_sha, trans->tr_sha,
sizeof(trans->tr_sha));
}
if (clog_mask_is_set(CT_LOG_SHA)) {
ct_sha1_encode(trans->tr_sha, shat);
CNDBG(CT_LOG_SHA, "extracting sha %s", shat);
}
trans->tr_state = TR_S_EX_SHA;
trans->tr_complete = ct_extract_complete_file_read;
trans->tr_dataslot = 0;
ct_ref_fnode(trans->tr_fl_node);
trans->tr_cleanup = ct_extract_cleanup_fnode;
ct_queue_first(state, trans);
break;
case XS_RET_FILE_END:
trans = ct_trans_realloc_local(state, trans);
if (ex_priv->doextract == 0 ||
ex_priv->fl_ex_node->fn_skip_file != 0) {
/* release our reference done with file */
if (ex_priv->fl_ex_node) {
ct_free_fnode(ex_priv->fl_ex_node);
ex_priv->fl_ex_node = NULL;
}
ct_trans_free(state, trans);
continue;
}
/* use saved fnode from state */
trans->tr_fl_node = ex_priv->fl_ex_node;
bcopy(ex_priv->xdr_ctx.xs_trl.cmt_sha, trans->tr_sha,
sizeof(trans->tr_sha));
trans->tr_state = TR_S_EX_FILE_END;
trans->tr_complete = ct_extract_complete_file_end;
trans->tr_cleanup = ct_extract_cleanup_fnode;
trans->tr_fl_node->fn_size =
ex_priv->xdr_ctx.xs_trl.cmt_orig_size;
/*
* no reference here since we give our reference to the
* last transaction on that file. We are done with it.
*/
ex_priv->fl_ex_node = NULL;
ct_queue_first(state, trans);
break;
case XS_RET_EOF:
CNDBG(CT_LOG_CTFILE, "Hit end of ctfile");
ctfile_parse_close(&ex_priv->xdr_ctx);
/* if rb tree and rb is empty, goto end state */
if ((ex_priv->haverb || ex_priv->fillrb) &&
ct_extract_rb_empty(&ex_priv->pending_tree)) {
/*
* Cleanup extract queue, in case we had files
* left.
*/
ct_extract_cleanup_queue(
&ex_priv->extract_head);
goto we_re_done_here;
}
if (!TAILQ_EMPTY(&ex_priv->extract_head)) {
/*
* if allfiles and this was the first pass.
* free the current match lists
* switch to rb tree mode
*/
if (ex_priv->fillrb) {
ct_match_unwind(ex_priv->inc_match);
if (ex_priv->ex_match)
ct_match_unwind(
ex_priv->ex_match);
ex_priv->ex_match = NULL;
ex_priv->inc_match = NULL;
ex_priv->haverb = 1;
ex_priv->fillrb = 0;
}
ct_trans_free(state, trans);
/* reinits ex_priv->xdr_ctx */
if ((ret =
ct_extract_open_next(&ex_priv->extract_head,
&ex_priv->xdr_ctx)) != 0) {
ct_fatal(state,
"Can't open next ctfile", ret);
goto dying;
}
state->ct_print_ctfile_info(
state->ct_print_state,
ex_priv->xdr_ctx.xs_filename,
&ex_priv->xdr_ctx.xs_gh);
/* poke file into action */
ct_wakeup_file(state->event_state);
} else {
/*
* If rb tree and it is still has entries,
* bitch about it
*/
/* XXX print out missing files */
if ((ex_priv->haverb || ex_priv->fillrb) &&
ct_extract_rb_empty(
&ex_priv->pending_tree)) {
CWARNX("out of ctfiles but some "
"files are not found");
}
we_re_done_here:
if (ex_priv->inc_match)
ct_match_unwind(ex_priv->inc_match);
if (ex_priv->ex_match)
ct_match_unwind(
ex_priv->ex_match);
ct_extract_pending_cleanup(
&ex_priv->pending_tree);
e_free(&ex_priv);
op->op_priv = NULL;
trans->tr_state = TR_S_DONE;
trans->tr_complete = ct_extract_complete_done;
trans->tr_cleanup = ct_extract_cleanup_done;
/*
* Technically this should be a local
* transaction. However, since we are done
* it doesn't really matter either way.
*/
ct_queue_first(state, trans);
CNDBG(CT_LOG_TRANS, "extract finished");
ct_set_file_state(state, CT_S_FINISHED);
}
return;
break;
case XS_RET_FAIL:
ct_fatal(state, "Failed to parse ctfile",
ex_priv->xdr_ctx.xs_errno);
goto dying;
break;
}
}
return;
dying:
/* only if we hadn't sent the final transaction yet */
if (ex_priv != NULL) {
ct_extract_cleanup_queue(&ex_priv->extract_head);
if (ex_priv->inc_match)
ct_match_unwind(ex_priv->inc_match);
if (ex_priv->ex_match)
ct_match_unwind(ex_priv->ex_match);
if (!ct_extract_rb_empty(&ex_priv->pending_tree)) {
ct_extract_pending_cleanup(&ex_priv->pending_tree);
}
if (ex_priv->fl_ex_node != NULL) {
ct_free_fnode(ex_priv->fl_ex_node);
}
/* XXX what about ex_priv->xdr_ctx ? */
e_free(&ex_priv);
op->op_priv = NULL;
/* if ex_priv is gone then the trans will clean this up */
if (state->extract_state)
ct_file_extract_cleanup(state->extract_state);
}
return;
}
int
ct_list(const char *file, char **flist, char **excludelist, int match_mode,
const char *ctfile_basedir, int strip_slash, int verbose)
{
struct ct_extract_state *ces;
struct ctfile_parse_state xs_ctx;
struct fnode fnodestore;
uint64_t reduction;
struct fnode *fnode = &fnodestore;
struct ct_match *match, *ex_match = NULL;
char *ct_next_filename;
char *sign;
int state;
int doprint = 0;
int ret;
int s_errno = 0, ct_errno = 0;
char shat[SHA_DIGEST_STRING_LENGTH];
char cshat[SHA_DIGEST_STRING_LENGTH];
char iv[CT_IV_LEN*2+1];
if ((ret = ct_file_extract_init(&ces, NULL, 1, 1, 0, NULL, NULL)) != 0)
CFATALX("failed to initialise extract state: %s",
ct_strerror(ret));
if ((ret = ct_match_compile(&match, match_mode, flist)) != 0)
CFATALX("failed to compile match pattern: %s",
ct_strerror(ret));
if (excludelist != NULL && (ret = ct_match_compile(&ex_match,
match_mode, excludelist)) != 0)
CFATALX("failed to compile exclude pattern: %s",
ct_strerror(ret));
verbose++; /* by default print something. */
ct_next_filename = NULL;
next_file:
ret = ctfile_parse_init(&xs_ctx, file, ctfile_basedir);
if (ret)
CFATALX("failed to open %s: %s", file, ct_strerror(ret));
ct_print_ctfile_info(&verbose, file, &xs_ctx.xs_gh);
if (ct_next_filename)
e_free(&ct_next_filename);
if (xs_ctx.xs_gh.cmg_prevlvl_filename) {
CNDBG(CT_LOG_CTFILE, "previous backup file %s\n",
xs_ctx.xs_gh.cmg_prevlvl_filename);
ct_next_filename = e_strdup(xs_ctx.xs_gh.cmg_prevlvl_filename);
}
bzero(&fnodestore, sizeof(fnodestore));
do {
ret = ctfile_parse(&xs_ctx);
switch (ret) {
case XS_RET_FILE:
ct_populate_fnode(ces, &xs_ctx, fnode, &state,
xs_ctx.xs_gh.cmg_flags & CT_MD_MLB_ALLFILES,
strip_slash);
doprint = !ct_match(match, fnode->fn_fullname);
if (doprint && ex_match != NULL &&
!ct_match(ex_match, fnode->fn_fullname))
doprint = 0;
if (doprint) {
ct_pr_fmt_file(&verbose, fnode);
if (!C_ISREG(xs_ctx.xs_hdr.cmh_type) ||
verbose > 2)
printf("\n");
}
if (fnode->fn_hlname)
e_free(&fnode->fn_hlname);
if (fnode->fn_fullname)
e_free(&fnode->fn_fullname);
break;
case XS_RET_FILE_END:
sign = " ";
if (xs_ctx.xs_trl.cmt_comp_size == 0)
reduction = 100;
else {
uint64_t orig, comp;
orig = xs_ctx.xs_trl.cmt_orig_size;
comp = xs_ctx.xs_trl.cmt_comp_size;
if (comp <= orig) {
reduction = 100 * (orig - comp) / orig;
} else {
reduction = 100 * (comp - orig) / orig;
if (reduction != 0)
sign = "-";
}
}
if (doprint && verbose > 1)
printf(" sz: %" PRIu64 " shas: %" PRIu64
" reduction: %s%" PRIu64 "%%\n",
xs_ctx.xs_trl.cmt_orig_size,
xs_ctx.xs_hdr.cmh_nr_shas,
sign, reduction);
else if (doprint)
printf("\n");
break;
case XS_RET_SHA:
if (!(doprint && verbose > 2)) {
if (ctfile_parse_seek(&xs_ctx)) {
CFATALX("seek failed");
}
} else {
int i;
ct_sha1_encode(xs_ctx.xs_sha, shat);
switch (xs_ctx.xs_gh.cmg_flags & CT_MD_CRYPTO) {
case 0:
printf(" sha %s\n", shat);
break;
case CT_MD_CRYPTO:
ct_sha1_encode(xs_ctx.xs_csha, cshat);
for (i = 0; i < CT_IV_LEN; i++)
snprintf(&iv[i * 2], 3, "%02x",
xs_ctx.xs_iv[i]);
printf(" sha %s csha %s iv %s\n",
shat, cshat, iv);
}
}
break;
case XS_RET_EOF:
break;
case XS_RET_FAIL:
s_errno = errno;
ct_errno = xs_ctx.xs_errno;
;
}
} while (ret != XS_RET_EOF && ret != XS_RET_FAIL);
ctfile_parse_close(&xs_ctx);
if (ret != XS_RET_EOF) {
errno = s_errno;
CWARNX("corrupt ctfile: %s", ct_strerror(ct_errno));
} else {
if (ct_next_filename) {
file = ct_next_filename;
goto next_file;
}
}
ct_match_unwind(match);
ct_file_extract_cleanup(ces);
return (0);
}
/*
* So that we can provide correct statistics we have to go through all ctfiles
* being extracted and sum the sizes to be extracted. This is kinda expensive,
* but not really avoidable if we want to provide the statistics.
*
* Failure means we have called ct fatal.
*/
int
ct_extract_calculate_total(struct ct_global_state *state,
struct ct_extract_args *cea, struct ct_match *inc_match,
struct ct_match *ex_match)
{
struct ct_extract_head extract_head;
struct ctfile_parse_state xdr_ctx;
struct ct_match *rb_match = NULL;
struct fnode *fnode;
int allfiles;
int fillrb = 0, haverb = 0;
int doextract = 0;
int tr_state;
int ret;
int retval = 1;
TAILQ_INIT(&extract_head);
if ((ret = ct_extract_setup(&extract_head,
&xdr_ctx, cea->cea_local_ctfile, cea->cea_ctfile_basedir,
&allfiles)) != 0) {
ct_fatal(state, "can't setup extract queue", ret);
goto done;
}
if (allfiles) {
char *nothing = NULL;
if ((ret = ct_match_compile(&rb_match,
CT_MATCH_RB, ¬hing)) != 0) {
ct_fatal(state, "Couldn't create match tree",
ret);
goto done;
}
fillrb = 1;
}
while (1) {
switch ((ret = ctfile_parse(&xdr_ctx))) {
case XS_RET_FILE:
if (fillrb == 0 && xdr_ctx.xs_hdr.cmh_nr_shas == -1) {
continue;
}
fnode = ct_alloc_fnode();
/* XXX need the fnode for the correct paths */
ct_populate_fnode(state->extract_state,
&xdr_ctx, fnode, &tr_state, allfiles,
cea->cea_strip_slash);
/* we don't care about individual shas */
if (C_ISREG(fnode->fn_type)) {
ctfile_parse_seek(&xdr_ctx);
}
doextract = !ct_match(inc_match,
fnode->fn_fullname);
if (doextract && ex_match != NULL &&
!ct_match(ex_match, fnode->fn_fullname))
doextract = 0;
/*
* If we're on the first ctfile in an allfiles backup
* put the matches with -1 on the rb tree so we'll
* remember to extract it from older files.
*/
if (doextract == 1 && fillrb &&
xdr_ctx.xs_hdr.cmh_nr_shas == -1) {
ct_match_insert_rb(rb_match, fnode->fn_fullname);
doextract = 0;
}
ct_free_fnode(fnode);
break;
case XS_RET_FILE_END:
if (doextract == 0)
continue;
/* update statistics */
state->ct_stats->st_bytes_tot +=
xdr_ctx.xs_trl.cmt_orig_size;
break;
case XS_RET_EOF:
ctfile_parse_close(&xdr_ctx);
/* if rb tree and rb is empty, goto end state */
if ((haverb && ct_match_rb_is_empty(inc_match)) ||
(fillrb && ct_match_rb_is_empty(rb_match))) {
retval = 0;
goto done;
}
if (!TAILQ_EMPTY(&extract_head)) {
/*
* if allfiles and this was the first pass.
* free the current match lists
* switch to rb tree mode
*/
if (fillrb) {
ex_match = NULL;
inc_match = rb_match;
rb_match = NULL;
haverb = 1;
fillrb = 0;
}
/* reinits xdr_ctx */
if ((ret = ct_extract_open_next(&extract_head,
&xdr_ctx)) != 0) {
ct_fatal(state,
"Can't open next ctfile", ret);
goto done;
}
continue;
}
retval = 0;
goto done;
break;
case XS_RET_FAIL:
ct_fatal(state, "Failed to parse ctfile",
xdr_ctx.xs_errno);
goto done;
break;
}
}
done:
/* empty unless we quit early */
ct_extract_cleanup_queue(&extract_head);
/* only have control of the rb tree we made */
if (haverb)
ct_match_unwind(inc_match);
if (rb_match != NULL)
ct_match_unwind(rb_match);
return (retval);
}
int
main(int argc, char *argv[])
{
struct ct_match *ctm = NULL;
uint32_t cflags;
int c, imode;
char *mode = NULL, *match = NULL;
clog_init(1);
cflags = CLOG_F_ENABLE | CLOG_F_STDERR;
cflags |= CLOG_F_DBGENABLE | CLOG_F_FILE | CLOG_F_FUNC | CLOG_F_LINE |
CLOG_F_DTIME;
if (clog_set_flags(cflags))
errx(1, "illegal clog flags");
CINFO("start");
while ((c = getopt(argc, argv, "m:M:")) != -1) {
switch (c) {
case 'm':
mode = optarg;
break;
case 'M':
match = optarg;
break;
default:
CFATALX("invalid option");
}
}
argc -= optind;
argv += optind;
if (mode) {
if (!strcmp(mode, "regex"))
imode = CT_MATCH_REGEX;
else if (!strcmp(mode, "rb"))
imode = CT_MATCH_RB;
else if (!strcmp(mode, "glob"))
imode = CT_MATCH_GLOB;
else
CFATALX("invalid mode %s", mode);
} else
CFATALX("no mode");
if (match == NULL)
CFATALX("no match");
CDBG("mode: %s", mode);
ct_match_compile(&ctm, imode, argv);
if (ct_match(ctm, match))
printf("%s not matched\n", match);
else
printf("%s matched\n", match);
ct_match_unwind(ctm);
e_check_memory();
return (0);
}
