void ct_extract(struct ct_global_state *state, struct ct_op *op) { struct ct_extract_args *cea = op->op_args; const char *ctfile = cea->cea_local_ctfile; char **filelist = cea->cea_filelist; int match_mode = cea->cea_matchmode; struct ct_extract_priv *ex_priv = op->op_priv; int ret; struct ct_trans *trans; char shat[SHA_DIGEST_STRING_LENGTH]; /* if we were woken up due to fatal, just clean up local state */ if (state->ct_dying != 0) goto dying; CNDBG(CT_LOG_TRANS, "entry"); switch (ct_get_file_state(state)) { case CT_S_STARTING: if (ex_priv == NULL) { ex_priv = e_calloc(1, sizeof(*ex_priv)); TAILQ_INIT(&ex_priv->extract_head); if ((ret = ct_match_compile(&ex_priv->inc_match, match_mode, filelist)) != 0) { ct_fatal(state, "failed to compile include pattern", ret); goto dying; } if (cea->cea_excllist != NULL && (ret = ct_match_compile(&ex_priv->ex_match, match_mode, cea->cea_excllist)) != 0) { ct_fatal(state, "failed to compile exclude pattern", ret); goto dying; } op->op_priv = ex_priv; RB_INIT(&ex_priv->pending_tree); } if ((ret = ct_file_extract_init(&state->extract_state, cea->cea_tdir, cea->cea_attr, cea->cea_follow_symlinks, ex_priv->allfiles, cea->cea_log_state, cea->cea_log_chown_failed)) != 0) { ct_fatal(state, "Can not initialize extract state", ret); goto dying; } if (ct_extract_calculate_total(state, cea, ex_priv->inc_match, ex_priv->ex_match) != 0) { CWARNX("failed to calculate stats"); goto dying; } if ((ret = ct_extract_setup(&ex_priv->extract_head, &ex_priv->xdr_ctx, ctfile, cea->cea_ctfile_basedir, &ex_priv->allfiles)) != 0) { ct_fatal(state, "can't setup extract queue", ret); goto dying; } state->ct_print_ctfile_info(state->ct_print_state, ex_priv->xdr_ctx.xs_filename, &ex_priv->xdr_ctx.xs_gh); /* XXX we should handle this better */ if (state->ct_max_block_size < ex_priv->xdr_ctx.xs_gh.cmg_chunk_size) CABORTX("block size negotiated with server %d is " "smaller than file max block size %d", state->ct_max_block_size, ex_priv->xdr_ctx.xs_gh.cmg_chunk_size); /* create rb tree head, prepare to start inserting */ if (ex_priv->allfiles) { ex_priv->fillrb = 1; } break; case CT_S_FINISHED: return; default: break; } ct_set_file_state(state, CT_S_RUNNING); while (1) { trans = ct_trans_alloc(state); if (trans == NULL) { /* system busy, return */ CNDBG(CT_LOG_TRANS, "ran out of transactions, waiting"); ct_set_file_state(state, CT_S_WAITING_TRANS); return; } trans->tr_statemachine = ct_state_extract; switch ((ret = ctfile_parse(&ex_priv->xdr_ctx))) { case XS_RET_FILE: if (ex_priv->fillrb == 0 && ex_priv->xdr_ctx.xs_hdr.cmh_nr_shas == -1) { if (ex_priv->allfiles == 0) CINFO("file %s has negative shas " "and backup is not allfiles", ex_priv->xdr_ctx.xs_hdr.cmh_filename); ex_priv->doextract = 0; goto skip; /* skip ze file for now */ } trans = ct_trans_realloc_local(state, trans); trans->tr_fl_node = ex_priv->fl_ex_node = ct_alloc_fnode(); ct_populate_fnode(state->extract_state, &ex_priv->xdr_ctx, trans->tr_fl_node, &trans->tr_state, ex_priv->allfiles, cea->cea_strip_slash); if (trans->tr_state == TR_S_EX_SPECIAL) { trans->tr_complete = ct_extract_complete_special; } else { trans->tr_complete = ct_extract_complete_file_start; } trans->tr_cleanup = ct_extract_cleanup_fnode; if (ex_priv->haverb) { struct ct_pending_file *cpf; if ((cpf = ct_extract_find_entry( &ex_priv->pending_tree, trans->tr_fl_node->fn_fullname)) != NULL) { struct fnode *hardlink; /* copy permissions over */ trans->tr_fl_node->fn_uid = cpf->cpf_uid; trans->tr_fl_node->fn_gid = cpf->cpf_gid; trans->tr_fl_node->fn_mode = cpf->cpf_mode; trans->tr_fl_node->fn_mtime = cpf->cpf_mtime; trans->tr_fl_node->fn_atime = cpf->cpf_atime; /* copy list of pending links over */ while ((hardlink = TAILQ_FIRST(&cpf->cpf_links))) { TAILQ_REMOVE(&cpf->cpf_links, hardlink, fn_list); TAILQ_INSERT_TAIL( &trans->tr_fl_node->fn_hardlinks, hardlink, fn_list); } ex_priv->doextract = 1; ct_extract_free_entry( &ex_priv->pending_tree, cpf); } else { ex_priv->doextract = 0; } } else { ex_priv->doextract = !ct_match(ex_priv->inc_match, trans->tr_fl_node->fn_fullname); if (ex_priv->doextract && ex_priv->ex_match != NULL && !ct_match(ex_priv->ex_match, trans->tr_fl_node->fn_fullname)) { ex_priv->doextract = 0; } } if (ex_priv->doextract && trans->tr_fl_node->fn_hardlink) { struct ct_pending_file *file; if ((file = ct_extract_find_entry( &ex_priv->pending_tree, trans->tr_fl_node->fn_hlname)) != NULL) { CNDBG(CT_LOG_FILE, "adding pending link for %s to %s", file->cpf_name, trans->tr_fl_node->fn_fullname); /* our reference to node passed */ ct_pending_file_add_link(file, trans->tr_fl_node); ex_priv->doextract = 0; goto skip; } } /* * If we're on the first ctfile in an allfiles backup * put the matches with -1 on the rb tree so we'll * remember to extract it from older files. */ if (ex_priv->doextract == 1 && ex_priv->fillrb && ex_priv->xdr_ctx.xs_hdr.cmh_nr_shas == -1) { ct_extract_insert_entry(&ex_priv->pending_tree, trans->tr_fl_node); ex_priv->doextract = 0; /* XXX reconsider the freeing */ } if (ex_priv->doextract == 0) { ct_free_fnode(trans->tr_fl_node); skip: ex_priv->fl_ex_node = NULL; ct_trans_free(state, trans); continue; } CNDBG(CT_LOG_CTFILE, "file %s numshas %" PRId64, trans->tr_fl_node->fn_fullname, ex_priv->xdr_ctx.xs_hdr.cmh_nr_shas); /* * special files we give our refcount up * regular files we need a new one since we need to * keep ours. */ if (trans->tr_state != TR_S_EX_SPECIAL) { ct_ref_fnode(trans->tr_fl_node); } else { ex_priv->fl_ex_node = NULL; } ct_queue_first(state, trans); break; case XS_RET_SHA: if (ex_priv->doextract == 0 || ex_priv->fl_ex_node->fn_skip_file != 0) { if (ctfile_parse_seek(&ex_priv->xdr_ctx)) { ct_fatal(state, "Can't seek past shas", ex_priv->xdr_ctx.xs_errno); goto dying; } ct_trans_free(state, trans); continue; } /* use saved fnode */ trans->tr_fl_node = ex_priv->fl_ex_node; if (memcmp(zerosha, ex_priv->xdr_ctx.xs_sha, SHA_DIGEST_LENGTH) == 0) { CWARNX("\"%s\" truncated during backup", trans->tr_fl_node->fn_fullname); if (ctfile_parse_seek(&ex_priv->xdr_ctx)) { ct_fatal(state, "Can't seek past " "truncation shas", ex_priv->xdr_ctx.xs_errno); goto dying; } ct_trans_free(state, trans); continue; } if (ex_priv->xdr_ctx.xs_gh.cmg_flags & CT_MD_CRYPTO) { /* * yes csha and sha are reversed, we want * to download csha, but putting it in sha * simplifies the code */ bcopy(ex_priv->xdr_ctx.xs_sha, trans->tr_csha, sizeof(trans->tr_csha)); bcopy(ex_priv->xdr_ctx.xs_csha, trans->tr_sha, sizeof(trans->tr_sha)); bcopy(ex_priv->xdr_ctx.xs_iv, trans->tr_iv, sizeof(trans->tr_iv)); } else { bcopy(ex_priv->xdr_ctx.xs_sha, trans->tr_sha, sizeof(trans->tr_sha)); } if (clog_mask_is_set(CT_LOG_SHA)) { ct_sha1_encode(trans->tr_sha, shat); CNDBG(CT_LOG_SHA, "extracting sha %s", shat); } trans->tr_state = TR_S_EX_SHA; trans->tr_complete = ct_extract_complete_file_read; trans->tr_dataslot = 0; ct_ref_fnode(trans->tr_fl_node); trans->tr_cleanup = ct_extract_cleanup_fnode; ct_queue_first(state, trans); break; case XS_RET_FILE_END: trans = ct_trans_realloc_local(state, trans); if (ex_priv->doextract == 0 || ex_priv->fl_ex_node->fn_skip_file != 0) { /* release our reference done with file */ if (ex_priv->fl_ex_node) { ct_free_fnode(ex_priv->fl_ex_node); ex_priv->fl_ex_node = NULL; } ct_trans_free(state, trans); continue; } /* use saved fnode from state */ trans->tr_fl_node = ex_priv->fl_ex_node; bcopy(ex_priv->xdr_ctx.xs_trl.cmt_sha, trans->tr_sha, sizeof(trans->tr_sha)); trans->tr_state = TR_S_EX_FILE_END; trans->tr_complete = ct_extract_complete_file_end; trans->tr_cleanup = ct_extract_cleanup_fnode; trans->tr_fl_node->fn_size = ex_priv->xdr_ctx.xs_trl.cmt_orig_size; /* * no reference here since we give our reference to the * last transaction on that file. We are done with it. */ ex_priv->fl_ex_node = NULL; ct_queue_first(state, trans); break; case XS_RET_EOF: CNDBG(CT_LOG_CTFILE, "Hit end of ctfile"); ctfile_parse_close(&ex_priv->xdr_ctx); /* if rb tree and rb is empty, goto end state */ if ((ex_priv->haverb || ex_priv->fillrb) && ct_extract_rb_empty(&ex_priv->pending_tree)) { /* * Cleanup extract queue, in case we had files * left. */ ct_extract_cleanup_queue( &ex_priv->extract_head); goto we_re_done_here; } if (!TAILQ_EMPTY(&ex_priv->extract_head)) { /* * if allfiles and this was the first pass. * free the current match lists * switch to rb tree mode */ if (ex_priv->fillrb) { ct_match_unwind(ex_priv->inc_match); if (ex_priv->ex_match) ct_match_unwind( ex_priv->ex_match); ex_priv->ex_match = NULL; ex_priv->inc_match = NULL; ex_priv->haverb = 1; ex_priv->fillrb = 0; } ct_trans_free(state, trans); /* reinits ex_priv->xdr_ctx */ if ((ret = ct_extract_open_next(&ex_priv->extract_head, &ex_priv->xdr_ctx)) != 0) { ct_fatal(state, "Can't open next ctfile", ret); goto dying; } state->ct_print_ctfile_info( state->ct_print_state, ex_priv->xdr_ctx.xs_filename, &ex_priv->xdr_ctx.xs_gh); /* poke file into action */ ct_wakeup_file(state->event_state); } else { /* * If rb tree and it is still has entries, * bitch about it */ /* XXX print out missing files */ if ((ex_priv->haverb || ex_priv->fillrb) && ct_extract_rb_empty( &ex_priv->pending_tree)) { CWARNX("out of ctfiles but some " "files are not found"); } we_re_done_here: if (ex_priv->inc_match) ct_match_unwind(ex_priv->inc_match); if (ex_priv->ex_match) ct_match_unwind( ex_priv->ex_match); ct_extract_pending_cleanup( &ex_priv->pending_tree); e_free(&ex_priv); op->op_priv = NULL; trans->tr_state = TR_S_DONE; trans->tr_complete = ct_extract_complete_done; trans->tr_cleanup = ct_extract_cleanup_done; /* * Technically this should be a local * transaction. However, since we are done * it doesn't really matter either way. */ ct_queue_first(state, trans); CNDBG(CT_LOG_TRANS, "extract finished"); ct_set_file_state(state, CT_S_FINISHED); } return; break; case XS_RET_FAIL: ct_fatal(state, "Failed to parse ctfile", ex_priv->xdr_ctx.xs_errno); goto dying; break; } } return; dying: /* only if we hadn't sent the final transaction yet */ if (ex_priv != NULL) { ct_extract_cleanup_queue(&ex_priv->extract_head); if (ex_priv->inc_match) ct_match_unwind(ex_priv->inc_match); if (ex_priv->ex_match) ct_match_unwind(ex_priv->ex_match); if (!ct_extract_rb_empty(&ex_priv->pending_tree)) { ct_extract_pending_cleanup(&ex_priv->pending_tree); } if (ex_priv->fl_ex_node != NULL) { ct_free_fnode(ex_priv->fl_ex_node); } /* XXX what about ex_priv->xdr_ctx ? */ e_free(&ex_priv); op->op_priv = NULL; /* if ex_priv is gone then the trans will clean this up */ if (state->extract_state) ct_file_extract_cleanup(state->extract_state); } return; }
int ct_list(const char *file, char **flist, char **excludelist, int match_mode, const char *ctfile_basedir, int strip_slash, int verbose) { struct ct_extract_state *ces; struct ctfile_parse_state xs_ctx; struct fnode fnodestore; uint64_t reduction; struct fnode *fnode = &fnodestore; struct ct_match *match, *ex_match = NULL; char *ct_next_filename; char *sign; int state; int doprint = 0; int ret; int s_errno = 0, ct_errno = 0; char shat[SHA_DIGEST_STRING_LENGTH]; char cshat[SHA_DIGEST_STRING_LENGTH]; char iv[CT_IV_LEN*2+1]; if ((ret = ct_file_extract_init(&ces, NULL, 1, 1, 0, NULL, NULL)) != 0) CFATALX("failed to initialise extract state: %s", ct_strerror(ret)); if ((ret = ct_match_compile(&match, match_mode, flist)) != 0) CFATALX("failed to compile match pattern: %s", ct_strerror(ret)); if (excludelist != NULL && (ret = ct_match_compile(&ex_match, match_mode, excludelist)) != 0) CFATALX("failed to compile exclude pattern: %s", ct_strerror(ret)); verbose++; /* by default print something. */ ct_next_filename = NULL; next_file: ret = ctfile_parse_init(&xs_ctx, file, ctfile_basedir); if (ret) CFATALX("failed to open %s: %s", file, ct_strerror(ret)); ct_print_ctfile_info(&verbose, file, &xs_ctx.xs_gh); if (ct_next_filename) e_free(&ct_next_filename); if (xs_ctx.xs_gh.cmg_prevlvl_filename) { CNDBG(CT_LOG_CTFILE, "previous backup file %s\n", xs_ctx.xs_gh.cmg_prevlvl_filename); ct_next_filename = e_strdup(xs_ctx.xs_gh.cmg_prevlvl_filename); } bzero(&fnodestore, sizeof(fnodestore)); do { ret = ctfile_parse(&xs_ctx); switch (ret) { case XS_RET_FILE: ct_populate_fnode(ces, &xs_ctx, fnode, &state, xs_ctx.xs_gh.cmg_flags & CT_MD_MLB_ALLFILES, strip_slash); doprint = !ct_match(match, fnode->fn_fullname); if (doprint && ex_match != NULL && !ct_match(ex_match, fnode->fn_fullname)) doprint = 0; if (doprint) { ct_pr_fmt_file(&verbose, fnode); if (!C_ISREG(xs_ctx.xs_hdr.cmh_type) || verbose > 2) printf("\n"); } if (fnode->fn_hlname) e_free(&fnode->fn_hlname); if (fnode->fn_fullname) e_free(&fnode->fn_fullname); break; case XS_RET_FILE_END: sign = " "; if (xs_ctx.xs_trl.cmt_comp_size == 0) reduction = 100; else { uint64_t orig, comp; orig = xs_ctx.xs_trl.cmt_orig_size; comp = xs_ctx.xs_trl.cmt_comp_size; if (comp <= orig) { reduction = 100 * (orig - comp) / orig; } else { reduction = 100 * (comp - orig) / orig; if (reduction != 0) sign = "-"; } } if (doprint && verbose > 1) printf(" sz: %" PRIu64 " shas: %" PRIu64 " reduction: %s%" PRIu64 "%%\n", xs_ctx.xs_trl.cmt_orig_size, xs_ctx.xs_hdr.cmh_nr_shas, sign, reduction); else if (doprint) printf("\n"); break; case XS_RET_SHA: if (!(doprint && verbose > 2)) { if (ctfile_parse_seek(&xs_ctx)) { CFATALX("seek failed"); } } else { int i; ct_sha1_encode(xs_ctx.xs_sha, shat); switch (xs_ctx.xs_gh.cmg_flags & CT_MD_CRYPTO) { case 0: printf(" sha %s\n", shat); break; case CT_MD_CRYPTO: ct_sha1_encode(xs_ctx.xs_csha, cshat); for (i = 0; i < CT_IV_LEN; i++) snprintf(&iv[i * 2], 3, "%02x", xs_ctx.xs_iv[i]); printf(" sha %s csha %s iv %s\n", shat, cshat, iv); } } break; case XS_RET_EOF: break; case XS_RET_FAIL: s_errno = errno; ct_errno = xs_ctx.xs_errno; ; } } while (ret != XS_RET_EOF && ret != XS_RET_FAIL); ctfile_parse_close(&xs_ctx); if (ret != XS_RET_EOF) { errno = s_errno; CWARNX("corrupt ctfile: %s", ct_strerror(ct_errno)); } else { if (ct_next_filename) { file = ct_next_filename; goto next_file; } } ct_match_unwind(match); ct_file_extract_cleanup(ces); return (0); }
/* * So that we can provide correct statistics we have to go through all ctfiles * being extracted and sum the sizes to be extracted. This is kinda expensive, * but not really avoidable if we want to provide the statistics. * * Failure means we have called ct fatal. */ int ct_extract_calculate_total(struct ct_global_state *state, struct ct_extract_args *cea, struct ct_match *inc_match, struct ct_match *ex_match) { struct ct_extract_head extract_head; struct ctfile_parse_state xdr_ctx; struct ct_match *rb_match = NULL; struct fnode *fnode; int allfiles; int fillrb = 0, haverb = 0; int doextract = 0; int tr_state; int ret; int retval = 1; TAILQ_INIT(&extract_head); if ((ret = ct_extract_setup(&extract_head, &xdr_ctx, cea->cea_local_ctfile, cea->cea_ctfile_basedir, &allfiles)) != 0) { ct_fatal(state, "can't setup extract queue", ret); goto done; } if (allfiles) { char *nothing = NULL; if ((ret = ct_match_compile(&rb_match, CT_MATCH_RB, ¬hing)) != 0) { ct_fatal(state, "Couldn't create match tree", ret); goto done; } fillrb = 1; } while (1) { switch ((ret = ctfile_parse(&xdr_ctx))) { case XS_RET_FILE: if (fillrb == 0 && xdr_ctx.xs_hdr.cmh_nr_shas == -1) { continue; } fnode = ct_alloc_fnode(); /* XXX need the fnode for the correct paths */ ct_populate_fnode(state->extract_state, &xdr_ctx, fnode, &tr_state, allfiles, cea->cea_strip_slash); /* we don't care about individual shas */ if (C_ISREG(fnode->fn_type)) { ctfile_parse_seek(&xdr_ctx); } doextract = !ct_match(inc_match, fnode->fn_fullname); if (doextract && ex_match != NULL && !ct_match(ex_match, fnode->fn_fullname)) doextract = 0; /* * If we're on the first ctfile in an allfiles backup * put the matches with -1 on the rb tree so we'll * remember to extract it from older files. */ if (doextract == 1 && fillrb && xdr_ctx.xs_hdr.cmh_nr_shas == -1) { ct_match_insert_rb(rb_match, fnode->fn_fullname); doextract = 0; } ct_free_fnode(fnode); break; case XS_RET_FILE_END: if (doextract == 0) continue; /* update statistics */ state->ct_stats->st_bytes_tot += xdr_ctx.xs_trl.cmt_orig_size; break; case XS_RET_EOF: ctfile_parse_close(&xdr_ctx); /* if rb tree and rb is empty, goto end state */ if ((haverb && ct_match_rb_is_empty(inc_match)) || (fillrb && ct_match_rb_is_empty(rb_match))) { retval = 0; goto done; } if (!TAILQ_EMPTY(&extract_head)) { /* * if allfiles and this was the first pass. * free the current match lists * switch to rb tree mode */ if (fillrb) { ex_match = NULL; inc_match = rb_match; rb_match = NULL; haverb = 1; fillrb = 0; } /* reinits xdr_ctx */ if ((ret = ct_extract_open_next(&extract_head, &xdr_ctx)) != 0) { ct_fatal(state, "Can't open next ctfile", ret); goto done; } continue; } retval = 0; goto done; break; case XS_RET_FAIL: ct_fatal(state, "Failed to parse ctfile", xdr_ctx.xs_errno); goto done; break; } } done: /* empty unless we quit early */ ct_extract_cleanup_queue(&extract_head); /* only have control of the rb tree we made */ if (haverb) ct_match_unwind(inc_match); if (rb_match != NULL) ct_match_unwind(rb_match); return (retval); }
int main(int argc, char *argv[]) { struct ct_match *ctm = NULL; uint32_t cflags; int c, imode; char *mode = NULL, *match = NULL; clog_init(1); cflags = CLOG_F_ENABLE | CLOG_F_STDERR; cflags |= CLOG_F_DBGENABLE | CLOG_F_FILE | CLOG_F_FUNC | CLOG_F_LINE | CLOG_F_DTIME; if (clog_set_flags(cflags)) errx(1, "illegal clog flags"); CINFO("start"); while ((c = getopt(argc, argv, "m:M:")) != -1) { switch (c) { case 'm': mode = optarg; break; case 'M': match = optarg; break; default: CFATALX("invalid option"); } } argc -= optind; argv += optind; if (mode) { if (!strcmp(mode, "regex")) imode = CT_MATCH_REGEX; else if (!strcmp(mode, "rb")) imode = CT_MATCH_RB; else if (!strcmp(mode, "glob")) imode = CT_MATCH_GLOB; else CFATALX("invalid mode %s", mode); } else CFATALX("no mode"); if (match == NULL) CFATALX("no match"); CDBG("mode: %s", mode); ct_match_compile(&ctm, imode, argv); if (ct_match(ctm, match)) printf("%s not matched\n", match); else printf("%s matched\n", match); ct_match_unwind(ctm); e_check_memory(); return (0); }