char *
ct_getloginbyuid(uid_t uid)
{
struct passwd *passwd;
struct ct_login_cache *entry, search;
search.lc_uid = uid;
entry = RB_FIND(ct_login_cache_tree, &ct_login_cache, &search);
if (entry != NULL) {
return entry->lc_name;
}
/* if the cache gets too big, dump all entries and refill. */
if (ct_login_cache_size > MAX_LC_CACHE_SIZE) {
ct_cleanup_login_cache();
}
/* yes, this even caches negative entries */
ct_login_cache_size++;
entry = e_calloc(1, sizeof(*entry));
entry->lc_uid = uid;
passwd = getpwuid(uid);
if (passwd)
entry->lc_name = e_strdup(passwd->pw_name);
else
entry->lc_name = NULL; /* entry not found cache NULL */
RB_INSERT(ct_login_cache_tree, &ct_login_cache, entry);
return entry->lc_name;
}
struct ct_io_queue *
ct_ioctx_alloc(void)
{
struct ct_io_queue *ioq;
ioq = e_calloc(1, sizeof(*ioq));
return ioq;
}
struct ct_header *
ct_header_alloc(void *vctx)
{
struct ct_header *hdr;
hdr = e_calloc(1, sizeof(*hdr));
return hdr;
}
/*
* filenames passed in remote mode are opaque tags for the backup.
* they are stored on the server and in remote mode in the form
* YYYYMMDD-HHMMSS-<strnvis(mname)>
*/
void
ctfile_find_for_extract(struct ct_global_state *state, struct ct_op *op)
{
struct ct_ctfile_find_args *ccfa = op->op_args;
const char *ctfile = ccfa->ccfa_tag;
struct ct_op *list_fakeop;
struct ct_ctfile_list_args *ccla;
if (state->ct_dying != 0) {
/* nothing to clean up */
return;
}
/* cook the ctfile so we only search for the actual tag */
if ((ctfile = ctfile_cook_name(ctfile)) == NULL) {
ct_fatal(state, ccfa->ccfa_tag, CTE_INVALID_CTFILE_NAME);
return;
}
list_fakeop = e_calloc(1, sizeof(*list_fakeop));
ccla = e_calloc(1, sizeof(*ccla));
list_fakeop->op_args = ccla;
ccla->ccla_search = e_calloc(2, sizeof(char **));
if (ctfile_is_fullname(ctfile)) {
/* use list as stat() for now */
*ccla->ccla_search = e_strdup(ctfile);
ccla->ccla_matchmode = CT_MATCH_GLOB;
} else {
e_asprintf(ccla->ccla_search,
"^[[:digit:]]{8}-[[:digit:]]{6}-%s$", ctfile);
ccla->ccla_matchmode = CT_MATCH_REGEX;
/*
* get the list of files matching this tag from the server.
* list returns an empty list if it found
* nothing and NULL upon failure.
*/
}
e_free(&ctfile);
CNDBG(CT_LOG_CTFILE, "looking for %s", ccla->ccla_search[0]);
op->op_priv = list_fakeop;
ctfile_list_start(state, list_fakeop);
return;
}
int
ct_match_compile(struct ct_match **matchp, int mode, char **flist)
{
struct ct_match *match;
int i, ret;
match = e_calloc(1, sizeof(*match));
match->cm_mode = mode;
switch (mode) {
case CT_MATCH_EVERYTHING:
ret = 0;
break;
case CT_MATCH_REGEX:
match->cm_regex = e_calloc(1, sizeof(regex_t));
ret = ct_regex_comp(match->cm_regex, flist);
break;
case CT_MATCH_RB:
match->cm_rb_head = e_calloc(1, sizeof(*match->cm_rb_head));
ret = ct_rb_comp(match->cm_rb_head, flist);
break;
case CT_MATCH_GLOB:
for (i = 0; flist[i] != NULL; i++)
if (flist[i] == NULL)
break;
if (i == 0) {
ret = 0;
break;
}
i++; /* extra NULL */
match->cm_glob = e_calloc(i, sizeof(char *));
for (i = 0; flist[i] != NULL; i++) {
if (flist[i] == NULL)
break;
match->cm_glob[i] = e_strdup(flist[i]);
}
ret = 0;
break;
default:
CABORTX("invalid match mode");
}
*matchp = match;
return (ret);
}
/*
* now the operation has completed we can kick off the next operation knowing
* that everything has been set up for it.
*/
int
ctfile_extract_nextop(struct ct_global_state *state, struct ct_op *op)
{
struct ct_ctfile_find_fileop_args *ccffa = op->op_args;
struct ct_ctfileop_args *cca;
char *cachename;
int ret = 0;
/*
* If this is an operation that needs the full incremental chain
* recursively fetch the next one in the chain till done.
*/
if (ccffa->ccffa_download_chain) {
/*
* download_next takes ownership of the pointers it is given,
* duplicate our copy.
*/
cca = e_calloc(1, sizeof(*cca));
if (ccffa->ccffa_base.cca_localname)
cca->cca_localname =
e_strdup(ccffa->ccffa_base.cca_localname);
if (ccffa->ccffa_base.cca_remotename)
cca->cca_remotename =
e_strdup(ccffa->ccffa_base.cca_remotename);
cca->cca_tdir = ccffa->ccffa_base.cca_tdir;
cca->cca_ctfile = ccffa->ccffa_base.cca_ctfile;
op->op_args = cca;
/* we give up ownership of cca here */
if ((ret = ctfile_download_next(state, op)) != 0)
goto out;
}
/*
* We now have the name of the file we wish to perform the main
* operation on, the nextop callback will add this operation
* to the operation list. Ownership of the allocated pointer
* passes to the child.
*/
if (ccffa->ccffa_base.cca_localname != NULL) {
cachename =
ctfile_get_cachename(ccffa->ccffa_base.cca_localname,
ccffa->ccffa_base.cca_tdir);
} else {
cachename = NULL;
}
ret = ccffa->ccffa_nextop(state, cachename, ccffa->ccffa_nextop_args);
out:
if (ccffa->ccffa_base.cca_localname)
e_free(&ccffa->ccffa_base.cca_localname);
if (ccffa->ccffa_base.cca_remotename)
e_free(&ccffa->ccffa_base.cca_remotename);
e_free(&ccffa);
return (ret);
}
void
ct_match_insert_rb(struct ct_match *match, char *string)
{
struct ct_match_node *n;
if (match->cm_mode != CT_MATCH_RB)
CABORTX("match mode %d is not rb", match->cm_mode);
n = e_calloc(1, sizeof(struct ct_match_node));
n->cmn_string = e_strdup(string);
if (RB_INSERT(ct_match_tree, match->cm_rb_head, n)) {
/* pattern already exists free it */
e_free(&n->cmn_string);
e_free(&n);
}
}
struct ct_op *
ct_add_operation_after(struct ct_global_state *state, struct ct_op *after,
ct_op_cb *start, ct_op_complete_cb *complete, void *args)
{
struct ct_op *op;
op = e_calloc(1, sizeof(*op));
op->op_start = start;
op->op_complete = complete;
op->op_args = args;
TAILQ_INSERT_AFTER(&state->ct_operations, after, op, op_link);
return (op);
}
int
ct_ssl_connect(struct ct_global_state *state)
{
struct ct_assl_io_ctx *ctx;
struct assl_context *c;
struct assl_connect_opts aco;
int ret;
ctx = e_calloc(1, sizeof (*ctx));
if ((c = assl_alloc_context(ASSL_M_TLSV1_CLIENT, 0)) == NULL) {
e_free(&ctx);
return (CTE_ASSL_CONTEXT);
}
if ((ret = ct_load_certs(state, c)) != 0) {
/* free assl thingy */
e_free(&ctx);
return (ret);
}
ct_assl_io_ctx_init(ctx, c, ct_handle_msg, ct_write_done,
state, ct_header_alloc, ct_header_free, ct_body_alloc,
ct_body_free, ct_ioctx_alloc, ct_ioctx_free);
aco.aco_flags = ASSL_F_NONBLOCK|ASSL_F_KEEPALIVE|ASSL_F_THROUGHPUT,
aco.aco_rcvbuf = state->ct_config->ct_sock_rcvbuf;
aco.aco_sndbuf = state->ct_config->ct_sock_sndbuf;
if (ct_assl_connect(ctx, state->ct_config->ct_host,
state->ct_config->ct_hostport, &aco,
ct_event_get_base(state->event_state))) {
ct_assl_disconnect(ctx);
e_free(&ctx);
ctx = NULL;
return (CTE_CONNECT_FAILED);
}
state->ct_assl_ctx = ctx;
if (state->ct_config->ct_io_bw_limit && ctx != NULL)
state->bw_limit =
ct_ssl_init_bw_lim(ct_event_get_base(state->event_state),
ctx, state->ct_config->ct_io_bw_limit);
return (0);
}
void
ctfile_find_for_operation(struct ct_global_state *state, char *tag,
ctfile_find_callback *nextop, void *nextop_args, int download_chain,
int empty_ok)
{
struct ct_ctfile_find_args *ccfa;
ccfa = e_calloc(1, sizeof(*ccfa));
ccfa->ccfa_tag = tag;
ccfa->ccfa_cachedir = state->ct_config->ct_ctfile_cachedir;
ccfa->ccfa_nextop = nextop;
ccfa->ccfa_nextop_args = nextop_args;
ccfa->ccfa_download_chain = download_chain;
ccfa->ccfa_empty_ok = empty_ok;
ct_add_operation(state, ctfile_find_for_extract,
ctfile_find_for_extract_complete, ccfa);
}
struct ctdb_state *
ctdb_setup(const char *path, int crypt_enabled)
{
struct ctdb_state *state;
if (path == NULL)
return (NULL);
state = e_calloc(1, sizeof(*state));
state->ctdb_genid = -1;
state->ctdb_crypt = crypt_enabled;
state->ctdb_dbfile = e_strdup(path);
if (ctdb_open(state) != 0) {
e_free(&state->ctdb_dbfile);
e_free(&state);
}
return (state);
}
void
ct_extract_insert_entry(struct ct_pending_files *head, struct fnode *fnode)
{
struct ct_pending_file *cpf;
CNDBG(CT_LOG_FILE, "%s: inserting %s", __func__, fnode->fn_fullname);
cpf = e_calloc(1, sizeof(*cpf));
cpf->cpf_name = e_strdup(fnode->fn_fullname);
cpf->cpf_uid = fnode->fn_uid;
cpf->cpf_gid = fnode->fn_gid;
cpf->cpf_mode = fnode->fn_mode;
cpf->cpf_mtime = fnode->fn_mtime;
cpf->cpf_atime = fnode->fn_atime;
TAILQ_INIT(&cpf->cpf_links);
RB_INSERT(ct_pending_files, head, cpf);
}
int
ct_rb_comp(struct ct_match_tree *head, char **flist)
{
int i;
struct ct_match_node *n;
for (i = 0; flist[i] != NULL; i++) {
if (flist[i] == NULL)
break;
n = e_calloc(1, sizeof(struct ct_match_node));
n->cmn_string = e_strdup(flist[i]);
if (RB_INSERT(ct_match_tree, head, n)) {
/* pattern already exists free it */
e_free(&n->cmn_string);
e_free(&n);
continue;
}
}
return (0);
}
int
ct_check_secrets_extract(struct ct_global_state *state, struct ct_op *op)
{
struct ct_ctfileop_args *cca;
if (!ct_file_on_server(state, "crypto.secrets"))
return (CTE_NO_SECRETS_ON_SERVER);
cca = e_calloc(1, sizeof(*cca));
/* XXX temporary name? */
cca->cca_localname = "cyphertite-server.secrets";
cca->cca_remotename = "crypto.secrets";
cca->cca_tdir = state->ct_config->ct_ctfile_cachedir;
cca->cca_cleartext = 0;
cca->cca_ctfile = 0;
ct_add_operation_after(state, op, ctfile_extract, ct_compare_secrets,
cca);
/* start download of secrets with finish file being the comparison */
return (0);
}
/* Do the main algorithm */
int unsignedhc_mcdist_mat2(
int num_genes,
int num_chromosomes,
int num_genomes,
int circular,
int verbose,
struct genome_struct *genome_list_in,
int nsegs,
int *seg_list,
int **weight_matrix,
int num_iterations,
struct genome_struct *out_genome
)
{
distmem_t distmem_mem;
struct uhc_mem uhcmem_mem, *uhcmem=&uhcmem_mem;
struct genome_struct uhcmem_g1, uhcmem_g2;
/* extreme upper bound on max distance possible */
/* TODO: improve for use with arbitrary weight matrices */
int max_dist_possible = num_genomes*(num_genomes-1)/2 * (num_genes+1);
int i,k,s1;
int cur_genome_num;
struct genome_struct *cur_genome = (struct genome_struct *) 0;
int *curseg;
int cur_len;
int num_options;
/**********************************************************************
* allocate memory
**********************************************************************/
uhcmem->max_dist_possible = max_dist_possible;
uhcmem->best_dist = max_dist_possible;
uhcmem->distmem = &distmem_mem;
uhcmem->run_no = 0;
uhcmem->genome_list =
(struct genome_struct *) e_malloc(num_genomes*sizeof(struct genome_struct),
"unsignedhc_mcdist_mat2: genome_list");
uhcmem->g1 = &uhcmem_g1;
alloc_simple_genome(num_genes, &uhcmem_g1);
uhcmem->g2 = &uhcmem_g2;
alloc_simple_genome(num_genes, &uhcmem_g2);
/* allocate space to work on a copy of all the genomes;
* keep the originals intact
*/
for (i=0; i<num_genomes; i++) {
uhcmem->genome_list[i].gnamePtr = genome_list_in[i].gnamePtr;
uhcmem->genome_list[i].genome_num = genome_list_in[i].genome_num;
uhcmem->genome_list[i].encoding = genome_list_in[i].encoding;
uhcmem->genome_list[i].genes =
(int *) e_malloc(num_genes * sizeof(int), "unsignedhc_mcdist_mat2: genes");
}
/* allocate memory for distance computations */
mcdist_allocmem(num_genes, num_chromosomes, uhcmem->distmem);
/* allocate memory for distance matrix */
uhcmem->dmat = (int *) e_malloc(num_genomes*num_genomes*sizeof(int),
"unsignedhc_mcdist_mat2: dmat");
/* allocate memory for modified row of distance matrix */
uhcmem->dmat_r = (int *) e_malloc(num_genomes*sizeof(int),
"unsignedhc_mcdist_mat2: dmat_r");
/* keep track of which flips have been tested and which have not */
uhcmem->options = (int *) e_malloc(2 * nsegs * sizeof(int),
"unsignedhc_mcdist_mat2: options");
/* allocate memory for statistical summary of results */
/* # runs with each score */
uhcmem->dist_counts = (int *) e_calloc(max_dist_possible+1, sizeof(int),
"unsignedhc_mcdist_mat2: dist_counts");
/* correlations */
uhcmem->dist_corr = (int *) e_calloc(nsegs * nsegs, sizeof(int),
"unsignedhc_mcdist_mat2: dist_corr");
uhcmem->nsegs = nsegs;
//printf("nsegs at alloc: %d\n", nsegs);
uhcmem->signs = (int *) e_malloc(nsegs * sizeof(int),
"unsignedhc_mcdist_mat2: signs");
uhcmem->bestsigns = (int *) e_malloc(nsegs * sizeof(int),
"unsignedhc_mcdist_mat2: bestsigns");
uhcmem->width_run = num_digits(num_iterations);
uhcmem->width_run2 = max2(uhcmem->width_run, 5);
uhcmem->width_score = num_digits(max_dist_possible);
uhcmem->width_score2 = max2(uhcmem->width_score, 5);
uhcmem->width_mat_entry = num_digits(num_genes+1);
uhcmem->width_segnum = num_digits(nsegs);
uhc_init_random_seed();
/**********************************************************************
* initialize list of allowed options
* 1. all segments individually
* 2. "antistrips" (just to increase the probability of finding them)
**********************************************************************/
/* individual segments */
num_options = 0;
for (s1=0; s1<nsegs; s1++) {
uhcmem->options[num_options++] = s1;
}
/* antistrips */
for (s1=0; s1<nsegs-1; s1++) {
curseg = USEG(seg_list,s1);
/* are segs s1 & s1+1 adjacent? skip if not. */
if (curseg[0] != curseg[3] /* genome numbers */
|| curseg[1]+curseg[2] != curseg[4] /* adjacent positions */
) {
/* not adjacent */
continue;
}
/* are s1 & s1-1 adjacent? skip if so (bigger than 2-strip) */
if (s1>0
&& curseg[0]==curseg[-3]
&& curseg[1]==curseg[-2]+curseg[-1]) {
/* adjacent, 3+ strip, skip */
continue;
}
uhcmem->options[num_options++] = s1 + nsegs;
}
uhcmem->num_options = num_options;
/**********************************************************************
* print list of segments
**********************************************************************/
if (verbose) {
//f//printf(outfile,"\nSegments:\n");
cur_genome_num = -1;
for (s1 = 0; s1 < nsegs; s1++) {
curseg = USEG(seg_list, s1);
/* print genome name if switched genomes */
if (curseg[0] != cur_genome_num) {
cur_genome_num = curseg[0];
cur_genome = &genome_list_in[cur_genome_num];
if (cur_genome->gnamePtr != (char *) 0) {
//f//printf(outfile, ">%s\n", cur_genome->gnamePtr);
} else {
//f//printf(outfile, ">genome%d\n", cur_genome_num + 1);
}
}
cur_len = curseg[2];
//f//printf(outfile, "S%-*d [", uhcmem->width_segnum, s1);
for (k=0; k<cur_len; k++) {
//f//printf(outfile, " %d", cur_genome->genes[cur_start+k]);
}
//f//printf(outfile," ]\n");
}
}
/**********************************************************************
* do lots of iterations
**********************************************************************/
/* if (verbose) {
//f//printf(outfile, "\nTrials:\n");
//f//printf(outfile, "%-*s %-*s matrix %*s signs\n",
uhcmem->width_run2, "run",
uhcmem->width_score2, "score",
((uhcmem->width_mat_entry+1)*num_genomes + 2)*num_genomes - 5, "");
}
*/
for (i=0; i<num_iterations; i++) {
uhc_run(num_genes,num_chromosomes,num_genomes,
circular,verbose,
genome_list_in,
nsegs, seg_list,
weight_matrix,
uhcmem);
}
/**********************************************************************
* report on results
**********************************************************************/
//if (verbose) {
uhc_summarize_results(num_genes, num_chromosomes, num_genomes,
circular,
genome_list_in,
nsegs, seg_list,
uhcmem, out_genome);
//}
/**********************************************************************
* cleanup memory
**********************************************************************/
free((void *) uhcmem->bestsigns);
free((void *) uhcmem->signs);
free((void *) uhcmem->options);
free((void *) uhcmem->dmat_r);
free((void *) uhcmem->dmat);
free((void *) uhcmem->dist_corr);
free((void *) uhcmem->dist_counts);
/* free memory for distance computations */
mcdist_freemem(uhcmem->distmem);
free_simple_genome((void *) uhcmem->g2);
free_simple_genome((void *) uhcmem->g1);
/* free memory for copy of genomes */
for (i=0; i<num_genomes; i++) {
free((void *) uhcmem->genome_list[i].genes);
}
free((void *) uhcmem->genome_list);
return uhcmem->best_dist;
}
/*
* Extract an individual file that has been passed into the op by op_priv.
*/
void
ct_extract_file(struct ct_global_state *state, struct ct_op *op)
{
struct ct_extract_file_args *cefa = op->op_args;
struct ct_file_extract_priv *ex_priv = op->op_priv;
const char *localfile = cefa->cefa_filename;
struct ct_trans *trans;
int ret;
char shat[SHA_DIGEST_STRING_LENGTH];
if (state->ct_dying != 0)
goto dying;
CNDBG(CT_LOG_TRANS, "entry");
switch (ct_get_file_state(state)) {
case CT_S_STARTING:
CNDBG(CT_LOG_TRANS, "starting");
ex_priv = e_calloc(1, sizeof(*ex_priv));
/* open file and seek to beginning of file */
if ((ret = ctfile_parse_init_at(&ex_priv->xdr_ctx,
cefa->cefa_ctfile, NULL, cefa->cefa_ctfile_off)) != 0) {
/* XXX add pathname */
ct_fatal(state, "Can't open ctfile", ret);
e_free(&ex_priv);
goto dying;
}
/* XXX we should handle this better */
if (state->ct_max_block_size <
ex_priv->xdr_ctx.xs_gh.cmg_chunk_size)
CABORTX("block size negotiated with server %d is "
"smaller than file max block size %d",
state->ct_max_block_size,
ex_priv->xdr_ctx.xs_gh.cmg_chunk_size);
if ((ret = ct_file_extract_init(&state->extract_state,
NULL, 0, 0, 0, NULL, NULL)) != 0) {
ct_fatal(state, "Can not initialise extract state",
ret);
e_free(&ex_priv);
goto dying;
}
op->op_priv = ex_priv;
break;
case CT_S_FINISHED:
return;
default:
break;
}
ct_set_file_state(state, CT_S_RUNNING);
while (1) {
if ((trans = ct_trans_alloc(state)) == NULL) {
CNDBG(CT_LOG_TRANS, "ran out of transactions, waiting");
ct_set_file_state(state, CT_S_WAITING_TRANS);
return;
}
trans->tr_statemachine = ct_state_extract;
if (ex_priv->done) {
CNDBG(CT_LOG_CTFILE, "Hit end of ctfile");
ctfile_parse_close(&ex_priv->xdr_ctx);
e_free(&ex_priv);
op->op_priv = NULL;
trans->tr_state = TR_S_DONE;
trans->tr_complete = ct_extract_complete_done;
trans->tr_cleanup = ct_extract_cleanup_done;
ct_queue_first(state, trans);
CNDBG(CT_LOG_TRANS, "extract finished");
ct_set_file_state(state, CT_S_FINISHED);
return;
}
/* unless start of file this is right */
trans->tr_fl_node = ex_priv->fl_ex_node;
switch ((ret = ctfile_parse(&ex_priv->xdr_ctx))) {
case XS_RET_FILE:
CNDBG(CT_LOG_CTFILE, "opening file");
if (ex_priv->xdr_ctx.xs_hdr.cmh_nr_shas == -1)
CABORTX("can't extract file with -1 shas");
trans = ct_trans_realloc_local(state, trans);
trans->tr_fl_node = ex_priv->fl_ex_node =
ct_alloc_fnode();
/* Make it local directory, it won't be set up right. */
ex_priv->xdr_ctx.xs_hdr.cmh_parent_dir = -1;
/*
* Allfiles doesn't matter, only processing one file.
* We have a full path to extract to so always strip
* slash.
*/
ct_populate_fnode(state->extract_state,
&ex_priv->xdr_ctx, trans->tr_fl_node,
&trans->tr_state, 0, 1);
if (trans->tr_state == TR_S_EX_SPECIAL) {
trans->tr_complete =
ct_extract_complete_special;
} else {
trans->tr_complete =
ct_extract_complete_file_start;
}
trans->tr_cleanup = ct_extract_cleanup_fnode;
e_free(&trans->tr_fl_node->fn_fullname);
trans->tr_fl_node->fn_fullname = e_strdup(localfile);
e_free(&trans->tr_fl_node->fn_name);
trans->tr_fl_node->fn_name = e_strdup(localfile);
/* Set name pointer to something else passed in */
CNDBG(CT_LOG_CTFILE, "file %s numshas %" PRId64,
trans->tr_fl_node->fn_fullname,
ex_priv->xdr_ctx.xs_hdr.cmh_nr_shas);
/*
* special files we give our refcount up
* regular files we need a new one since we need to
* keep ours.
*/
if (trans->tr_state != TR_S_EX_SPECIAL) {
ct_ref_fnode(trans->tr_fl_node);
} else {
ex_priv->fl_ex_node = NULL;
}
break;
case XS_RET_SHA:
CNDBG(CT_LOG_SHA, "sha!");
if (ex_priv->xdr_ctx.xs_gh.cmg_flags & CT_MD_CRYPTO) {
/*
* yes csha and sha are reversed, we want
* to download csha, but putting it in sha
* simplifies the code
*/
bcopy(ex_priv->xdr_ctx.xs_sha, trans->tr_csha,
sizeof(trans->tr_csha));
bcopy(ex_priv->xdr_ctx.xs_csha, trans->tr_sha,
sizeof(trans->tr_sha));
bcopy(ex_priv->xdr_ctx.xs_iv, trans->tr_iv,
sizeof(trans->tr_iv));
} else {
bcopy(ex_priv->xdr_ctx.xs_sha, trans->tr_sha,
sizeof(trans->tr_sha));
}
if (clog_mask_is_set(CT_LOG_SHA)) {
ct_sha1_encode(trans->tr_sha, shat);
CNDBG(CT_LOG_SHA, "extracting sha %s", shat);
}
trans->tr_state = TR_S_EX_SHA;
trans->tr_complete = ct_extract_complete_file_read;
trans->tr_cleanup = ct_extract_cleanup_fnode;
trans->tr_dataslot = 0;
ct_ref_fnode(trans->tr_fl_node);
break;
case XS_RET_FILE_END:
trans = ct_trans_realloc_local(state, trans);
trans->tr_fl_node = ex_priv->fl_ex_node; /* reload */
CNDBG(CT_LOG_CTFILE, "file end!");
bcopy(ex_priv->xdr_ctx.xs_trl.cmt_sha, trans->tr_sha,
sizeof(trans->tr_sha));
trans->tr_state = TR_S_EX_FILE_END;
trans->tr_complete = ct_extract_complete_file_end;
trans->tr_cleanup = ct_extract_cleanup_fnode;
trans->tr_fl_node->fn_size =
ex_priv->xdr_ctx.xs_trl.cmt_orig_size;
/* Done now, don't parse further. */
ex_priv->done = 1;
/*
* no reference here since we give our reference to the
* last transaction on that file.
*/
ex_priv->fl_ex_node = NULL;
break;
case XS_RET_FAIL:
ct_fatal(state, "Failed to parse ctfile",
ex_priv->xdr_ctx.xs_errno);
goto dying;
break;
default:
CABORTX("%s: invalid state %d", __func__, ret);
}
ct_queue_first(state, trans);
}
return;
dying:
if (ex_priv) {
ctfile_parse_close(&ex_priv->xdr_ctx);
if (ex_priv->fl_ex_node != NULL) {
ct_free_fnode(ex_priv->fl_ex_node);
}
e_free(&ex_priv);
/* will be cleaned up by trans if ex_priv already gone */
if (state->extract_state)
ct_file_extract_cleanup(state->extract_state);
}
return;
}
void
ct_extract(struct ct_global_state *state, struct ct_op *op)
{
struct ct_extract_args *cea = op->op_args;
const char *ctfile = cea->cea_local_ctfile;
char **filelist = cea->cea_filelist;
int match_mode = cea->cea_matchmode;
struct ct_extract_priv *ex_priv = op->op_priv;
int ret;
struct ct_trans *trans;
char shat[SHA_DIGEST_STRING_LENGTH];
/* if we were woken up due to fatal, just clean up local state */
if (state->ct_dying != 0)
goto dying;
CNDBG(CT_LOG_TRANS, "entry");
switch (ct_get_file_state(state)) {
case CT_S_STARTING:
if (ex_priv == NULL) {
ex_priv = e_calloc(1, sizeof(*ex_priv));
TAILQ_INIT(&ex_priv->extract_head);
if ((ret = ct_match_compile(&ex_priv->inc_match,
match_mode, filelist)) != 0) {
ct_fatal(state,
"failed to compile include pattern", ret);
goto dying;
}
if (cea->cea_excllist != NULL &&
(ret = ct_match_compile(&ex_priv->ex_match,
match_mode, cea->cea_excllist)) != 0) {
ct_fatal(state,
"failed to compile exclude pattern", ret);
goto dying;
}
op->op_priv = ex_priv;
RB_INIT(&ex_priv->pending_tree);
}
if ((ret = ct_file_extract_init(&state->extract_state,
cea->cea_tdir, cea->cea_attr, cea->cea_follow_symlinks,
ex_priv->allfiles, cea->cea_log_state,
cea->cea_log_chown_failed)) != 0) {
ct_fatal(state, "Can not initialize extract state",
ret);
goto dying;
}
if (ct_extract_calculate_total(state, cea, ex_priv->inc_match,
ex_priv->ex_match) != 0) {
CWARNX("failed to calculate stats");
goto dying;
}
if ((ret = ct_extract_setup(&ex_priv->extract_head,
&ex_priv->xdr_ctx, ctfile, cea->cea_ctfile_basedir,
&ex_priv->allfiles)) != 0) {
ct_fatal(state, "can't setup extract queue", ret);
goto dying;
}
state->ct_print_ctfile_info(state->ct_print_state,
ex_priv->xdr_ctx.xs_filename, &ex_priv->xdr_ctx.xs_gh);
/* XXX we should handle this better */
if (state->ct_max_block_size <
ex_priv->xdr_ctx.xs_gh.cmg_chunk_size)
CABORTX("block size negotiated with server %d is "
"smaller than file max block size %d",
state->ct_max_block_size,
ex_priv->xdr_ctx.xs_gh.cmg_chunk_size);
/* create rb tree head, prepare to start inserting */
if (ex_priv->allfiles) {
ex_priv->fillrb = 1;
}
break;
case CT_S_FINISHED:
return;
default:
break;
}
ct_set_file_state(state, CT_S_RUNNING);
while (1) {
trans = ct_trans_alloc(state);
if (trans == NULL) {
/* system busy, return */
CNDBG(CT_LOG_TRANS, "ran out of transactions, waiting");
ct_set_file_state(state, CT_S_WAITING_TRANS);
return;
}
trans->tr_statemachine = ct_state_extract;
switch ((ret = ctfile_parse(&ex_priv->xdr_ctx))) {
case XS_RET_FILE:
if (ex_priv->fillrb == 0 &&
ex_priv->xdr_ctx.xs_hdr.cmh_nr_shas == -1) {
if (ex_priv->allfiles == 0)
CINFO("file %s has negative shas "
"and backup is not allfiles",
ex_priv->xdr_ctx.xs_hdr.cmh_filename);
ex_priv->doextract = 0;
goto skip; /* skip ze file for now */
}
trans = ct_trans_realloc_local(state, trans);
trans->tr_fl_node = ex_priv->fl_ex_node =
ct_alloc_fnode();
ct_populate_fnode(state->extract_state,
&ex_priv->xdr_ctx, trans->tr_fl_node,
&trans->tr_state, ex_priv->allfiles,
cea->cea_strip_slash);
if (trans->tr_state == TR_S_EX_SPECIAL) {
trans->tr_complete =
ct_extract_complete_special;
} else {
trans->tr_complete =
ct_extract_complete_file_start;
}
trans->tr_cleanup = ct_extract_cleanup_fnode;
if (ex_priv->haverb) {
struct ct_pending_file *cpf;
if ((cpf = ct_extract_find_entry(
&ex_priv->pending_tree,
trans->tr_fl_node->fn_fullname)) != NULL) {
struct fnode *hardlink;
/* copy permissions over */
trans->tr_fl_node->fn_uid =
cpf->cpf_uid;
trans->tr_fl_node->fn_gid =
cpf->cpf_gid;
trans->tr_fl_node->fn_mode =
cpf->cpf_mode;
trans->tr_fl_node->fn_mtime =
cpf->cpf_mtime;
trans->tr_fl_node->fn_atime =
cpf->cpf_atime;
/* copy list of pending links over */
while ((hardlink =
TAILQ_FIRST(&cpf->cpf_links))) {
TAILQ_REMOVE(&cpf->cpf_links,
hardlink, fn_list);
TAILQ_INSERT_TAIL(
&trans->tr_fl_node->fn_hardlinks,
hardlink, fn_list);
}
ex_priv->doextract = 1;
ct_extract_free_entry(
&ex_priv->pending_tree, cpf);
} else {
ex_priv->doextract = 0;
}
} else {
ex_priv->doextract =
!ct_match(ex_priv->inc_match,
trans->tr_fl_node->fn_fullname);
if (ex_priv->doextract &&
ex_priv->ex_match != NULL &&
!ct_match(ex_priv->ex_match,
trans->tr_fl_node->fn_fullname)) {
ex_priv->doextract = 0;
}
}
if (ex_priv->doextract &&
trans->tr_fl_node->fn_hardlink) {
struct ct_pending_file *file;
if ((file = ct_extract_find_entry(
&ex_priv->pending_tree,
trans->tr_fl_node->fn_hlname)) != NULL) {
CNDBG(CT_LOG_FILE,
"adding pending link for %s to %s",
file->cpf_name,
trans->tr_fl_node->fn_fullname);
/* our reference to node passed */
ct_pending_file_add_link(file,
trans->tr_fl_node);
ex_priv->doextract = 0;
goto skip;
}
}
/*
* If we're on the first ctfile in an allfiles backup
* put the matches with -1 on the rb tree so we'll
* remember to extract it from older files.
*/
if (ex_priv->doextract == 1 && ex_priv->fillrb &&
ex_priv->xdr_ctx.xs_hdr.cmh_nr_shas == -1) {
ct_extract_insert_entry(&ex_priv->pending_tree,
trans->tr_fl_node);
ex_priv->doextract = 0;
/* XXX reconsider the freeing */
}
if (ex_priv->doextract == 0) {
ct_free_fnode(trans->tr_fl_node);
skip:
ex_priv->fl_ex_node = NULL;
ct_trans_free(state, trans);
continue;
}
CNDBG(CT_LOG_CTFILE,
"file %s numshas %" PRId64,
trans->tr_fl_node->fn_fullname,
ex_priv->xdr_ctx.xs_hdr.cmh_nr_shas);
/*
* special files we give our refcount up
* regular files we need a new one since we need to
* keep ours.
*/
if (trans->tr_state != TR_S_EX_SPECIAL) {
ct_ref_fnode(trans->tr_fl_node);
} else {
ex_priv->fl_ex_node = NULL;
}
ct_queue_first(state, trans);
break;
case XS_RET_SHA:
if (ex_priv->doextract == 0 ||
ex_priv->fl_ex_node->fn_skip_file != 0) {
if (ctfile_parse_seek(&ex_priv->xdr_ctx)) {
ct_fatal(state, "Can't seek past shas",
ex_priv->xdr_ctx.xs_errno);
goto dying;
}
ct_trans_free(state, trans);
continue;
}
/* use saved fnode */
trans->tr_fl_node = ex_priv->fl_ex_node;
if (memcmp(zerosha, ex_priv->xdr_ctx.xs_sha,
SHA_DIGEST_LENGTH) == 0) {
CWARNX("\"%s\" truncated during backup",
trans->tr_fl_node->fn_fullname);
if (ctfile_parse_seek(&ex_priv->xdr_ctx)) {
ct_fatal(state, "Can't seek past "
"truncation shas",
ex_priv->xdr_ctx.xs_errno);
goto dying;
}
ct_trans_free(state, trans);
continue;
}
if (ex_priv->xdr_ctx.xs_gh.cmg_flags & CT_MD_CRYPTO) {
/*
* yes csha and sha are reversed, we want
* to download csha, but putting it in sha
* simplifies the code
*/
bcopy(ex_priv->xdr_ctx.xs_sha, trans->tr_csha,
sizeof(trans->tr_csha));
bcopy(ex_priv->xdr_ctx.xs_csha, trans->tr_sha,
sizeof(trans->tr_sha));
bcopy(ex_priv->xdr_ctx.xs_iv, trans->tr_iv,
sizeof(trans->tr_iv));
} else {
bcopy(ex_priv->xdr_ctx.xs_sha, trans->tr_sha,
sizeof(trans->tr_sha));
}
if (clog_mask_is_set(CT_LOG_SHA)) {
ct_sha1_encode(trans->tr_sha, shat);
CNDBG(CT_LOG_SHA, "extracting sha %s", shat);
}
trans->tr_state = TR_S_EX_SHA;
trans->tr_complete = ct_extract_complete_file_read;
trans->tr_dataslot = 0;
ct_ref_fnode(trans->tr_fl_node);
trans->tr_cleanup = ct_extract_cleanup_fnode;
ct_queue_first(state, trans);
break;
case XS_RET_FILE_END:
trans = ct_trans_realloc_local(state, trans);
if (ex_priv->doextract == 0 ||
ex_priv->fl_ex_node->fn_skip_file != 0) {
/* release our reference done with file */
if (ex_priv->fl_ex_node) {
ct_free_fnode(ex_priv->fl_ex_node);
ex_priv->fl_ex_node = NULL;
}
ct_trans_free(state, trans);
continue;
}
/* use saved fnode from state */
trans->tr_fl_node = ex_priv->fl_ex_node;
bcopy(ex_priv->xdr_ctx.xs_trl.cmt_sha, trans->tr_sha,
sizeof(trans->tr_sha));
trans->tr_state = TR_S_EX_FILE_END;
trans->tr_complete = ct_extract_complete_file_end;
trans->tr_cleanup = ct_extract_cleanup_fnode;
trans->tr_fl_node->fn_size =
ex_priv->xdr_ctx.xs_trl.cmt_orig_size;
/*
* no reference here since we give our reference to the
* last transaction on that file. We are done with it.
*/
ex_priv->fl_ex_node = NULL;
ct_queue_first(state, trans);
break;
case XS_RET_EOF:
CNDBG(CT_LOG_CTFILE, "Hit end of ctfile");
ctfile_parse_close(&ex_priv->xdr_ctx);
/* if rb tree and rb is empty, goto end state */
if ((ex_priv->haverb || ex_priv->fillrb) &&
ct_extract_rb_empty(&ex_priv->pending_tree)) {
/*
* Cleanup extract queue, in case we had files
* left.
*/
ct_extract_cleanup_queue(
&ex_priv->extract_head);
goto we_re_done_here;
}
if (!TAILQ_EMPTY(&ex_priv->extract_head)) {
/*
* if allfiles and this was the first pass.
* free the current match lists
* switch to rb tree mode
*/
if (ex_priv->fillrb) {
ct_match_unwind(ex_priv->inc_match);
if (ex_priv->ex_match)
ct_match_unwind(
ex_priv->ex_match);
ex_priv->ex_match = NULL;
ex_priv->inc_match = NULL;
ex_priv->haverb = 1;
ex_priv->fillrb = 0;
}
ct_trans_free(state, trans);
/* reinits ex_priv->xdr_ctx */
if ((ret =
ct_extract_open_next(&ex_priv->extract_head,
&ex_priv->xdr_ctx)) != 0) {
ct_fatal(state,
"Can't open next ctfile", ret);
goto dying;
}
state->ct_print_ctfile_info(
state->ct_print_state,
ex_priv->xdr_ctx.xs_filename,
&ex_priv->xdr_ctx.xs_gh);
/* poke file into action */
ct_wakeup_file(state->event_state);
} else {
/*
* If rb tree and it is still has entries,
* bitch about it
*/
/* XXX print out missing files */
if ((ex_priv->haverb || ex_priv->fillrb) &&
ct_extract_rb_empty(
&ex_priv->pending_tree)) {
CWARNX("out of ctfiles but some "
"files are not found");
}
we_re_done_here:
if (ex_priv->inc_match)
ct_match_unwind(ex_priv->inc_match);
if (ex_priv->ex_match)
ct_match_unwind(
ex_priv->ex_match);
ct_extract_pending_cleanup(
&ex_priv->pending_tree);
e_free(&ex_priv);
op->op_priv = NULL;
trans->tr_state = TR_S_DONE;
trans->tr_complete = ct_extract_complete_done;
trans->tr_cleanup = ct_extract_cleanup_done;
/*
* Technically this should be a local
* transaction. However, since we are done
* it doesn't really matter either way.
*/
ct_queue_first(state, trans);
CNDBG(CT_LOG_TRANS, "extract finished");
ct_set_file_state(state, CT_S_FINISHED);
}
return;
break;
case XS_RET_FAIL:
ct_fatal(state, "Failed to parse ctfile",
ex_priv->xdr_ctx.xs_errno);
goto dying;
break;
}
}
return;
dying:
/* only if we hadn't sent the final transaction yet */
if (ex_priv != NULL) {
ct_extract_cleanup_queue(&ex_priv->extract_head);
if (ex_priv->inc_match)
ct_match_unwind(ex_priv->inc_match);
if (ex_priv->ex_match)
ct_match_unwind(ex_priv->ex_match);
if (!ct_extract_rb_empty(&ex_priv->pending_tree)) {
ct_extract_pending_cleanup(&ex_priv->pending_tree);
}
if (ex_priv->fl_ex_node != NULL) {
ct_free_fnode(ex_priv->fl_ex_node);
}
/* XXX what about ex_priv->xdr_ctx ? */
e_free(&ex_priv);
op->op_priv = NULL;
/* if ex_priv is gone then the trans will clean this up */
if (state->extract_state)
ct_file_extract_cleanup(state->extract_state);
}
return;
}
/*
* Perform EXISTS checking on every sha in a ctfile chain.
*
* We don't do any filtering. It is assumed that the localdb has been
* flushed/made good before this operation starts so that we can trust lookups.
*/
void
ct_exists_file(struct ct_global_state *state, struct ct_op *op)
{
struct ct_exists_args *ce = op->op_args;
struct ct_exists_priv *ex_priv = op->op_priv;
struct ct_trans *trans;
int ret, allfiles;
/* if we were woken up due to fatal, just clean up local state */
if (state->ct_dying != 0)
goto dying;
CNDBG(CT_LOG_TRANS, "entry");
switch (ct_get_file_state(state)) {
case CT_S_STARTING:
if (ex_priv == NULL) {
ex_priv = e_calloc(1, sizeof(*ex_priv));
TAILQ_INIT(&ex_priv->extract_head);
op->op_priv = ex_priv;
}
if ((ret = ct_extract_setup(&ex_priv->extract_head,
&ex_priv->xdr_ctx, ce->ce_ctfile, ce->ce_ctfile_basedir,
&allfiles)) != 0) {
ct_fatal(state, "can't setup extract queue", ret);
goto dying;
}
break;
case CT_S_FINISHED:
return;
default:
break;
}
ct_set_file_state(state, CT_S_RUNNING);
while (1) {
if ((trans = ct_trans_alloc(state)) == NULL) {
CNDBG(CT_LOG_TRANS, "ran out of transactions, waiting");
ct_set_file_state(state, CT_S_WAITING_TRANS);
return;
}
trans->tr_statemachine = ct_state_exists;
switch ((ret = ctfile_parse(&ex_priv->xdr_ctx))) {
case XS_RET_FILE:
case XS_RET_FILE_END:
ct_trans_free(state, trans);
break;
case XS_RET_SHA:
if (memcmp(zerosha, ex_priv->xdr_ctx.xs_sha,
SHA_DIGEST_LENGTH) == 0) {
if (ctfile_parse_seek(&ex_priv->xdr_ctx)) {
ct_fatal(state, "Can't seek past "
"truncation shas",
ex_priv->xdr_ctx.xs_errno);
goto dying;
}
ct_trans_free(state, trans);
continue;
}
if (ex_priv->xdr_ctx.xs_gh.cmg_flags & CT_MD_CRYPTO) {
/*
* yes csha and sha are reversed, we want
* to download csha, but putting it in sha
* simplifies the code
*/
bcopy(ex_priv->xdr_ctx.xs_sha, trans->tr_sha,
sizeof(trans->tr_csha));
bcopy(ex_priv->xdr_ctx.xs_csha, trans->tr_csha,
sizeof(trans->tr_sha));
bcopy(ex_priv->xdr_ctx.xs_iv, trans->tr_iv,
sizeof(trans->tr_iv));
trans->tr_state = TR_S_COMPSHA_ED;
} else {
trans->tr_state = TR_S_UNCOMPSHA_ED;
bcopy(ex_priv->xdr_ctx.xs_sha, trans->tr_sha,
sizeof(trans->tr_sha));
}
if (clog_mask_is_set(CT_LOG_SHA)) {
char shat[SHA_DIGEST_STRING_LENGTH];
ct_sha1_encode(trans->tr_sha, shat);
CNDBG(CT_LOG_SHA, "EXISTSing sha %s", shat);
}
trans->tr_old_genid = -1; /* XXX */
if (ctdb_lookup_sha(state->ct_db_state, trans->tr_sha,
trans->tr_csha, trans->tr_iv,
&trans->tr_old_genid)) {
CNDBG(CT_LOG_SHA, "sha already in localdb");
state->ct_stats->st_bytes_exists +=
trans->tr_chsize;
ct_trans_free(state, trans);
continue;
}
trans->tr_complete = ct_exists_complete;
trans->tr_cleanup = NULL;
trans->tr_dataslot = 0;
ct_queue_first(state, trans);
break;
case XS_RET_EOF:
CNDBG(CT_LOG_CTFILE, "Hit end of ctfile");
ctfile_parse_close(&ex_priv->xdr_ctx);
if (!TAILQ_EMPTY(&ex_priv->extract_head)) {
/*
* if allfiles and this was the first pass.
* free the current match lists
* switch to rb tree mode
*/
ct_trans_free(state, trans);
/* reinits ex_priv->xdr_ctx */
if ((ret =
ct_extract_open_next(&ex_priv->extract_head,
&ex_priv->xdr_ctx)) != 0) {
ct_fatal(state,
"Can't open next ctfile", ret);
goto dying;
}
} else {
e_free(&ex_priv);
op->op_priv = NULL;
trans->tr_state = TR_S_DONE;
trans->tr_complete = ct_exists_complete_done;
trans->tr_cleanup = NULL;
/*
* Technically this should be a local
* transaction. However, since we are done
* it doesn't really matter either way.
*/
ct_queue_first(state, trans);
CNDBG(CT_LOG_TRANS, "extract finished");
ct_set_file_state(state, CT_S_FINISHED);
}
return;
break;
case XS_RET_FAIL:
ct_fatal(state, "Failed to parse ctfile",
ex_priv->xdr_ctx.xs_errno);
goto dying;
break;
}
}
return;
dying:
/* only if we hadn't sent the final transaction yet */
if (ex_priv != NULL) {
ct_extract_cleanup_queue(&ex_priv->extract_head);
/* XXX what about ex_priv->xdr_ctx ? */
e_free(&ex_priv);
op->op_priv = NULL;
}
return;
}
/*
* Main guts of ctd_build_version_tree. Factored out to avoid deep nesting.
* Insert or update an entry in the tree with the information received from
* the ctfile.
*/
static int
ct_vertree_add(struct ct_vertree_dnode_cache *dnode_cache,
struct ct_vertree_entry *head, struct ctfile_parse_state *parse_state,
struct ct_vertree_ctfile *ctfile, off_t fileoffset, int allfiles)
{
struct ctfile_header *hdr = &parse_state->xs_hdr;
struct ctfile_header *hdrlnk= &parse_state->xs_lnkhdr;
struct dnode *dnode;
struct ct_vertree_dnode *fb_dnode;
struct ct_vertree_entry *parent = NULL, sentry, *entry;
struct ct_vertree_ver *lastver, *ver;
struct ct_vertree_file *file;
struct ct_vertree_spec *spec;
struct ct_vertree_link *linkver;
size_t sz;
bool root_dnode = false;
entry = NULL;
/* First find parent directory if any */
if (hdr->cmh_parent_dir != -1 && hdr->cmh_parent_dir != -2) {
if ((dnode = ctfile_parse_finddir(parse_state,
hdr->cmh_parent_dir)) == NULL) {
CNDBG(CT_LOG_VERTREE, "can't find dir %" PRId64,
hdr->cmh_parent_dir);
return (CTE_CTFILE_CORRUPT);
}
fb_dnode = (struct ct_vertree_dnode *)dnode;
if (fb_dnode == dnode_cache->root_dnode) {
// If we have the root dnode, store in head.
parent = head;
} else {
parent = fb_dnode->cvd_dir;
}
} else {
parent = head;
}
if (parent == head && strcmp(hdr->cmh_filename, CT_PATHSEP_STR) == 0) {
root_dnode = true;
}
/*
* Have parent node, search children to see if we already exist.
* Else make a new one and insert.
*/
sentry.cve_name = hdr->cmh_filename;
if ((entry = RB_FIND(ct_vertree_entries, &parent->cve_children,
&sentry)) == NULL) {
/* new name, insert node */
entry = e_calloc(1, sizeof(*entry));
TAILQ_INIT(&entry->cve_versions);
RB_INIT(&entry->cve_children);
entry->cve_parent = parent;
entry->cve_name = e_strdup(sentry.cve_name);
/* don't insert root dnodes, just do dnode dance */
if (root_dnode) {
goto rootdir;
}
if (RB_INSERT(ct_vertree_entries, &parent->cve_children,
entry) != NULL) {
CNDBG(CT_LOG_VERTREE, "entry %s already exists",
sentry.cve_name);
e_free(&sentry.cve_name);
goto err;
}
}
/*
* then check version tags -> head/tail if mtime and type match, we're
* good else prepare version entry.
*/
if (allfiles) {
lastver = TAILQ_FIRST(&entry->cve_versions);
} else {
lastver = TAILQ_LAST(&entry->cve_versions, ct_vertree_vers);
}
/* Don't check atime, it doesn't matter */
if (lastver != NULL && lastver->cvv_type == hdr->cmh_type &&
lastver->cvv_mtime == hdr->cmh_mtime &&
lastver->cvv_uid == hdr->cmh_uid &&
lastver->cvv_gid == hdr->cmh_gid &&
lastver->cvv_mode == hdr->cmh_mode) {
ver = lastver;
} else { /* something changed. make a new one */
if (C_ISDIR(hdr->cmh_type)) {
sz = sizeof(struct ct_vertree_dir);
} else if (C_ISBLK(hdr->cmh_type) ||
C_ISCHR(hdr->cmh_type)) {
sz = sizeof(struct ct_vertree_spec);
} else if (C_ISLINK(hdr->cmh_type)) {
sz = sizeof(struct ct_vertree_link);
} else if (C_ISREG(hdr->cmh_type)) {
sz = sizeof(struct ct_vertree_file);
} else {
CNDBG(CT_LOG_VERTREE, "invalid type %d", hdr->cmh_type);
goto err;
}
ver = e_calloc(1, sz);
ver->cvv_type = hdr->cmh_type;
ver->cvv_mtime = hdr->cmh_mtime;
ver->cvv_atime = hdr->cmh_atime;
ver->cvv_uid = hdr->cmh_uid;
ver->cvv_gid = hdr->cmh_gid;
ver->cvv_mode = hdr->cmh_mode;
/* dir handled below */
if (C_ISBLK(hdr->cmh_type) || C_ISCHR(hdr->cmh_type)) {
spec = (struct ct_vertree_spec *)ver;
spec->cvs_rdev = hdr->cmh_rdev;
} else if (C_ISLINK(hdr->cmh_type)) {
/* hardlink/symlink */
linkver = (struct ct_vertree_link *)ver;
linkver->cvl_linkname = e_strdup(hdrlnk->cmh_filename);
linkver->cvl_hardlink = !C_ISLINK(hdrlnk->cmh_type);
} else if (C_ISREG(hdr->cmh_type)) {
file = (struct ct_vertree_file *)ver;
file->cvf_nr_shas = -1;
}
if (allfiles) {
TAILQ_INSERT_HEAD(&entry->cve_versions, ver, cvv_link);
} else {
TAILQ_INSERT_TAIL(&entry->cve_versions, ver, cvv_link);
}
}
/*
* Each ctfile only has each directory referenced once, so put it
* in the cache regardless of whether it was known of before, that
* will be a previous run and the cache will have been wiped since
* then.
*/
if (C_ISDIR(hdr->cmh_type)) {
rootdir:
fb_dnode = e_calloc(1, sizeof(*fb_dnode));
fb_dnode->cvd_dnode.d_name = e_strdup(entry->cve_name);
/*
* in the root_dnode case this will be a bad pointer but it
* will never be derefed.
*/
fb_dnode->cvd_dir = entry;
if ((dnode = ctfile_parse_insertdir(parse_state,
&fb_dnode->cvd_dnode)) != NULL)
CABORTX("duplicate dentry");
TAILQ_INSERT_TAIL(&dnode_cache->cache, fb_dnode, cvd_link);
if (root_dnode) {
dnode_cache->root_dnode = fb_dnode;
}
} else if (C_ISREG(hdr->cmh_type)) {
/*
* Allfiles ctfiles may have shas == -1, so in some cases we
* may wish to update an existing file when we find the actual
* shas. It is an error to have a file node with -1 for shas
* after all metadata have been parsed. it means one was
* missing.
*/
file = (struct ct_vertree_file *)ver;
/*
* bugs in previous editions with incremental selection and
* off_t on linux mean that there are ctfiles in the wild which
* provide a list of shas in a later level when the file is
* defined in an earlier level file, also. For example for the
* same filename and date we have level 0: 3 shas, level 1: -1
* shas (i.e. in a previous level), level 2: 3 shas (same as
* level * 0). In that case we just assume that if we already
* have sha data for a file * then it is correct and we skip
* previous versions.
*/
if (file->cvf_nr_shas != -1) {
goto out;
}
/*
* previous linux off_t bugs with files over 2gb mean that there
* are sign extended ctfiles in the wild, so we count those as
* zero length for purposes of the version tree.
*/
if (hdr->cmh_nr_shas < -1) {
hdr->cmh_nr_shas = 0;
}
if (hdr->cmh_nr_shas != -1) {
file->cvf_nr_shas = hdr->cmh_nr_shas;
file->cvf_sha_offs = fileoffset;
file->cvf_file = ctfile;
if (ctfile_parse_seek(parse_state)) {
CNDBG(CT_LOG_VERTREE,
"failed to skip shas in %s",
ctfile->cvc_path);
goto err;
}
}
if (ctfile_parse(parse_state) != XS_RET_FILE_END) {
CNDBG(CT_LOG_VERTREE, "no file trailer found");
goto err;
}
file->cvf_file_size = parse_state->xs_trl.cmt_orig_size;
}
out:
/*
* If we're an explicit "/" entry then we don't want to be added to
* the tree. all our children will be added to the root entry.
*/
if (root_dnode) {
e_free(&entry);
}
return (0);
err:
if (entry != NULL)
e_free(&entry);
return (CTE_CTFILE_CORRUPT);
}
int
ct_version_tree_build(const char *filename, const char *ctfile_basedir,
struct ct_version_tree **version_tree)
{
struct ct_version_tree *tree = NULL;
struct ct_extract_head extract_head;
struct ctfile_parse_state parse_state;
struct ct_vertree_dnode_cache dnode_cache;
struct ct_vertree_dnode *dnode_entry;
struct ct_vertree_ctfile *ctfile = NULL;
struct ct_vertree_dir *root_dir;
struct ct_vertree_ver *root_version;
off_t offset;
int allfiles;
int rv = 0;
TAILQ_INIT(&extract_head);
TAILQ_INIT(&dnode_cache.cache);
dnode_cache.root_dnode = NULL;
if ((rv = ct_extract_setup(&extract_head, &parse_state, filename,
ctfile_basedir, &allfiles))) {
CNDBG(CT_LOG_VERTREE,
"failed to setup extract for filename %s: %s",
filename, ct_strerror(rv));
goto out;
}
/* Create and init ctfile cache */
tree = e_calloc(1, sizeof(*tree));
TAILQ_INIT(&tree->cvt_ctfiles);
TAILQ_INIT(&tree->cvt_head.cve_versions);
RB_INIT(&tree->cvt_head.cve_children);
tree->cvt_head.cve_name = e_strdup("/");
nextfile:
root_dir = e_calloc(1, sizeof(*root_dir));
root_version = &root_dir->cvd_base;
root_version->cvv_type = C_TY_DIR;
root_version->cvv_uid = 0;
root_version->cvv_gid = 0;
root_version->cvv_mode = 0777;
root_version->cvv_atime = parse_state.xs_gh.cmg_created;
root_version->cvv_mtime = parse_state.xs_gh.cmg_created;
TAILQ_INSERT_HEAD(&tree->cvt_head.cve_versions, root_version, cvv_link);
/*
* Create only one struct for each ctfile. Each entry in the version
* tree references the appropriate one. These are added to a cache list
* so they can be freed during tree cleanup.
*/
ctfile = e_calloc(1, sizeof(*ctfile));
strlcpy(ctfile->cvc_path, parse_state.xs_filename,
sizeof(ctfile->cvc_path));
offset = ctfile_parse_tell(&parse_state);
TAILQ_INSERT_TAIL(&tree->cvt_ctfiles, ctfile, cvc_link);
while (((rv = ctfile_parse(&parse_state)) != XS_RET_EOF) &&
(rv != XS_RET_FAIL)) {
switch(rv) {
case XS_RET_FILE:
if ((rv = ct_vertree_add(&dnode_cache, &tree->cvt_head,
&parse_state, ctfile, offset, allfiles)) != 0) {
goto out;
}
break;
case XS_RET_FILE_END:
break;
case XS_RET_SHA:
if ((rv = ctfile_parse_seek(&parse_state))) {
goto out;
}
break;
default:
rv = CTE_CTFILE_CORRUPT;
goto out;
}
offset = ctfile_parse_tell(&parse_state);
}
if (rv == XS_RET_EOF) {
ctfile_parse_close(&parse_state);
if (!TAILQ_EMPTY(&extract_head)) {
/* XXX do we need to zero root dnode? */
ct_extract_open_next(&extract_head, &parse_state);
goto nextfile;
}
rv = 0;
/* free state */
} else {
rv = CTE_CTFILE_CORRUPT;
goto out;
}
*version_tree = tree;
out:
/* Free dnode_cache entries. */
while ((dnode_entry = TAILQ_FIRST(&dnode_cache.cache)) != NULL) {
TAILQ_REMOVE(&dnode_cache.cache, dnode_entry, cvd_link);
if (dnode_entry->cvd_dnode.d_name != NULL)
e_free(&dnode_entry->cvd_dnode.d_name);
e_free(&dnode_entry);
}
return rv;
}
/*
* wrap the event code in this file.
*/
struct ct_event_state *
ct_event_init(struct ct_global_state *state,
void (*reconn_cb)(evutil_socket_t, short, void *),
void (*info_cb)(evutil_socket_t, short, void *))
{
struct ct_event_state *ev_st;
ev_st = e_calloc(1, sizeof(*ev_st));
ev_st->ct_evt_base = event_base_new();
if (ev_st->ct_evt_base == NULL) {
ct_event_cleanup(ev_st);
return (NULL);
}
/* cache siginfo */
if (info_cb != NULL) {
#if defined(SIGINFO) && SIGINFO != SIGUSR1
ev_st->ct_ev_sig_info = evsignal_new(ev_st->ct_evt_base,
SIGINFO, info_cb, state);
if (ev_st->ct_ev_sig_info == NULL) {
ct_event_cleanup(ev_st);
return (NULL);
}
evsignal_add(ev_st->ct_ev_sig_info, NULL);
#endif
#if defined(SIGUSR1)
ev_st->ct_ev_sig_usr1 = evsignal_new(ev_st->ct_evt_base,
SIGUSR1, info_cb, state);
if (ev_st->ct_ev_sig_usr1 == NULL) {
ct_event_cleanup(ev_st);
return (NULL);
}
evsignal_add(ev_st->ct_ev_sig_usr1, NULL);
#endif
}
#if defined(SIGPIPE)
ev_st->ct_ev_sig_pipe = evsignal_new(ev_st->ct_evt_base, SIGPIPE,
ct_pipe_sig, state);
if (ev_st->ct_ev_sig_pipe == NULL) {
ct_event_cleanup(ev_st);
return (NULL);
}
evsignal_add(ev_st->ct_ev_sig_pipe, NULL);
#endif
ev_st->recon_ev = evtimer_new(ev_st->ct_evt_base, reconn_cb, state);
if (ev_st->recon_ev == NULL) {
ct_event_cleanup(ev_st);
return (NULL);
}
ev_st->keepalive_ev = evtimer_new(ev_st->ct_evt_base, ct_keepalive,
state);
if (ev_st->keepalive_ev == NULL) {
ct_event_cleanup(ev_st);
return (NULL);
}
ct_set_keepalive_timeout(ev_st, CT_KEEPALIVE_TIMEOUT);
return (ev_st);
}
int
ct_assl_negotiate_poll(struct ct_global_state *state)
{
void *body;
struct ct_header hdr;
ssize_t sz;
int rv = 1;
int payload_sz;
uint8_t buf[20];
/* send server request */
if ((rv = ct_create_neg(&hdr, &body, state->ct_max_trans,
state->ct_max_block_size)) != 0)
goto done;
payload_sz = hdr.c_size;
ct_wire_header(&hdr);
if (ct_assl_io_write_poll(state->ct_assl_ctx, &hdr, sizeof hdr,
ASSL_TIMEOUT) != sizeof hdr) {
rv = CTE_SHORT_WRITE;
goto done;
}
if (ct_assl_io_write_poll(state->ct_assl_ctx, body, payload_sz,
ASSL_TIMEOUT) != payload_sz) {
rv = CTE_SHORT_WRITE;
goto done;
}
/* get server reply */
sz = ct_assl_io_read_poll(state->ct_assl_ctx, &hdr, sizeof hdr,
ASSL_TIMEOUT);
if (sz != sizeof hdr) {
rv = CTE_SHORT_READ;
CWARNX("invalid header size %ld", (long) sz);
goto done;
}
ct_unwire_header(&hdr);
/* negotiate reply is the same size as the request, so reuse the body */
if (hdr.c_size != payload_sz) {
rv = CTE_INVALID_REPLY_LEN;
CWARNX("invalid negotiate reply size %d", hdr.c_size);
goto done;
}
if (ct_assl_io_read_poll(state->ct_assl_ctx, buf,
hdr.c_size, ASSL_TIMEOUT) != hdr.c_size) {
CWARNX("couldn't read neg parameters");
rv = CTE_SHORT_READ;
goto done;
}
if ((rv = ct_parse_neg_reply(&hdr, buf, &state->ct_max_trans,
&state->ct_max_block_size)) != 0) {
goto done;
}
e_free(&body);
CNDBG(CT_LOG_NET, "negotiated queue depth: %u max chunk size: %u",
state->ct_max_trans, state->ct_max_block_size);
if ((rv = ct_create_login(&hdr, &body, state->ct_config->ct_username,
state->ct_config->ct_password)) != 0)
goto done;
payload_sz = hdr.c_size;
ct_wire_header(&hdr);
if (ct_assl_io_write_poll(state->ct_assl_ctx, &hdr, sizeof hdr,
ASSL_TIMEOUT) != sizeof hdr) {
rv = CTE_SHORT_WRITE;
goto done;
}
if (ct_assl_io_write_poll(state->ct_assl_ctx, body, payload_sz,
ASSL_TIMEOUT) != payload_sz) {
rv = CTE_SHORT_WRITE;
goto done;
}
/* get server reply */
sz = ct_assl_io_read_poll(state->ct_assl_ctx, &hdr, sizeof hdr,
ASSL_TIMEOUT);
if (sz != sizeof hdr) {
rv = CTE_SHORT_READ;
goto done;
}
e_free(&body);
ct_unwire_header(&hdr);
/* XXX need a way to get error crud out, right now the function warns
for us. */
if ((rv = ct_parse_login_reply(&hdr, NULL)) != 0)
goto done;
if (ct_skip_xml_negotiate) {
goto out;
}
if ((rv = ct_create_xml_negotiate(&hdr, &body,
ctdb_get_genid(state->ct_db_state))) != 0) {
goto done;
}
payload_sz = hdr.c_size;
ct_wire_header(&hdr);
if (ct_assl_io_write_poll(state->ct_assl_ctx, &hdr, sizeof hdr,
ASSL_TIMEOUT) != sizeof hdr) {
rv = CTE_SHORT_WRITE;
goto done;
}
if (ct_assl_io_write_poll(state->ct_assl_ctx, body, payload_sz,
ASSL_TIMEOUT)
!= payload_sz) {
rv = CTE_SHORT_WRITE;
CWARNX("could not write body");
goto done;
}
e_free(&body);
/* get server reply */
sz = ct_assl_io_read_poll(state->ct_assl_ctx, &hdr, sizeof hdr,
ASSL_TIMEOUT);
if (sz != sizeof hdr) {
rv = CTE_SHORT_READ;
CWARNX("invalid header size %" PRId64, (int64_t)sz);
goto done;
}
ct_unwire_header(&hdr);
if (hdr.c_size == 0) {
goto out;
}
/* get server reply body */
body = e_calloc(1, hdr.c_size);
sz = ct_assl_io_read_poll(state->ct_assl_ctx, body, hdr.c_size,
ASSL_TIMEOUT);
if (sz != hdr.c_size) {
rv = CTE_SHORT_READ;
goto done;
}
/* XXX check xml data */
if ((rv = ct_parse_xml_negotiate_reply(&hdr, body,
state->ct_db_state)) != 0) {
e_free(&body);
goto done;
}
e_free(&body);
out:
CNDBG(CT_LOG_NET, "login successful");
rv = 0;
done:
return (rv);
}
/*
* List has completed.
*
* Select the best filename for download, and download it if missing.
*/
int
ctfile_find_for_extract_complete(struct ct_global_state *state,
struct ct_op *op)
{
struct ct_ctfile_find_args *ccfa = op->op_args;
struct ct_ctfile_find_fileop_args *ccffa;
struct ct_op *list_fakeop = op->op_priv;
struct ct_ctfile_list_args *ccla = list_fakeop->op_args;
struct ctfile_list_tree result;
struct ctfile_list_file *tmp;
char *best = NULL;
int ret = 0;
RB_INIT(&result);
ctfile_list_complete(&state->ctfile_list_files, ccla->ccla_matchmode,
ccla->ccla_search, ccla->ccla_exclude, &result);
e_free(ccla->ccla_search);
e_free(&ccla->ccla_search);
e_free(&ccla);
e_free(&list_fakeop);
/*
* Prepare arguments for next operation.
* either we'll download the next file, or skip straight to
* the callback for after the download, either way we need the nextop
*/
ccffa = e_calloc(1, sizeof(*ccffa));
ccffa->ccffa_nextop = ccfa->ccfa_nextop;
ccffa->ccffa_nextop_args = ccfa->ccfa_nextop_args;
ccffa->ccffa_download_chain = ccfa->ccfa_download_chain;
/* grab the newest one */
if ((tmp = RB_MAX(ctfile_list_tree, &result)) == NULL) {
if (ccfa->ccfa_empty_ok)
goto do_operation;
else {
CWARNX("%s: %s", ccfa->ccfa_tag,
ct_strerror(CTE_NO_SUCH_BACKUP));
ret = CTE_NO_SUCH_BACKUP;
e_free(&ccffa);
goto out;
}
}
/* pick the newest one */
best = e_strdup(tmp->mlf_name);
CNDBG(CT_LOG_CTFILE, "backup file is %s", best);
while ((tmp = RB_ROOT(&result)) != NULL) {
RB_REMOVE(ctfile_list_tree, &result, tmp);
e_free(&tmp);
}
/*
* if the metadata file is not in the cache directory then we
* need to download it first. if we need to recursively download
* an incremental chain then that code will handle scheduling
* those operations too. If we have it, we still need to check
* that all others in the chain exist, however.
*/
if (!ctfile_in_cache(best, ccfa->ccfa_cachedir)) {
ccffa->ccffa_base.cca_localname = best;
ccffa->ccffa_base.cca_tdir = ccfa->ccfa_cachedir;
ccffa->ccffa_base.cca_remotename = e_strdup(best);
ccffa->ccffa_base.cca_ctfile = 1;
ct_add_operation(state, ctfile_extract, ctfile_extract_nextop,
ccffa);
} else {
do_operation:
/*
* No download needed, fake the next operation callback
* to see if we need anymore.
*/
ccffa->ccffa_base.cca_localname = best;
ccffa->ccffa_base.cca_tdir = ccfa->ccfa_cachedir;
ccffa->ccffa_base.cca_ctfile = 1;
op->op_args = ccffa;
ctfile_extract_nextop(state, op);
}
out:
e_free(&ccfa);
return (ret);
}
/*
* Download all dependent ctfiles of the current ctfile.
* (called repeatedly until all are fetched).
*/
int
ctfile_download_next(struct ct_global_state *state, struct ct_op *op)
{
struct ct_ctfileop_args *cca = op->op_args, *nextcca;
const char *ctfile = cca->cca_localname;
const char *rfile = cca->cca_remotename;
char *prevfile;
char *cookedname;
int ret = 0;
again:
CNDBG(CT_LOG_CTFILE, "ctfile %s", ctfile);
if ((ret = ctfile_get_previous(ctfile, cca->cca_tdir,
&prevfile)) != 0) {
CWARNX("can not get previous filename for %s", ctfile);
/* error output will happen when even loop returns */
goto out;
}
if (prevfile == NULL) /* done with this chain */
goto out;
if (prevfile[0] != '\0') {
if ((cookedname = ctfile_cook_name(prevfile)) == NULL) {
CWARNX("%s: %s", prevfile,
ct_strerror(CTE_INVALID_CTFILE_NAME));
ret = CTE_INVALID_CTFILE_NAME;
e_free(&prevfile);
goto out;
}
CNDBG(CT_LOG_CTFILE, "prev file %s cookedname %s", prevfile,
cookedname);
if (!ctfile_in_cache(cookedname, cca->cca_tdir)) {
nextcca = e_calloc(1, sizeof(*nextcca));
nextcca->cca_localname = cookedname;
nextcca->cca_remotename = e_strdup(cookedname);
nextcca->cca_tdir = cca->cca_tdir;
nextcca->cca_ctfile = 1;
ct_add_operation_after(state, op, ctfile_extract,
ctfile_download_next, nextcca);
} else {
if (ctfile)
e_free(&ctfile);
if (rfile)
e_free(&rfile);
e_free(&cookedname);
ctfile = prevfile;
goto again;
}
} else
e_free(&prevfile);
out:
if (ctfile)
e_free(&ctfile);
if (rfile)
e_free(&rfile);
e_free(&cca);
return (ret);
}
int
ctfile_nextop_archive(struct ct_global_state *state, char *basis, void *args)
{
struct ct_archive_args *caa = args;
struct ct_ctfileop_args *cca;
char *ctfile;
char buf[TIMEDATA_LEN], *fullname, *cachename;
time_t now;
CNDBG(CT_LOG_CTFILE, "setting basisname %s", basis ? basis : "<none>");
caa->caa_basis = basis;
/*
* We now have the basis found for us, cook and prepare the tag
* we wish to create then add the operation.
*/
if ((ctfile = ctfile_cook_name(caa->caa_tag)) == NULL) {
CWARNX("%s: %s", caa->caa_tag,
ct_strerror(CTE_INVALID_CTFILE_NAME));
return (CTE_INVALID_CTFILE_NAME);
}
if (ctfile_is_fullname(ctfile) != 0) {
CWARNX("%s", ct_strerror(CTE_ARCHIVE_FULLNAME));
e_free(&ctfile);
return (CTE_ARCHIVE_FULLNAME);
}
now = time(NULL);
if (strftime(buf, TIMEDATA_LEN, "%Y%m%d-%H%M%S",
localtime(&now)) == 0)
CABORTX("can't format time");
e_asprintf(&fullname, "%s-%s", buf, ctfile);
CNDBG(CT_LOG_CTFILE, "backup file is %s", fullname);
/* check it isn't already in the cache */
cachename = ctfile_get_cachename(fullname,
state->ct_config->ct_ctfile_cachedir);
if (ctfile_in_cache(fullname, state->ct_config->ct_ctfile_cachedir)) {
CWARNX("%s: %s", fullname,
ct_strerror(CTE_BACKUP_ALREADY_EXISTS));
e_free(&ctfile);
e_free(&fullname);
e_free(&cachename);
return (CTE_BACKUP_ALREADY_EXISTS);
}
e_free(&ctfile);
e_free(&fullname);
caa->caa_local_ctfile = cachename;
ct_add_operation(state, ct_archive, NULL, caa);
/*
* set up an additional operation to upload the newly created
* ctfile after the archive is completed.
*/
cca = e_calloc(1, sizeof(*cca));
cca->cca_localname = cachename;
cca->cca_cleartext = 0;
cca->cca_ctfile = 1;
ct_add_operation(state, ctfile_archive, ctfile_nextop_archive_cleanup,
cca);
return (0);
}
