int ctdb_update_sha(struct ctdb_state *state, uint8_t *sha, int32_t genid) { sqlite3_stmt *stmt; char shat[SHA_DIGEST_STRING_LENGTH]; int rv = 0; rv = 0; if (state == NULL || state->ctdb_db == NULL) return rv; stmt = state->ctdb_stmt_update; ct_sha1_encode(sha, shat); if (state->ctdb_in_transaction == 0) { if (ctdb_begin_transaction(state) != 0) return (rv); state->ctdb_trans_commit_rem = OPS_PER_TRANSACTION; } if (clog_mask_is_set(CT_LOG_DB)) { ct_sha1_encode(sha, shat); CNDBG(CT_LOG_DB, "updating %s to genid %d ", shat, genid); } if (sqlite3_bind_int(stmt, 1, genid)) { CNDBG(CT_LOG_DB, "could not bind genid"); return rv; } if (sqlite3_bind_blob(stmt, 2, sha, SHA_DIGEST_LENGTH, SQLITE_STATIC)) { CNDBG(CT_LOG_DB, "could not bind sha"); return rv; } if (sqlite3_step(stmt) == SQLITE_DONE) { CNDBG(CT_LOG_DB, "update completed"); rv = 1; } else { CNDBG(CT_LOG_DB, "update failed %d [%s]", sqlite3_extended_errcode(state->ctdb_db), sqlite3_errmsg(state->ctdb_db)); } sqlite3_reset(stmt); /* inserts are more 'costly' than reads */ state->ctdb_trans_commit_rem -= 4; if (state->ctdb_trans_commit_rem <= 0) ctdb_end_transaction(state); return rv; }
int ct_extract_complete_file_read(struct ct_global_state *state, struct ct_trans *trans) { int slot, ret; state->ct_stats->st_chunks_completed++; if (trans->tr_errno != 0) { char *errstr; char shat[SHA_DIGEST_STRING_LENGTH]; /* any other read failure is bad */ ct_sha1_encode(trans->tr_sha, shat); e_asprintf(&errstr, "Data missing on server: " "file %s, sha %s", trans->tr_fl_node ? trans->tr_fl_node->fn_fullname : "unknown", shat); ct_fatal(state, errstr, trans->tr_errno); free(errstr); return (0); } if (trans->tr_fl_node->fn_skip_file == 0) { slot = trans->tr_dataslot; ct_sha1_add(trans->tr_data[slot], &trans->tr_fl_node->fn_shactx, trans->tr_size[slot]); if ((ret = ct_file_extract_write(state->extract_state, trans->tr_fl_node, trans->tr_data[slot], trans->tr_size[slot])) != 0) { /* * XXX really this shouldn't be fatal, just make us skip * the file in future and CWARNX. */ ct_fatal(state, "Failed to write file", ret); return (0); } state->ct_stats->st_bytes_written += trans->tr_size[slot]; } return (0); }
int ctdb_insert_sha(struct ctdb_state *state, uint8_t *sha_k, uint8_t *sha_v, uint8_t *iv, int32_t genid) { char shatk[SHA_DIGEST_STRING_LENGTH]; char shatv[SHA_DIGEST_STRING_LENGTH]; int rv, rc; sqlite3_stmt *stmt; rv = 0; if (state == NULL || state->ctdb_db == NULL) return rv; stmt = state->ctdb_stmt_insert; if (state->ctdb_in_transaction == 0) { if (ctdb_begin_transaction(state) != 0) return (rv); state->ctdb_trans_commit_rem = OPS_PER_TRANSACTION; } if (clog_mask_is_set(CT_LOG_DB)) { ct_sha1_encode(sha_k, shatk); if (sha_v == NULL) shatv[0] = '\0'; else ct_sha1_encode(sha_v, shatv); CNDBG(CT_LOG_DB, "inserting for bin %s, %s", shatk, shatv); } if (sqlite3_bind_blob(stmt, 1, sha_k, SHA_DIGEST_LENGTH, SQLITE_STATIC)) { CNDBG(CT_LOG_DB, "could not bind sha_k"); return rv; } if (state->ctdb_crypt) { if (sha_v == NULL || iv == NULL) CABORTX("crypt mode, but no sha_v/iv"); if (sqlite3_bind_blob(stmt, 2, sha_v, SHA_DIGEST_LENGTH, SQLITE_STATIC)) { CNDBG(CT_LOG_DB, "could not bind sha_v"); sqlite3_reset(stmt); return rv; } if (sqlite3_bind_blob(stmt, 3, iv, CT_IV_LEN, SQLITE_STATIC)) { CNDBG(CT_LOG_DB, "could not bind iv "); sqlite3_reset(stmt); return rv; } if (sqlite3_bind_int(stmt, 4, genid) != 0) { CNDBG(CT_LOG_DB, "could not bind genid"); sqlite3_reset(stmt); return rv; } } else { if (sqlite3_bind_int(stmt, 2, genid) != 0) { CNDBG(CT_LOG_DB, "could not bind genid"); sqlite3_reset(stmt); return rv; } } rc = sqlite3_step(stmt); if (rc == SQLITE_DONE) { CNDBG(CT_LOG_DB, "insert completed"); rv = 1; } else if (rc != SQLITE_CONSTRAINT) { CNDBG(CT_LOG_DB, "insert failed %d %d [%s]", rc, sqlite3_extended_errcode(state->ctdb_db), sqlite3_errmsg(state->ctdb_db)); } else { CNDBG(CT_LOG_DB, "sha already exists"); } sqlite3_reset(stmt); /* inserts are more 'costly' than reads */ state->ctdb_trans_commit_rem -= 4; if (state->ctdb_trans_commit_rem <= 0) ctdb_end_transaction(state); return rv; }
enum ctdb_lookup ctdb_lookup_sha(struct ctdb_state *state, uint8_t *sha_k, uint8_t *sha_v, uint8_t *iv, int32_t *old_genid) { char shat[SHA_DIGEST_STRING_LENGTH]; int rv, rc; int32_t genid; uint8_t *p; sqlite3_stmt *stmt; rv = CTDB_SHA_NEXISTS; *old_genid = -1; if (state == NULL || state->ctdb_db == NULL) return rv; stmt = state->ctdb_stmt_lookup; if (state->ctdb_in_transaction == 0) { if (ctdb_begin_transaction(state) != 0) return (rv); state->ctdb_trans_commit_rem = OPS_PER_TRANSACTION; } if (clog_mask_is_set(CT_LOG_DB)) { ct_sha1_encode(sha_k, shat); CNDBG(CT_LOG_DB, "looking for bin %s", shat); } if (sqlite3_bind_blob(stmt, 1, sha_k, SHA_DIGEST_LENGTH, SQLITE_STATIC)) { CNDBG(CT_LOG_DB, "could not sha"); return (CTDB_SHA_NEXISTS); } rc = sqlite3_step(stmt); if (rc == SQLITE_DONE) { CNDBG(CT_LOG_DB, "not found"); sqlite3_reset(stmt); return CTDB_SHA_NEXISTS; } else if (rc != SQLITE_ROW) { CNDBG(CT_LOG_DB, "could not step(%d) %d %d %s", __LINE__, rc, sqlite3_extended_errcode(state->ctdb_db), sqlite3_errmsg(state->ctdb_db)); sqlite3_reset(stmt); return CTDB_SHA_NEXISTS; } CNDBG(CT_LOG_DB, "found"); p = (uint8_t *)sqlite3_column_blob(stmt, 0); if (p) { if (sqlite3_column_bytes(stmt, 0) != SHA_DIGEST_LENGTH) { CNDBG(CT_LOG_DB, "invalid blob size"); sqlite3_reset(stmt); return rv; } if (clog_mask_is_set(CT_LOG_DB)) { ct_sha1_encode(p, shat); CNDBG(CT_LOG_DB, "found bin %s", shat); } rv = CTDB_SHA_EXISTS; bcopy (p, sha_v, SHA_DIGEST_LENGTH); } else { CNDBG(CT_LOG_DB, "no bin found"); } if (state->ctdb_crypt) { p = (uint8_t *)sqlite3_column_blob(stmt, 1); if (p) { if (sqlite3_column_bytes(stmt, 1) != CT_IV_LEN) { CNDBG(CT_LOG_DB, "invalid blob size"); sqlite3_reset(stmt); rv = CTDB_SHA_NEXISTS; return rv; } if (clog_mask_is_set(CT_LOG_DB)) { ct_sha1_encode(p, shat); CNDBG(CT_LOG_DB, "found iv (prefix) %s", shat); } bcopy (p, iv, CT_IV_LEN); } else { CNDBG(CT_LOG_DB, "no iv found"); rv = CTDB_SHA_NEXISTS; } genid = sqlite3_column_int(stmt, 2); } else { genid = sqlite3_column_int(stmt, 1); } sqlite3_reset(stmt); if (genid < state->ctdb_genid) { ct_sha1_encode(sha_k, shat); rv = CTDB_SHA_MAYBE_EXISTS; *old_genid = genid; } else if (genid > state->ctdb_genid) { /* XXX Abort? */ CWARNX("WARNING: sha with higher genid than database!"); } state->ctdb_trans_commit_rem--; if (state->ctdb_trans_commit_rem <= 0) { ctdb_end_transaction(state); } return rv; }
/* * Extract an individual file that has been passed into the op by op_priv. */ void ct_extract_file(struct ct_global_state *state, struct ct_op *op) { struct ct_extract_file_args *cefa = op->op_args; struct ct_file_extract_priv *ex_priv = op->op_priv; const char *localfile = cefa->cefa_filename; struct ct_trans *trans; int ret; char shat[SHA_DIGEST_STRING_LENGTH]; if (state->ct_dying != 0) goto dying; CNDBG(CT_LOG_TRANS, "entry"); switch (ct_get_file_state(state)) { case CT_S_STARTING: CNDBG(CT_LOG_TRANS, "starting"); ex_priv = e_calloc(1, sizeof(*ex_priv)); /* open file and seek to beginning of file */ if ((ret = ctfile_parse_init_at(&ex_priv->xdr_ctx, cefa->cefa_ctfile, NULL, cefa->cefa_ctfile_off)) != 0) { /* XXX add pathname */ ct_fatal(state, "Can't open ctfile", ret); e_free(&ex_priv); goto dying; } /* XXX we should handle this better */ if (state->ct_max_block_size < ex_priv->xdr_ctx.xs_gh.cmg_chunk_size) CABORTX("block size negotiated with server %d is " "smaller than file max block size %d", state->ct_max_block_size, ex_priv->xdr_ctx.xs_gh.cmg_chunk_size); if ((ret = ct_file_extract_init(&state->extract_state, NULL, 0, 0, 0, NULL, NULL)) != 0) { ct_fatal(state, "Can not initialise extract state", ret); e_free(&ex_priv); goto dying; } op->op_priv = ex_priv; break; case CT_S_FINISHED: return; default: break; } ct_set_file_state(state, CT_S_RUNNING); while (1) { if ((trans = ct_trans_alloc(state)) == NULL) { CNDBG(CT_LOG_TRANS, "ran out of transactions, waiting"); ct_set_file_state(state, CT_S_WAITING_TRANS); return; } trans->tr_statemachine = ct_state_extract; if (ex_priv->done) { CNDBG(CT_LOG_CTFILE, "Hit end of ctfile"); ctfile_parse_close(&ex_priv->xdr_ctx); e_free(&ex_priv); op->op_priv = NULL; trans->tr_state = TR_S_DONE; trans->tr_complete = ct_extract_complete_done; trans->tr_cleanup = ct_extract_cleanup_done; ct_queue_first(state, trans); CNDBG(CT_LOG_TRANS, "extract finished"); ct_set_file_state(state, CT_S_FINISHED); return; } /* unless start of file this is right */ trans->tr_fl_node = ex_priv->fl_ex_node; switch ((ret = ctfile_parse(&ex_priv->xdr_ctx))) { case XS_RET_FILE: CNDBG(CT_LOG_CTFILE, "opening file"); if (ex_priv->xdr_ctx.xs_hdr.cmh_nr_shas == -1) CABORTX("can't extract file with -1 shas"); trans = ct_trans_realloc_local(state, trans); trans->tr_fl_node = ex_priv->fl_ex_node = ct_alloc_fnode(); /* Make it local directory, it won't be set up right. */ ex_priv->xdr_ctx.xs_hdr.cmh_parent_dir = -1; /* * Allfiles doesn't matter, only processing one file. * We have a full path to extract to so always strip * slash. */ ct_populate_fnode(state->extract_state, &ex_priv->xdr_ctx, trans->tr_fl_node, &trans->tr_state, 0, 1); if (trans->tr_state == TR_S_EX_SPECIAL) { trans->tr_complete = ct_extract_complete_special; } else { trans->tr_complete = ct_extract_complete_file_start; } trans->tr_cleanup = ct_extract_cleanup_fnode; e_free(&trans->tr_fl_node->fn_fullname); trans->tr_fl_node->fn_fullname = e_strdup(localfile); e_free(&trans->tr_fl_node->fn_name); trans->tr_fl_node->fn_name = e_strdup(localfile); /* Set name pointer to something else passed in */ CNDBG(CT_LOG_CTFILE, "file %s numshas %" PRId64, trans->tr_fl_node->fn_fullname, ex_priv->xdr_ctx.xs_hdr.cmh_nr_shas); /* * special files we give our refcount up * regular files we need a new one since we need to * keep ours. */ if (trans->tr_state != TR_S_EX_SPECIAL) { ct_ref_fnode(trans->tr_fl_node); } else { ex_priv->fl_ex_node = NULL; } break; case XS_RET_SHA: CNDBG(CT_LOG_SHA, "sha!"); if (ex_priv->xdr_ctx.xs_gh.cmg_flags & CT_MD_CRYPTO) { /* * yes csha and sha are reversed, we want * to download csha, but putting it in sha * simplifies the code */ bcopy(ex_priv->xdr_ctx.xs_sha, trans->tr_csha, sizeof(trans->tr_csha)); bcopy(ex_priv->xdr_ctx.xs_csha, trans->tr_sha, sizeof(trans->tr_sha)); bcopy(ex_priv->xdr_ctx.xs_iv, trans->tr_iv, sizeof(trans->tr_iv)); } else { bcopy(ex_priv->xdr_ctx.xs_sha, trans->tr_sha, sizeof(trans->tr_sha)); } if (clog_mask_is_set(CT_LOG_SHA)) { ct_sha1_encode(trans->tr_sha, shat); CNDBG(CT_LOG_SHA, "extracting sha %s", shat); } trans->tr_state = TR_S_EX_SHA; trans->tr_complete = ct_extract_complete_file_read; trans->tr_cleanup = ct_extract_cleanup_fnode; trans->tr_dataslot = 0; ct_ref_fnode(trans->tr_fl_node); break; case XS_RET_FILE_END: trans = ct_trans_realloc_local(state, trans); trans->tr_fl_node = ex_priv->fl_ex_node; /* reload */ CNDBG(CT_LOG_CTFILE, "file end!"); bcopy(ex_priv->xdr_ctx.xs_trl.cmt_sha, trans->tr_sha, sizeof(trans->tr_sha)); trans->tr_state = TR_S_EX_FILE_END; trans->tr_complete = ct_extract_complete_file_end; trans->tr_cleanup = ct_extract_cleanup_fnode; trans->tr_fl_node->fn_size = ex_priv->xdr_ctx.xs_trl.cmt_orig_size; /* Done now, don't parse further. */ ex_priv->done = 1; /* * no reference here since we give our reference to the * last transaction on that file. */ ex_priv->fl_ex_node = NULL; break; case XS_RET_FAIL: ct_fatal(state, "Failed to parse ctfile", ex_priv->xdr_ctx.xs_errno); goto dying; break; default: CABORTX("%s: invalid state %d", __func__, ret); } ct_queue_first(state, trans); } return; dying: if (ex_priv) { ctfile_parse_close(&ex_priv->xdr_ctx); if (ex_priv->fl_ex_node != NULL) { ct_free_fnode(ex_priv->fl_ex_node); } e_free(&ex_priv); /* will be cleaned up by trans if ex_priv already gone */ if (state->extract_state) ct_file_extract_cleanup(state->extract_state); } return; }
void ct_extract(struct ct_global_state *state, struct ct_op *op) { struct ct_extract_args *cea = op->op_args; const char *ctfile = cea->cea_local_ctfile; char **filelist = cea->cea_filelist; int match_mode = cea->cea_matchmode; struct ct_extract_priv *ex_priv = op->op_priv; int ret; struct ct_trans *trans; char shat[SHA_DIGEST_STRING_LENGTH]; /* if we were woken up due to fatal, just clean up local state */ if (state->ct_dying != 0) goto dying; CNDBG(CT_LOG_TRANS, "entry"); switch (ct_get_file_state(state)) { case CT_S_STARTING: if (ex_priv == NULL) { ex_priv = e_calloc(1, sizeof(*ex_priv)); TAILQ_INIT(&ex_priv->extract_head); if ((ret = ct_match_compile(&ex_priv->inc_match, match_mode, filelist)) != 0) { ct_fatal(state, "failed to compile include pattern", ret); goto dying; } if (cea->cea_excllist != NULL && (ret = ct_match_compile(&ex_priv->ex_match, match_mode, cea->cea_excllist)) != 0) { ct_fatal(state, "failed to compile exclude pattern", ret); goto dying; } op->op_priv = ex_priv; RB_INIT(&ex_priv->pending_tree); } if ((ret = ct_file_extract_init(&state->extract_state, cea->cea_tdir, cea->cea_attr, cea->cea_follow_symlinks, ex_priv->allfiles, cea->cea_log_state, cea->cea_log_chown_failed)) != 0) { ct_fatal(state, "Can not initialize extract state", ret); goto dying; } if (ct_extract_calculate_total(state, cea, ex_priv->inc_match, ex_priv->ex_match) != 0) { CWARNX("failed to calculate stats"); goto dying; } if ((ret = ct_extract_setup(&ex_priv->extract_head, &ex_priv->xdr_ctx, ctfile, cea->cea_ctfile_basedir, &ex_priv->allfiles)) != 0) { ct_fatal(state, "can't setup extract queue", ret); goto dying; } state->ct_print_ctfile_info(state->ct_print_state, ex_priv->xdr_ctx.xs_filename, &ex_priv->xdr_ctx.xs_gh); /* XXX we should handle this better */ if (state->ct_max_block_size < ex_priv->xdr_ctx.xs_gh.cmg_chunk_size) CABORTX("block size negotiated with server %d is " "smaller than file max block size %d", state->ct_max_block_size, ex_priv->xdr_ctx.xs_gh.cmg_chunk_size); /* create rb tree head, prepare to start inserting */ if (ex_priv->allfiles) { ex_priv->fillrb = 1; } break; case CT_S_FINISHED: return; default: break; } ct_set_file_state(state, CT_S_RUNNING); while (1) { trans = ct_trans_alloc(state); if (trans == NULL) { /* system busy, return */ CNDBG(CT_LOG_TRANS, "ran out of transactions, waiting"); ct_set_file_state(state, CT_S_WAITING_TRANS); return; } trans->tr_statemachine = ct_state_extract; switch ((ret = ctfile_parse(&ex_priv->xdr_ctx))) { case XS_RET_FILE: if (ex_priv->fillrb == 0 && ex_priv->xdr_ctx.xs_hdr.cmh_nr_shas == -1) { if (ex_priv->allfiles == 0) CINFO("file %s has negative shas " "and backup is not allfiles", ex_priv->xdr_ctx.xs_hdr.cmh_filename); ex_priv->doextract = 0; goto skip; /* skip ze file for now */ } trans = ct_trans_realloc_local(state, trans); trans->tr_fl_node = ex_priv->fl_ex_node = ct_alloc_fnode(); ct_populate_fnode(state->extract_state, &ex_priv->xdr_ctx, trans->tr_fl_node, &trans->tr_state, ex_priv->allfiles, cea->cea_strip_slash); if (trans->tr_state == TR_S_EX_SPECIAL) { trans->tr_complete = ct_extract_complete_special; } else { trans->tr_complete = ct_extract_complete_file_start; } trans->tr_cleanup = ct_extract_cleanup_fnode; if (ex_priv->haverb) { struct ct_pending_file *cpf; if ((cpf = ct_extract_find_entry( &ex_priv->pending_tree, trans->tr_fl_node->fn_fullname)) != NULL) { struct fnode *hardlink; /* copy permissions over */ trans->tr_fl_node->fn_uid = cpf->cpf_uid; trans->tr_fl_node->fn_gid = cpf->cpf_gid; trans->tr_fl_node->fn_mode = cpf->cpf_mode; trans->tr_fl_node->fn_mtime = cpf->cpf_mtime; trans->tr_fl_node->fn_atime = cpf->cpf_atime; /* copy list of pending links over */ while ((hardlink = TAILQ_FIRST(&cpf->cpf_links))) { TAILQ_REMOVE(&cpf->cpf_links, hardlink, fn_list); TAILQ_INSERT_TAIL( &trans->tr_fl_node->fn_hardlinks, hardlink, fn_list); } ex_priv->doextract = 1; ct_extract_free_entry( &ex_priv->pending_tree, cpf); } else { ex_priv->doextract = 0; } } else { ex_priv->doextract = !ct_match(ex_priv->inc_match, trans->tr_fl_node->fn_fullname); if (ex_priv->doextract && ex_priv->ex_match != NULL && !ct_match(ex_priv->ex_match, trans->tr_fl_node->fn_fullname)) { ex_priv->doextract = 0; } } if (ex_priv->doextract && trans->tr_fl_node->fn_hardlink) { struct ct_pending_file *file; if ((file = ct_extract_find_entry( &ex_priv->pending_tree, trans->tr_fl_node->fn_hlname)) != NULL) { CNDBG(CT_LOG_FILE, "adding pending link for %s to %s", file->cpf_name, trans->tr_fl_node->fn_fullname); /* our reference to node passed */ ct_pending_file_add_link(file, trans->tr_fl_node); ex_priv->doextract = 0; goto skip; } } /* * If we're on the first ctfile in an allfiles backup * put the matches with -1 on the rb tree so we'll * remember to extract it from older files. */ if (ex_priv->doextract == 1 && ex_priv->fillrb && ex_priv->xdr_ctx.xs_hdr.cmh_nr_shas == -1) { ct_extract_insert_entry(&ex_priv->pending_tree, trans->tr_fl_node); ex_priv->doextract = 0; /* XXX reconsider the freeing */ } if (ex_priv->doextract == 0) { ct_free_fnode(trans->tr_fl_node); skip: ex_priv->fl_ex_node = NULL; ct_trans_free(state, trans); continue; } CNDBG(CT_LOG_CTFILE, "file %s numshas %" PRId64, trans->tr_fl_node->fn_fullname, ex_priv->xdr_ctx.xs_hdr.cmh_nr_shas); /* * special files we give our refcount up * regular files we need a new one since we need to * keep ours. */ if (trans->tr_state != TR_S_EX_SPECIAL) { ct_ref_fnode(trans->tr_fl_node); } else { ex_priv->fl_ex_node = NULL; } ct_queue_first(state, trans); break; case XS_RET_SHA: if (ex_priv->doextract == 0 || ex_priv->fl_ex_node->fn_skip_file != 0) { if (ctfile_parse_seek(&ex_priv->xdr_ctx)) { ct_fatal(state, "Can't seek past shas", ex_priv->xdr_ctx.xs_errno); goto dying; } ct_trans_free(state, trans); continue; } /* use saved fnode */ trans->tr_fl_node = ex_priv->fl_ex_node; if (memcmp(zerosha, ex_priv->xdr_ctx.xs_sha, SHA_DIGEST_LENGTH) == 0) { CWARNX("\"%s\" truncated during backup", trans->tr_fl_node->fn_fullname); if (ctfile_parse_seek(&ex_priv->xdr_ctx)) { ct_fatal(state, "Can't seek past " "truncation shas", ex_priv->xdr_ctx.xs_errno); goto dying; } ct_trans_free(state, trans); continue; } if (ex_priv->xdr_ctx.xs_gh.cmg_flags & CT_MD_CRYPTO) { /* * yes csha and sha are reversed, we want * to download csha, but putting it in sha * simplifies the code */ bcopy(ex_priv->xdr_ctx.xs_sha, trans->tr_csha, sizeof(trans->tr_csha)); bcopy(ex_priv->xdr_ctx.xs_csha, trans->tr_sha, sizeof(trans->tr_sha)); bcopy(ex_priv->xdr_ctx.xs_iv, trans->tr_iv, sizeof(trans->tr_iv)); } else { bcopy(ex_priv->xdr_ctx.xs_sha, trans->tr_sha, sizeof(trans->tr_sha)); } if (clog_mask_is_set(CT_LOG_SHA)) { ct_sha1_encode(trans->tr_sha, shat); CNDBG(CT_LOG_SHA, "extracting sha %s", shat); } trans->tr_state = TR_S_EX_SHA; trans->tr_complete = ct_extract_complete_file_read; trans->tr_dataslot = 0; ct_ref_fnode(trans->tr_fl_node); trans->tr_cleanup = ct_extract_cleanup_fnode; ct_queue_first(state, trans); break; case XS_RET_FILE_END: trans = ct_trans_realloc_local(state, trans); if (ex_priv->doextract == 0 || ex_priv->fl_ex_node->fn_skip_file != 0) { /* release our reference done with file */ if (ex_priv->fl_ex_node) { ct_free_fnode(ex_priv->fl_ex_node); ex_priv->fl_ex_node = NULL; } ct_trans_free(state, trans); continue; } /* use saved fnode from state */ trans->tr_fl_node = ex_priv->fl_ex_node; bcopy(ex_priv->xdr_ctx.xs_trl.cmt_sha, trans->tr_sha, sizeof(trans->tr_sha)); trans->tr_state = TR_S_EX_FILE_END; trans->tr_complete = ct_extract_complete_file_end; trans->tr_cleanup = ct_extract_cleanup_fnode; trans->tr_fl_node->fn_size = ex_priv->xdr_ctx.xs_trl.cmt_orig_size; /* * no reference here since we give our reference to the * last transaction on that file. We are done with it. */ ex_priv->fl_ex_node = NULL; ct_queue_first(state, trans); break; case XS_RET_EOF: CNDBG(CT_LOG_CTFILE, "Hit end of ctfile"); ctfile_parse_close(&ex_priv->xdr_ctx); /* if rb tree and rb is empty, goto end state */ if ((ex_priv->haverb || ex_priv->fillrb) && ct_extract_rb_empty(&ex_priv->pending_tree)) { /* * Cleanup extract queue, in case we had files * left. */ ct_extract_cleanup_queue( &ex_priv->extract_head); goto we_re_done_here; } if (!TAILQ_EMPTY(&ex_priv->extract_head)) { /* * if allfiles and this was the first pass. * free the current match lists * switch to rb tree mode */ if (ex_priv->fillrb) { ct_match_unwind(ex_priv->inc_match); if (ex_priv->ex_match) ct_match_unwind( ex_priv->ex_match); ex_priv->ex_match = NULL; ex_priv->inc_match = NULL; ex_priv->haverb = 1; ex_priv->fillrb = 0; } ct_trans_free(state, trans); /* reinits ex_priv->xdr_ctx */ if ((ret = ct_extract_open_next(&ex_priv->extract_head, &ex_priv->xdr_ctx)) != 0) { ct_fatal(state, "Can't open next ctfile", ret); goto dying; } state->ct_print_ctfile_info( state->ct_print_state, ex_priv->xdr_ctx.xs_filename, &ex_priv->xdr_ctx.xs_gh); /* poke file into action */ ct_wakeup_file(state->event_state); } else { /* * If rb tree and it is still has entries, * bitch about it */ /* XXX print out missing files */ if ((ex_priv->haverb || ex_priv->fillrb) && ct_extract_rb_empty( &ex_priv->pending_tree)) { CWARNX("out of ctfiles but some " "files are not found"); } we_re_done_here: if (ex_priv->inc_match) ct_match_unwind(ex_priv->inc_match); if (ex_priv->ex_match) ct_match_unwind( ex_priv->ex_match); ct_extract_pending_cleanup( &ex_priv->pending_tree); e_free(&ex_priv); op->op_priv = NULL; trans->tr_state = TR_S_DONE; trans->tr_complete = ct_extract_complete_done; trans->tr_cleanup = ct_extract_cleanup_done; /* * Technically this should be a local * transaction. However, since we are done * it doesn't really matter either way. */ ct_queue_first(state, trans); CNDBG(CT_LOG_TRANS, "extract finished"); ct_set_file_state(state, CT_S_FINISHED); } return; break; case XS_RET_FAIL: ct_fatal(state, "Failed to parse ctfile", ex_priv->xdr_ctx.xs_errno); goto dying; break; } } return; dying: /* only if we hadn't sent the final transaction yet */ if (ex_priv != NULL) { ct_extract_cleanup_queue(&ex_priv->extract_head); if (ex_priv->inc_match) ct_match_unwind(ex_priv->inc_match); if (ex_priv->ex_match) ct_match_unwind(ex_priv->ex_match); if (!ct_extract_rb_empty(&ex_priv->pending_tree)) { ct_extract_pending_cleanup(&ex_priv->pending_tree); } if (ex_priv->fl_ex_node != NULL) { ct_free_fnode(ex_priv->fl_ex_node); } /* XXX what about ex_priv->xdr_ctx ? */ e_free(&ex_priv); op->op_priv = NULL; /* if ex_priv is gone then the trans will clean this up */ if (state->extract_state) ct_file_extract_cleanup(state->extract_state); } return; }
/* * Perform EXISTS checking on every sha in a ctfile chain. * * We don't do any filtering. It is assumed that the localdb has been * flushed/made good before this operation starts so that we can trust lookups. */ void ct_exists_file(struct ct_global_state *state, struct ct_op *op) { struct ct_exists_args *ce = op->op_args; struct ct_exists_priv *ex_priv = op->op_priv; struct ct_trans *trans; int ret, allfiles; /* if we were woken up due to fatal, just clean up local state */ if (state->ct_dying != 0) goto dying; CNDBG(CT_LOG_TRANS, "entry"); switch (ct_get_file_state(state)) { case CT_S_STARTING: if (ex_priv == NULL) { ex_priv = e_calloc(1, sizeof(*ex_priv)); TAILQ_INIT(&ex_priv->extract_head); op->op_priv = ex_priv; } if ((ret = ct_extract_setup(&ex_priv->extract_head, &ex_priv->xdr_ctx, ce->ce_ctfile, ce->ce_ctfile_basedir, &allfiles)) != 0) { ct_fatal(state, "can't setup extract queue", ret); goto dying; } break; case CT_S_FINISHED: return; default: break; } ct_set_file_state(state, CT_S_RUNNING); while (1) { if ((trans = ct_trans_alloc(state)) == NULL) { CNDBG(CT_LOG_TRANS, "ran out of transactions, waiting"); ct_set_file_state(state, CT_S_WAITING_TRANS); return; } trans->tr_statemachine = ct_state_exists; switch ((ret = ctfile_parse(&ex_priv->xdr_ctx))) { case XS_RET_FILE: case XS_RET_FILE_END: ct_trans_free(state, trans); break; case XS_RET_SHA: if (memcmp(zerosha, ex_priv->xdr_ctx.xs_sha, SHA_DIGEST_LENGTH) == 0) { if (ctfile_parse_seek(&ex_priv->xdr_ctx)) { ct_fatal(state, "Can't seek past " "truncation shas", ex_priv->xdr_ctx.xs_errno); goto dying; } ct_trans_free(state, trans); continue; } if (ex_priv->xdr_ctx.xs_gh.cmg_flags & CT_MD_CRYPTO) { /* * yes csha and sha are reversed, we want * to download csha, but putting it in sha * simplifies the code */ bcopy(ex_priv->xdr_ctx.xs_sha, trans->tr_sha, sizeof(trans->tr_csha)); bcopy(ex_priv->xdr_ctx.xs_csha, trans->tr_csha, sizeof(trans->tr_sha)); bcopy(ex_priv->xdr_ctx.xs_iv, trans->tr_iv, sizeof(trans->tr_iv)); trans->tr_state = TR_S_COMPSHA_ED; } else { trans->tr_state = TR_S_UNCOMPSHA_ED; bcopy(ex_priv->xdr_ctx.xs_sha, trans->tr_sha, sizeof(trans->tr_sha)); } if (clog_mask_is_set(CT_LOG_SHA)) { char shat[SHA_DIGEST_STRING_LENGTH]; ct_sha1_encode(trans->tr_sha, shat); CNDBG(CT_LOG_SHA, "EXISTSing sha %s", shat); } trans->tr_old_genid = -1; /* XXX */ if (ctdb_lookup_sha(state->ct_db_state, trans->tr_sha, trans->tr_csha, trans->tr_iv, &trans->tr_old_genid)) { CNDBG(CT_LOG_SHA, "sha already in localdb"); state->ct_stats->st_bytes_exists += trans->tr_chsize; ct_trans_free(state, trans); continue; } trans->tr_complete = ct_exists_complete; trans->tr_cleanup = NULL; trans->tr_dataslot = 0; ct_queue_first(state, trans); break; case XS_RET_EOF: CNDBG(CT_LOG_CTFILE, "Hit end of ctfile"); ctfile_parse_close(&ex_priv->xdr_ctx); if (!TAILQ_EMPTY(&ex_priv->extract_head)) { /* * if allfiles and this was the first pass. * free the current match lists * switch to rb tree mode */ ct_trans_free(state, trans); /* reinits ex_priv->xdr_ctx */ if ((ret = ct_extract_open_next(&ex_priv->extract_head, &ex_priv->xdr_ctx)) != 0) { ct_fatal(state, "Can't open next ctfile", ret); goto dying; } } else { e_free(&ex_priv); op->op_priv = NULL; trans->tr_state = TR_S_DONE; trans->tr_complete = ct_exists_complete_done; trans->tr_cleanup = NULL; /* * Technically this should be a local * transaction. However, since we are done * it doesn't really matter either way. */ ct_queue_first(state, trans); CNDBG(CT_LOG_TRANS, "extract finished"); ct_set_file_state(state, CT_S_FINISHED); } return; break; case XS_RET_FAIL: ct_fatal(state, "Failed to parse ctfile", ex_priv->xdr_ctx.xs_errno); goto dying; break; } } return; dying: /* only if we hadn't sent the final transaction yet */ if (ex_priv != NULL) { ct_extract_cleanup_queue(&ex_priv->extract_head); /* XXX what about ex_priv->xdr_ctx ? */ e_free(&ex_priv); op->op_priv = NULL; } return; }
int ct_list(const char *file, char **flist, char **excludelist, int match_mode, const char *ctfile_basedir, int strip_slash, int verbose) { struct ct_extract_state *ces; struct ctfile_parse_state xs_ctx; struct fnode fnodestore; uint64_t reduction; struct fnode *fnode = &fnodestore; struct ct_match *match, *ex_match = NULL; char *ct_next_filename; char *sign; int state; int doprint = 0; int ret; int s_errno = 0, ct_errno = 0; char shat[SHA_DIGEST_STRING_LENGTH]; char cshat[SHA_DIGEST_STRING_LENGTH]; char iv[CT_IV_LEN*2+1]; if ((ret = ct_file_extract_init(&ces, NULL, 1, 1, 0, NULL, NULL)) != 0) CFATALX("failed to initialise extract state: %s", ct_strerror(ret)); if ((ret = ct_match_compile(&match, match_mode, flist)) != 0) CFATALX("failed to compile match pattern: %s", ct_strerror(ret)); if (excludelist != NULL && (ret = ct_match_compile(&ex_match, match_mode, excludelist)) != 0) CFATALX("failed to compile exclude pattern: %s", ct_strerror(ret)); verbose++; /* by default print something. */ ct_next_filename = NULL; next_file: ret = ctfile_parse_init(&xs_ctx, file, ctfile_basedir); if (ret) CFATALX("failed to open %s: %s", file, ct_strerror(ret)); ct_print_ctfile_info(&verbose, file, &xs_ctx.xs_gh); if (ct_next_filename) e_free(&ct_next_filename); if (xs_ctx.xs_gh.cmg_prevlvl_filename) { CNDBG(CT_LOG_CTFILE, "previous backup file %s\n", xs_ctx.xs_gh.cmg_prevlvl_filename); ct_next_filename = e_strdup(xs_ctx.xs_gh.cmg_prevlvl_filename); } bzero(&fnodestore, sizeof(fnodestore)); do { ret = ctfile_parse(&xs_ctx); switch (ret) { case XS_RET_FILE: ct_populate_fnode(ces, &xs_ctx, fnode, &state, xs_ctx.xs_gh.cmg_flags & CT_MD_MLB_ALLFILES, strip_slash); doprint = !ct_match(match, fnode->fn_fullname); if (doprint && ex_match != NULL && !ct_match(ex_match, fnode->fn_fullname)) doprint = 0; if (doprint) { ct_pr_fmt_file(&verbose, fnode); if (!C_ISREG(xs_ctx.xs_hdr.cmh_type) || verbose > 2) printf("\n"); } if (fnode->fn_hlname) e_free(&fnode->fn_hlname); if (fnode->fn_fullname) e_free(&fnode->fn_fullname); break; case XS_RET_FILE_END: sign = " "; if (xs_ctx.xs_trl.cmt_comp_size == 0) reduction = 100; else { uint64_t orig, comp; orig = xs_ctx.xs_trl.cmt_orig_size; comp = xs_ctx.xs_trl.cmt_comp_size; if (comp <= orig) { reduction = 100 * (orig - comp) / orig; } else { reduction = 100 * (comp - orig) / orig; if (reduction != 0) sign = "-"; } } if (doprint && verbose > 1) printf(" sz: %" PRIu64 " shas: %" PRIu64 " reduction: %s%" PRIu64 "%%\n", xs_ctx.xs_trl.cmt_orig_size, xs_ctx.xs_hdr.cmh_nr_shas, sign, reduction); else if (doprint) printf("\n"); break; case XS_RET_SHA: if (!(doprint && verbose > 2)) { if (ctfile_parse_seek(&xs_ctx)) { CFATALX("seek failed"); } } else { int i; ct_sha1_encode(xs_ctx.xs_sha, shat); switch (xs_ctx.xs_gh.cmg_flags & CT_MD_CRYPTO) { case 0: printf(" sha %s\n", shat); break; case CT_MD_CRYPTO: ct_sha1_encode(xs_ctx.xs_csha, cshat); for (i = 0; i < CT_IV_LEN; i++) snprintf(&iv[i * 2], 3, "%02x", xs_ctx.xs_iv[i]); printf(" sha %s csha %s iv %s\n", shat, cshat, iv); } } break; case XS_RET_EOF: break; case XS_RET_FAIL: s_errno = errno; ct_errno = xs_ctx.xs_errno; ; } } while (ret != XS_RET_EOF && ret != XS_RET_FAIL); ctfile_parse_close(&xs_ctx); if (ret != XS_RET_EOF) { errno = s_errno; CWARNX("corrupt ctfile: %s", ct_strerror(ct_errno)); } else { if (ct_next_filename) { file = ct_next_filename; goto next_file; } } ct_match_unwind(match); ct_file_extract_cleanup(ces); return (0); }