static int compact_closerename(struct backup **originalp, struct backup **compactp, time_t now) { struct backup *original = *originalp; struct backup *compact = *compactp; struct buf ts_data_fname = BUF_INITIALIZER; struct buf ts_index_fname = BUF_INITIALIZER; int r; buf_printf(&ts_data_fname, "%s.%ld", original->data_fname, now); buf_printf(&ts_index_fname, "%s.%ld", original->index_fname, now); /* link original files into timestamped names */ r = link(original->data_fname, buf_cstring(&ts_data_fname)); if (!r) link(original->index_fname, buf_cstring(&ts_index_fname)); if (r) { /* on error, trash the new links and bail out */ unlink(buf_cstring(&ts_data_fname)); unlink(buf_cstring(&ts_index_fname)); goto done; } /* replace original files with compacted files */ r = rename(compact->data_fname, original->data_fname); if (!r) r = rename(compact->index_fname, original->index_fname); if (r) { /* on error, put original files back */ unlink(original->data_fname); unlink(original->index_fname); link(buf_cstring(&ts_data_fname), original->data_fname); link(buf_cstring(&ts_index_fname), original->index_fname); goto done; } /* finally, clean up the timestamped ones */ if (!config_getswitch(IMAPOPT_BACKUP_KEEP_PREVIOUS)) { unlink(buf_cstring(&ts_data_fname)); unlink(buf_cstring(&ts_index_fname)); } /* release our locks */ backup_close(originalp); backup_close(compactp); done: buf_free(&ts_data_fname); buf_free(&ts_index_fname); return r; }
static int compact_open(const char *name, struct backup **originalp, struct backup **compactp, enum backup_open_nonblock nonblock) { struct backup *original = NULL; struct backup *compact = NULL; struct buf original_data_fname = BUF_INITIALIZER; struct buf original_index_fname = BUF_INITIALIZER; struct buf compact_data_fname = BUF_INITIALIZER; struct buf compact_index_fname = BUF_INITIALIZER; int r; buf_printf(&original_data_fname, "%s", name); buf_printf(&original_index_fname, "%s.index", name); buf_printf(&compact_data_fname, "%s.new", name); buf_printf(&compact_index_fname, "%s.index.new", name); r = backup_real_open(&original, buf_cstring(&original_data_fname), buf_cstring(&original_index_fname), BACKUP_OPEN_NOREINDEX, nonblock, BACKUP_OPEN_NOCREATE); if (r) goto done; r = backup_real_open(&compact, buf_cstring(&compact_data_fname), buf_cstring(&compact_index_fname), BACKUP_OPEN_NOREINDEX, BACKUP_OPEN_NONBLOCK, // FIXME think about this BACKUP_OPEN_CREATE_EXCL); if (r) { backup_close(&original); goto done; } *originalp = original; *compactp = compact; done: buf_free(&original_data_fname); buf_free(&original_index_fname); buf_free(&compact_data_fname); buf_free(&compact_index_fname); return r; }
int main(int argc, char **argv) { save_argv0(argv[0]); struct cyrbu_cmd_options options = {0}; enum cyrbu_mode mode = CYRBU_MODE_UNSPECIFIED; enum cyrbu_cmd cmd = CYRBU_CMD_UNSPECIFIED; const char *alt_config = NULL; const char *backup_name = NULL; const char *command = NULL; const char *subcommand = NULL; struct backup *backup = NULL; mbname_t *mbname = NULL; int i, opt, r = 0; while ((opt = getopt(argc, argv, "C:fmuv")) != EOF) { switch (opt) { case 'C': alt_config = optarg; break; case 'f': if (mode != CYRBU_MODE_UNSPECIFIED) usage(); mode = CYRBU_MODE_FILENAME; break; case 'm': if (mode != CYRBU_MODE_UNSPECIFIED) usage(); mode = CYRBU_MODE_MBOXNAME; break; case 'u': if (mode != CYRBU_MODE_UNSPECIFIED) usage(); mode = CYRBU_MODE_USERNAME; break; case 'v': options.verbose++; break; default: usage(); break; } } /* default mode is username */ if (mode == CYRBU_MODE_UNSPECIFIED) mode = CYRBU_MODE_USERNAME; /* get the backup name */ if (optind == argc) usage(); backup_name = argv[optind++]; /* get the command */ if (optind == argc) usage(); command = argv[optind++]; /* get the subcommand */ if (optind == argc) usage(); subcommand = argv[optind++]; /* parse the command and subcommand */ cmd = parse_cmd_string(command, subcommand); /* check remaining arguments based on command */ switch (cmd) { case CYRBU_CMD_LIST_ALL: case CYRBU_CMD_LIST_CHUNKS: case CYRBU_CMD_LIST_MAILBOXES: case CYRBU_CMD_LIST_MESSAGES: /* these want no more arguments */ if (optind != argc) usage(); break; case CYRBU_CMD_SHOW_CHUNKS: case CYRBU_CMD_SHOW_MAILBOXES: case CYRBU_CMD_SHOW_MESSAGES: /* these need at least one more argument */ if (optind == argc) usage(); break; case CYRBU_CMD_DUMP_CHUNK: case CYRBU_CMD_DUMP_MAILBOX: case CYRBU_CMD_DUMP_MESSAGE: /* these need exactly one more argument */ if (argc - optind != 1) usage(); break; default: usage(); break; } /* build a nice args list */ options.argv = strarray_new(); for (i = optind; i < argc; i++) { strarray_add(options.argv, argv[i]); } // FIXME finish parsing options cyrus_init(alt_config, "cyr_backup", 0, 0); /* open backup */ switch (mode) { case CYRBU_MODE_FILENAME: r = backup_open_paths(&backup, backup_name, NULL, BACKUP_OPEN_NONBLOCK, BACKUP_OPEN_NOCREATE); break; case CYRBU_MODE_MBOXNAME: mbname = mbname_from_intname(backup_name); if (!mbname) usage(); r = backup_open(&backup, mbname, BACKUP_OPEN_NONBLOCK, BACKUP_OPEN_NOCREATE); break; case CYRBU_MODE_USERNAME: mbname = mbname_from_userid(backup_name); if (!mbname) usage(); r = backup_open(&backup, mbname, BACKUP_OPEN_NONBLOCK, BACKUP_OPEN_NOCREATE); break; default: usage(); break; } /* run command */ if (!r && cmd_func[cmd]) r = cmd_func[cmd](backup, &options); if (r) fprintf(stderr, "%s: %s\n", backup_name, error_message(r)); /* close backup */ if (backup) backup_close(&backup); /* clean up and exit */ backup_cleanup_staging_path(); cyrus_done(); strarray_free(options.argv); exit(r ? EC_TEMPFAIL : EC_OK); }
int main(int argc, char **argv) { save_argv0(argv[0]); const char *alt_config = NULL; const char *input_file = NULL; const char *backup_name = NULL; const char *servername = NULL; enum restore_mode mode = RESTORE_MODE_UNSPECIFIED; int local_only = 0; int wait = 0; int do_nothing = 0; int do_all_mailboxes = 0; struct restore_options options = {0}; options.expunged_mode = RESTORE_EXPUNGED_OKAY; options.trim_deletedprefix = 1; struct backup *backup = NULL; mbname_t *mbname = NULL; struct backup_mailbox_list *mailbox_list = NULL; struct sync_folder_list *reserve_folder_list = NULL; struct sync_reserve_list *reserve_list = NULL; struct buf tagbuf = BUF_INITIALIZER; struct backend *backend = NULL; struct dlist *upload = NULL; int opt, r; while ((opt = getopt(argc, argv, ":A:C:DF:LM:P:UXaf:m:nru:vw:xz")) != EOF) { switch (opt) { case 'A': if (options.keep_uidvalidity) usage(); options.override_acl = optarg; break; case 'C': alt_config = optarg; break; case 'D': /* XXX does this clash with keep_uidvalidity? */ options.trim_deletedprefix = 0; break; case 'F': if (do_all_mailboxes) usage(); input_file = optarg; break; case 'L': local_only = 1; break; case 'M': if (options.keep_uidvalidity) usage(); options.override_mboxname = optarg; break; case 'P': if (options.keep_uidvalidity) usage(); options.override_partition = optarg; break; case 'U': if (options.override_acl || options.override_mboxname || options.override_partition) usage(); options.keep_uidvalidity = 1; break; case 'X': if (options.expunged_mode != RESTORE_EXPUNGED_OKAY) usage(); options.expunged_mode = RESTORE_EXPUNGED_EXCLUDE; break; case 'a': if (input_file) usage(); do_all_mailboxes = 1; break; case 'f': if (mode != RESTORE_MODE_UNSPECIFIED) usage(); mode = RESTORE_MODE_FILENAME; backup_name = optarg; break; case 'm': if (mode != RESTORE_MODE_UNSPECIFIED) usage(); mode = RESTORE_MODE_MBOXNAME; backup_name = optarg; break; case 'n': do_nothing = 1; break; case 'r': options.do_submailboxes = 1; break; case 'u': if (mode != RESTORE_MODE_UNSPECIFIED) usage(); mode = RESTORE_MODE_USERNAME; backup_name = optarg; break; case 'v': options.verbose++; break; case 'w': wait = atoi(optarg); if (wait < 0) usage(); break; case 'x': if (options.expunged_mode != RESTORE_EXPUNGED_OKAY) usage(); options.expunged_mode = RESTORE_EXPUNGED_ONLY; break; case 'z': options.require_compression = 1; break; case ':': if (optopt == 'A') options.override_acl = ""; else usage(); break; default: usage(); break; } } /* we need a server name */ if (optind == argc) usage(); servername = argv[optind++]; /* we need a source of backup data */ if (mode == RESTORE_MODE_UNSPECIFIED) { if (optind == argc) usage(); backup_name = argv[optind++]; mode = RESTORE_MODE_USERNAME; } /* we need either an input file or some objects to restore */ if (!do_all_mailboxes && !input_file && optind == argc) usage(); /* and we can't have both because i said */ if ((input_file || do_all_mailboxes) && optind < argc) usage(); /* okay, arguments seem sane, we are go */ cyrus_init(alt_config, "restore", 0, 0); /* load the SASL plugins */ global_sasl_init(1, 0, mysasl_cb); /* wait here for gdb attach */ if (wait) { fprintf(stderr, "Waiting for %d seconds for gdb attach...\n", wait); sleep(wait); } /* open backup */ switch (mode) { case RESTORE_MODE_FILENAME: r = backup_open_paths(&backup, backup_name, NULL, BACKUP_OPEN_NONBLOCK, BACKUP_OPEN_NOCREATE); break; case RESTORE_MODE_MBOXNAME: mbname = mbname_from_intname(backup_name); if (!mbname) usage(); r = backup_open(&backup, mbname, BACKUP_OPEN_NONBLOCK, BACKUP_OPEN_NOCREATE); mbname_free(&mbname); break; case RESTORE_MODE_USERNAME: mbname = mbname_from_userid(backup_name); if (!mbname) usage(); r = backup_open(&backup, mbname, BACKUP_OPEN_NONBLOCK, BACKUP_OPEN_NOCREATE); mbname_free(&mbname); break; default: usage(); break; } if (r) goto done; /* scan for objects to restore: * mailboxes will have all messages added, modulo expunged_mode * messages will be added individually with appropriate folder */ mailbox_list = xzmalloc(sizeof *mailbox_list); reserve_folder_list = restore_make_reserve_folder_list(backup); reserve_list = sync_reserve_list_create(SYNC_MSGID_LIST_HASH_SIZE); if (input_file) { char buf[MAX_MAILBOX_NAME + 2]; // \n\0 size_t len; FILE *f; if (0 != strcmp(input_file, "-")) { f = fopen(input_file, "r"); if (!f) { fprintf(stderr, "fopen %s: %s\n", input_file, strerror(errno)); goto done;// FIXME shut_down? } } else { f = stdin; } while (fgets(buf, sizeof(buf), f)) { len = strlen(buf); if (len > 0 && buf[len - 1] == '\n') buf[--len] = '\0'; if (len == 0 || buf[0] == '#') continue; r = restore_add_object(buf, &options, backup, mailbox_list, reserve_folder_list, reserve_list); // FIXME r } fclose(f); } else if (do_all_mailboxes) { struct backup_mailbox_list *all_mailboxes; struct backup_mailbox *mailbox; all_mailboxes = backup_get_mailboxes(backup, 0, BACKUP_MAILBOX_ALL_RECORDS); for (mailbox = all_mailboxes->head; mailbox; mailbox = mailbox->next) { restore_add_mailbox(mailbox, &options, mailbox_list, reserve_folder_list, reserve_list); } backup_mailbox_list_empty(all_mailboxes); free(all_mailboxes); } else { int i; for (i = optind; i < argc; i++) { r = restore_add_object(argv[i], &options, backup, mailbox_list, reserve_folder_list, reserve_list); // FIXME r } } if (do_nothing) { if (options.verbose) fprintf(stderr, "do nothing (-n) specified, exiting now\n"); goto done; } /* connect to destination */ backend = restore_connect(servername, &tagbuf, &options); if (!backend) { // FIXME r = -1; goto done; } /* do the restore */ struct sync_reserve *reserve; for (reserve = reserve_list->head; reserve; reserve = reserve->next) { /* send APPLY RESERVE and parse missing lists */ r = sync_reserve_partition(reserve->part, reserve_folder_list, reserve->list, backend); if (r) goto done; /* send APPLY MESSAGEs */ r = backup_prepare_message_upload(backup, reserve->part, reserve->list, &sync_msgid_lookup, &upload); if (r) goto done; /* upload in small(ish) blocks to avoid timeouts */ while (upload->head) { struct dlist *block = dlist_splice(upload, 1024); sync_send_apply(block, backend->out); r = sync_parse_response("MESSAGE", backend->in, NULL); dlist_unlink_files(block); dlist_free(&block); if (r) goto done; } } /* sync_prepare_dlists needs to upload messages per-mailbox, because * it needs the mailbox to find the filename for the message. but * we have no such limitation, so we can upload messages while * looping over the reserve_list instead, above. * * alternatively, if we do it on a per-mailbox basis then we can limit * the hit on the staging directory to only a mailbox worth of messages * at a time. but we don't have a logical grouping that would make * this coherent */ /* send RESTORE MAILBOXes */ struct backup_mailbox *mailbox; for (mailbox = mailbox_list->head; mailbox; mailbox = mailbox->next) { /* XXX filter the mailbox records based on reserve/missing/upload */ /* XXX does this sensibly handle mailbox objs with empty values? */ struct dlist *dl = backup_mailbox_to_dlist(mailbox); if (!dl) continue; if (local_only) { free(dl->name); dl->name = xstrdup("LOCAL_MAILBOX"); } sync_send_restore(dl, backend->out); r = sync_parse_response("MAILBOX", backend->in, NULL); dlist_free(&dl); if (r) goto done; } done: if (r) fprintf(stderr, "%s: %s\n", backup_name, error_message(r)); /* release lock asap */ if (backup) backup_close(&backup); if (backend) backend_disconnect(backend); if (upload) { dlist_unlink_files(upload); dlist_free(&upload); } if (mailbox_list) { backup_mailbox_list_empty(mailbox_list); free(mailbox_list); } if (reserve_folder_list) sync_folder_list_free(&reserve_folder_list); if (reserve_list) sync_reserve_list_free(&reserve_list); buf_free(&tagbuf); backup_cleanup_staging_path(); cyrus_done(); exit(r ? EX_TEMPFAIL : EX_OK); }
/* returns: * 0 on success * 1 if compact was not needed * negative on error */ EXPORTED int backup_compact(const char *name, enum backup_open_nonblock nonblock, int force, int verbose, FILE *out) { struct backup *original = NULL; struct backup *compact = NULL; struct backup_chunk_list *all_chunks = NULL; struct backup_chunk_list *keep_chunks = NULL; struct backup_chunk *chunk = NULL; struct sync_msgid_list *keep_message_guids = NULL; struct gzuncat *gzuc = NULL; struct protstream *in = NULL; time_t since, chunk_start_time, ts; int r; compact_readconfig(); r = compact_open(name, &original, &compact, nonblock); if (r) return r; /* calculate current time after obtaining locks, in case of a wait */ const time_t now = time(NULL); const int retention_days = config_getint(IMAPOPT_BACKUP_RETENTION_DAYS); if (retention_days > 0) { since = now - (retention_days * 24 * 60 * 60); } else { /* zero or negative retention days means "keep forever" */ since = -1; } all_chunks = backup_get_chunks(original); if (!all_chunks) goto error; keep_chunks = backup_get_live_chunks(original, since); if (!keep_chunks) goto error; if (!force && !compact_required(all_chunks, keep_chunks)) { /* nothing to do */ backup_chunk_list_free(&all_chunks); backup_chunk_list_free(&keep_chunks); backup_unlink(&compact); backup_close(&original); return 1; } if (verbose) { fprintf(out, "keeping " SIZE_T_FMT " chunks:\n", keep_chunks->count); for (chunk = keep_chunks->head; chunk; chunk = chunk->next) { fprintf(out, " %d", chunk->id); } fprintf(out, "\n"); } gzuc = gzuc_new(original->fd); if (!gzuc) goto error; chunk_start_time = -1; ts = 0; struct buf cmd = BUF_INITIALIZER; for (chunk = keep_chunks->head; chunk; chunk = chunk->next) { keep_message_guids = sync_msgid_list_create(0); r = backup_message_foreach(original, chunk->id, &since, _keep_message_guids_cb, keep_message_guids); if (r) goto error; gzuc_member_start_from(gzuc, chunk->offset); in = prot_readcb(_prot_fill_cb, gzuc); while (1) { struct dlist *dl = NULL; int c = parse_backup_line(in, &ts, &cmd, &dl); if (c == EOF) { const char *error = prot_error(in); if (error && 0 != strcmp(error, PROT_EOF_STRING)) { syslog(LOG_ERR, "IOERROR: %s: error reading chunk at offset %jd, byte %i: %s\n", name, chunk->offset, prot_bytes_in(in), error); if (out) fprintf(out, "error reading chunk at offset %jd, byte %i: %s\n", chunk->offset, prot_bytes_in(in), error); r = IMAP_IOERROR; goto error; } break; } if (chunk_start_time == -1) { r = backup_append_start(compact, &ts, BACKUP_APPEND_NOFLUSH); if (r) goto error; chunk_start_time = ts; } // XXX if this line is worth keeping if (want_append(original, chunk->id, dl, keep_message_guids)) { // FIXME if message is removed due to unneeded chunk, // subsequent mailbox lines for it will fail here // so we need to be able to tell which lines apply to messages we don't want anymore r = backup_append(compact, dl, &ts, BACKUP_APPEND_NOFLUSH); if (r) goto error; } dlist_unlink_files(dl); dlist_free(&dl); // if this line put us over compact_maxsize if (want_split(chunk, &compact->append_state->wrote)) { r = backup_append_end(compact, &ts); chunk_start_time = -1; if (verbose) { fprintf(out, "splitting chunk %d\n", chunk->id); } } } // if we're due to start a new chunk if (compact->append_state && compact->append_state->mode) { if (!want_combine(compact->append_state->wrote, chunk->next)) { r = backup_append_end(compact, &ts); chunk_start_time = -1; } else if (verbose) { fprintf(out, "combining chunks %d and %d\n", chunk->id, chunk->next->id); } } prot_free(in); in = NULL; gzuc_member_end(gzuc, NULL); sync_msgid_list_free(&keep_message_guids); } buf_free(&cmd); if (compact->append_state && compact->append_state->mode) backup_append_end(compact, &ts); gzuc_free(&gzuc); backup_chunk_list_free(&keep_chunks); /* if we get here okay, then the compact succeeded */ r = compact_closerename(&original, &compact, now); if (r) goto error; return 0; error: if (in) prot_free(in); if (gzuc) gzuc_free(&gzuc); if (keep_message_guids) sync_msgid_list_free(&keep_message_guids); if (all_chunks) backup_chunk_list_free(&all_chunks); if (keep_chunks) backup_chunk_list_free(&keep_chunks); if (compact) backup_unlink(&compact); if (original) backup_close(&original); return r ? r : -1; }