static int compact_closerename(struct backup **originalp,
struct backup **compactp,
time_t now)
{
struct backup *original = *originalp;
struct backup *compact = *compactp;
struct buf ts_data_fname = BUF_INITIALIZER;
struct buf ts_index_fname = BUF_INITIALIZER;
int r;
buf_printf(&ts_data_fname, "%s.%ld", original->data_fname, now);
buf_printf(&ts_index_fname, "%s.%ld", original->index_fname, now);
/* link original files into timestamped names */
r = link(original->data_fname, buf_cstring(&ts_data_fname));
if (!r) link(original->index_fname, buf_cstring(&ts_index_fname));
if (r) {
/* on error, trash the new links and bail out */
unlink(buf_cstring(&ts_data_fname));
unlink(buf_cstring(&ts_index_fname));
goto done;
}
/* replace original files with compacted files */
r = rename(compact->data_fname, original->data_fname);
if (!r) r = rename(compact->index_fname, original->index_fname);
if (r) {
/* on error, put original files back */
unlink(original->data_fname);
unlink(original->index_fname);
link(buf_cstring(&ts_data_fname), original->data_fname);
link(buf_cstring(&ts_index_fname), original->index_fname);
goto done;
}
/* finally, clean up the timestamped ones */
if (!config_getswitch(IMAPOPT_BACKUP_KEEP_PREVIOUS)) {
unlink(buf_cstring(&ts_data_fname));
unlink(buf_cstring(&ts_index_fname));
}
/* release our locks */
backup_close(originalp);
backup_close(compactp);
done:
buf_free(&ts_data_fname);
buf_free(&ts_index_fname);
return r;
}
static int compact_open(const char *name,
struct backup **originalp,
struct backup **compactp,
enum backup_open_nonblock nonblock)
{
struct backup *original = NULL;
struct backup *compact = NULL;
struct buf original_data_fname = BUF_INITIALIZER;
struct buf original_index_fname = BUF_INITIALIZER;
struct buf compact_data_fname = BUF_INITIALIZER;
struct buf compact_index_fname = BUF_INITIALIZER;
int r;
buf_printf(&original_data_fname, "%s", name);
buf_printf(&original_index_fname, "%s.index", name);
buf_printf(&compact_data_fname, "%s.new", name);
buf_printf(&compact_index_fname, "%s.index.new", name);
r = backup_real_open(&original,
buf_cstring(&original_data_fname),
buf_cstring(&original_index_fname),
BACKUP_OPEN_NOREINDEX,
nonblock,
BACKUP_OPEN_NOCREATE);
if (r) goto done;
r = backup_real_open(&compact,
buf_cstring(&compact_data_fname),
buf_cstring(&compact_index_fname),
BACKUP_OPEN_NOREINDEX,
BACKUP_OPEN_NONBLOCK, // FIXME think about this
BACKUP_OPEN_CREATE_EXCL);
if (r) {
backup_close(&original);
goto done;
}
*originalp = original;
*compactp = compact;
done:
buf_free(&original_data_fname);
buf_free(&original_index_fname);
buf_free(&compact_data_fname);
buf_free(&compact_index_fname);
return r;
}
int main(int argc, char **argv)
{
save_argv0(argv[0]);
struct cyrbu_cmd_options options = {0};
enum cyrbu_mode mode = CYRBU_MODE_UNSPECIFIED;
enum cyrbu_cmd cmd = CYRBU_CMD_UNSPECIFIED;
const char *alt_config = NULL;
const char *backup_name = NULL;
const char *command = NULL;
const char *subcommand = NULL;
struct backup *backup = NULL;
mbname_t *mbname = NULL;
int i, opt, r = 0;
while ((opt = getopt(argc, argv, "C:fmuv")) != EOF) {
switch (opt) {
case 'C':
alt_config = optarg;
break;
case 'f':
if (mode != CYRBU_MODE_UNSPECIFIED) usage();
mode = CYRBU_MODE_FILENAME;
break;
case 'm':
if (mode != CYRBU_MODE_UNSPECIFIED) usage();
mode = CYRBU_MODE_MBOXNAME;
break;
case 'u':
if (mode != CYRBU_MODE_UNSPECIFIED) usage();
mode = CYRBU_MODE_USERNAME;
break;
case 'v':
options.verbose++;
break;
default:
usage();
break;
}
}
/* default mode is username */
if (mode == CYRBU_MODE_UNSPECIFIED)
mode = CYRBU_MODE_USERNAME;
/* get the backup name */
if (optind == argc) usage();
backup_name = argv[optind++];
/* get the command */
if (optind == argc) usage();
command = argv[optind++];
/* get the subcommand */
if (optind == argc) usage();
subcommand = argv[optind++];
/* parse the command and subcommand */
cmd = parse_cmd_string(command, subcommand);
/* check remaining arguments based on command */
switch (cmd) {
case CYRBU_CMD_LIST_ALL:
case CYRBU_CMD_LIST_CHUNKS:
case CYRBU_CMD_LIST_MAILBOXES:
case CYRBU_CMD_LIST_MESSAGES:
/* these want no more arguments */
if (optind != argc) usage();
break;
case CYRBU_CMD_SHOW_CHUNKS:
case CYRBU_CMD_SHOW_MAILBOXES:
case CYRBU_CMD_SHOW_MESSAGES:
/* these need at least one more argument */
if (optind == argc) usage();
break;
case CYRBU_CMD_DUMP_CHUNK:
case CYRBU_CMD_DUMP_MAILBOX:
case CYRBU_CMD_DUMP_MESSAGE:
/* these need exactly one more argument */
if (argc - optind != 1) usage();
break;
default:
usage();
break;
}
/* build a nice args list */
options.argv = strarray_new();
for (i = optind; i < argc; i++) {
strarray_add(options.argv, argv[i]);
}
// FIXME finish parsing options
cyrus_init(alt_config, "cyr_backup", 0, 0);
/* open backup */
switch (mode) {
case CYRBU_MODE_FILENAME:
r = backup_open_paths(&backup, backup_name, NULL,
BACKUP_OPEN_NONBLOCK, BACKUP_OPEN_NOCREATE);
break;
case CYRBU_MODE_MBOXNAME:
mbname = mbname_from_intname(backup_name);
if (!mbname) usage();
r = backup_open(&backup, mbname,
BACKUP_OPEN_NONBLOCK, BACKUP_OPEN_NOCREATE);
break;
case CYRBU_MODE_USERNAME:
mbname = mbname_from_userid(backup_name);
if (!mbname) usage();
r = backup_open(&backup, mbname,
BACKUP_OPEN_NONBLOCK, BACKUP_OPEN_NOCREATE);
break;
default:
usage();
break;
}
/* run command */
if (!r && cmd_func[cmd])
r = cmd_func[cmd](backup, &options);
if (r)
fprintf(stderr, "%s: %s\n", backup_name, error_message(r));
/* close backup */
if (backup)
backup_close(&backup);
/* clean up and exit */
backup_cleanup_staging_path();
cyrus_done();
strarray_free(options.argv);
exit(r ? EC_TEMPFAIL : EC_OK);
}
int main(int argc, char **argv)
{
save_argv0(argv[0]);
const char *alt_config = NULL;
const char *input_file = NULL;
const char *backup_name = NULL;
const char *servername = NULL;
enum restore_mode mode = RESTORE_MODE_UNSPECIFIED;
int local_only = 0;
int wait = 0;
int do_nothing = 0;
int do_all_mailboxes = 0;
struct restore_options options = {0};
options.expunged_mode = RESTORE_EXPUNGED_OKAY;
options.trim_deletedprefix = 1;
struct backup *backup = NULL;
mbname_t *mbname = NULL;
struct backup_mailbox_list *mailbox_list = NULL;
struct sync_folder_list *reserve_folder_list = NULL;
struct sync_reserve_list *reserve_list = NULL;
struct buf tagbuf = BUF_INITIALIZER;
struct backend *backend = NULL;
struct dlist *upload = NULL;
int opt, r;
while ((opt = getopt(argc, argv, ":A:C:DF:LM:P:UXaf:m:nru:vw:xz")) != EOF) {
switch (opt) {
case 'A':
if (options.keep_uidvalidity) usage();
options.override_acl = optarg;
break;
case 'C':
alt_config = optarg;
break;
case 'D':
/* XXX does this clash with keep_uidvalidity? */
options.trim_deletedprefix = 0;
break;
case 'F':
if (do_all_mailboxes) usage();
input_file = optarg;
break;
case 'L':
local_only = 1;
break;
case 'M':
if (options.keep_uidvalidity) usage();
options.override_mboxname = optarg;
break;
case 'P':
if (options.keep_uidvalidity) usage();
options.override_partition = optarg;
break;
case 'U':
if (options.override_acl || options.override_mboxname || options.override_partition)
usage();
options.keep_uidvalidity = 1;
break;
case 'X':
if (options.expunged_mode != RESTORE_EXPUNGED_OKAY) usage();
options.expunged_mode = RESTORE_EXPUNGED_EXCLUDE;
break;
case 'a':
if (input_file) usage();
do_all_mailboxes = 1;
break;
case 'f':
if (mode != RESTORE_MODE_UNSPECIFIED) usage();
mode = RESTORE_MODE_FILENAME;
backup_name = optarg;
break;
case 'm':
if (mode != RESTORE_MODE_UNSPECIFIED) usage();
mode = RESTORE_MODE_MBOXNAME;
backup_name = optarg;
break;
case 'n':
do_nothing = 1;
break;
case 'r':
options.do_submailboxes = 1;
break;
case 'u':
if (mode != RESTORE_MODE_UNSPECIFIED) usage();
mode = RESTORE_MODE_USERNAME;
backup_name = optarg;
break;
case 'v':
options.verbose++;
break;
case 'w':
wait = atoi(optarg);
if (wait < 0) usage();
break;
case 'x':
if (options.expunged_mode != RESTORE_EXPUNGED_OKAY) usage();
options.expunged_mode = RESTORE_EXPUNGED_ONLY;
break;
case 'z':
options.require_compression = 1;
break;
case ':':
if (optopt == 'A') options.override_acl = "";
else usage();
break;
default:
usage();
break;
}
}
/* we need a server name */
if (optind == argc) usage();
servername = argv[optind++];
/* we need a source of backup data */
if (mode == RESTORE_MODE_UNSPECIFIED) {
if (optind == argc) usage();
backup_name = argv[optind++];
mode = RESTORE_MODE_USERNAME;
}
/* we need either an input file or some objects to restore */
if (!do_all_mailboxes && !input_file && optind == argc) usage();
/* and we can't have both because i said */
if ((input_file || do_all_mailboxes) && optind < argc) usage();
/* okay, arguments seem sane, we are go */
cyrus_init(alt_config, "restore", 0, 0);
/* load the SASL plugins */
global_sasl_init(1, 0, mysasl_cb);
/* wait here for gdb attach */
if (wait) {
fprintf(stderr, "Waiting for %d seconds for gdb attach...\n", wait);
sleep(wait);
}
/* open backup */
switch (mode) {
case RESTORE_MODE_FILENAME:
r = backup_open_paths(&backup, backup_name, NULL,
BACKUP_OPEN_NONBLOCK, BACKUP_OPEN_NOCREATE);
break;
case RESTORE_MODE_MBOXNAME:
mbname = mbname_from_intname(backup_name);
if (!mbname) usage();
r = backup_open(&backup, mbname,
BACKUP_OPEN_NONBLOCK, BACKUP_OPEN_NOCREATE);
mbname_free(&mbname);
break;
case RESTORE_MODE_USERNAME:
mbname = mbname_from_userid(backup_name);
if (!mbname) usage();
r = backup_open(&backup, mbname,
BACKUP_OPEN_NONBLOCK, BACKUP_OPEN_NOCREATE);
mbname_free(&mbname);
break;
default:
usage();
break;
}
if (r) goto done;
/* scan for objects to restore:
* mailboxes will have all messages added, modulo expunged_mode
* messages will be added individually with appropriate folder
*/
mailbox_list = xzmalloc(sizeof *mailbox_list);
reserve_folder_list = restore_make_reserve_folder_list(backup);
reserve_list = sync_reserve_list_create(SYNC_MSGID_LIST_HASH_SIZE);
if (input_file) {
char buf[MAX_MAILBOX_NAME + 2]; // \n\0
size_t len;
FILE *f;
if (0 != strcmp(input_file, "-")) {
f = fopen(input_file, "r");
if (!f) {
fprintf(stderr, "fopen %s: %s\n", input_file, strerror(errno));
goto done;// FIXME shut_down?
}
}
else {
f = stdin;
}
while (fgets(buf, sizeof(buf), f)) {
len = strlen(buf);
if (len > 0 && buf[len - 1] == '\n')
buf[--len] = '\0';
if (len == 0 || buf[0] == '#')
continue;
r = restore_add_object(buf, &options, backup, mailbox_list,
reserve_folder_list, reserve_list);
// FIXME r
}
fclose(f);
}
else if (do_all_mailboxes) {
struct backup_mailbox_list *all_mailboxes;
struct backup_mailbox *mailbox;
all_mailboxes = backup_get_mailboxes(backup, 0,
BACKUP_MAILBOX_ALL_RECORDS);
for (mailbox = all_mailboxes->head; mailbox; mailbox = mailbox->next) {
restore_add_mailbox(mailbox, &options, mailbox_list,
reserve_folder_list, reserve_list);
}
backup_mailbox_list_empty(all_mailboxes);
free(all_mailboxes);
}
else {
int i;
for (i = optind; i < argc; i++) {
r = restore_add_object(argv[i], &options, backup, mailbox_list,
reserve_folder_list, reserve_list);
// FIXME r
}
}
if (do_nothing) {
if (options.verbose)
fprintf(stderr, "do nothing (-n) specified, exiting now\n");
goto done;
}
/* connect to destination */
backend = restore_connect(servername, &tagbuf, &options);
if (!backend) {
// FIXME
r = -1;
goto done;
}
/* do the restore */
struct sync_reserve *reserve;
for (reserve = reserve_list->head; reserve; reserve = reserve->next) {
/* send APPLY RESERVE and parse missing lists */
r = sync_reserve_partition(reserve->part,
reserve_folder_list,
reserve->list,
backend);
if (r) goto done;
/* send APPLY MESSAGEs */
r = backup_prepare_message_upload(backup,
reserve->part,
reserve->list,
&sync_msgid_lookup,
&upload);
if (r) goto done;
/* upload in small(ish) blocks to avoid timeouts */
while (upload->head) {
struct dlist *block = dlist_splice(upload, 1024);
sync_send_apply(block, backend->out);
r = sync_parse_response("MESSAGE", backend->in, NULL);
dlist_unlink_files(block);
dlist_free(&block);
if (r) goto done;
}
}
/* sync_prepare_dlists needs to upload messages per-mailbox, because
* it needs the mailbox to find the filename for the message. but
* we have no such limitation, so we can upload messages while
* looping over the reserve_list instead, above.
*
* alternatively, if we do it on a per-mailbox basis then we can limit
* the hit on the staging directory to only a mailbox worth of messages
* at a time. but we don't have a logical grouping that would make
* this coherent
*/
/* send RESTORE MAILBOXes */
struct backup_mailbox *mailbox;
for (mailbox = mailbox_list->head; mailbox; mailbox = mailbox->next) {
/* XXX filter the mailbox records based on reserve/missing/upload */
/* XXX does this sensibly handle mailbox objs with empty values? */
struct dlist *dl = backup_mailbox_to_dlist(mailbox);
if (!dl) continue;
if (local_only) {
free(dl->name);
dl->name = xstrdup("LOCAL_MAILBOX");
}
sync_send_restore(dl, backend->out);
r = sync_parse_response("MAILBOX", backend->in, NULL);
dlist_free(&dl);
if (r) goto done;
}
done:
if (r)
fprintf(stderr, "%s: %s\n", backup_name, error_message(r));
/* release lock asap */
if (backup)
backup_close(&backup);
if (backend)
backend_disconnect(backend);
if (upload) {
dlist_unlink_files(upload);
dlist_free(&upload);
}
if (mailbox_list) {
backup_mailbox_list_empty(mailbox_list);
free(mailbox_list);
}
if (reserve_folder_list)
sync_folder_list_free(&reserve_folder_list);
if (reserve_list)
sync_reserve_list_free(&reserve_list);
buf_free(&tagbuf);
backup_cleanup_staging_path();
cyrus_done();
exit(r ? EX_TEMPFAIL : EX_OK);
}
/* returns:
* 0 on success
* 1 if compact was not needed
* negative on error
*/
EXPORTED int backup_compact(const char *name,
enum backup_open_nonblock nonblock,
int force, int verbose, FILE *out)
{
struct backup *original = NULL;
struct backup *compact = NULL;
struct backup_chunk_list *all_chunks = NULL;
struct backup_chunk_list *keep_chunks = NULL;
struct backup_chunk *chunk = NULL;
struct sync_msgid_list *keep_message_guids = NULL;
struct gzuncat *gzuc = NULL;
struct protstream *in = NULL;
time_t since, chunk_start_time, ts;
int r;
compact_readconfig();
r = compact_open(name, &original, &compact, nonblock);
if (r) return r;
/* calculate current time after obtaining locks, in case of a wait */
const time_t now = time(NULL);
const int retention_days = config_getint(IMAPOPT_BACKUP_RETENTION_DAYS);
if (retention_days > 0) {
since = now - (retention_days * 24 * 60 * 60);
}
else {
/* zero or negative retention days means "keep forever" */
since = -1;
}
all_chunks = backup_get_chunks(original);
if (!all_chunks) goto error;
keep_chunks = backup_get_live_chunks(original, since);
if (!keep_chunks) goto error;
if (!force && !compact_required(all_chunks, keep_chunks)) {
/* nothing to do */
backup_chunk_list_free(&all_chunks);
backup_chunk_list_free(&keep_chunks);
backup_unlink(&compact);
backup_close(&original);
return 1;
}
if (verbose) {
fprintf(out, "keeping " SIZE_T_FMT " chunks:\n", keep_chunks->count);
for (chunk = keep_chunks->head; chunk; chunk = chunk->next) {
fprintf(out, " %d", chunk->id);
}
fprintf(out, "\n");
}
gzuc = gzuc_new(original->fd);
if (!gzuc) goto error;
chunk_start_time = -1;
ts = 0;
struct buf cmd = BUF_INITIALIZER;
for (chunk = keep_chunks->head; chunk; chunk = chunk->next) {
keep_message_guids = sync_msgid_list_create(0);
r = backup_message_foreach(original, chunk->id, &since,
_keep_message_guids_cb, keep_message_guids);
if (r) goto error;
gzuc_member_start_from(gzuc, chunk->offset);
in = prot_readcb(_prot_fill_cb, gzuc);
while (1) {
struct dlist *dl = NULL;
int c = parse_backup_line(in, &ts, &cmd, &dl);
if (c == EOF) {
const char *error = prot_error(in);
if (error && 0 != strcmp(error, PROT_EOF_STRING)) {
syslog(LOG_ERR,
"IOERROR: %s: error reading chunk at offset %jd, byte %i: %s\n",
name, chunk->offset, prot_bytes_in(in), error);
if (out)
fprintf(out, "error reading chunk at offset %jd, byte %i: %s\n",
chunk->offset, prot_bytes_in(in), error);
r = IMAP_IOERROR;
goto error;
}
break;
}
if (chunk_start_time == -1) {
r = backup_append_start(compact, &ts, BACKUP_APPEND_NOFLUSH);
if (r) goto error;
chunk_start_time = ts;
}
// XXX if this line is worth keeping
if (want_append(original, chunk->id, dl, keep_message_guids)) {
// FIXME if message is removed due to unneeded chunk,
// subsequent mailbox lines for it will fail here
// so we need to be able to tell which lines apply to messages we don't want anymore
r = backup_append(compact, dl, &ts, BACKUP_APPEND_NOFLUSH);
if (r) goto error;
}
dlist_unlink_files(dl);
dlist_free(&dl);
// if this line put us over compact_maxsize
if (want_split(chunk, &compact->append_state->wrote)) {
r = backup_append_end(compact, &ts);
chunk_start_time = -1;
if (verbose) {
fprintf(out, "splitting chunk %d\n", chunk->id);
}
}
}
// if we're due to start a new chunk
if (compact->append_state && compact->append_state->mode) {
if (!want_combine(compact->append_state->wrote, chunk->next)) {
r = backup_append_end(compact, &ts);
chunk_start_time = -1;
}
else if (verbose) {
fprintf(out, "combining chunks %d and %d\n",
chunk->id, chunk->next->id);
}
}
prot_free(in);
in = NULL;
gzuc_member_end(gzuc, NULL);
sync_msgid_list_free(&keep_message_guids);
}
buf_free(&cmd);
if (compact->append_state && compact->append_state->mode)
backup_append_end(compact, &ts);
gzuc_free(&gzuc);
backup_chunk_list_free(&keep_chunks);
/* if we get here okay, then the compact succeeded */
r = compact_closerename(&original, &compact, now);
if (r) goto error;
return 0;
error:
if (in) prot_free(in);
if (gzuc) gzuc_free(&gzuc);
if (keep_message_guids) sync_msgid_list_free(&keep_message_guids);
if (all_chunks) backup_chunk_list_free(&all_chunks);
if (keep_chunks) backup_chunk_list_free(&keep_chunks);
if (compact) backup_unlink(&compact);
if (original) backup_close(&original);
return r ? r : -1;
}
