static struct sync_folder_list *restore_make_reserve_folder_list(
struct backup *backup)
{
struct sync_folder_list *folder_list = sync_folder_list_create();
struct backup_mailbox_list *mailboxes = NULL;
struct backup_mailbox *iter;
mailboxes = backup_get_mailboxes(backup, 0, BACKUP_MAILBOX_NO_RECORDS);
if (mailboxes) {
for (iter = mailboxes->head; iter; iter = iter->next) {
const struct synccrcs synccrcs = { 0, 0 };
/* we only care about mboxname here */
sync_folder_list_add(folder_list, NULL, iter->mboxname,
0, NULL, NULL, 0, 0, 0, 0, synccrcs,
0, 0, 0, 0, NULL, 0, 0, 0);
}
backup_mailbox_list_empty(mailboxes);
free(mailboxes);
}
return folder_list;
}
static int cmd_show_messages(struct backup *backup,
const struct cyrbu_cmd_options *options)
{
struct backup_message *message = NULL;
struct backup_mailbox_list *mailboxes = NULL;
struct backup_mailbox *mailbox;
struct message_guid want_guid;
int i;
for (i = 0; i < strarray_size(options->argv); i++) {
if (!message_guid_decode(&want_guid, strarray_nth(options->argv, i)))
continue;
message = backup_get_message(backup, &want_guid);
if (!message)
continue;
fprintf(stdout, "guid:\t%s\n", message_guid_encode(message->guid));
mailboxes = backup_get_mailboxes_by_message(backup, message, 0);
if (mailboxes) {
fprintf(stdout, "mailboxes:\n");
for (mailbox = mailboxes->head; mailbox; mailbox = mailbox->next) {
fprintf(stdout, "\t%s\t%s\n",
mailbox->uniqueid,
mailbox->mboxname);
}
backup_mailbox_list_empty(mailboxes);
free(mailboxes);
}
fprintf(stdout, "headers:\n");
backup_read_message_data(backup, message,
show_message_headers, stdout);
fprintf(stdout, "\n");
backup_message_free(&message);
}
return 0;
}
/* verify that the matching MAILBOX exists within the claimed chunk
* for each mailbox or mailbox_message in the index
*/
static int verify_chunk_mailbox_links(struct backup *backup, struct backup_chunk *chunk,
struct gzuncat *gzuc, int verbose, FILE *out)
{
/*
* get list of mailboxes in chunk
* get list of mailbox_messages in chunk
* index mailboxes list by uniqueid
* index mailbox_messages list by uniqueid:uid
* open chunk
* foreach line in chunk
* read dlist
* skip if it's not a mailbox
* if details in dlist match details in mailbox
* remove from mailbox list/index
* foreach record in dlist
* if details in dlist match details in mailbox_message
* remove from mailbox_message list/index
* failed if either list of mailboxes or list of mailbox_messages is not empty
*/
struct backup_mailbox_list *mailbox_list = NULL;
struct backup_mailbox_message_list *mailbox_message_list = NULL;
hash_table mailbox_list_index = HASH_TABLE_INITIALIZER;
hash_table mailbox_message_list_index = HASH_TABLE_INITIALIZER;
struct backup_mailbox *mailbox = NULL;
struct backup_mailbox_message *mailbox_message = NULL;
int r;
if (out && verbose)
fprintf(out, "checking chunk %d mailbox links...\n", chunk->id);
mailbox_list = backup_get_mailboxes(backup, chunk->id, 0);
mailbox_message_list = backup_get_mailbox_messages(backup, chunk->id);
if (mailbox_list->count == 0 && mailbox_message_list->count == 0) {
/* nothing we care about in this chunk */
free(mailbox_list);
free(mailbox_message_list);
if (out && verbose)
fprintf(out, "ok\n");
return 0;
}
/* XXX consider whether the two hashes should use pools */
if (mailbox_list->count) {
/* build an index of the mailbox list */
construct_hash_table(&mailbox_list_index, mailbox_list->count, 0);
mailbox = mailbox_list->head;
while (mailbox) {
hash_insert(mailbox->uniqueid, mailbox, &mailbox_list_index);
mailbox = mailbox->next;
}
}
if (mailbox_message_list->count) {
/* build an index of the mailbox message list */
construct_hash_table(&mailbox_message_list_index,
mailbox_message_list->count, 0);
mailbox_message = mailbox_message_list->head;
while (mailbox_message) {
char keybuf[1024]; // FIXME whatever
snprintf(keybuf, sizeof(keybuf), "%s:%d",
mailbox_message->mailbox_uniqueid, mailbox_message->uid);
hash_insert(keybuf, mailbox_message, &mailbox_message_list_index);
mailbox_message = mailbox_message->next;
}
}
r = gzuc_member_start_from(gzuc, chunk->offset);
if (r) {
syslog(LOG_ERR, "%s: error reading chunk %i at offset %jd: %s",
__func__, chunk->id, chunk->offset, zError(r));
if (out)
fprintf(out, "error reading chunk %i at offset %jd: %s",
chunk->id, chunk->offset, zError(r));
goto done;
}
struct protstream *ps = prot_readcb(_prot_fill_cb, gzuc);
prot_setisclient(ps, 1); /* don't sync literals */
struct buf cmd = BUF_INITIALIZER;
while (1) {
struct dlist *dl = NULL;
struct dlist *record = NULL;
struct dlist *di = NULL;
const char *uniqueid = NULL;
int c = parse_backup_line(ps, NULL, &cmd, &dl);
if (c == EOF) {
const char *error = prot_error(ps);
if (error && 0 != strcmp(error, PROT_EOF_STRING)) {
syslog(LOG_ERR,
"%s: error reading chunk %i data at offset %jd, byte %i: %s",
__func__, chunk->id, chunk->offset, prot_bytes_in(ps), error);
if (out)
fprintf(out, "error reading chunk %i data at offset %jd, byte %i: %s",
chunk->id, chunk->offset, prot_bytes_in(ps), error);
r = EOF;
}
break;
}
if (strcmp(buf_cstring(&cmd), "APPLY") != 0)
goto next_line;
if (strcmp(dl->name, "MAILBOX") != 0)
goto next_line;
if (!dlist_getatom(dl, "UNIQUEID", &uniqueid))
goto next_line;
if (mailbox_list->count) {
mailbox = (struct backup_mailbox *) hash_lookup(uniqueid, &mailbox_list_index);
if (mailbox && mailbox_matches(mailbox, dl)) {
backup_mailbox_list_remove(mailbox_list, mailbox);
hash_del(uniqueid, &mailbox_list_index);
backup_mailbox_free(&mailbox);
}
}
if (mailbox_message_list->count) {
if (!dlist_getlist(dl, "RECORD", &record))
goto next_line;
for (di = record->head; di; di = di->next) {
char keybuf[1024]; // FIXME whatever
uint32_t uid;
if (!dlist_getnum32(di, "UID", &uid))
continue;
snprintf(keybuf, sizeof(keybuf), "%s:%d", uniqueid, uid);
mailbox_message = (struct backup_mailbox_message *) hash_lookup(
keybuf, &mailbox_message_list_index);
if (!mailbox_message)
continue;
if (!mailbox_message_matches(mailbox_message, di))
continue;
backup_mailbox_message_list_remove(mailbox_message_list, mailbox_message);
hash_del(keybuf, &mailbox_message_list_index);
backup_mailbox_message_free(&mailbox_message);
}
}
next_line:
if (dl) {
dlist_unlink_files(dl);
dlist_free(&dl);
}
}
buf_free(&cmd);
prot_free(ps);
gzuc_member_end(gzuc, NULL);
/* anything left in either of the lists is missing from the chunk data. bad! */
mailbox = mailbox_list->head;
while (mailbox) {
syslog(LOG_DEBUG, "%s: chunk %d missing mailbox data for %s (%s)\n",
__func__, chunk->id, mailbox->uniqueid, mailbox->mboxname);
if (out)
fprintf(out, "chunk %d missing mailbox data for %s (%s)\n",
chunk->id, mailbox->uniqueid, mailbox->mboxname);
mailbox = mailbox->next;
}
mailbox_message = mailbox_message_list->head;
while (mailbox_message) {
syslog(LOG_DEBUG, "%s: chunk %d missing mailbox_message data for %s uid %u\n",
__func__, chunk->id, mailbox_message->mailbox_uniqueid,
mailbox_message->uid);
if (out)
fprintf(out, "chunk %d missing mailbox_message data for %s uid %u\n",
chunk->id, mailbox_message->mailbox_uniqueid,
mailbox_message->uid);
mailbox_message = mailbox_message->next;
}
if (!r) r = mailbox_list->count || mailbox_message_list->count ? -1 : 0;
done:
free_hash_table(&mailbox_list_index, NULL);
free_hash_table(&mailbox_message_list_index, NULL);
backup_mailbox_list_empty(mailbox_list);
free(mailbox_list);
backup_mailbox_message_list_empty(mailbox_message_list);
free(mailbox_message_list);
syslog(LOG_DEBUG, "%s: chunk %d %s!\n", __func__, chunk->id,
r ? "failed" : "passed");
if (out && verbose)
fprintf(out, "%s\n", r ? "error" : "ok");
return r;
}
static int restore_add_object(const char *object_name,
const struct restore_options *options,
struct backup *backup,
struct backup_mailbox_list *mailbox_list,
struct sync_folder_list *reserve_folder_list,
struct sync_reserve_list *reserve_list)
{
struct backup_mailbox *mailbox = NULL;
struct backup_message *message = NULL;
struct message_guid tmp_guid;
size_t len;
int r;
/* try to work out what we're restoring */
len = strlen(object_name);
if (len == 24 && strspn(object_name, HEX_DIGITS) == len) {
/* looks like a non-libuuid uniqueid */
mailbox = backup_get_mailbox_by_uniqueid(backup, object_name,
BACKUP_MAILBOX_ALL_RECORDS);
}
else if (len == 36 && strspn(object_name, "-" HEX_DIGITS) == len) {
/* looks like a libuuid uniqueid */
mailbox = backup_get_mailbox_by_uniqueid(backup, object_name,
BACKUP_MAILBOX_ALL_RECORDS);
}
else if (message_guid_decode(&tmp_guid, object_name)) {
/* looks like it's a message guid */
message = backup_get_message(backup, &tmp_guid);
}
else if (strchr(object_name, '.')) {
/* has a dot, might be an mboxname */
mbname_t *mbname = mbname_from_intname(object_name);
mailbox = backup_get_mailbox_by_name(backup, mbname,
BACKUP_MAILBOX_ALL_RECORDS);
mbname_free(&mbname);
}
else {
/* not sure what it is, guess mboxname? */
mbname_t *mbname = mbname_from_intname(object_name);
mailbox = backup_get_mailbox_by_name(backup, mbname,
BACKUP_MAILBOX_ALL_RECORDS);
mbname_free(&mbname);
}
/* add it to the restore lists */
if (mailbox) {
r = restore_add_mailbox(mailbox, options, mailbox_list,
reserve_folder_list, reserve_list);
if (!r && options->do_submailboxes) {
char prefix[MAX_MAILBOX_NAME + 1];
int len;
len = snprintf(prefix, sizeof(prefix), "%s.", mailbox->mboxname);
/* can only be submailboxes if parent's name is short enough... */
if (len < MAX_MAILBOX_NAME) {
struct submailbox_rock rock = {
prefix,
strlen(prefix),
options,
mailbox_list,
reserve_folder_list,
reserve_list,
};
r = backup_mailbox_foreach(backup, 0,
BACKUP_MAILBOX_ALL_RECORDS,
submailbox_cb, &rock);
}
}
backup_mailbox_free(&mailbox);
}
else if (message) {
struct backup_mailbox_list *mailboxes = NULL;
if (!options->override_mboxname)
mailboxes = backup_get_mailboxes_by_message(backup, message,
BACKUP_MAILBOX_MATCH_RECORDS);
r = restore_add_message(message, mailboxes, options, mailbox_list,
reserve_folder_list, reserve_list);
if (mailboxes) {
backup_mailbox_list_empty(mailboxes);
free(mailboxes);
}
backup_message_free(&message);
}
else {
r = IMAP_MAILBOX_NONEXISTENT;
}
return r;
}
int main(int argc, char **argv)
{
save_argv0(argv[0]);
const char *alt_config = NULL;
const char *input_file = NULL;
const char *backup_name = NULL;
const char *servername = NULL;
enum restore_mode mode = RESTORE_MODE_UNSPECIFIED;
int local_only = 0;
int wait = 0;
int do_nothing = 0;
int do_all_mailboxes = 0;
struct restore_options options = {0};
options.expunged_mode = RESTORE_EXPUNGED_OKAY;
options.trim_deletedprefix = 1;
struct backup *backup = NULL;
mbname_t *mbname = NULL;
struct backup_mailbox_list *mailbox_list = NULL;
struct sync_folder_list *reserve_folder_list = NULL;
struct sync_reserve_list *reserve_list = NULL;
struct buf tagbuf = BUF_INITIALIZER;
struct backend *backend = NULL;
struct dlist *upload = NULL;
int opt, r;
while ((opt = getopt(argc, argv, ":A:C:DF:LM:P:UXaf:m:nru:vw:xz")) != EOF) {
switch (opt) {
case 'A':
if (options.keep_uidvalidity) usage();
options.override_acl = optarg;
break;
case 'C':
alt_config = optarg;
break;
case 'D':
/* XXX does this clash with keep_uidvalidity? */
options.trim_deletedprefix = 0;
break;
case 'F':
if (do_all_mailboxes) usage();
input_file = optarg;
break;
case 'L':
local_only = 1;
break;
case 'M':
if (options.keep_uidvalidity) usage();
options.override_mboxname = optarg;
break;
case 'P':
if (options.keep_uidvalidity) usage();
options.override_partition = optarg;
break;
case 'U':
if (options.override_acl || options.override_mboxname || options.override_partition)
usage();
options.keep_uidvalidity = 1;
break;
case 'X':
if (options.expunged_mode != RESTORE_EXPUNGED_OKAY) usage();
options.expunged_mode = RESTORE_EXPUNGED_EXCLUDE;
break;
case 'a':
if (input_file) usage();
do_all_mailboxes = 1;
break;
case 'f':
if (mode != RESTORE_MODE_UNSPECIFIED) usage();
mode = RESTORE_MODE_FILENAME;
backup_name = optarg;
break;
case 'm':
if (mode != RESTORE_MODE_UNSPECIFIED) usage();
mode = RESTORE_MODE_MBOXNAME;
backup_name = optarg;
break;
case 'n':
do_nothing = 1;
break;
case 'r':
options.do_submailboxes = 1;
break;
case 'u':
if (mode != RESTORE_MODE_UNSPECIFIED) usage();
mode = RESTORE_MODE_USERNAME;
backup_name = optarg;
break;
case 'v':
options.verbose++;
break;
case 'w':
wait = atoi(optarg);
if (wait < 0) usage();
break;
case 'x':
if (options.expunged_mode != RESTORE_EXPUNGED_OKAY) usage();
options.expunged_mode = RESTORE_EXPUNGED_ONLY;
break;
case 'z':
options.require_compression = 1;
break;
case ':':
if (optopt == 'A') options.override_acl = "";
else usage();
break;
default:
usage();
break;
}
}
/* we need a server name */
if (optind == argc) usage();
servername = argv[optind++];
/* we need a source of backup data */
if (mode == RESTORE_MODE_UNSPECIFIED) {
if (optind == argc) usage();
backup_name = argv[optind++];
mode = RESTORE_MODE_USERNAME;
}
/* we need either an input file or some objects to restore */
if (!do_all_mailboxes && !input_file && optind == argc) usage();
/* and we can't have both because i said */
if ((input_file || do_all_mailboxes) && optind < argc) usage();
/* okay, arguments seem sane, we are go */
cyrus_init(alt_config, "restore", 0, 0);
/* load the SASL plugins */
global_sasl_init(1, 0, mysasl_cb);
/* wait here for gdb attach */
if (wait) {
fprintf(stderr, "Waiting for %d seconds for gdb attach...\n", wait);
sleep(wait);
}
/* open backup */
switch (mode) {
case RESTORE_MODE_FILENAME:
r = backup_open_paths(&backup, backup_name, NULL,
BACKUP_OPEN_NONBLOCK, BACKUP_OPEN_NOCREATE);
break;
case RESTORE_MODE_MBOXNAME:
mbname = mbname_from_intname(backup_name);
if (!mbname) usage();
r = backup_open(&backup, mbname,
BACKUP_OPEN_NONBLOCK, BACKUP_OPEN_NOCREATE);
mbname_free(&mbname);
break;
case RESTORE_MODE_USERNAME:
mbname = mbname_from_userid(backup_name);
if (!mbname) usage();
r = backup_open(&backup, mbname,
BACKUP_OPEN_NONBLOCK, BACKUP_OPEN_NOCREATE);
mbname_free(&mbname);
break;
default:
usage();
break;
}
if (r) goto done;
/* scan for objects to restore:
* mailboxes will have all messages added, modulo expunged_mode
* messages will be added individually with appropriate folder
*/
mailbox_list = xzmalloc(sizeof *mailbox_list);
reserve_folder_list = restore_make_reserve_folder_list(backup);
reserve_list = sync_reserve_list_create(SYNC_MSGID_LIST_HASH_SIZE);
if (input_file) {
char buf[MAX_MAILBOX_NAME + 2]; // \n\0
size_t len;
FILE *f;
if (0 != strcmp(input_file, "-")) {
f = fopen(input_file, "r");
if (!f) {
fprintf(stderr, "fopen %s: %s\n", input_file, strerror(errno));
goto done;// FIXME shut_down?
}
}
else {
f = stdin;
}
while (fgets(buf, sizeof(buf), f)) {
len = strlen(buf);
if (len > 0 && buf[len - 1] == '\n')
buf[--len] = '\0';
if (len == 0 || buf[0] == '#')
continue;
r = restore_add_object(buf, &options, backup, mailbox_list,
reserve_folder_list, reserve_list);
// FIXME r
}
fclose(f);
}
else if (do_all_mailboxes) {
struct backup_mailbox_list *all_mailboxes;
struct backup_mailbox *mailbox;
all_mailboxes = backup_get_mailboxes(backup, 0,
BACKUP_MAILBOX_ALL_RECORDS);
for (mailbox = all_mailboxes->head; mailbox; mailbox = mailbox->next) {
restore_add_mailbox(mailbox, &options, mailbox_list,
reserve_folder_list, reserve_list);
}
backup_mailbox_list_empty(all_mailboxes);
free(all_mailboxes);
}
else {
int i;
for (i = optind; i < argc; i++) {
r = restore_add_object(argv[i], &options, backup, mailbox_list,
reserve_folder_list, reserve_list);
// FIXME r
}
}
if (do_nothing) {
if (options.verbose)
fprintf(stderr, "do nothing (-n) specified, exiting now\n");
goto done;
}
/* connect to destination */
backend = restore_connect(servername, &tagbuf, &options);
if (!backend) {
// FIXME
r = -1;
goto done;
}
/* do the restore */
struct sync_reserve *reserve;
for (reserve = reserve_list->head; reserve; reserve = reserve->next) {
/* send APPLY RESERVE and parse missing lists */
r = sync_reserve_partition(reserve->part,
reserve_folder_list,
reserve->list,
backend);
if (r) goto done;
/* send APPLY MESSAGEs */
r = backup_prepare_message_upload(backup,
reserve->part,
reserve->list,
&sync_msgid_lookup,
&upload);
if (r) goto done;
/* upload in small(ish) blocks to avoid timeouts */
while (upload->head) {
struct dlist *block = dlist_splice(upload, 1024);
sync_send_apply(block, backend->out);
r = sync_parse_response("MESSAGE", backend->in, NULL);
dlist_unlink_files(block);
dlist_free(&block);
if (r) goto done;
}
}
/* sync_prepare_dlists needs to upload messages per-mailbox, because
* it needs the mailbox to find the filename for the message. but
* we have no such limitation, so we can upload messages while
* looping over the reserve_list instead, above.
*
* alternatively, if we do it on a per-mailbox basis then we can limit
* the hit on the staging directory to only a mailbox worth of messages
* at a time. but we don't have a logical grouping that would make
* this coherent
*/
/* send RESTORE MAILBOXes */
struct backup_mailbox *mailbox;
for (mailbox = mailbox_list->head; mailbox; mailbox = mailbox->next) {
/* XXX filter the mailbox records based on reserve/missing/upload */
/* XXX does this sensibly handle mailbox objs with empty values? */
struct dlist *dl = backup_mailbox_to_dlist(mailbox);
if (!dl) continue;
if (local_only) {
free(dl->name);
dl->name = xstrdup("LOCAL_MAILBOX");
}
sync_send_restore(dl, backend->out);
r = sync_parse_response("MAILBOX", backend->in, NULL);
dlist_free(&dl);
if (r) goto done;
}
done:
if (r)
fprintf(stderr, "%s: %s\n", backup_name, error_message(r));
/* release lock asap */
if (backup)
backup_close(&backup);
if (backend)
backend_disconnect(backend);
if (upload) {
dlist_unlink_files(upload);
dlist_free(&upload);
}
if (mailbox_list) {
backup_mailbox_list_empty(mailbox_list);
free(mailbox_list);
}
if (reserve_folder_list)
sync_folder_list_free(&reserve_folder_list);
if (reserve_list)
sync_reserve_list_free(&reserve_list);
buf_free(&tagbuf);
backup_cleanup_staging_path();
cyrus_done();
exit(r ? EX_TEMPFAIL : EX_OK);
}
