static struct sync_folder_list *restore_make_reserve_folder_list( struct backup *backup) { struct sync_folder_list *folder_list = sync_folder_list_create(); struct backup_mailbox_list *mailboxes = NULL; struct backup_mailbox *iter; mailboxes = backup_get_mailboxes(backup, 0, BACKUP_MAILBOX_NO_RECORDS); if (mailboxes) { for (iter = mailboxes->head; iter; iter = iter->next) { const struct synccrcs synccrcs = { 0, 0 }; /* we only care about mboxname here */ sync_folder_list_add(folder_list, NULL, iter->mboxname, 0, NULL, NULL, 0, 0, 0, 0, synccrcs, 0, 0, 0, 0, NULL, 0, 0, 0); } backup_mailbox_list_empty(mailboxes); free(mailboxes); } return folder_list; }
static int cmd_show_messages(struct backup *backup, const struct cyrbu_cmd_options *options) { struct backup_message *message = NULL; struct backup_mailbox_list *mailboxes = NULL; struct backup_mailbox *mailbox; struct message_guid want_guid; int i; for (i = 0; i < strarray_size(options->argv); i++) { if (!message_guid_decode(&want_guid, strarray_nth(options->argv, i))) continue; message = backup_get_message(backup, &want_guid); if (!message) continue; fprintf(stdout, "guid:\t%s\n", message_guid_encode(message->guid)); mailboxes = backup_get_mailboxes_by_message(backup, message, 0); if (mailboxes) { fprintf(stdout, "mailboxes:\n"); for (mailbox = mailboxes->head; mailbox; mailbox = mailbox->next) { fprintf(stdout, "\t%s\t%s\n", mailbox->uniqueid, mailbox->mboxname); } backup_mailbox_list_empty(mailboxes); free(mailboxes); } fprintf(stdout, "headers:\n"); backup_read_message_data(backup, message, show_message_headers, stdout); fprintf(stdout, "\n"); backup_message_free(&message); } return 0; }
/* verify that the matching MAILBOX exists within the claimed chunk * for each mailbox or mailbox_message in the index */ static int verify_chunk_mailbox_links(struct backup *backup, struct backup_chunk *chunk, struct gzuncat *gzuc, int verbose, FILE *out) { /* * get list of mailboxes in chunk * get list of mailbox_messages in chunk * index mailboxes list by uniqueid * index mailbox_messages list by uniqueid:uid * open chunk * foreach line in chunk * read dlist * skip if it's not a mailbox * if details in dlist match details in mailbox * remove from mailbox list/index * foreach record in dlist * if details in dlist match details in mailbox_message * remove from mailbox_message list/index * failed if either list of mailboxes or list of mailbox_messages is not empty */ struct backup_mailbox_list *mailbox_list = NULL; struct backup_mailbox_message_list *mailbox_message_list = NULL; hash_table mailbox_list_index = HASH_TABLE_INITIALIZER; hash_table mailbox_message_list_index = HASH_TABLE_INITIALIZER; struct backup_mailbox *mailbox = NULL; struct backup_mailbox_message *mailbox_message = NULL; int r; if (out && verbose) fprintf(out, "checking chunk %d mailbox links...\n", chunk->id); mailbox_list = backup_get_mailboxes(backup, chunk->id, 0); mailbox_message_list = backup_get_mailbox_messages(backup, chunk->id); if (mailbox_list->count == 0 && mailbox_message_list->count == 0) { /* nothing we care about in this chunk */ free(mailbox_list); free(mailbox_message_list); if (out && verbose) fprintf(out, "ok\n"); return 0; } /* XXX consider whether the two hashes should use pools */ if (mailbox_list->count) { /* build an index of the mailbox list */ construct_hash_table(&mailbox_list_index, mailbox_list->count, 0); mailbox = mailbox_list->head; while (mailbox) { hash_insert(mailbox->uniqueid, mailbox, &mailbox_list_index); mailbox = mailbox->next; } } if (mailbox_message_list->count) { /* build an index of the mailbox message list */ construct_hash_table(&mailbox_message_list_index, mailbox_message_list->count, 0); mailbox_message = mailbox_message_list->head; while (mailbox_message) { char keybuf[1024]; // FIXME whatever snprintf(keybuf, sizeof(keybuf), "%s:%d", mailbox_message->mailbox_uniqueid, mailbox_message->uid); hash_insert(keybuf, mailbox_message, &mailbox_message_list_index); mailbox_message = mailbox_message->next; } } r = gzuc_member_start_from(gzuc, chunk->offset); if (r) { syslog(LOG_ERR, "%s: error reading chunk %i at offset %jd: %s", __func__, chunk->id, chunk->offset, zError(r)); if (out) fprintf(out, "error reading chunk %i at offset %jd: %s", chunk->id, chunk->offset, zError(r)); goto done; } struct protstream *ps = prot_readcb(_prot_fill_cb, gzuc); prot_setisclient(ps, 1); /* don't sync literals */ struct buf cmd = BUF_INITIALIZER; while (1) { struct dlist *dl = NULL; struct dlist *record = NULL; struct dlist *di = NULL; const char *uniqueid = NULL; int c = parse_backup_line(ps, NULL, &cmd, &dl); if (c == EOF) { const char *error = prot_error(ps); if (error && 0 != strcmp(error, PROT_EOF_STRING)) { syslog(LOG_ERR, "%s: error reading chunk %i data at offset %jd, byte %i: %s", __func__, chunk->id, chunk->offset, prot_bytes_in(ps), error); if (out) fprintf(out, "error reading chunk %i data at offset %jd, byte %i: %s", chunk->id, chunk->offset, prot_bytes_in(ps), error); r = EOF; } break; } if (strcmp(buf_cstring(&cmd), "APPLY") != 0) goto next_line; if (strcmp(dl->name, "MAILBOX") != 0) goto next_line; if (!dlist_getatom(dl, "UNIQUEID", &uniqueid)) goto next_line; if (mailbox_list->count) { mailbox = (struct backup_mailbox *) hash_lookup(uniqueid, &mailbox_list_index); if (mailbox && mailbox_matches(mailbox, dl)) { backup_mailbox_list_remove(mailbox_list, mailbox); hash_del(uniqueid, &mailbox_list_index); backup_mailbox_free(&mailbox); } } if (mailbox_message_list->count) { if (!dlist_getlist(dl, "RECORD", &record)) goto next_line; for (di = record->head; di; di = di->next) { char keybuf[1024]; // FIXME whatever uint32_t uid; if (!dlist_getnum32(di, "UID", &uid)) continue; snprintf(keybuf, sizeof(keybuf), "%s:%d", uniqueid, uid); mailbox_message = (struct backup_mailbox_message *) hash_lookup( keybuf, &mailbox_message_list_index); if (!mailbox_message) continue; if (!mailbox_message_matches(mailbox_message, di)) continue; backup_mailbox_message_list_remove(mailbox_message_list, mailbox_message); hash_del(keybuf, &mailbox_message_list_index); backup_mailbox_message_free(&mailbox_message); } } next_line: if (dl) { dlist_unlink_files(dl); dlist_free(&dl); } } buf_free(&cmd); prot_free(ps); gzuc_member_end(gzuc, NULL); /* anything left in either of the lists is missing from the chunk data. bad! */ mailbox = mailbox_list->head; while (mailbox) { syslog(LOG_DEBUG, "%s: chunk %d missing mailbox data for %s (%s)\n", __func__, chunk->id, mailbox->uniqueid, mailbox->mboxname); if (out) fprintf(out, "chunk %d missing mailbox data for %s (%s)\n", chunk->id, mailbox->uniqueid, mailbox->mboxname); mailbox = mailbox->next; } mailbox_message = mailbox_message_list->head; while (mailbox_message) { syslog(LOG_DEBUG, "%s: chunk %d missing mailbox_message data for %s uid %u\n", __func__, chunk->id, mailbox_message->mailbox_uniqueid, mailbox_message->uid); if (out) fprintf(out, "chunk %d missing mailbox_message data for %s uid %u\n", chunk->id, mailbox_message->mailbox_uniqueid, mailbox_message->uid); mailbox_message = mailbox_message->next; } if (!r) r = mailbox_list->count || mailbox_message_list->count ? -1 : 0; done: free_hash_table(&mailbox_list_index, NULL); free_hash_table(&mailbox_message_list_index, NULL); backup_mailbox_list_empty(mailbox_list); free(mailbox_list); backup_mailbox_message_list_empty(mailbox_message_list); free(mailbox_message_list); syslog(LOG_DEBUG, "%s: chunk %d %s!\n", __func__, chunk->id, r ? "failed" : "passed"); if (out && verbose) fprintf(out, "%s\n", r ? "error" : "ok"); return r; }
static int restore_add_object(const char *object_name, const struct restore_options *options, struct backup *backup, struct backup_mailbox_list *mailbox_list, struct sync_folder_list *reserve_folder_list, struct sync_reserve_list *reserve_list) { struct backup_mailbox *mailbox = NULL; struct backup_message *message = NULL; struct message_guid tmp_guid; size_t len; int r; /* try to work out what we're restoring */ len = strlen(object_name); if (len == 24 && strspn(object_name, HEX_DIGITS) == len) { /* looks like a non-libuuid uniqueid */ mailbox = backup_get_mailbox_by_uniqueid(backup, object_name, BACKUP_MAILBOX_ALL_RECORDS); } else if (len == 36 && strspn(object_name, "-" HEX_DIGITS) == len) { /* looks like a libuuid uniqueid */ mailbox = backup_get_mailbox_by_uniqueid(backup, object_name, BACKUP_MAILBOX_ALL_RECORDS); } else if (message_guid_decode(&tmp_guid, object_name)) { /* looks like it's a message guid */ message = backup_get_message(backup, &tmp_guid); } else if (strchr(object_name, '.')) { /* has a dot, might be an mboxname */ mbname_t *mbname = mbname_from_intname(object_name); mailbox = backup_get_mailbox_by_name(backup, mbname, BACKUP_MAILBOX_ALL_RECORDS); mbname_free(&mbname); } else { /* not sure what it is, guess mboxname? */ mbname_t *mbname = mbname_from_intname(object_name); mailbox = backup_get_mailbox_by_name(backup, mbname, BACKUP_MAILBOX_ALL_RECORDS); mbname_free(&mbname); } /* add it to the restore lists */ if (mailbox) { r = restore_add_mailbox(mailbox, options, mailbox_list, reserve_folder_list, reserve_list); if (!r && options->do_submailboxes) { char prefix[MAX_MAILBOX_NAME + 1]; int len; len = snprintf(prefix, sizeof(prefix), "%s.", mailbox->mboxname); /* can only be submailboxes if parent's name is short enough... */ if (len < MAX_MAILBOX_NAME) { struct submailbox_rock rock = { prefix, strlen(prefix), options, mailbox_list, reserve_folder_list, reserve_list, }; r = backup_mailbox_foreach(backup, 0, BACKUP_MAILBOX_ALL_RECORDS, submailbox_cb, &rock); } } backup_mailbox_free(&mailbox); } else if (message) { struct backup_mailbox_list *mailboxes = NULL; if (!options->override_mboxname) mailboxes = backup_get_mailboxes_by_message(backup, message, BACKUP_MAILBOX_MATCH_RECORDS); r = restore_add_message(message, mailboxes, options, mailbox_list, reserve_folder_list, reserve_list); if (mailboxes) { backup_mailbox_list_empty(mailboxes); free(mailboxes); } backup_message_free(&message); } else { r = IMAP_MAILBOX_NONEXISTENT; } return r; }
int main(int argc, char **argv) { save_argv0(argv[0]); const char *alt_config = NULL; const char *input_file = NULL; const char *backup_name = NULL; const char *servername = NULL; enum restore_mode mode = RESTORE_MODE_UNSPECIFIED; int local_only = 0; int wait = 0; int do_nothing = 0; int do_all_mailboxes = 0; struct restore_options options = {0}; options.expunged_mode = RESTORE_EXPUNGED_OKAY; options.trim_deletedprefix = 1; struct backup *backup = NULL; mbname_t *mbname = NULL; struct backup_mailbox_list *mailbox_list = NULL; struct sync_folder_list *reserve_folder_list = NULL; struct sync_reserve_list *reserve_list = NULL; struct buf tagbuf = BUF_INITIALIZER; struct backend *backend = NULL; struct dlist *upload = NULL; int opt, r; while ((opt = getopt(argc, argv, ":A:C:DF:LM:P:UXaf:m:nru:vw:xz")) != EOF) { switch (opt) { case 'A': if (options.keep_uidvalidity) usage(); options.override_acl = optarg; break; case 'C': alt_config = optarg; break; case 'D': /* XXX does this clash with keep_uidvalidity? */ options.trim_deletedprefix = 0; break; case 'F': if (do_all_mailboxes) usage(); input_file = optarg; break; case 'L': local_only = 1; break; case 'M': if (options.keep_uidvalidity) usage(); options.override_mboxname = optarg; break; case 'P': if (options.keep_uidvalidity) usage(); options.override_partition = optarg; break; case 'U': if (options.override_acl || options.override_mboxname || options.override_partition) usage(); options.keep_uidvalidity = 1; break; case 'X': if (options.expunged_mode != RESTORE_EXPUNGED_OKAY) usage(); options.expunged_mode = RESTORE_EXPUNGED_EXCLUDE; break; case 'a': if (input_file) usage(); do_all_mailboxes = 1; break; case 'f': if (mode != RESTORE_MODE_UNSPECIFIED) usage(); mode = RESTORE_MODE_FILENAME; backup_name = optarg; break; case 'm': if (mode != RESTORE_MODE_UNSPECIFIED) usage(); mode = RESTORE_MODE_MBOXNAME; backup_name = optarg; break; case 'n': do_nothing = 1; break; case 'r': options.do_submailboxes = 1; break; case 'u': if (mode != RESTORE_MODE_UNSPECIFIED) usage(); mode = RESTORE_MODE_USERNAME; backup_name = optarg; break; case 'v': options.verbose++; break; case 'w': wait = atoi(optarg); if (wait < 0) usage(); break; case 'x': if (options.expunged_mode != RESTORE_EXPUNGED_OKAY) usage(); options.expunged_mode = RESTORE_EXPUNGED_ONLY; break; case 'z': options.require_compression = 1; break; case ':': if (optopt == 'A') options.override_acl = ""; else usage(); break; default: usage(); break; } } /* we need a server name */ if (optind == argc) usage(); servername = argv[optind++]; /* we need a source of backup data */ if (mode == RESTORE_MODE_UNSPECIFIED) { if (optind == argc) usage(); backup_name = argv[optind++]; mode = RESTORE_MODE_USERNAME; } /* we need either an input file or some objects to restore */ if (!do_all_mailboxes && !input_file && optind == argc) usage(); /* and we can't have both because i said */ if ((input_file || do_all_mailboxes) && optind < argc) usage(); /* okay, arguments seem sane, we are go */ cyrus_init(alt_config, "restore", 0, 0); /* load the SASL plugins */ global_sasl_init(1, 0, mysasl_cb); /* wait here for gdb attach */ if (wait) { fprintf(stderr, "Waiting for %d seconds for gdb attach...\n", wait); sleep(wait); } /* open backup */ switch (mode) { case RESTORE_MODE_FILENAME: r = backup_open_paths(&backup, backup_name, NULL, BACKUP_OPEN_NONBLOCK, BACKUP_OPEN_NOCREATE); break; case RESTORE_MODE_MBOXNAME: mbname = mbname_from_intname(backup_name); if (!mbname) usage(); r = backup_open(&backup, mbname, BACKUP_OPEN_NONBLOCK, BACKUP_OPEN_NOCREATE); mbname_free(&mbname); break; case RESTORE_MODE_USERNAME: mbname = mbname_from_userid(backup_name); if (!mbname) usage(); r = backup_open(&backup, mbname, BACKUP_OPEN_NONBLOCK, BACKUP_OPEN_NOCREATE); mbname_free(&mbname); break; default: usage(); break; } if (r) goto done; /* scan for objects to restore: * mailboxes will have all messages added, modulo expunged_mode * messages will be added individually with appropriate folder */ mailbox_list = xzmalloc(sizeof *mailbox_list); reserve_folder_list = restore_make_reserve_folder_list(backup); reserve_list = sync_reserve_list_create(SYNC_MSGID_LIST_HASH_SIZE); if (input_file) { char buf[MAX_MAILBOX_NAME + 2]; // \n\0 size_t len; FILE *f; if (0 != strcmp(input_file, "-")) { f = fopen(input_file, "r"); if (!f) { fprintf(stderr, "fopen %s: %s\n", input_file, strerror(errno)); goto done;// FIXME shut_down? } } else { f = stdin; } while (fgets(buf, sizeof(buf), f)) { len = strlen(buf); if (len > 0 && buf[len - 1] == '\n') buf[--len] = '\0'; if (len == 0 || buf[0] == '#') continue; r = restore_add_object(buf, &options, backup, mailbox_list, reserve_folder_list, reserve_list); // FIXME r } fclose(f); } else if (do_all_mailboxes) { struct backup_mailbox_list *all_mailboxes; struct backup_mailbox *mailbox; all_mailboxes = backup_get_mailboxes(backup, 0, BACKUP_MAILBOX_ALL_RECORDS); for (mailbox = all_mailboxes->head; mailbox; mailbox = mailbox->next) { restore_add_mailbox(mailbox, &options, mailbox_list, reserve_folder_list, reserve_list); } backup_mailbox_list_empty(all_mailboxes); free(all_mailboxes); } else { int i; for (i = optind; i < argc; i++) { r = restore_add_object(argv[i], &options, backup, mailbox_list, reserve_folder_list, reserve_list); // FIXME r } } if (do_nothing) { if (options.verbose) fprintf(stderr, "do nothing (-n) specified, exiting now\n"); goto done; } /* connect to destination */ backend = restore_connect(servername, &tagbuf, &options); if (!backend) { // FIXME r = -1; goto done; } /* do the restore */ struct sync_reserve *reserve; for (reserve = reserve_list->head; reserve; reserve = reserve->next) { /* send APPLY RESERVE and parse missing lists */ r = sync_reserve_partition(reserve->part, reserve_folder_list, reserve->list, backend); if (r) goto done; /* send APPLY MESSAGEs */ r = backup_prepare_message_upload(backup, reserve->part, reserve->list, &sync_msgid_lookup, &upload); if (r) goto done; /* upload in small(ish) blocks to avoid timeouts */ while (upload->head) { struct dlist *block = dlist_splice(upload, 1024); sync_send_apply(block, backend->out); r = sync_parse_response("MESSAGE", backend->in, NULL); dlist_unlink_files(block); dlist_free(&block); if (r) goto done; } } /* sync_prepare_dlists needs to upload messages per-mailbox, because * it needs the mailbox to find the filename for the message. but * we have no such limitation, so we can upload messages while * looping over the reserve_list instead, above. * * alternatively, if we do it on a per-mailbox basis then we can limit * the hit on the staging directory to only a mailbox worth of messages * at a time. but we don't have a logical grouping that would make * this coherent */ /* send RESTORE MAILBOXes */ struct backup_mailbox *mailbox; for (mailbox = mailbox_list->head; mailbox; mailbox = mailbox->next) { /* XXX filter the mailbox records based on reserve/missing/upload */ /* XXX does this sensibly handle mailbox objs with empty values? */ struct dlist *dl = backup_mailbox_to_dlist(mailbox); if (!dl) continue; if (local_only) { free(dl->name); dl->name = xstrdup("LOCAL_MAILBOX"); } sync_send_restore(dl, backend->out); r = sync_parse_response("MAILBOX", backend->in, NULL); dlist_free(&dl); if (r) goto done; } done: if (r) fprintf(stderr, "%s: %s\n", backup_name, error_message(r)); /* release lock asap */ if (backup) backup_close(&backup); if (backend) backend_disconnect(backend); if (upload) { dlist_unlink_files(upload); dlist_free(&upload); } if (mailbox_list) { backup_mailbox_list_empty(mailbox_list); free(mailbox_list); } if (reserve_folder_list) sync_folder_list_free(&reserve_folder_list); if (reserve_list) sync_reserve_list_free(&reserve_list); buf_free(&tagbuf); backup_cleanup_staging_path(); cyrus_done(); exit(r ? EX_TEMPFAIL : EX_OK); }