static int want_append_message(struct dlist *dlist, struct sync_msgid_list *keep_message_guids) { struct dlist *di, *next; for (di = dlist->head; di; di = next) { struct message_guid *guid = NULL; /* save next pointer now in case we need to unstitch */ next = di->next; if (!dlist_tofile(di, NULL, &guid, NULL, NULL)) continue; if (!sync_msgid_lookup(keep_message_guids, guid)) { syslog(LOG_DEBUG, "%s: MESSAGE no longer needed: %s", __func__, message_guid_encode(guid)); dlist_unstitch(dlist, di); dlist_unlink_files(di); dlist_free(&di); } } if (dlist->head) { syslog(LOG_DEBUG, "%s: keeping MESSAGE line", __func__); return 1; } syslog(LOG_DEBUG, "%s: MESSAGE line has no more messages", __func__); return 0; }
EXPORTED void dlist_unlink_files(struct dlist *dl) { struct dlist *i; if (!dl) return; for (i = dl->head; i; i = i->next) { dlist_unlink_files(i); } if (dl->type != DL_FILE) return; if (!dl->sval) return; syslog(LOG_DEBUG, "%s: unlinking %s", __func__, dl->sval); unlink(dl->sval); }
/* verify that each message exists within the chunk the index claims */ static int verify_chunk_messages(struct backup *backup, struct backup_chunk *chunk, struct gzuncat *gzuc, unsigned level, int verbose, FILE *out) { int r; struct verify_message_rock vmrock = { gzuc, (level & BACKUP_VERIFY_MESSAGE_GUIDS), NULL, 0, verbose, out, }; if (out && verbose) fprintf(out, "checking chunk %d messages...\n", chunk->id); r = gzuc_member_start_from(gzuc, chunk->offset); if (!r) { r = backup_message_foreach(backup, chunk->id, NULL, _verify_message_cb, &vmrock); gzuc_member_end(gzuc, NULL); } if (vmrock.cached_dlist) { dlist_unlink_files(vmrock.cached_dlist); dlist_free(&vmrock.cached_dlist); } syslog(LOG_DEBUG, "%s: chunk %d %s!\n", __func__, chunk->id, r ? "failed" : "passed"); if (out && verbose) fprintf(out, "%s\n", r ? "error" : "ok"); return r; }
/* verify that the matching MAILBOX exists within the claimed chunk * for each mailbox or mailbox_message in the index */ static int verify_chunk_mailbox_links(struct backup *backup, struct backup_chunk *chunk, struct gzuncat *gzuc, int verbose, FILE *out) { /* * get list of mailboxes in chunk * get list of mailbox_messages in chunk * index mailboxes list by uniqueid * index mailbox_messages list by uniqueid:uid * open chunk * foreach line in chunk * read dlist * skip if it's not a mailbox * if details in dlist match details in mailbox * remove from mailbox list/index * foreach record in dlist * if details in dlist match details in mailbox_message * remove from mailbox_message list/index * failed if either list of mailboxes or list of mailbox_messages is not empty */ struct backup_mailbox_list *mailbox_list = NULL; struct backup_mailbox_message_list *mailbox_message_list = NULL; hash_table mailbox_list_index = HASH_TABLE_INITIALIZER; hash_table mailbox_message_list_index = HASH_TABLE_INITIALIZER; struct backup_mailbox *mailbox = NULL; struct backup_mailbox_message *mailbox_message = NULL; int r; if (out && verbose) fprintf(out, "checking chunk %d mailbox links...\n", chunk->id); mailbox_list = backup_get_mailboxes(backup, chunk->id, 0); mailbox_message_list = backup_get_mailbox_messages(backup, chunk->id); if (mailbox_list->count == 0 && mailbox_message_list->count == 0) { /* nothing we care about in this chunk */ free(mailbox_list); free(mailbox_message_list); if (out && verbose) fprintf(out, "ok\n"); return 0; } /* XXX consider whether the two hashes should use pools */ if (mailbox_list->count) { /* build an index of the mailbox list */ construct_hash_table(&mailbox_list_index, mailbox_list->count, 0); mailbox = mailbox_list->head; while (mailbox) { hash_insert(mailbox->uniqueid, mailbox, &mailbox_list_index); mailbox = mailbox->next; } } if (mailbox_message_list->count) { /* build an index of the mailbox message list */ construct_hash_table(&mailbox_message_list_index, mailbox_message_list->count, 0); mailbox_message = mailbox_message_list->head; while (mailbox_message) { char keybuf[1024]; // FIXME whatever snprintf(keybuf, sizeof(keybuf), "%s:%d", mailbox_message->mailbox_uniqueid, mailbox_message->uid); hash_insert(keybuf, mailbox_message, &mailbox_message_list_index); mailbox_message = mailbox_message->next; } } r = gzuc_member_start_from(gzuc, chunk->offset); if (r) { syslog(LOG_ERR, "%s: error reading chunk %i at offset %jd: %s", __func__, chunk->id, chunk->offset, zError(r)); if (out) fprintf(out, "error reading chunk %i at offset %jd: %s", chunk->id, chunk->offset, zError(r)); goto done; } struct protstream *ps = prot_readcb(_prot_fill_cb, gzuc); prot_setisclient(ps, 1); /* don't sync literals */ struct buf cmd = BUF_INITIALIZER; while (1) { struct dlist *dl = NULL; struct dlist *record = NULL; struct dlist *di = NULL; const char *uniqueid = NULL; int c = parse_backup_line(ps, NULL, &cmd, &dl); if (c == EOF) { const char *error = prot_error(ps); if (error && 0 != strcmp(error, PROT_EOF_STRING)) { syslog(LOG_ERR, "%s: error reading chunk %i data at offset %jd, byte %i: %s", __func__, chunk->id, chunk->offset, prot_bytes_in(ps), error); if (out) fprintf(out, "error reading chunk %i data at offset %jd, byte %i: %s", chunk->id, chunk->offset, prot_bytes_in(ps), error); r = EOF; } break; } if (strcmp(buf_cstring(&cmd), "APPLY") != 0) goto next_line; if (strcmp(dl->name, "MAILBOX") != 0) goto next_line; if (!dlist_getatom(dl, "UNIQUEID", &uniqueid)) goto next_line; if (mailbox_list->count) { mailbox = (struct backup_mailbox *) hash_lookup(uniqueid, &mailbox_list_index); if (mailbox && mailbox_matches(mailbox, dl)) { backup_mailbox_list_remove(mailbox_list, mailbox); hash_del(uniqueid, &mailbox_list_index); backup_mailbox_free(&mailbox); } } if (mailbox_message_list->count) { if (!dlist_getlist(dl, "RECORD", &record)) goto next_line; for (di = record->head; di; di = di->next) { char keybuf[1024]; // FIXME whatever uint32_t uid; if (!dlist_getnum32(di, "UID", &uid)) continue; snprintf(keybuf, sizeof(keybuf), "%s:%d", uniqueid, uid); mailbox_message = (struct backup_mailbox_message *) hash_lookup( keybuf, &mailbox_message_list_index); if (!mailbox_message) continue; if (!mailbox_message_matches(mailbox_message, di)) continue; backup_mailbox_message_list_remove(mailbox_message_list, mailbox_message); hash_del(keybuf, &mailbox_message_list_index); backup_mailbox_message_free(&mailbox_message); } } next_line: if (dl) { dlist_unlink_files(dl); dlist_free(&dl); } } buf_free(&cmd); prot_free(ps); gzuc_member_end(gzuc, NULL); /* anything left in either of the lists is missing from the chunk data. bad! */ mailbox = mailbox_list->head; while (mailbox) { syslog(LOG_DEBUG, "%s: chunk %d missing mailbox data for %s (%s)\n", __func__, chunk->id, mailbox->uniqueid, mailbox->mboxname); if (out) fprintf(out, "chunk %d missing mailbox data for %s (%s)\n", chunk->id, mailbox->uniqueid, mailbox->mboxname); mailbox = mailbox->next; } mailbox_message = mailbox_message_list->head; while (mailbox_message) { syslog(LOG_DEBUG, "%s: chunk %d missing mailbox_message data for %s uid %u\n", __func__, chunk->id, mailbox_message->mailbox_uniqueid, mailbox_message->uid); if (out) fprintf(out, "chunk %d missing mailbox_message data for %s uid %u\n", chunk->id, mailbox_message->mailbox_uniqueid, mailbox_message->uid); mailbox_message = mailbox_message->next; } if (!r) r = mailbox_list->count || mailbox_message_list->count ? -1 : 0; done: free_hash_table(&mailbox_list_index, NULL); free_hash_table(&mailbox_message_list_index, NULL); backup_mailbox_list_empty(mailbox_list); free(mailbox_list); backup_mailbox_message_list_empty(mailbox_message_list); free(mailbox_message_list); syslog(LOG_DEBUG, "%s: chunk %d %s!\n", __func__, chunk->id, r ? "failed" : "passed"); if (out && verbose) fprintf(out, "%s\n", r ? "error" : "ok"); return r; }
static int _verify_message_cb(const struct backup_message *message, void *rock) { struct verify_message_rock *vmrock = (struct verify_message_rock *) rock; struct dlist *dl = NULL; struct dlist *di = NULL; FILE *out = vmrock->out; int r; /* cache the dlist so that multiple reads from the same offset don't * cause expensive reverse seeks in decompression stream */ if (!vmrock->cached_dlist || vmrock->cached_offset != message->offset) { if (vmrock->cached_dlist) { dlist_unlink_files(vmrock->cached_dlist); dlist_free(&vmrock->cached_dlist); } r = gzuc_seekto(vmrock->gzuc, message->offset); if (r) return r; struct protstream *ps = prot_readcb(_prot_fill_cb, vmrock->gzuc); prot_setisclient(ps, 1); /* don't sync literals */ r = parse_backup_line(ps, NULL, NULL, &dl); prot_free(ps); if (r == EOF) { const char *error = prot_error(ps); if (error && 0 != strcmp(error, PROT_EOF_STRING)) { syslog(LOG_ERR, "%s: error reading message %i at offset %jd, byte %i: %s", __func__, message->id, message->offset, prot_bytes_in(ps), error); if (out) fprintf(out, "error reading message %i at offset %jd, byte %i: %s", message->id, message->offset, prot_bytes_in(ps), error); } return r; } vmrock->cached_dlist = dl; vmrock->cached_offset = message->offset; } else { dl = vmrock->cached_dlist; } r = strcmp(dl->name, "MESSAGE"); if (r) return r; r = -1; for (di = dl->head; di; di = di->next) { struct message_guid *guid = NULL; const char *fname = NULL; if (!dlist_tofile(di, NULL, &guid, NULL, &fname)) continue; r = message_guid_cmp(guid, message->guid); if (!r) { if (vmrock->verify_guid) { const char *msg_base = NULL; size_t msg_len = 0; struct message_guid computed_guid; int fd; fd = open(fname, O_RDWR); if (fd != -1) { map_refresh(fd, 1, &msg_base, &msg_len, MAP_UNKNOWN_LEN, fname, NULL); message_guid_generate(&computed_guid, msg_base, msg_len); r = message_guid_cmp(&computed_guid, message->guid); if (r && out) fprintf(out, "guid mismatch for message %i\n", message->id); map_free(&msg_base, &msg_len); close(fd); } else { syslog(LOG_ERR, "IOERROR: %s open %s: %m", __func__, fname); if (out) fprintf(out, "error reading staging file for message %i\n", message->id); r = -1; } } break; } } return r; }
int main(int argc, char **argv) { save_argv0(argv[0]); const char *alt_config = NULL; const char *input_file = NULL; const char *backup_name = NULL; const char *servername = NULL; enum restore_mode mode = RESTORE_MODE_UNSPECIFIED; int local_only = 0; int wait = 0; int do_nothing = 0; int do_all_mailboxes = 0; struct restore_options options = {0}; options.expunged_mode = RESTORE_EXPUNGED_OKAY; options.trim_deletedprefix = 1; struct backup *backup = NULL; mbname_t *mbname = NULL; struct backup_mailbox_list *mailbox_list = NULL; struct sync_folder_list *reserve_folder_list = NULL; struct sync_reserve_list *reserve_list = NULL; struct buf tagbuf = BUF_INITIALIZER; struct backend *backend = NULL; struct dlist *upload = NULL; int opt, r; while ((opt = getopt(argc, argv, ":A:C:DF:LM:P:UXaf:m:nru:vw:xz")) != EOF) { switch (opt) { case 'A': if (options.keep_uidvalidity) usage(); options.override_acl = optarg; break; case 'C': alt_config = optarg; break; case 'D': /* XXX does this clash with keep_uidvalidity? */ options.trim_deletedprefix = 0; break; case 'F': if (do_all_mailboxes) usage(); input_file = optarg; break; case 'L': local_only = 1; break; case 'M': if (options.keep_uidvalidity) usage(); options.override_mboxname = optarg; break; case 'P': if (options.keep_uidvalidity) usage(); options.override_partition = optarg; break; case 'U': if (options.override_acl || options.override_mboxname || options.override_partition) usage(); options.keep_uidvalidity = 1; break; case 'X': if (options.expunged_mode != RESTORE_EXPUNGED_OKAY) usage(); options.expunged_mode = RESTORE_EXPUNGED_EXCLUDE; break; case 'a': if (input_file) usage(); do_all_mailboxes = 1; break; case 'f': if (mode != RESTORE_MODE_UNSPECIFIED) usage(); mode = RESTORE_MODE_FILENAME; backup_name = optarg; break; case 'm': if (mode != RESTORE_MODE_UNSPECIFIED) usage(); mode = RESTORE_MODE_MBOXNAME; backup_name = optarg; break; case 'n': do_nothing = 1; break; case 'r': options.do_submailboxes = 1; break; case 'u': if (mode != RESTORE_MODE_UNSPECIFIED) usage(); mode = RESTORE_MODE_USERNAME; backup_name = optarg; break; case 'v': options.verbose++; break; case 'w': wait = atoi(optarg); if (wait < 0) usage(); break; case 'x': if (options.expunged_mode != RESTORE_EXPUNGED_OKAY) usage(); options.expunged_mode = RESTORE_EXPUNGED_ONLY; break; case 'z': options.require_compression = 1; break; case ':': if (optopt == 'A') options.override_acl = ""; else usage(); break; default: usage(); break; } } /* we need a server name */ if (optind == argc) usage(); servername = argv[optind++]; /* we need a source of backup data */ if (mode == RESTORE_MODE_UNSPECIFIED) { if (optind == argc) usage(); backup_name = argv[optind++]; mode = RESTORE_MODE_USERNAME; } /* we need either an input file or some objects to restore */ if (!do_all_mailboxes && !input_file && optind == argc) usage(); /* and we can't have both because i said */ if ((input_file || do_all_mailboxes) && optind < argc) usage(); /* okay, arguments seem sane, we are go */ cyrus_init(alt_config, "restore", 0, 0); /* load the SASL plugins */ global_sasl_init(1, 0, mysasl_cb); /* wait here for gdb attach */ if (wait) { fprintf(stderr, "Waiting for %d seconds for gdb attach...\n", wait); sleep(wait); } /* open backup */ switch (mode) { case RESTORE_MODE_FILENAME: r = backup_open_paths(&backup, backup_name, NULL, BACKUP_OPEN_NONBLOCK, BACKUP_OPEN_NOCREATE); break; case RESTORE_MODE_MBOXNAME: mbname = mbname_from_intname(backup_name); if (!mbname) usage(); r = backup_open(&backup, mbname, BACKUP_OPEN_NONBLOCK, BACKUP_OPEN_NOCREATE); mbname_free(&mbname); break; case RESTORE_MODE_USERNAME: mbname = mbname_from_userid(backup_name); if (!mbname) usage(); r = backup_open(&backup, mbname, BACKUP_OPEN_NONBLOCK, BACKUP_OPEN_NOCREATE); mbname_free(&mbname); break; default: usage(); break; } if (r) goto done; /* scan for objects to restore: * mailboxes will have all messages added, modulo expunged_mode * messages will be added individually with appropriate folder */ mailbox_list = xzmalloc(sizeof *mailbox_list); reserve_folder_list = restore_make_reserve_folder_list(backup); reserve_list = sync_reserve_list_create(SYNC_MSGID_LIST_HASH_SIZE); if (input_file) { char buf[MAX_MAILBOX_NAME + 2]; // \n\0 size_t len; FILE *f; if (0 != strcmp(input_file, "-")) { f = fopen(input_file, "r"); if (!f) { fprintf(stderr, "fopen %s: %s\n", input_file, strerror(errno)); goto done;// FIXME shut_down? } } else { f = stdin; } while (fgets(buf, sizeof(buf), f)) { len = strlen(buf); if (len > 0 && buf[len - 1] == '\n') buf[--len] = '\0'; if (len == 0 || buf[0] == '#') continue; r = restore_add_object(buf, &options, backup, mailbox_list, reserve_folder_list, reserve_list); // FIXME r } fclose(f); } else if (do_all_mailboxes) { struct backup_mailbox_list *all_mailboxes; struct backup_mailbox *mailbox; all_mailboxes = backup_get_mailboxes(backup, 0, BACKUP_MAILBOX_ALL_RECORDS); for (mailbox = all_mailboxes->head; mailbox; mailbox = mailbox->next) { restore_add_mailbox(mailbox, &options, mailbox_list, reserve_folder_list, reserve_list); } backup_mailbox_list_empty(all_mailboxes); free(all_mailboxes); } else { int i; for (i = optind; i < argc; i++) { r = restore_add_object(argv[i], &options, backup, mailbox_list, reserve_folder_list, reserve_list); // FIXME r } } if (do_nothing) { if (options.verbose) fprintf(stderr, "do nothing (-n) specified, exiting now\n"); goto done; } /* connect to destination */ backend = restore_connect(servername, &tagbuf, &options); if (!backend) { // FIXME r = -1; goto done; } /* do the restore */ struct sync_reserve *reserve; for (reserve = reserve_list->head; reserve; reserve = reserve->next) { /* send APPLY RESERVE and parse missing lists */ r = sync_reserve_partition(reserve->part, reserve_folder_list, reserve->list, backend); if (r) goto done; /* send APPLY MESSAGEs */ r = backup_prepare_message_upload(backup, reserve->part, reserve->list, &sync_msgid_lookup, &upload); if (r) goto done; /* upload in small(ish) blocks to avoid timeouts */ while (upload->head) { struct dlist *block = dlist_splice(upload, 1024); sync_send_apply(block, backend->out); r = sync_parse_response("MESSAGE", backend->in, NULL); dlist_unlink_files(block); dlist_free(&block); if (r) goto done; } } /* sync_prepare_dlists needs to upload messages per-mailbox, because * it needs the mailbox to find the filename for the message. but * we have no such limitation, so we can upload messages while * looping over the reserve_list instead, above. * * alternatively, if we do it on a per-mailbox basis then we can limit * the hit on the staging directory to only a mailbox worth of messages * at a time. but we don't have a logical grouping that would make * this coherent */ /* send RESTORE MAILBOXes */ struct backup_mailbox *mailbox; for (mailbox = mailbox_list->head; mailbox; mailbox = mailbox->next) { /* XXX filter the mailbox records based on reserve/missing/upload */ /* XXX does this sensibly handle mailbox objs with empty values? */ struct dlist *dl = backup_mailbox_to_dlist(mailbox); if (!dl) continue; if (local_only) { free(dl->name); dl->name = xstrdup("LOCAL_MAILBOX"); } sync_send_restore(dl, backend->out); r = sync_parse_response("MAILBOX", backend->in, NULL); dlist_free(&dl); if (r) goto done; } done: if (r) fprintf(stderr, "%s: %s\n", backup_name, error_message(r)); /* release lock asap */ if (backup) backup_close(&backup); if (backend) backend_disconnect(backend); if (upload) { dlist_unlink_files(upload); dlist_free(&upload); } if (mailbox_list) { backup_mailbox_list_empty(mailbox_list); free(mailbox_list); } if (reserve_folder_list) sync_folder_list_free(&reserve_folder_list); if (reserve_list) sync_reserve_list_free(&reserve_list); buf_free(&tagbuf); backup_cleanup_staging_path(); cyrus_done(); exit(r ? EX_TEMPFAIL : EX_OK); }
/* returns: * 0 on success * 1 if compact was not needed * negative on error */ EXPORTED int backup_compact(const char *name, enum backup_open_nonblock nonblock, int force, int verbose, FILE *out) { struct backup *original = NULL; struct backup *compact = NULL; struct backup_chunk_list *all_chunks = NULL; struct backup_chunk_list *keep_chunks = NULL; struct backup_chunk *chunk = NULL; struct sync_msgid_list *keep_message_guids = NULL; struct gzuncat *gzuc = NULL; struct protstream *in = NULL; time_t since, chunk_start_time, ts; int r; compact_readconfig(); r = compact_open(name, &original, &compact, nonblock); if (r) return r; /* calculate current time after obtaining locks, in case of a wait */ const time_t now = time(NULL); const int retention_days = config_getint(IMAPOPT_BACKUP_RETENTION_DAYS); if (retention_days > 0) { since = now - (retention_days * 24 * 60 * 60); } else { /* zero or negative retention days means "keep forever" */ since = -1; } all_chunks = backup_get_chunks(original); if (!all_chunks) goto error; keep_chunks = backup_get_live_chunks(original, since); if (!keep_chunks) goto error; if (!force && !compact_required(all_chunks, keep_chunks)) { /* nothing to do */ backup_chunk_list_free(&all_chunks); backup_chunk_list_free(&keep_chunks); backup_unlink(&compact); backup_close(&original); return 1; } if (verbose) { fprintf(out, "keeping " SIZE_T_FMT " chunks:\n", keep_chunks->count); for (chunk = keep_chunks->head; chunk; chunk = chunk->next) { fprintf(out, " %d", chunk->id); } fprintf(out, "\n"); } gzuc = gzuc_new(original->fd); if (!gzuc) goto error; chunk_start_time = -1; ts = 0; struct buf cmd = BUF_INITIALIZER; for (chunk = keep_chunks->head; chunk; chunk = chunk->next) { keep_message_guids = sync_msgid_list_create(0); r = backup_message_foreach(original, chunk->id, &since, _keep_message_guids_cb, keep_message_guids); if (r) goto error; gzuc_member_start_from(gzuc, chunk->offset); in = prot_readcb(_prot_fill_cb, gzuc); while (1) { struct dlist *dl = NULL; int c = parse_backup_line(in, &ts, &cmd, &dl); if (c == EOF) { const char *error = prot_error(in); if (error && 0 != strcmp(error, PROT_EOF_STRING)) { syslog(LOG_ERR, "IOERROR: %s: error reading chunk at offset %jd, byte %i: %s\n", name, chunk->offset, prot_bytes_in(in), error); if (out) fprintf(out, "error reading chunk at offset %jd, byte %i: %s\n", chunk->offset, prot_bytes_in(in), error); r = IMAP_IOERROR; goto error; } break; } if (chunk_start_time == -1) { r = backup_append_start(compact, &ts, BACKUP_APPEND_NOFLUSH); if (r) goto error; chunk_start_time = ts; } // XXX if this line is worth keeping if (want_append(original, chunk->id, dl, keep_message_guids)) { // FIXME if message is removed due to unneeded chunk, // subsequent mailbox lines for it will fail here // so we need to be able to tell which lines apply to messages we don't want anymore r = backup_append(compact, dl, &ts, BACKUP_APPEND_NOFLUSH); if (r) goto error; } dlist_unlink_files(dl); dlist_free(&dl); // if this line put us over compact_maxsize if (want_split(chunk, &compact->append_state->wrote)) { r = backup_append_end(compact, &ts); chunk_start_time = -1; if (verbose) { fprintf(out, "splitting chunk %d\n", chunk->id); } } } // if we're due to start a new chunk if (compact->append_state && compact->append_state->mode) { if (!want_combine(compact->append_state->wrote, chunk->next)) { r = backup_append_end(compact, &ts); chunk_start_time = -1; } else if (verbose) { fprintf(out, "combining chunks %d and %d\n", chunk->id, chunk->next->id); } } prot_free(in); in = NULL; gzuc_member_end(gzuc, NULL); sync_msgid_list_free(&keep_message_guids); } buf_free(&cmd); if (compact->append_state && compact->append_state->mode) backup_append_end(compact, &ts); gzuc_free(&gzuc); backup_chunk_list_free(&keep_chunks); /* if we get here okay, then the compact succeeded */ r = compact_closerename(&original, &compact, now); if (r) goto error; return 0; error: if (in) prot_free(in); if (gzuc) gzuc_free(&gzuc); if (keep_message_guids) sync_msgid_list_free(&keep_message_guids); if (all_chunks) backup_chunk_list_free(&all_chunks); if (keep_chunks) backup_chunk_list_free(&keep_chunks); if (compact) backup_unlink(&compact); if (original) backup_close(&original); return r ? r : -1; }
static int want_append_mailbox(struct backup *orig_backup, int orig_chunk_id, struct dlist *dlist) { struct dlist *record = NULL; const char *uniqueid = NULL; struct backup_mailbox *mailbox = NULL; int mailbox_last_chunk_id = 0; if (!dlist_getatom(dlist, "UNIQUEID", &uniqueid)) { syslog(LOG_DEBUG, "%s: MAILBOX line with no UNIQUEID", __func__); return 1; /* better keep it for now */ } dlist_getlist(dlist, "RECORD", &record); if (record && record->head) { struct dlist *ki = NULL, *next = NULL; int keep = 0; /* keep MAILBOX lines that contain the last RECORD for any message, */ /* pruning out stale RECORDs */ for (ki = record->head; ki; ki = next) { const char *guid = NULL; struct backup_mailbox_message *mailbox_message = NULL; /* save next pointer now in case we need to unstitch */ next = ki->next; if (!dlist_getatom(ki, "GUID", &guid)) { syslog(LOG_DEBUG, "%s: MAILBOX RECORD with no GUID", __func__); keep = 1; /* better keep it for now */ continue; } mailbox_message = backup_get_mailbox_message(orig_backup, uniqueid, guid); if (mailbox_message) { int mailbox_message_last_chunk_id = mailbox_message->last_chunk_id; backup_mailbox_message_free(&mailbox_message); if (mailbox_message_last_chunk_id == orig_chunk_id) { syslog(LOG_DEBUG, "%s: keeping MAILBOX line containing last RECORD for guid %s", __func__, guid); keep = 1; continue; } } /* don't need this record */ syslog(LOG_DEBUG, "%s: pruning stale MAILBOX RECORD for guid %s", __func__, guid); dlist_unstitch(record, ki); dlist_unlink_files(ki); dlist_free(&ki); } if (keep) return 1; } mailbox = backup_get_mailbox_by_uniqueid(orig_backup, uniqueid, BACKUP_MAILBOX_NO_RECORDS); if (!mailbox) { /* what? */ syslog(LOG_DEBUG, "%s: couldn't find mailbox entry for uniqueid %s", __func__, uniqueid); return 1; /* better keep it for now */ } mailbox_last_chunk_id = mailbox->last_chunk_id; backup_mailbox_free(&mailbox); if (mailbox_last_chunk_id == orig_chunk_id) { /* keep all mailbox lines from the chunk recorded as its last */ syslog(LOG_DEBUG, "%s: keeping MAILBOX line from its last known chunk", __func__); return 1; } syslog(LOG_DEBUG, "%s: discarding stale MAILBOX line (chunk %d, last %d, uniqueid %s)", __func__, orig_chunk_id, mailbox_last_chunk_id, uniqueid); return 0; }