Example #1
0
static int _verify_message_cb(const struct backup_message *message, void *rock)
{
    struct verify_message_rock *vmrock = (struct verify_message_rock *) rock;
    struct dlist *dl = NULL;
    struct dlist *di = NULL;
    FILE *out = vmrock->out;
    int r;

    /* cache the dlist so that multiple reads from the same offset don't
     * cause expensive reverse seeks in decompression stream
     */
    if (!vmrock->cached_dlist || vmrock->cached_offset != message->offset) {
        if (vmrock->cached_dlist) {
            dlist_unlink_files(vmrock->cached_dlist);
            dlist_free(&vmrock->cached_dlist);
        }

        r = gzuc_seekto(vmrock->gzuc, message->offset);
        if (r) return r;

        struct protstream *ps = prot_readcb(_prot_fill_cb, vmrock->gzuc);
        prot_setisclient(ps, 1); /* don't sync literals */
        r = parse_backup_line(ps, NULL, NULL, &dl);
        prot_free(ps);

        if (r == EOF) {
            const char *error = prot_error(ps);
            if (error && 0 != strcmp(error, PROT_EOF_STRING)) {
                syslog(LOG_ERR,
                       "%s: error reading message %i at offset %jd, byte %i: %s",
                       __func__, message->id, message->offset, prot_bytes_in(ps), error);
                if (out)
                    fprintf(out, "error reading message %i at offset %jd, byte %i: %s",
                            message->id, message->offset, prot_bytes_in(ps), error);
            }
            return r;
        }

        vmrock->cached_dlist = dl;
        vmrock->cached_offset = message->offset;
    }
    else {
        dl = vmrock->cached_dlist;
    }

    r = strcmp(dl->name, "MESSAGE");
    if (r) return r;

    r = -1;
    for (di = dl->head; di; di = di->next) {
        struct message_guid *guid = NULL;
        const char *fname = NULL;

        if (!dlist_tofile(di, NULL, &guid, NULL, &fname))
            continue;

        r = message_guid_cmp(guid, message->guid);
        if (!r) {
            if (vmrock->verify_guid) {
                const char *msg_base = NULL;
                size_t msg_len = 0;
                struct message_guid computed_guid;
                int fd;

                fd = open(fname, O_RDWR);
                if (fd != -1) {
                    map_refresh(fd, 1, &msg_base, &msg_len, MAP_UNKNOWN_LEN, fname, NULL);

                    message_guid_generate(&computed_guid, msg_base, msg_len);
                    r = message_guid_cmp(&computed_guid, message->guid);
                    if (r && out)
                        fprintf(out, "guid mismatch for message %i\n", message->id);

                    map_free(&msg_base, &msg_len);
                    close(fd);
                }
                else {
                    syslog(LOG_ERR, "IOERROR: %s open %s: %m", __func__, fname);
                    if (out)
                        fprintf(out, "error reading staging file for message %i\n", message->id);
                    r = -1;
                }
            }
            break;
        }
    }

    return r;
}
Example #2
0
/* verify that the matching MAILBOX exists within the claimed chunk
 * for each mailbox or mailbox_message in the index
 */
static int verify_chunk_mailbox_links(struct backup *backup, struct backup_chunk *chunk,
                                      struct gzuncat *gzuc, int verbose, FILE *out)
{
    /*
     *   get list of mailboxes in chunk
     *   get list of mailbox_messages in chunk
     *   index mailboxes list by uniqueid
     *   index mailbox_messages list by uniqueid:uid
     *   open chunk
     *   foreach line in chunk
     *     read dlist
     *     skip if it's not a mailbox
     *     if details in dlist match details in mailbox
     *       remove from mailbox list/index
     *     foreach record in dlist
     *       if details in dlist match details in mailbox_message
     *       remove from mailbox_message list/index
     *   failed if either list of mailboxes or list of mailbox_messages is not empty
     */

    struct backup_mailbox_list *mailbox_list = NULL;
    struct backup_mailbox_message_list *mailbox_message_list = NULL;
    hash_table mailbox_list_index = HASH_TABLE_INITIALIZER;
    hash_table mailbox_message_list_index = HASH_TABLE_INITIALIZER;
    struct backup_mailbox *mailbox = NULL;
    struct backup_mailbox_message *mailbox_message = NULL;
    int r;

    if (out && verbose)
        fprintf(out, "checking chunk %d mailbox links...\n", chunk->id);

    mailbox_list = backup_get_mailboxes(backup, chunk->id, 0);
    mailbox_message_list = backup_get_mailbox_messages(backup, chunk->id);

    if (mailbox_list->count == 0 && mailbox_message_list->count == 0) {
        /* nothing we care about in this chunk */
        free(mailbox_list);
        free(mailbox_message_list);
        if (out && verbose)
            fprintf(out, "ok\n");
        return 0;
    }

    /* XXX consider whether the two hashes should use pools */

    if (mailbox_list->count) {
        /* build an index of the mailbox list */
        construct_hash_table(&mailbox_list_index, mailbox_list->count, 0);
        mailbox = mailbox_list->head;
        while (mailbox) {
            hash_insert(mailbox->uniqueid, mailbox, &mailbox_list_index);
            mailbox = mailbox->next;
        }
    }

    if (mailbox_message_list->count) {
        /* build an index of the mailbox message list */
        construct_hash_table(&mailbox_message_list_index,
                             mailbox_message_list->count, 0);
        mailbox_message = mailbox_message_list->head;
        while (mailbox_message) {
            char keybuf[1024]; // FIXME whatever
            snprintf(keybuf, sizeof(keybuf), "%s:%d",
                     mailbox_message->mailbox_uniqueid, mailbox_message->uid);
            hash_insert(keybuf, mailbox_message, &mailbox_message_list_index);
            mailbox_message = mailbox_message->next;
        }
    }

    r = gzuc_member_start_from(gzuc, chunk->offset);
    if (r) {
        syslog(LOG_ERR, "%s: error reading chunk %i at offset %jd: %s",
                        __func__, chunk->id, chunk->offset, zError(r));
        if (out)
            fprintf(out, "error reading chunk %i at offset %jd: %s",
                    chunk->id, chunk->offset, zError(r));
        goto done;
    }
    struct protstream *ps = prot_readcb(_prot_fill_cb, gzuc);
    prot_setisclient(ps, 1); /* don't sync literals */

    struct buf cmd = BUF_INITIALIZER;
    while (1) {
        struct dlist *dl = NULL;
        struct dlist *record = NULL;
        struct dlist *di = NULL;
        const char *uniqueid = NULL;

        int c = parse_backup_line(ps, NULL, &cmd, &dl);
        if (c == EOF) {
            const char *error = prot_error(ps);
            if (error && 0 != strcmp(error, PROT_EOF_STRING)) {
                syslog(LOG_ERR,
                       "%s: error reading chunk %i data at offset %jd, byte %i: %s",
                       __func__, chunk->id, chunk->offset, prot_bytes_in(ps), error);
                if (out)
                    fprintf(out, "error reading chunk %i data at offset %jd, byte %i: %s",
                            chunk->id, chunk->offset, prot_bytes_in(ps), error);
                r = EOF;
            }
            break;
        }

        if (strcmp(buf_cstring(&cmd), "APPLY") != 0)
            goto next_line;

        if (strcmp(dl->name, "MAILBOX") != 0)
            goto next_line;

        if (!dlist_getatom(dl, "UNIQUEID", &uniqueid))
            goto next_line;

        if (mailbox_list->count) {
            mailbox = (struct backup_mailbox *) hash_lookup(uniqueid, &mailbox_list_index);

            if (mailbox && mailbox_matches(mailbox, dl)) {
                backup_mailbox_list_remove(mailbox_list, mailbox);
                hash_del(uniqueid, &mailbox_list_index);
                backup_mailbox_free(&mailbox);
            }
        }

        if (mailbox_message_list->count) {
            if (!dlist_getlist(dl, "RECORD", &record))
                goto next_line;

            for (di = record->head; di; di = di->next) {
                char keybuf[1024]; // FIXME whatever
                uint32_t uid;

                if (!dlist_getnum32(di, "UID", &uid))
                    continue;

                snprintf(keybuf, sizeof(keybuf), "%s:%d", uniqueid, uid);
                mailbox_message = (struct backup_mailbox_message *) hash_lookup(
                    keybuf, &mailbox_message_list_index);

                if (!mailbox_message)
                    continue;

                if (!mailbox_message_matches(mailbox_message, di))
                    continue;

                backup_mailbox_message_list_remove(mailbox_message_list, mailbox_message);
                hash_del(keybuf, &mailbox_message_list_index);
                backup_mailbox_message_free(&mailbox_message);
            }
        }

next_line:
        if (dl) {
            dlist_unlink_files(dl);
            dlist_free(&dl);
        }
    }
    buf_free(&cmd);

    prot_free(ps);
    gzuc_member_end(gzuc, NULL);

    /* anything left in either of the lists is missing from the chunk data. bad! */
    mailbox = mailbox_list->head;
    while (mailbox) {
        syslog(LOG_DEBUG, "%s: chunk %d missing mailbox data for %s (%s)\n",
                __func__, chunk->id, mailbox->uniqueid, mailbox->mboxname);
        if (out)
            fprintf(out, "chunk %d missing mailbox data for %s (%s)\n",
                    chunk->id, mailbox->uniqueid, mailbox->mboxname);
        mailbox = mailbox->next;
    }

    mailbox_message = mailbox_message_list->head;
    while (mailbox_message) {
        syslog(LOG_DEBUG, "%s: chunk %d missing mailbox_message data for %s uid %u\n",
                __func__, chunk->id, mailbox_message->mailbox_uniqueid,
                mailbox_message->uid);
        if (out)
            fprintf(out, "chunk %d missing mailbox_message data for %s uid %u\n",
                    chunk->id, mailbox_message->mailbox_uniqueid,
                    mailbox_message->uid);
        mailbox_message = mailbox_message->next;
    }

    if (!r) r = mailbox_list->count || mailbox_message_list->count ? -1 : 0;

done:
    free_hash_table(&mailbox_list_index, NULL);
    free_hash_table(&mailbox_message_list_index, NULL);

    backup_mailbox_list_empty(mailbox_list);
    free(mailbox_list);

    backup_mailbox_message_list_empty(mailbox_message_list);
    free(mailbox_message_list);

    syslog(LOG_DEBUG, "%s: chunk %d %s!\n", __func__, chunk->id,
            r ? "failed" : "passed");
    if (out && verbose)
        fprintf(out, "%s\n", r ? "error" : "ok");
    return r;
}
Example #3
0
/* returns:
 *   0 on success
 *   1 if compact was not needed
 *   negative on error
 */
EXPORTED int backup_compact(const char *name,
                            enum backup_open_nonblock nonblock,
                            int force, int verbose, FILE *out)
{
    struct backup *original = NULL;
    struct backup *compact = NULL;
    struct backup_chunk_list *all_chunks = NULL;
    struct backup_chunk_list *keep_chunks = NULL;
    struct backup_chunk *chunk = NULL;
    struct sync_msgid_list *keep_message_guids = NULL;
    struct gzuncat *gzuc = NULL;
    struct protstream *in = NULL;
    time_t since, chunk_start_time, ts;
    int r;

    compact_readconfig();

    r = compact_open(name, &original, &compact, nonblock);
    if (r) return r;

    /* calculate current time after obtaining locks, in case of a wait */
    const time_t now = time(NULL);

    const int retention_days = config_getint(IMAPOPT_BACKUP_RETENTION_DAYS);
    if (retention_days > 0) {
        since = now - (retention_days * 24 * 60 * 60);
    }
    else {
        /* zero or negative retention days means "keep forever" */
        since = -1;
    }

    all_chunks = backup_get_chunks(original);
    if (!all_chunks) goto error;

    keep_chunks = backup_get_live_chunks(original, since);
    if (!keep_chunks) goto error;

    if (!force && !compact_required(all_chunks, keep_chunks)) {
        /* nothing to do */
        backup_chunk_list_free(&all_chunks);
        backup_chunk_list_free(&keep_chunks);
        backup_unlink(&compact);
        backup_close(&original);
        return 1;
    }

    if (verbose) {
        fprintf(out, "keeping " SIZE_T_FMT " chunks:\n", keep_chunks->count);

        for (chunk = keep_chunks->head; chunk; chunk = chunk->next) {
            fprintf(out, " %d", chunk->id);
        }

        fprintf(out, "\n");
    }

    gzuc = gzuc_new(original->fd);
    if (!gzuc) goto error;

    chunk_start_time = -1;
    ts = 0;
    struct buf cmd = BUF_INITIALIZER;
    for (chunk = keep_chunks->head; chunk; chunk = chunk->next) {
        keep_message_guids = sync_msgid_list_create(0);
        r = backup_message_foreach(original, chunk->id, &since,
                                   _keep_message_guids_cb, keep_message_guids);
        if (r) goto error;

        gzuc_member_start_from(gzuc, chunk->offset);

        in = prot_readcb(_prot_fill_cb, gzuc);

        while (1) {
            struct dlist *dl = NULL;

            int c = parse_backup_line(in, &ts, &cmd, &dl);

            if (c == EOF) {
                const char *error = prot_error(in);
                if (error && 0 != strcmp(error, PROT_EOF_STRING)) {
                    syslog(LOG_ERR,
                           "IOERROR: %s: error reading chunk at offset %jd, byte %i: %s\n",
                           name, chunk->offset, prot_bytes_in(in), error);

                    if (out)
                        fprintf(out, "error reading chunk at offset %jd, byte %i: %s\n",
                                chunk->offset, prot_bytes_in(in), error);

                    r = IMAP_IOERROR;
                    goto error;
                }

                break;
            }

            if (chunk_start_time == -1) {
                r = backup_append_start(compact, &ts, BACKUP_APPEND_NOFLUSH);
                if (r) goto error;
                chunk_start_time = ts;
            }

            // XXX if this line is worth keeping
            if (want_append(original, chunk->id, dl, keep_message_guids)) {
                // FIXME if message is removed due to unneeded chunk,
                // subsequent mailbox lines for it will fail here
                // so we need to be able to tell which lines apply to messages we don't want anymore
                r = backup_append(compact, dl, &ts, BACKUP_APPEND_NOFLUSH);
                if (r) goto error;
            }

            dlist_unlink_files(dl);
            dlist_free(&dl);

            // if this line put us over compact_maxsize
            if (want_split(chunk, &compact->append_state->wrote)) {
                r = backup_append_end(compact, &ts);
                chunk_start_time = -1;

                if (verbose) {
                    fprintf(out, "splitting chunk %d\n", chunk->id);
                }
            }
        }

        // if we're due to start a new chunk
        if (compact->append_state && compact->append_state->mode) {
            if (!want_combine(compact->append_state->wrote, chunk->next)) {
                r = backup_append_end(compact, &ts);
                chunk_start_time = -1;
            }
            else if (verbose) {
                fprintf(out, "combining chunks %d and %d\n",
                             chunk->id, chunk->next->id);
            }
        }

        prot_free(in);
        in = NULL;
        gzuc_member_end(gzuc, NULL);

        sync_msgid_list_free(&keep_message_guids);
    }
    buf_free(&cmd);

    if (compact->append_state && compact->append_state->mode)
        backup_append_end(compact, &ts);

    gzuc_free(&gzuc);

    backup_chunk_list_free(&keep_chunks);

    /* if we get here okay, then the compact succeeded */
    r = compact_closerename(&original, &compact, now);
    if (r) goto error;

    return 0;

error:
    if (in) prot_free(in);
    if (gzuc) gzuc_free(&gzuc);
    if (keep_message_guids) sync_msgid_list_free(&keep_message_guids);
    if (all_chunks) backup_chunk_list_free(&all_chunks);
    if (keep_chunks) backup_chunk_list_free(&keep_chunks);
    if (compact) backup_unlink(&compact);
    if (original) backup_close(&original);

    return r ? r : -1;
}