static int _index_expunge(struct backup *backup, struct dlist *dl,
time_t ts, off_t dl_offset)
{
syslog(LOG_DEBUG, "indexing EXPUNGE at " OFF_T_FMT "...\n", dl_offset);
const char *mboxname;
const char *uniqueid;
struct dlist *uidl;
struct dlist *di;
struct backup_mailbox *mailbox = NULL;
int r = 0;
if (!dlist_getatom(dl, "MBOXNAME", &mboxname))
return IMAP_PROTOCOL_BAD_PARAMETERS;
if (!dlist_getatom(dl, "UNIQUEID", &uniqueid))
return IMAP_PROTOCOL_BAD_PARAMETERS;
if (!dlist_getlist(dl, "UID", &uidl))
return IMAP_PROTOCOL_BAD_PARAMETERS;
mbname_t *mbname = mbname_from_intname(mboxname);
mailbox = backup_get_mailbox_by_name(backup, mbname, 0);
mbname_free(&mbname);
if (!mailbox)
return IMAP_MAILBOX_NONEXISTENT;
/* verify that uniqueid matches */
if (strcmp(mailbox->uniqueid, uniqueid) != 0) {
syslog(LOG_ERR, "%s: uniqueid mismatch for %s: %s on wire, %s in index",
__func__, mboxname, uniqueid, mailbox->uniqueid);
r = IMAP_PROTOCOL_BAD_PARAMETERS;
}
for (di = uidl->head; di && !r; di = di->next) {
struct sqldb_bindval bval[] = {
{ ":mailbox_id", SQLITE_INTEGER, { .i = mailbox->id } },
/*
* Parse a quota database entry, which is formatted as a string
* containing multiple space-separated fields, into a struct quota.
* Returns: 0 on success or an IMAP error code.
*/
static int quota_parseval(const char *data, size_t datalen,
struct quota *quota, int iswrite)
{
strarray_t *fields = NULL;
int r = IMAP_MAILBOX_BADFORMAT;
int i = 0;
int res = QUOTA_STORAGE;
struct dlist *dl = NULL;
quota_t temp;
/* new dlist format */
if (data[0] == '%') {
if (dlist_parsemap(&dl, 0, data, datalen))
goto out;
for (res = 0; res < QUOTA_NUMRESOURCES; res++) {
struct dlist *val;
struct dlist *item = dlist_getchild(dl, quota_db_names[res]);
if (!item) continue;
val = dlist_getchildn(item, 0);
if (val) quota->useds[res] = dlist_num(val);
val = dlist_getchildn(item, 1);
if (val) quota->limits[res] = dlist_num(val);
}
/* only read the SCAN stuff if it's a write lock */
if (iswrite) {
struct dlist *scan = dlist_getchild(dl, "SCAN");
const char *mboxname = NULL;
if (!scan) goto done;
if (!dlist_getatom(scan, "MBOX", &mboxname)) goto done;
quota->scanmbox = xstrdup(mboxname);
for (res = 0; res < QUOTA_NUMRESOURCES; res++) {
struct dlist *val = dlist_getchild(scan, quota_db_names[res]);
if (val) quota->scanuseds[res] = dlist_num(val);
}
}
goto done;
}
/* parse historical formats */
fields = strarray_split(data, NULL, 0);
for (;;) {
if (i+2 > fields->count)
goto out; /* need at least 2 more fields */
if (sscanf(fields->data[i++], QUOTA_T_FMT, "a->useds[res]) != 1)
goto out;
if (sscanf(fields->data[i++], QUOTA_T_FMT, "a->limits[res]) != 1)
goto out;
/* skip over temporary extra used data from failed quota -f runs */
if (i < fields->count &&
sscanf(fields->data[i], QUOTA_T_FMT, &temp) == 1) {
i++;
}
if (i == fields->count)
break; /* successfully parsed whole line */
for (res = 0 ; res < QUOTA_NUMRESOURCES ; res++) {
if (quota_db_names[res] && !strcasecmp(fields->data[i], quota_db_names[res]))
break;
}
if (res == QUOTA_NUMRESOURCES)
goto out;
i++;
}
done:
r = 0;
out:
dlist_free(&dl);
strarray_free(fields);
return r;
}
static int mailbox_matches(const struct backup_mailbox *mailbox,
struct dlist *dlist)
{
const char *mboxname = NULL;
uint32_t last_uid = 0;
modseq_t highestmodseq = 0;
uint32_t recentuid = 0;
time_t recenttime = 0;
time_t last_appenddate = 0;
uint32_t uidvalidity = 0;
const char *partition = NULL;
const char *acl = NULL;
const char *options = NULL;
modseq_t xconvmodseq = 0;
struct synccrcs synccrcs = { 0, 0 };
if (!dlist_getatom(dlist, "MBOXNAME", &mboxname)
|| strcmp(mboxname, mailbox->mboxname) != 0)
return 0;
if (!dlist_getnum32(dlist, "LAST_UID", &last_uid)
|| last_uid != mailbox->last_uid)
return 0;
if (!dlist_getnum64(dlist, "HIGHESTMODSEQ", &highestmodseq)
|| highestmodseq != mailbox->highestmodseq)
return 0;
if (!dlist_getnum32(dlist, "RECENTUID", &recentuid)
|| recentuid != mailbox->recentuid)
return 0;
if (!dlist_getdate(dlist, "RECENTTIME", &recenttime)
|| recenttime != mailbox->recenttime)
return 0;
if (!dlist_getdate(dlist, "LAST_APPENDDATE", &last_appenddate)
|| last_appenddate != mailbox->last_appenddate)
return 0;
if (!dlist_getnum32(dlist, "UIDVALIDITY", &uidvalidity)
|| uidvalidity != mailbox->uidvalidity)
return 0;
if (!dlist_getatom(dlist, "PARTITION", &partition)
|| strcmp(partition, mailbox->partition) != 0)
return 0;
if (!dlist_getatom(dlist, "ACL", &acl)
|| strcmp(acl, mailbox->acl) != 0)
return 0;
if (!dlist_getatom(dlist, "OPTIONS", &options)
|| strcmp(options, mailbox->options) != 0)
return 0;
/* optional */
dlist_getnum64(dlist, "XCONVMODSEQ", &xconvmodseq);
if (xconvmodseq != mailbox->xconvmodseq)
return 0;
/* CRCs */
dlist_getnum32(dlist, "SYNC_CRC", &synccrcs.basic);
dlist_getnum32(dlist, "SYNC_CRC_ANNOT", &synccrcs.annot);
if (synccrcs.basic != mailbox->sync_crc)
return 0;
if (synccrcs.annot != mailbox->sync_crc_annot)
return 0;
syslog(LOG_DEBUG, "%s: %s matches!\n", __func__, mailbox->uniqueid);
return 1;
}
/* verify that the matching MAILBOX exists within the claimed chunk
* for each mailbox or mailbox_message in the index
*/
static int verify_chunk_mailbox_links(struct backup *backup, struct backup_chunk *chunk,
struct gzuncat *gzuc, int verbose, FILE *out)
{
/*
* get list of mailboxes in chunk
* get list of mailbox_messages in chunk
* index mailboxes list by uniqueid
* index mailbox_messages list by uniqueid:uid
* open chunk
* foreach line in chunk
* read dlist
* skip if it's not a mailbox
* if details in dlist match details in mailbox
* remove from mailbox list/index
* foreach record in dlist
* if details in dlist match details in mailbox_message
* remove from mailbox_message list/index
* failed if either list of mailboxes or list of mailbox_messages is not empty
*/
struct backup_mailbox_list *mailbox_list = NULL;
struct backup_mailbox_message_list *mailbox_message_list = NULL;
hash_table mailbox_list_index = HASH_TABLE_INITIALIZER;
hash_table mailbox_message_list_index = HASH_TABLE_INITIALIZER;
struct backup_mailbox *mailbox = NULL;
struct backup_mailbox_message *mailbox_message = NULL;
int r;
if (out && verbose)
fprintf(out, "checking chunk %d mailbox links...\n", chunk->id);
mailbox_list = backup_get_mailboxes(backup, chunk->id, 0);
mailbox_message_list = backup_get_mailbox_messages(backup, chunk->id);
if (mailbox_list->count == 0 && mailbox_message_list->count == 0) {
/* nothing we care about in this chunk */
free(mailbox_list);
free(mailbox_message_list);
if (out && verbose)
fprintf(out, "ok\n");
return 0;
}
/* XXX consider whether the two hashes should use pools */
if (mailbox_list->count) {
/* build an index of the mailbox list */
construct_hash_table(&mailbox_list_index, mailbox_list->count, 0);
mailbox = mailbox_list->head;
while (mailbox) {
hash_insert(mailbox->uniqueid, mailbox, &mailbox_list_index);
mailbox = mailbox->next;
}
}
if (mailbox_message_list->count) {
/* build an index of the mailbox message list */
construct_hash_table(&mailbox_message_list_index,
mailbox_message_list->count, 0);
mailbox_message = mailbox_message_list->head;
while (mailbox_message) {
char keybuf[1024]; // FIXME whatever
snprintf(keybuf, sizeof(keybuf), "%s:%d",
mailbox_message->mailbox_uniqueid, mailbox_message->uid);
hash_insert(keybuf, mailbox_message, &mailbox_message_list_index);
mailbox_message = mailbox_message->next;
}
}
r = gzuc_member_start_from(gzuc, chunk->offset);
if (r) {
syslog(LOG_ERR, "%s: error reading chunk %i at offset %jd: %s",
__func__, chunk->id, chunk->offset, zError(r));
if (out)
fprintf(out, "error reading chunk %i at offset %jd: %s",
chunk->id, chunk->offset, zError(r));
goto done;
}
struct protstream *ps = prot_readcb(_prot_fill_cb, gzuc);
prot_setisclient(ps, 1); /* don't sync literals */
struct buf cmd = BUF_INITIALIZER;
while (1) {
struct dlist *dl = NULL;
struct dlist *record = NULL;
struct dlist *di = NULL;
const char *uniqueid = NULL;
int c = parse_backup_line(ps, NULL, &cmd, &dl);
if (c == EOF) {
const char *error = prot_error(ps);
if (error && 0 != strcmp(error, PROT_EOF_STRING)) {
syslog(LOG_ERR,
"%s: error reading chunk %i data at offset %jd, byte %i: %s",
__func__, chunk->id, chunk->offset, prot_bytes_in(ps), error);
if (out)
fprintf(out, "error reading chunk %i data at offset %jd, byte %i: %s",
chunk->id, chunk->offset, prot_bytes_in(ps), error);
r = EOF;
}
break;
}
if (strcmp(buf_cstring(&cmd), "APPLY") != 0)
goto next_line;
if (strcmp(dl->name, "MAILBOX") != 0)
goto next_line;
if (!dlist_getatom(dl, "UNIQUEID", &uniqueid))
goto next_line;
if (mailbox_list->count) {
mailbox = (struct backup_mailbox *) hash_lookup(uniqueid, &mailbox_list_index);
if (mailbox && mailbox_matches(mailbox, dl)) {
backup_mailbox_list_remove(mailbox_list, mailbox);
hash_del(uniqueid, &mailbox_list_index);
backup_mailbox_free(&mailbox);
}
}
if (mailbox_message_list->count) {
if (!dlist_getlist(dl, "RECORD", &record))
goto next_line;
for (di = record->head; di; di = di->next) {
char keybuf[1024]; // FIXME whatever
uint32_t uid;
if (!dlist_getnum32(di, "UID", &uid))
continue;
snprintf(keybuf, sizeof(keybuf), "%s:%d", uniqueid, uid);
mailbox_message = (struct backup_mailbox_message *) hash_lookup(
keybuf, &mailbox_message_list_index);
if (!mailbox_message)
continue;
if (!mailbox_message_matches(mailbox_message, di))
continue;
backup_mailbox_message_list_remove(mailbox_message_list, mailbox_message);
hash_del(keybuf, &mailbox_message_list_index);
backup_mailbox_message_free(&mailbox_message);
}
}
next_line:
if (dl) {
dlist_unlink_files(dl);
dlist_free(&dl);
}
}
buf_free(&cmd);
prot_free(ps);
gzuc_member_end(gzuc, NULL);
/* anything left in either of the lists is missing from the chunk data. bad! */
mailbox = mailbox_list->head;
while (mailbox) {
syslog(LOG_DEBUG, "%s: chunk %d missing mailbox data for %s (%s)\n",
__func__, chunk->id, mailbox->uniqueid, mailbox->mboxname);
if (out)
fprintf(out, "chunk %d missing mailbox data for %s (%s)\n",
chunk->id, mailbox->uniqueid, mailbox->mboxname);
mailbox = mailbox->next;
}
mailbox_message = mailbox_message_list->head;
while (mailbox_message) {
syslog(LOG_DEBUG, "%s: chunk %d missing mailbox_message data for %s uid %u\n",
__func__, chunk->id, mailbox_message->mailbox_uniqueid,
mailbox_message->uid);
if (out)
fprintf(out, "chunk %d missing mailbox_message data for %s uid %u\n",
chunk->id, mailbox_message->mailbox_uniqueid,
mailbox_message->uid);
mailbox_message = mailbox_message->next;
}
if (!r) r = mailbox_list->count || mailbox_message_list->count ? -1 : 0;
done:
free_hash_table(&mailbox_list_index, NULL);
free_hash_table(&mailbox_message_list_index, NULL);
backup_mailbox_list_empty(mailbox_list);
free(mailbox_list);
backup_mailbox_message_list_empty(mailbox_message_list);
free(mailbox_message_list);
syslog(LOG_DEBUG, "%s: chunk %d %s!\n", __func__, chunk->id,
r ? "failed" : "passed");
if (out && verbose)
fprintf(out, "%s\n", r ? "error" : "ok");
return r;
}
static int want_append_mailbox(struct backup *orig_backup,
int orig_chunk_id,
struct dlist *dlist)
{
struct dlist *record = NULL;
const char *uniqueid = NULL;
struct backup_mailbox *mailbox = NULL;
int mailbox_last_chunk_id = 0;
if (!dlist_getatom(dlist, "UNIQUEID", &uniqueid)) {
syslog(LOG_DEBUG, "%s: MAILBOX line with no UNIQUEID", __func__);
return 1; /* better keep it for now */
}
dlist_getlist(dlist, "RECORD", &record);
if (record && record->head) {
struct dlist *ki = NULL, *next = NULL;
int keep = 0;
/* keep MAILBOX lines that contain the last RECORD for any message, */
/* pruning out stale RECORDs */
for (ki = record->head; ki; ki = next) {
const char *guid = NULL;
struct backup_mailbox_message *mailbox_message = NULL;
/* save next pointer now in case we need to unstitch */
next = ki->next;
if (!dlist_getatom(ki, "GUID", &guid)) {
syslog(LOG_DEBUG, "%s: MAILBOX RECORD with no GUID", __func__);
keep = 1; /* better keep it for now */
continue;
}
mailbox_message = backup_get_mailbox_message(orig_backup, uniqueid, guid);
if (mailbox_message) {
int mailbox_message_last_chunk_id = mailbox_message->last_chunk_id;
backup_mailbox_message_free(&mailbox_message);
if (mailbox_message_last_chunk_id == orig_chunk_id) {
syslog(LOG_DEBUG, "%s: keeping MAILBOX line containing last RECORD for guid %s",
__func__, guid);
keep = 1;
continue;
}
}
/* don't need this record */
syslog(LOG_DEBUG, "%s: pruning stale MAILBOX RECORD for guid %s",
__func__, guid);
dlist_unstitch(record, ki);
dlist_unlink_files(ki);
dlist_free(&ki);
}
if (keep) return 1;
}
mailbox = backup_get_mailbox_by_uniqueid(orig_backup, uniqueid,
BACKUP_MAILBOX_NO_RECORDS);
if (!mailbox) {
/* what? */
syslog(LOG_DEBUG, "%s: couldn't find mailbox entry for uniqueid %s", __func__, uniqueid);
return 1; /* better keep it for now */
}
mailbox_last_chunk_id = mailbox->last_chunk_id;
backup_mailbox_free(&mailbox);
if (mailbox_last_chunk_id == orig_chunk_id) {
/* keep all mailbox lines from the chunk recorded as its last */
syslog(LOG_DEBUG, "%s: keeping MAILBOX line from its last known chunk", __func__);
return 1;
}
syslog(LOG_DEBUG, "%s: discarding stale MAILBOX line (chunk %d, last %d, uniqueid %s)",
__func__, orig_chunk_id, mailbox_last_chunk_id, uniqueid);
return 0;
}
