int mail_index_sync_keywords_reset(struct mail_index_sync_map_ctx *ctx, const struct mail_transaction_header *hdr, const struct mail_transaction_keyword_reset *r) { struct mail_index_map *map = ctx->view->map; struct mail_index_record *rec; const struct mail_index_ext *ext; const struct mail_transaction_keyword_reset *end; uint32_t ext_map_idx, seq1, seq2; if (!mail_index_map_lookup_ext(map, MAIL_INDEX_EXT_KEYWORDS, &ext_map_idx)) { /* nothing to do */ return 1; } ext = array_idx(&map->extensions, ext_map_idx); end = CONST_PTR_OFFSET(r, hdr->size); for (; r != end; r++) { if (!mail_index_lookup_seq_range(ctx->view, r->uid1, r->uid2, &seq1, &seq2)) continue; mail_index_modseq_reset_keywords(ctx->modseq_ctx, seq1, seq2); for (; seq1 <= seq2; seq1++) { rec = MAIL_INDEX_REC_AT_SEQ(map, seq1); memset(PTR_OFFSET(rec, ext->record_offset), 0, ext->record_size); } } return 1; }
int mail_index_map_parse_extensions(struct mail_index_map *map) { struct mail_index *index = map->index; const struct mail_index_ext_header *ext_hdr; unsigned int i, old_count, offset; const char *name, *error; uint32_t ext_id, ext_map_idx, ext_offset; /* extension headers always start from 64bit offsets, so if base header doesn't happen to be 64bit aligned we'll skip some bytes */ offset = MAIL_INDEX_HEADER_SIZE_ALIGN(map->hdr.base_header_size); if (offset >= map->hdr.header_size && map->extension_pool == NULL) { /* nothing to do, skip allocatations and all */ return 0; } old_count = array_count(&index->extensions); mail_index_map_init_extbufs(map, old_count + 5); ext_id = (uint32_t)-1; for (i = 0; i < old_count; i++) array_append(&map->ext_id_map, &ext_id, 1); for (i = 0; offset < map->hdr.header_size; i++) { ext_offset = offset; if (mail_index_map_ext_get_next(map, &offset, &ext_hdr, &name) < 0) { mail_index_set_error(index, "Corrupted index file %s: " "Header extension #%d (%s) goes outside header", index->filepath, i, name); return -1; } if (mail_index_map_ext_hdr_check(&map->hdr, ext_hdr, name, &error) < 0) { mail_index_set_error(index, "Corrupted index file %s: " "Broken extension #%d (%s): %s", index->filepath, i, name, error); return -1; } if (mail_index_map_lookup_ext(map, name, &ext_map_idx)) { mail_index_set_error(index, "Corrupted index file %s: " "Duplicate header extension %s", index->filepath, name); return -1; } (void)mail_index_map_register_ext(map, name, ext_offset, ext_hdr); } return 0; }
int mail_index_sync_keywords(struct mail_index_sync_map_ctx *ctx, const struct mail_transaction_header *hdr, const struct mail_transaction_keyword_update *rec) { struct mail_index_view *view = ctx->view; const char *keyword_name; const struct mail_index_ext *ext; const uint32_t *uid, *end; uint32_t seqset_offset, ext_map_idx; unsigned int keyword_idx; int ret; i_assert(rec->name_size > 0); seqset_offset = sizeof(*rec) + rec->name_size; if ((seqset_offset % 4) != 0) seqset_offset += 4 - (seqset_offset % 4); i_assert(seqset_offset < hdr->size); uid = CONST_PTR_OFFSET(rec, seqset_offset); end = CONST_PTR_OFFSET(rec, hdr->size); keyword_name = t_strndup(rec + 1, rec->name_size); if (!keyword_lookup(ctx, keyword_name, &keyword_idx)) keywords_header_add(ctx, keyword_name, &keyword_idx); /* if the keyword wasn't found, the "keywords" extension was created. if it was found, the record size should already be correct, but in case it isn't just fix it ourself. */ if (!mail_index_map_lookup_ext(view->map, MAIL_INDEX_EXT_KEYWORDS, &ext_map_idx)) i_unreached(); ext = array_idx(&view->map->extensions, ext_map_idx); if (keyword_idx / CHAR_BIT >= ext->record_size) { if (rec->modify_type == MODIFY_REMOVE) { /* nothing to do */ return 1; } /* grow the record size */ keywords_ext_register(ctx, ext_map_idx, ext->reset_id, ext->hdr_size, array_count(&view->map->keyword_idx_map)); if (!mail_index_map_lookup_ext(view->map, MAIL_INDEX_EXT_KEYWORDS, &ext_map_idx)) i_unreached(); ext = array_idx(&view->map->extensions, ext_map_idx); } while (uid+2 <= end) { ret = keywords_update_records(ctx, ext, keyword_idx, rec->modify_type, uid[0], uid[1]); if (ret <= 0) return ret; uid += 2; } return 1; }
static void keywords_header_add(struct mail_index_sync_map_ctx *ctx, const char *keyword_name, unsigned int *keyword_idx_r) { struct mail_index_map *map; const struct mail_index_ext *ext = NULL; struct mail_index_keyword_header *kw_hdr; struct mail_index_keyword_header_rec kw_rec; uint32_t ext_map_idx; buffer_t *buf = NULL; size_t keyword_len, rec_offset, name_offset, name_offset_root; unsigned int keywords_count; /* if we crash in the middle of writing the header, the keywords are more or less corrupted. avoid that by making sure the header is updated atomically. */ map = mail_index_sync_get_atomic_map(ctx); if (!mail_index_map_lookup_ext(map, MAIL_INDEX_EXT_KEYWORDS, &ext_map_idx)) ext_map_idx = (uint32_t)-1; else { /* update existing header */ ext = array_idx(&map->extensions, ext_map_idx); buf = keywords_get_header_buf(map, ext, 1, &keywords_count, &rec_offset, &name_offset_root, &name_offset); } if (buf == NULL) { /* create new / replace broken header */ const unsigned int initial_keywords_count = 1; buf = buffer_create_dynamic(pool_datastack_create(), 512); kw_hdr = buffer_append_space_unsafe(buf, sizeof(*kw_hdr)); kw_hdr->keywords_count = initial_keywords_count; keywords_count = kw_hdr->keywords_count; rec_offset = buf->used; name_offset_root = rec_offset + initial_keywords_count * sizeof(kw_rec); name_offset = 0; } /* add the keyword */ memset(&kw_rec, 0, sizeof(kw_rec)); kw_rec.name_offset = name_offset; keyword_len = strlen(keyword_name) + 1; buffer_write(buf, rec_offset, &kw_rec, sizeof(kw_rec)); buffer_write(buf, name_offset_root, keyword_name, keyword_len); rec_offset += sizeof(kw_rec); kw_rec.name_offset += keyword_len; name_offset_root += keyword_len; if ((buf->used % 4) != 0) buffer_append_zero(buf, 4 - (buf->used % 4)); if (ext == NULL || buf->used > ext->hdr_size || (uint32_t)ext->record_size * CHAR_BIT < keywords_count) { /* if we need to grow the buffer, add some padding */ buffer_append_zero(buf, 128); keywords_ext_register(ctx, ext_map_idx, ext == NULL ? 0 : ext->reset_id, buf->used, keywords_count); /* map may have changed */ map = ctx->view->map; if (!mail_index_map_lookup_ext(map, MAIL_INDEX_EXT_KEYWORDS, &ext_map_idx)) i_unreached(); ext = array_idx(&map->extensions, ext_map_idx); i_assert(ext->hdr_size == buf->used); } buffer_copy(map->hdr_copy_buf, ext->hdr_offset, buf, 0, buf->used); map->hdr_base = map->hdr_copy_buf->data; i_assert(map->hdr_copy_buf->used == map->hdr.header_size); if (mail_index_map_parse_keywords(map) < 0) i_panic("Keyword update corrupted keywords header"); *keyword_idx_r = keywords_count - 1; i_assert(*keyword_idx_r / CHAR_BIT < ext->record_size); }
int mail_index_map_parse_keywords(struct mail_index_map *map) { struct mail_index *index = map->index; const struct mail_index_ext *ext; const struct mail_index_keyword_header *kw_hdr; const struct mail_index_keyword_header_rec *kw_rec; const char *name; unsigned int i, name_area_end_offset, old_count; uint32_t idx; if (!mail_index_map_lookup_ext(map, MAIL_INDEX_EXT_KEYWORDS, &idx)) { if (array_is_created(&map->keyword_idx_map)) array_clear(&map->keyword_idx_map); return 0; } ext = array_idx(&map->extensions, idx); /* Extension header contains: - struct mail_index_keyword_header - struct mail_index_keyword_header_rec * keywords_count - const char names[] * keywords_count */ i_assert(ext->hdr_offset < map->hdr.header_size); kw_hdr = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset); kw_rec = (const void *)(kw_hdr + 1); name = (const char *)(kw_rec + kw_hdr->keywords_count); old_count = !array_is_created(&map->keyword_idx_map) ? 0 : array_count(&map->keyword_idx_map); /* Keywords can only be added into same mapping. Removing requires a new mapping (recreating the index file) */ if (kw_hdr->keywords_count == old_count) { /* nothing changed */ return 0; } /* make sure the header is valid */ if (kw_hdr->keywords_count < old_count) { mail_index_set_error(index, "Corrupted index file %s: " "Keywords removed unexpectedly", index->filepath); return -1; } if ((size_t)(name - (const char *)kw_hdr) > ext->hdr_size) { mail_index_set_error(index, "Corrupted index file %s: " "keywords_count larger than header size", index->filepath); return -1; } name_area_end_offset = (const char *)kw_hdr + ext->hdr_size - name; for (i = 0; i < kw_hdr->keywords_count; i++) { if (kw_rec[i].name_offset > name_area_end_offset) { mail_index_set_error(index, "Corrupted index file %s: " "name_offset points outside allocated header", index->filepath); return -1; } } if (name[name_area_end_offset-1] != '\0') { mail_index_set_error(index, "Corrupted index file %s: " "Keyword header doesn't end with NUL", index->filepath); return -1; } /* create file -> index mapping */ if (!array_is_created(&map->keyword_idx_map)) i_array_init(&map->keyword_idx_map, kw_hdr->keywords_count); #ifdef DEBUG /* Check that existing headers are still the same. It's behind DEBUG since it's pretty useless waste of CPU normally. */ for (i = 0; i < array_count(&map->keyword_idx_map); i++) { const char *keyword = name + kw_rec[i].name_offset; const unsigned int *old_idx; unsigned int kw_idx; old_idx = array_idx(&map->keyword_idx_map, i); if (!mail_index_keyword_lookup(index, keyword, &kw_idx) || kw_idx != *old_idx) { mail_index_set_error(index, "Corrupted index file %s: " "Keywords changed unexpectedly", index->filepath); return -1; } } #endif /* Register the newly seen keywords */ i = array_count(&map->keyword_idx_map); for (; i < kw_hdr->keywords_count; i++) { const char *keyword = name + kw_rec[i].name_offset; unsigned int kw_idx; if (*keyword == '\0') { mail_index_set_error(index, "Corrupted index file %s: " "Empty keyword name in header", index->filepath); return -1; } mail_index_keyword_lookup_or_create(index, keyword, &kw_idx); array_append(&map->keyword_idx_map, &kw_idx, 1); } return 0; }