void mdbox_update_header(struct mdbox_mailbox *mbox,
struct mail_index_transaction *trans,
const struct mailbox_update *update)
{
struct mdbox_index_header hdr, new_hdr;
bool need_resize;
if (mdbox_read_header(mbox, &hdr, &need_resize) < 0) {
memset(&hdr, 0, sizeof(hdr));
need_resize = TRUE;
}
new_hdr = hdr;
if (update != NULL && !guid_128_is_empty(update->mailbox_guid)) {
memcpy(new_hdr.mailbox_guid, update->mailbox_guid,
sizeof(new_hdr.mailbox_guid));
} else if (guid_128_is_empty(new_hdr.mailbox_guid)) {
guid_128_generate(new_hdr.mailbox_guid);
}
new_hdr.map_uid_validity =
mdbox_map_get_uid_validity(mbox->storage->map);
if (need_resize) {
mail_index_ext_resize_hdr(trans, mbox->hdr_ext_id,
sizeof(new_hdr));
}
if (memcmp(&hdr, &new_hdr, sizeof(hdr)) != 0) {
mail_index_update_header_ext(trans, mbox->hdr_ext_id, 0,
&new_hdr, sizeof(new_hdr));
}
}
static void sdbox_update_header(struct sdbox_mailbox *mbox,
struct mail_index_transaction *trans,
const struct mailbox_update *update)
{
struct sdbox_index_header hdr, new_hdr;
bool need_resize;
if (sdbox_read_header(mbox, &hdr, TRUE, &need_resize) < 0) {
i_zero(&hdr);
need_resize = TRUE;
}
new_hdr = hdr;
if (update != NULL && !guid_128_is_empty(update->mailbox_guid)) {
memcpy(new_hdr.mailbox_guid, update->mailbox_guid,
sizeof(new_hdr.mailbox_guid));
} else if (guid_128_is_empty(new_hdr.mailbox_guid)) {
guid_128_generate(new_hdr.mailbox_guid);
}
if (need_resize) {
mail_index_ext_resize_hdr(trans, mbox->hdr_ext_id,
sizeof(new_hdr));
}
if (memcmp(&hdr, &new_hdr, sizeof(hdr)) != 0) {
mail_index_update_header_ext(trans, mbox->hdr_ext_id, 0,
&new_hdr, sizeof(new_hdr));
}
memcpy(mbox->mailbox_guid, new_hdr.mailbox_guid,
sizeof(mbox->mailbox_guid));
}
static int dsync_mailbox_tree_add(struct dsync_mailbox_tree *tree,
const struct mailbox_info *info,
const guid_128_t box_guid,
enum mail_error *error_r)
{
struct dsync_mailbox_node *node;
struct mailbox *box;
struct mailbox_metadata metadata;
struct mailbox_status status;
const char *errstr;
enum mail_error error;
int ret = 0;
if ((info->flags & MAILBOX_NONEXISTENT) != 0)
return 0;
if ((info->flags & MAILBOX_NOSELECT) != 0) {
return !guid_128_is_empty(box_guid) ? 0 :
dsync_mailbox_tree_add_exists_node(tree, info, &node, error_r);
}
/* get GUID and UIDVALIDITY for selectable mailbox */
box = mailbox_alloc(info->ns->list, info->vname, MAILBOX_FLAG_READONLY);
if (dsync_mailbox_tree_get_selectable(box, &metadata, &status) < 0) {
errstr = mailbox_get_last_internal_error(box, &error);
switch (error) {
case MAIL_ERROR_NOTFOUND:
/* mailbox was just deleted? */
break;
case MAIL_ERROR_NOTPOSSIBLE:
/* invalid mbox files? ignore */
break;
default:
i_error("Failed to access mailbox %s: %s",
info->vname, errstr);
*error_r = error;
ret = -1;
}
mailbox_free(&box);
return ret;
}
mailbox_free(&box);
if (!guid_128_is_empty(box_guid) &&
!guid_128_equals(box_guid, metadata.guid)) {
/* unwanted mailbox */
return 0;
}
if (dsync_mailbox_tree_add_exists_node(tree, info, &node, error_r) < 0)
return -1;
memcpy(node->mailbox_guid, metadata.guid,
sizeof(node->mailbox_guid));
node->uid_validity = status.uidvalidity;
node->uid_next = status.uidnext;
return 0;
}
static bool
maildir_expunge_is_valid_guid(struct maildir_index_sync_context *ctx,
uint32_t uid, const char *filename,
guid_128_t expunged_guid_128)
{
guid_128_t guid_128;
const char *guid;
if (guid_128_is_empty(expunged_guid_128)) {
/* no GUID associated with expunge */
return TRUE;
}
T_BEGIN {
guid = maildir_uidlist_lookup_ext(ctx->mbox->uidlist, uid,
MAILDIR_UIDLIST_REC_EXT_GUID);
if (guid == NULL)
guid = t_strcut(filename, ':');
mail_generate_guid_128_hash(guid, guid_128);
} T_END;
if (memcmp(guid_128, expunged_guid_128, sizeof(guid_128)) == 0)
return TRUE;
mail_storage_set_critical(&ctx->mbox->storage->storage,
"Mailbox %s: Expunged GUID mismatch for UID %u: %s vs %s",
ctx->mbox->box.vname, ctx->uid,
guid_128_to_string(guid_128),
guid_128_to_string(expunged_guid_128));
return FALSE;
}
static int
mbox_mailbox_get_guid(struct mbox_mailbox *mbox, guid_128_t guid_r)
{
const char *errstr;
if (mail_index_is_in_memory(mbox->box.index)) {
errstr = "Mailbox GUIDs are not permanent without index files";
if (mbox->storage->set->mbox_min_index_size != 0) {
errstr = t_strconcat(errstr,
" (mbox_min_index_size is non-zero)", NULL);
}
mail_storage_set_error(mbox->box.storage,
MAIL_ERROR_NOTPOSSIBLE, errstr);
return -1;
}
if (mbox_sync_header_refresh(mbox) < 0)
return -1;
if (!guid_128_is_empty(mbox->mbox_hdr.mailbox_guid)) {
/* we have the GUID */
} else if (mbox_file_open(mbox) < 0)
return -1;
else if (mbox->backend_readonly) {
mail_storage_set_error(mbox->box.storage, MAIL_ERROR_PERM,
"Can't set mailbox GUID to a read-only mailbox");
return -1;
} else {
/* create another mailbox and sync */
struct mailbox *box2;
struct mbox_mailbox *mbox2;
int ret;
i_assert(mbox->mbox_lock_type == F_UNLCK);
box2 = mailbox_alloc(mbox->box.list, mbox->box.vname, 0);
ret = mailbox_sync(box2, 0);
mbox2 = (struct mbox_mailbox *)box2;
memcpy(guid_r, mbox2->mbox_hdr.mailbox_guid, GUID_128_SIZE);
mailbox_free(&box2);
return ret;
}
memcpy(guid_r, mbox->mbox_hdr.mailbox_guid, GUID_128_SIZE);
return 0;
}
static int
dbox_sync_verify_expunge_guid(struct mdbox_sync_context *ctx, uint32_t seq,
const guid_128_t guid_128)
{
const void *data;
uint32_t uid;
mail_index_lookup_uid(ctx->sync_view, seq, &uid);
mail_index_lookup_ext(ctx->sync_view, seq,
ctx->mbox->guid_ext_id, &data, NULL);
if (guid_128_is_empty(guid_128) ||
memcmp(data, guid_128, GUID_128_SIZE) == 0)
return 0;
mail_storage_set_critical(&ctx->mbox->storage->storage.storage,
"Mailbox %s: Expunged GUID mismatch for UID %u: %s vs %s",
ctx->mbox->box.vname, uid,
guid_128_to_string(data), guid_128_to_string(guid_128));
mdbox_storage_set_corrupted(ctx->mbox->storage);
return -1;
}
int sdbox_read_header(struct sdbox_mailbox *mbox,
struct sdbox_index_header *hdr, bool log_error,
bool *need_resize_r)
{
struct mail_index_view *view;
const void *data;
size_t data_size;
int ret = 0;
i_assert(mbox->box.opened);
view = mail_index_view_open(mbox->box.index);
mail_index_get_header_ext(view, mbox->hdr_ext_id,
&data, &data_size);
if (data_size < SDBOX_INDEX_HEADER_MIN_SIZE &&
(!mbox->box.creating || data_size != 0)) {
if (log_error) {
mail_storage_set_critical(
&mbox->storage->storage.storage,
"sdbox %s: Invalid dbox header size",
mailbox_get_path(&mbox->box));
}
ret = -1;
} else {
memset(hdr, 0, sizeof(*hdr));
memcpy(hdr, data, I_MIN(data_size, sizeof(*hdr)));
if (guid_128_is_empty(hdr->mailbox_guid))
ret = -1;
else {
/* data is valid. remember it in case mailbox
is being reset */
mail_index_set_ext_init_data(mbox->box.index,
mbox->hdr_ext_id,
hdr, sizeof(*hdr));
}
}
mail_index_view_close(&view);
*need_resize_r = data_size < sizeof(*hdr);
return ret;
}
static int
mbox_mailbox_update(struct mailbox *box, const struct mailbox_update *update)
{
struct mbox_mailbox *mbox = (struct mbox_mailbox *)box;
int ret = 0;
if (!box->opened) {
if (mailbox_open(box) < 0)
return -1;
}
if (update->uid_validity != 0 || update->min_next_uid != 0 ||
!guid_128_is_empty(update->mailbox_guid)) {
mbox->sync_hdr_update = update;
ret = mbox_sync(mbox, MBOX_SYNC_HEADER | MBOX_SYNC_FORCE_SYNC |
MBOX_SYNC_REWRITE);
mbox->sync_hdr_update = NULL;
}
if (ret == 0)
ret = index_storage_mailbox_update(box, update);
return ret;
}
static int
dsync_mailbox_tree_get_selectable(struct mailbox *box,
struct mailbox_metadata *metadata_r,
struct mailbox_status *status_r)
{
/* try the fast path */
if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, metadata_r) < 0)
return -1;
if (mailbox_get_status(box, STATUS_UIDVALIDITY | STATUS_UIDNEXT, status_r) < 0)
return -1;
i_assert(!guid_128_is_empty(metadata_r->guid));
if (status_r->uidvalidity != 0)
return 0;
/* no UIDVALIDITY assigned yet. syncing a mailbox should add it. */
if (mailbox_sync(box, 0) < 0)
return -1;
if (mailbox_get_status(box, STATUS_UIDVALIDITY | STATUS_UIDNEXT, status_r) < 0)
return -1;
i_assert(status_r->uidvalidity != 0);
return 0;
}
static int
dsync_brain_try_next_mailbox(struct dsync_brain *brain, struct mailbox **box_r,
struct dsync_mailbox *dsync_box_r)
{
enum mailbox_flags flags = 0;
struct dsync_mailbox dsync_box;
struct mailbox *box;
struct dsync_mailbox_node *node;
const char *vname = NULL;
bool synced = FALSE;
int ret;
*box_r = NULL;
while (dsync_mailbox_tree_iter_next(brain->local_tree_iter, &vname, &node)) {
if (node->existence == DSYNC_MAILBOX_NODE_EXISTS &&
!guid_128_is_empty(node->mailbox_guid))
break;
vname = NULL;
}
if (vname == NULL) {
/* no more mailboxes */
dsync_mailbox_tree_iter_deinit(&brain->local_tree_iter);
return -1;
}
if (brain->backup_send) {
/* make sure mailbox isn't modified */
flags |= MAILBOX_FLAG_READONLY;
}
box = mailbox_alloc(node->ns->list, vname, flags);
for (;;) {
if ((ret = dsync_box_get(box, &dsync_box)) <= 0) {
if (ret < 0)
brain->failed = TRUE;
mailbox_free(&box);
return ret;
}
/* if mailbox's last_common_* state equals the current state,
we can skip the mailbox */
if (!dsync_brain_has_mailbox_state_changed(brain, &dsync_box)) {
mailbox_free(&box);
return 0;
}
if (synced) {
/* ok, the mailbox really changed */
break;
}
/* mailbox appears to have changed. do a full sync here and get the
state again */
if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) {
i_error("Can't sync mailbox %s: %s",
mailbox_get_vname(box),
mailbox_get_last_error(box, NULL));
brain->failed = TRUE;
mailbox_free(&box);
return -1;
}
synced = TRUE;
}
*box_r = box;
*dsync_box_r = dsync_box;
return 1;
}
static void
dsync_brain_mailbox_tree_add_delete(struct dsync_mailbox_tree *tree,
struct dsync_mailbox_tree *other_tree,
const struct dsync_mailbox_delete *other_del,
const struct dsync_mailbox_node **node_r,
const char **status_r)
{
const struct dsync_mailbox_node *node;
struct dsync_mailbox_node *other_node, *old_node;
const char *name;
/* see if we can find the deletion based on mailbox tree that should
still have the mailbox */
node = *node_r = dsync_mailbox_tree_find_delete(tree, other_del);
if (node == NULL) {
*status_r = "not found";
return;
}
switch (other_del->type) {
case DSYNC_MAILBOX_DELETE_TYPE_MAILBOX:
/* mailbox is always deleted */
break;
case DSYNC_MAILBOX_DELETE_TYPE_DIR:
if (other_del->timestamp <= node->last_renamed_or_created) {
/* we don't want to delete this directory, we already
have a newer timestamp for it */
*status_r = "keep directory, we have a newer timestamp";
return;
}
break;
case DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE:
if (other_del->timestamp <= node->last_subscription_change) {
/* we don't want to unsubscribe, since we already have
a newer subscription timestamp */
*status_r = "keep subscription, we have a newer timestamp";
return;
}
break;
}
/* make a node for it in the other mailbox tree */
name = dsync_mailbox_node_get_full_name(tree, node);
other_node = dsync_mailbox_tree_get(other_tree, name);
if (other_node->existence == DSYNC_MAILBOX_NODE_EXISTS &&
(!guid_128_is_empty(other_node->mailbox_guid) ||
other_del->type != DSYNC_MAILBOX_DELETE_TYPE_MAILBOX)) {
/* other side has already created a new mailbox or
directory with this name, we can't delete it */
*status_r = "name has already been recreated";
return;
}
/* ok, mark the other node deleted */
if (other_del->type == DSYNC_MAILBOX_DELETE_TYPE_MAILBOX) {
memcpy(other_node->mailbox_guid, node->mailbox_guid,
sizeof(other_node->mailbox_guid));
}
if (other_node->ns != node->ns && other_node->ns != NULL) {
/* namespace mismatch for this node. this shouldn't happen
normally, but especially during some misconfigurations it's
possible that one side has created mailboxes that conflict
with another namespace's prefix. since we're here because
one of the mailboxes was deleted, we'll just ignore this. */
*status_r = "namespace mismatch";
return;
}
other_node->ns = node->ns;
if (other_del->type != DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE) {
other_node->existence = DSYNC_MAILBOX_NODE_DELETED;
*status_r = "marked as deleted";
} else {
other_node->last_subscription_change = other_del->timestamp;
other_node->subscribed = FALSE;
*status_r = "marked as unsubscribed";
}
if (dsync_mailbox_tree_guid_hash_add(other_tree, other_node,
&old_node) < 0)
i_unreached();
}
static bool
cmd_mailbox_dsync_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
{
struct dsync_cmd_context *ctx = (struct dsync_cmd_context *)_ctx;
const char *str;
switch (c) {
case '1':
ctx->oneway = TRUE;
ctx->backup = TRUE;
break;
case 'd':
ctx->default_replica_location = TRUE;
break;
case 'E':
/* dsync wrapper detection flag */
legacy_dsync = TRUE;
break;
case 'f':
ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_FULL;
break;
case 'g':
if (optarg[0] == '\0')
ctx->no_mail_sync = TRUE;
else if (guid_128_from_string(optarg, ctx->mailbox_guid) < 0 ||
guid_128_is_empty(ctx->mailbox_guid))
i_fatal("Invalid -g parameter: %s", optarg);
break;
case 'l':
ctx->lock = TRUE;
if (str_to_uint(optarg, &ctx->lock_timeout) < 0)
i_fatal("Invalid -l parameter: %s", optarg);
break;
case 'm':
if (optarg[0] == '\0')
ctx->no_mail_sync = TRUE;
else
ctx->mailbox = optarg;
break;
case 'x':
str = optarg;
array_append(&ctx->exclude_mailboxes, &str, 1);
break;
case 'n':
ctx->namespace_prefix = optarg;
break;
case 'N':
ctx->sync_visible_namespaces = TRUE;
break;
case 'P':
ctx->purge_remote = TRUE;
break;
case 'r':
ctx->rawlog_path = optarg;
break;
case 'R':
ctx->reverse_backup = TRUE;
break;
case 's':
if (ctx->sync_type != DSYNC_BRAIN_SYNC_TYPE_FULL &&
*optarg != '\0')
ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_STATE;
ctx->state_input = optarg;
break;
case 'U':
ctx->replicator_notify = TRUE;
break;
default:
return FALSE;
}
return TRUE;
}
static void
dsync_brain_mailbox_tree_add_delete(struct dsync_mailbox_tree *tree,
struct dsync_mailbox_tree *other_tree,
const struct dsync_mailbox_delete *other_del)
{
const struct dsync_mailbox_node *node;
struct dsync_mailbox_node *other_node, *old_node;
const char *name;
/* see if we can find the deletion based on mailbox tree that should
still have the mailbox */
node = dsync_mailbox_tree_find_delete(tree, other_del);
if (node == NULL)
return;
switch (other_del->type) {
case DSYNC_MAILBOX_DELETE_TYPE_MAILBOX:
/* mailbox is always deleted */
break;
case DSYNC_MAILBOX_DELETE_TYPE_DIR:
if (other_del->timestamp <= node->last_renamed_or_created) {
/* we don't want to delete this directory, we already
have a newer timestamp for it */
return;
}
break;
case DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE:
if (other_del->timestamp <= node->last_subscription_change) {
/* we don't want to unsubscribe, since we already have
a newer subscription timestamp */
return;
}
break;
}
/* make a node for it in the other mailbox tree */
name = dsync_mailbox_node_get_full_name(tree, node);
other_node = dsync_mailbox_tree_get(other_tree, name);
if (other_node->existence == DSYNC_MAILBOX_NODE_EXISTS &&
(!guid_128_is_empty(other_node->mailbox_guid) ||
other_del->type != DSYNC_MAILBOX_DELETE_TYPE_MAILBOX)) {
/* other side has already created a new mailbox or
directory with this name, we can't delete it */
return;
}
/* ok, mark the other node deleted */
if (other_del->type == DSYNC_MAILBOX_DELETE_TYPE_MAILBOX) {
memcpy(other_node->mailbox_guid, node->mailbox_guid,
sizeof(other_node->mailbox_guid));
}
i_assert(other_node->ns == NULL || other_node->ns == node->ns);
other_node->ns = node->ns;
if (other_del->type != DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE)
other_node->existence = DSYNC_MAILBOX_NODE_DELETED;
else {
other_node->last_subscription_change = other_del->timestamp;
other_node->subscribed = FALSE;
}
if (dsync_mailbox_tree_guid_hash_add(other_tree, other_node,
&old_node) < 0)
i_unreached();
}
void test_guid(void)
{
static const guid_128_t test_guid =
{ 0x01, 0x23, 0x45, 0x67, 0x89,
0xab, 0xcd, 0xef,
0xAB, 0xCD, 0xEF,
0x00, 0x00, 0x00, 0x00, 0x00 };
guid_128_t guid1, guid2, guid3;
const char *str;
char guidbuf[GUID_128_SIZE*2 + 2];
unsigned int i;
test_begin("guid_128_generate()");
guid_128_generate(guid1);
guid_128_generate(guid2);
test_assert(!guid_128_equals(guid1, guid2));
test_assert(guid_128_cmp(guid1, guid2) != 0);
test_end();
test_begin("guid_128_is_empty()");
test_assert(!guid_128_is_empty(guid1));
test_assert(!guid_128_is_empty(guid2));
guid_128_generate(guid3);
guid_128_empty(guid3);
test_assert(guid_128_is_empty(guid3));
test_end();
test_begin("guid_128_copy()");
guid_128_copy(guid3, guid1);
test_assert(guid_128_equals(guid3, guid1));
test_assert(!guid_128_equals(guid3, guid2));
guid_128_copy(guid3, guid2);
test_assert(!guid_128_equals(guid3, guid1));
test_assert(guid_128_equals(guid3, guid2));
test_end();
test_begin("guid_128_to_string()");
str = guid_128_to_string(guid1);
test_assert(guid_128_from_string(str, guid3) == 0);
test_assert(guid_128_equals(guid3, guid1));
test_end();
test_begin("guid_128_from_string()");
/* empty */
memset(guidbuf, '0', GUID_128_SIZE*2);
guidbuf[GUID_128_SIZE*2] = '\0';
guidbuf[GUID_128_SIZE*2+1] = '\0';
test_assert(guid_128_from_string(guidbuf, guid3) == 0);
test_assert(guid_128_is_empty(guid3));
/* too large */
guidbuf[GUID_128_SIZE*2] = '0';
test_assert(guid_128_from_string(guidbuf, guid3) < 0);
/* too small */
guidbuf[GUID_128_SIZE*2-1] = '\0';
test_assert(guid_128_from_string(guidbuf, guid3) < 0);
/* reset to normal */
guidbuf[GUID_128_SIZE*2-1] = '0';
guidbuf[GUID_128_SIZE*2] = '\0';
test_assert(guid_128_from_string(guidbuf, guid3) == 0);
/* upper + lowercase hex chars */
i_assert(GUID_128_SIZE*2 > 16 + 6);
for (i = 0; i < 10; i++)
guidbuf[i] = '0' + i;
for (i = 0; i < 6; i++)
guidbuf[10 + i] = 'a' + i;
for (i = 0; i < 6; i++)
guidbuf[16 + i] = 'A' + i;
test_assert(guid_128_from_string(guidbuf, guid3) == 0);
test_assert(guid_128_equals(guid3, test_guid));
/* non-hex chars */
guidbuf[0] = 'g';
test_assert(guid_128_from_string(guidbuf, guid3) < 0);
guidbuf[0] = ' ';
test_assert(guid_128_from_string(guidbuf, guid3) < 0);
test_end();
}