dovecot-2.2: dsync: If incremental dsync finds that its state is...
dovecot at dovecot.org
dovecot at dovecot.org
Mon Apr 28 17:14:34 UTC 2014
details: http://hg.dovecot.org/dovecot-2.2/rev/0d34a90b2760
changeset: 17283:0d34a90b2760
user: Timo Sirainen <tss at iki.fi>
date: Mon Apr 28 20:14:03 2014 +0300
description:
dsync: If incremental dsync finds that its state is stale, retry (later) with full sync.
This works only when the master itself notices such changes locally. Remote
changes aren't noticed, because the master doesn't send the state to remote
dsync and the master is also the one that decides which mailboxes are
synced. The fix to this probably means sending the state string to remote
dsync and have it check if the state is still valid (= protocol change).
diffstat:
src/doveadm/dsync/dsync-brain-mailbox.c | 52 +++++++++++++++++++++-----
src/doveadm/dsync/dsync-brain-mails.c | 14 +++++-
src/doveadm/dsync/dsync-brain-private.h | 3 +-
src/doveadm/dsync/dsync-transaction-log-scan.c | 22 ++++++++---
4 files changed, 71 insertions(+), 20 deletions(-)
diffs (261 lines):
diff -r 9669c9a8984f -r 0d34a90b2760 src/doveadm/dsync/dsync-brain-mailbox.c
--- a/src/doveadm/dsync/dsync-brain-mailbox.c Mon Apr 28 20:10:56 2014 +0300
+++ b/src/doveadm/dsync/dsync-brain-mailbox.c Mon Apr 28 20:14:03 2014 +0300
@@ -196,9 +196,11 @@
int dsync_brain_sync_mailbox_open(struct dsync_brain *brain,
const struct dsync_mailbox *remote_dsync_box)
{
+ struct mailbox_status status;
enum dsync_mailbox_exporter_flags exporter_flags = 0;
uint32_t last_common_uid, highest_wanted_uid;
uint64_t last_common_modseq, last_common_pvt_modseq;
+ int ret;
i_assert(brain->log_scan == NULL);
i_assert(brain->box_exporter == NULL);
@@ -208,18 +210,33 @@
last_common_pvt_modseq = brain->mailbox_state.last_common_pvt_modseq;
highest_wanted_uid = last_common_uid == 0 ?
(uint32_t)-1 : last_common_uid;
- if (dsync_transaction_log_scan_init(brain->box->view,
- brain->box->view_pvt,
- highest_wanted_uid,
- last_common_modseq,
- last_common_pvt_modseq,
- &brain->log_scan) < 0) {
+ ret = dsync_transaction_log_scan_init(brain->box->view,
+ brain->box->view_pvt,
+ highest_wanted_uid,
+ last_common_modseq,
+ last_common_pvt_modseq,
+ &brain->log_scan);
+ if (ret < 0) {
i_error("Failed to read transaction log for mailbox %s",
mailbox_get_vname(brain->box));
brain->failed = TRUE;
return -1;
}
+ if (last_common_uid != 0) {
+ mailbox_get_open_status(brain->box, STATUS_UIDNEXT |
+ STATUS_HIGHESTMODSEQ |
+ STATUS_HIGHESTPVTMODSEQ, &status);
+ if (status.uidnext < last_common_uid ||
+ status.highest_modseq < last_common_modseq ||
+ status.highest_pvt_modseq < last_common_pvt_modseq) {
+ /* last_common_* is higher than our current ones.
+ incremental sync state is stale, we need to do
+ a full resync */
+ ret = 0;
+ }
+ }
+
if (!brain->mail_requests)
exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS;
if (remote_dsync_box->have_save_guids &&
@@ -232,13 +249,25 @@
last_common_uid,
exporter_flags);
dsync_brain_sync_mailbox_init_remote(brain, remote_dsync_box);
- return 0;
+ if (ret == 0) {
+ i_warning("Failed to do incremental sync for mailbox %s, "
+ "retry with a full sync",
+ mailbox_get_vname(brain->box));
+ brain->changes_during_sync = TRUE;
+ brain->require_full_resync = TRUE;
+ return 0;
+ }
+ return 1;
}
void dsync_brain_sync_mailbox_deinit(struct dsync_brain *brain)
{
i_assert(brain->box != NULL);
+ if (brain->require_full_resync) {
+ brain->mailbox_state.last_uidvalidity = 0;
+ brain->require_full_resync = FALSE;
+ }
array_append(&brain->remote_mailbox_states, &brain->mailbox_state, 1);
if (brain->box_exporter != NULL) {
const char *error;
@@ -544,13 +573,14 @@
}
}
-void dsync_brain_mailbox_update_pre(struct dsync_brain *brain,
+bool dsync_brain_mailbox_update_pre(struct dsync_brain *brain,
struct mailbox *box,
const struct dsync_mailbox *local_box,
const struct dsync_mailbox *remote_box)
{
struct mailbox_update update;
const struct dsync_mailbox_state *state;
+ bool ret = TRUE;
memset(&update, 0, sizeof(update));
@@ -568,7 +598,7 @@
session, because the other side already started
sending mailbox changes, but not for all mails. */
dsync_mailbox_state_remove(brain, local_box->mailbox_guid);
- // FIXME: handle this properly
+ ret = FALSE;
}
}
@@ -577,7 +607,7 @@
if (update.uid_validity == 0 &&
update.cache_updates == NULL) {
/* no changes */
- return;
+ return ret;
}
if (mailbox_update(box, &update) < 0) {
@@ -586,6 +616,7 @@
mailbox_get_last_error(box, NULL));
brain->failed = TRUE;
}
+ return ret;
}
static void
@@ -614,6 +645,7 @@
struct mailbox *box;
const char *error;
int ret;
+ bool resync;
i_assert(!brain->master_brain);
i_assert(brain->box == NULL);
diff -r 9669c9a8984f -r 0d34a90b2760 src/doveadm/dsync/dsync-brain-mails.c
--- a/src/doveadm/dsync/dsync-brain-mails.c Mon Apr 28 20:10:56 2014 +0300
+++ b/src/doveadm/dsync/dsync-brain-mails.c Mon Apr 28 20:14:03 2014 +0300
@@ -22,6 +22,7 @@
{
const struct dsync_mailbox *dsync_box;
enum dsync_ibc_recv_ret ret;
+ bool resync;
i_assert(brain->master_brain);
@@ -42,19 +43,26 @@
if (dsync_box->mailbox_lost) {
/* remote lost the mailbox. it's probably already deleted, but
verify it on next sync just to be sure */
+ brain->changes_during_sync = TRUE;
+ brain->require_full_resync = TRUE;
dsync_brain_sync_mailbox_deinit(brain);
return TRUE;
}
- dsync_brain_mailbox_update_pre(brain, brain->box,
- &brain->local_dsync_box, dsync_box);
+ resync = !dsync_brain_mailbox_update_pre(brain, brain->box,
+ &brain->local_dsync_box,
+ dsync_box);
if (!dsync_boxes_need_sync(brain, &brain->local_dsync_box, dsync_box)) {
/* no fields appear to have changed, skip this mailbox */
dsync_brain_sync_mailbox_deinit(brain);
return TRUE;
}
- if (dsync_brain_sync_mailbox_open(brain, dsync_box) < 0)
+ if ((ret = dsync_brain_sync_mailbox_open(brain, dsync_box)) < 0)
return TRUE;
+ if (ret == 0 || resync) {
+ brain->changes_during_sync = TRUE;
+ brain->require_full_resync = TRUE;
+ }
dsync_brain_sync_init_box_states(brain);
return TRUE;
}
diff -r 9669c9a8984f -r 0d34a90b2760 src/doveadm/dsync/dsync-brain-private.h
--- a/src/doveadm/dsync/dsync-brain-private.h Mon Apr 28 20:10:56 2014 +0300
+++ b/src/doveadm/dsync/dsync-brain-private.h Mon Apr 28 20:14:03 2014 +0300
@@ -99,6 +99,7 @@
unsigned int no_mail_sync:1;
unsigned int no_backup_overwrite:1;
unsigned int changes_during_sync:1;
+ unsigned int require_full_resync:1;
unsigned int verbose_proctitle:1;
unsigned int failed:1;
};
@@ -116,7 +117,7 @@
void dsync_brain_sync_mailbox_deinit(struct dsync_brain *brain);
int dsync_brain_mailbox_alloc(struct dsync_brain *brain, const guid_128_t guid,
struct mailbox **box_r, const char **error_r);
-void dsync_brain_mailbox_update_pre(struct dsync_brain *brain,
+bool dsync_brain_mailbox_update_pre(struct dsync_brain *brain,
struct mailbox *box,
const struct dsync_mailbox *local_box,
const struct dsync_mailbox *remote_box);
diff -r 9669c9a8984f -r 0d34a90b2760 src/doveadm/dsync/dsync-transaction-log-scan.c
--- a/src/doveadm/dsync/dsync-transaction-log-scan.c Mon Apr 28 20:10:56 2014 +0300
+++ b/src/doveadm/dsync/dsync-transaction-log-scan.c Mon Apr 28 20:14:03 2014 +0300
@@ -399,7 +399,13 @@
view->index->filepath, log_seq, end_seq);
ret = -1;
}
- return ret < 0 ? -1 : 0;
+ if (ret < 0)
+ return -1;
+ if (modseq != 0) {
+ /* we didn't see all the changes that we wanted to */
+ return 0;
+ }
+ return 1;
}
static int
@@ -412,9 +418,10 @@
uint32_t file_seq, max_seq;
uoff_t file_offset, max_offset;
uint64_t cur_modseq;
+ int ret;
log_view = mail_transaction_log_view_open(view->index->log);
- if (dsync_log_set(ctx, view, pvt_scan, log_view, modseq) < 0) {
+ if ((ret = dsync_log_set(ctx, view, pvt_scan, log_view, modseq)) < 0) {
mail_transaction_log_view_close(&log_view);
return -1;
}
@@ -475,7 +482,7 @@
ctx->last_log_offset = file_offset;
}
mail_transaction_log_view_close(&log_view);
- return 0;
+ return ret;
}
static int
@@ -503,6 +510,7 @@
{
struct dsync_transaction_log_scan *ctx;
pool_t pool;
+ int ret, ret2;
pool = pool_alloconly_create(MEMPOOL_GROWING"dsync transaction log scan",
10240);
@@ -515,15 +523,17 @@
ctx->view = view;
ctx->highest_wanted_uid = highest_wanted_uid;
- if (dsync_log_scan(ctx, view, modseq, FALSE) < 0)
+ if ((ret = dsync_log_scan(ctx, view, modseq, FALSE)) < 0)
return -1;
if (pvt_view != NULL) {
- if (dsync_log_scan(ctx, pvt_view, pvt_modseq, TRUE) < 0)
+ if ((ret2 = dsync_log_scan(ctx, pvt_view, pvt_modseq, TRUE)) < 0)
return -1;
+ if (ret2 == 0)
+ ret = 0;
}
*scan_r = ctx;
- return 0;
+ return ret;
}
HASH_TABLE_TYPE(dsync_uid_mail_change)
More information about the dovecot-cvs
mailing list