dovecot-2.2: Initial version of dsync rewrite.
dovecot at dovecot.org
dovecot at dovecot.org
Tue May 22 23:17:45 EEST 2012
details: http://hg.dovecot.org/dovecot-2.2/rev/b2076acc3715
changeset: 14584:b2076acc3715
user: Timo Sirainen <tss at iki.fi>
date: Tue May 22 23:17:31 2012 +0300
description:
Initial version of dsync rewrite.
* doveadm backup not implemented at all yet
* syncing mailbox renames is somewhat broken (at least renaming \noselect
mailboxes)
* saving/restoring "state" is implemented by dsync brain, but not by
doveadm. this should be easy to do, just need to figure out how the
replication code wants it.
diffstat:
src/doveadm/doveadm-settings.c | 11 +-
src/doveadm/dsync/Makefile.am | 83 +-
src/doveadm/dsync/doveadm-dsync.c | 233 +-
src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c | 150 +
src/doveadm/dsync/dsync-brain-mailbox-tree.c | 394 ++++
src/doveadm/dsync/dsync-brain-mailbox.c | 587 ++++++
src/doveadm/dsync/dsync-brain-mails.c | 274 ++
src/doveadm/dsync/dsync-brain-msgs-new.c | 393 ----
src/doveadm/dsync/dsync-brain-msgs.c | 543 -----
src/doveadm/dsync/dsync-brain-private.h | 209 +-
src/doveadm/dsync/dsync-brain.c | 1095 ++---------
src/doveadm/dsync/dsync-brain.h | 44 +-
src/doveadm/dsync/dsync-data.c | 146 -
src/doveadm/dsync/dsync-data.h | 85 -
src/doveadm/dsync/dsync-deserializer.c | 193 ++
src/doveadm/dsync/dsync-deserializer.h | 27 +
src/doveadm/dsync/dsync-mail.c | 84 +
src/doveadm/dsync/dsync-mail.h | 74 +
src/doveadm/dsync/dsync-mailbox-export.c | 692 +++++++
src/doveadm/dsync/dsync-mailbox-export.h | 26 +
src/doveadm/dsync/dsync-mailbox-import.c | 1461 +++++++++++++++
src/doveadm/dsync/dsync-mailbox-import.h | 36 +
src/doveadm/dsync/dsync-mailbox-state-export.h | 14 +
src/doveadm/dsync/dsync-mailbox-state.c | 85 +
src/doveadm/dsync/dsync-mailbox-state.h | 19 +
src/doveadm/dsync/dsync-mailbox-tree-fill.c | 211 ++
src/doveadm/dsync/dsync-mailbox-tree-private.h | 23 +
src/doveadm/dsync/dsync-mailbox-tree-sync.c | 603 ++++++
src/doveadm/dsync/dsync-mailbox-tree.c | 349 +++
src/doveadm/dsync/dsync-mailbox-tree.h | 139 +
src/doveadm/dsync/dsync-mailbox.h | 17 +
src/doveadm/dsync/dsync-proxy-client.c | 1191 -------------
src/doveadm/dsync/dsync-proxy-server-cmd.c | 608 ------
src/doveadm/dsync/dsync-proxy-server.c | 205 --
src/doveadm/dsync/dsync-proxy-server.h | 46 -
src/doveadm/dsync/dsync-proxy.c | 410 ----
src/doveadm/dsync/dsync-proxy.h | 55 -
src/doveadm/dsync/dsync-serializer.c | 117 +
src/doveadm/dsync/dsync-serializer.h | 18 +
src/doveadm/dsync/dsync-slave-io.c | 1517 ++++++++++++++++
src/doveadm/dsync/dsync-slave-pipe.c | 485 +++++
src/doveadm/dsync/dsync-slave-private.h | 78 +
src/doveadm/dsync/dsync-slave.c | 202 ++
src/doveadm/dsync/dsync-slave.h | 118 +
src/doveadm/dsync/dsync-transaction-log-scan.c | 467 +++++
src/doveadm/dsync/dsync-transaction-log-scan.h | 22 +
src/doveadm/dsync/dsync-worker-local.c | 1922 ---------------------
src/doveadm/dsync/dsync-worker-private.h | 105 -
src/doveadm/dsync/dsync-worker.c | 285 ---
src/doveadm/dsync/dsync-worker.h | 165 -
src/doveadm/dsync/test-dsync-brain-msgs.c | 670 -------
src/doveadm/dsync/test-dsync-brain.c | 289 ---
src/doveadm/dsync/test-dsync-common.c | 80 -
src/doveadm/dsync/test-dsync-common.h | 18 -
src/doveadm/dsync/test-dsync-proxy-server-cmd.c | 484 -----
src/doveadm/dsync/test-dsync-proxy.c | 184 --
src/doveadm/dsync/test-dsync-worker.c | 481 -----
src/doveadm/dsync/test-dsync-worker.h | 96 -
58 files changed, 8944 insertions(+), 9674 deletions(-)
diffs (truncated from 19099 to 300 lines):
diff -r 5b17dc1f8313 -r b2076acc3715 src/doveadm/doveadm-settings.c
--- a/src/doveadm/doveadm-settings.c Fri May 04 05:35:36 2012 +0300
+++ b/src/doveadm/doveadm-settings.c Tue May 22 23:17:31 2012 +0300
@@ -111,15 +111,18 @@
}
/* <settings checks> */
-static bool doveadm_settings_check(void *_set ATTR_UNUSED,
- pool_t pool ATTR_UNUSED,
- const char **error_r ATTR_UNUSED)
+static bool doveadm_settings_check(void *_set, pool_t pool ATTR_UNUSED,
+ const char **error_r)
{
-#ifndef CONFIG_BINARY
struct doveadm_settings *set = _set;
+#ifndef CONFIG_BINARY
fix_base_path(set, pool, &set->doveadm_socket_path);
#endif
+ if (*set->dsync_alt_char == '\0') {
+ *error_r = "dsync_alt_char must not be empty";
+ return FALSE;
+ }
return TRUE;
}
/* </settings checks> */
diff -r 5b17dc1f8313 -r b2076acc3715 src/doveadm/dsync/Makefile.am
--- a/src/doveadm/dsync/Makefile.am Fri May 04 05:35:36 2012 +0300
+++ b/src/doveadm/dsync/Makefile.am Tue May 22 23:17:31 2012 +0300
@@ -14,65 +14,34 @@
libdsync_a_SOURCES = \
doveadm-dsync.c \
dsync-brain.c \
- dsync-brain-msgs.c \
- dsync-brain-msgs-new.c \
- dsync-data.c \
- dsync-proxy.c \
- dsync-proxy-client.c \
- dsync-proxy-server.c \
- dsync-proxy-server-cmd.c \
- dsync-worker.c \
- dsync-worker-local.c
+ dsync-brain-mailbox.c \
+ dsync-brain-mailbox-tree.c \
+ dsync-brain-mailbox-tree-sync.c \
+ dsync-brain-mails.c \
+ dsync-deserializer.c \
+ dsync-mail.c \
+ dsync-mailbox-import.c \
+ dsync-mailbox-export.c \
+ dsync-mailbox-state.c \
+ dsync-mailbox-tree.c \
+ dsync-mailbox-tree-fill.c \
+ dsync-mailbox-tree-sync.c \
+ dsync-serializer.c \
+ dsync-slave.c \
+ dsync-slave-io.c \
+ dsync-slave-pipe.c \
+ dsync-transaction-log-scan.c
noinst_HEADERS = \
doveadm-dsync.h \
dsync-brain.h \
dsync-brain-private.h \
- dsync-data.h \
- dsync-proxy.h \
- dsync-proxy-server.h \
- dsync-worker.h \
- dsync-worker-private.h \
- test-dsync-common.h \
- test-dsync-worker.h
-
-test_programs = \
- test-dsync-brain \
- test-dsync-brain-msgs \
- test-dsync-proxy \
- test-dsync-proxy-server-cmd
-
-noinst_PROGRAMS = $(test_programs)
-
-test_libs = \
- ../../lib-test/libtest.la \
- ../../lib-mail/libmail.la \
- ../../lib-imap/libimap.la \
- ../../lib-charset/libcharset.la \
- ../../lib/liblib.la
-
-test_ldadd = \
- $(test_libs) \
- $(LIBICONV)
-
-test_dsync_brain_SOURCES = test-dsync-brain.c test-dsync-worker.c test-dsync-common.c
-test_dsync_brain_LDADD = dsync-data.o dsync-brain.o dsync-worker.o $(test_ldadd)
-test_dsync_brain_DEPENDENCIES = dsync-data.o dsync-brain.o dsync-worker.o $(test_libs)
-
-test_dsync_brain_msgs_SOURCES = test-dsync-brain-msgs.c test-dsync-worker.c test-dsync-common.c
-test_dsync_brain_msgs_LDADD = dsync-data.o dsync-brain-msgs.o dsync-worker.o $(test_ldadd)
-test_dsync_brain_msgs_DEPENDENCIES = dsync-data.o dsync-brain-msgs.o dsync-worker.o $(test_libs)
-
-test_dsync_proxy_SOURCES = test-dsync-proxy.c test-dsync-common.c
-test_dsync_proxy_LDADD = dsync-proxy.o dsync-data.o $(test_ldadd)
-test_dsync_proxy_DEPENDENCIES = dsync-proxy.o dsync-data.o $(test_libs)
-
-test_dsync_proxy_server_cmd_SOURCES = test-dsync-proxy-server-cmd.c test-dsync-worker.c test-dsync-common.c
-test_dsync_proxy_server_cmd_LDADD = dsync-worker.o dsync-proxy.o dsync-proxy-server-cmd.o dsync-data.o $(test_ldadd)
-test_dsync_proxy_server_cmd_DEPENDENCIES = dsync-worker.o dsync-proxy.o dsync-proxy-server-cmd.o dsync-data.o $(test_libs)
-
-check: check-am check-test
-check-test: all-am
- for bin in $(test_programs); do \
- if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \
- done
+ dsync-mailbox-import.h \
+ dsync-mailbox-export.h \
+ dsync-mailbox-state.h \
+ dsync-mailbox-tree.h \
+ dsync-serializer.h \
+ dsync-deserializer.h \
+ dsync-slave.h \
+ dsync-slave-private.h \
+ dsync-transaction-log-scan.h
diff -r 5b17dc1f8313 -r b2076acc3715 src/doveadm/dsync/doveadm-dsync.c
--- a/src/doveadm/dsync/doveadm-dsync.c Fri May 04 05:35:36 2012 +0300
+++ b/src/doveadm/dsync/doveadm-dsync.c Tue May 22 23:17:31 2012 +0300
@@ -11,11 +11,11 @@
#include "mail-storage-service.h"
#include "mail-user.h"
#include "mail-namespace.h"
+#include "mailbox-list.h"
#include "doveadm-settings.h"
#include "doveadm-mail.h"
#include "dsync-brain.h"
-#include "dsync-worker.h"
-#include "dsync-proxy-server.h"
+#include "dsync-slave.h"
#include "doveadm-dsync.h"
#include <stdio.h>
@@ -23,13 +23,12 @@
#include <unistd.h>
#include <ctype.h>
-#define DSYNC_LOCK_FILENAME ".dovecot-sync.lock"
-
struct dsync_cmd_context {
struct doveadm_mail_cmd_context ctx;
- enum dsync_brain_flags brain_flags;
+ enum dsync_brain_sync_type sync_type;
const char *mailbox, *namespace_prefix;
+ const char *remote_name;
const char *local_location;
int fd_in, fd_out, fd_err;
@@ -39,7 +38,7 @@
unsigned int lock:1;
unsigned int default_replica_location:1;
- unsigned int reverse_workers:1;
+ unsigned int reverse_backup:1; //FIXME
unsigned int remote:1;
};
@@ -227,17 +226,18 @@
return TRUE;
}
-static struct dsync_worker *
-cmd_dsync_run_local(struct dsync_cmd_context *ctx, struct mail_user *user)
+static struct dsync_slave *
+cmd_dsync_run_local(struct dsync_cmd_context *ctx, struct mail_user *user,
+ struct dsync_brain *brain, struct dsync_slave *slave2)
{
+ struct dsync_brain *brain2;
struct mail_user *user2;
- struct dsync_worker *worker2;
struct setting_parser_context *set_parser;
const char *set_line, *path1, *path2;
+ bool brain1_running, brain2_running, changed1, changed2;
i_assert(ctx->local_location != NULL);
- ctx->brain_flags |= DSYNC_BRAIN_FLAG_LOCAL;
i_set_failure_prefix(t_strdup_printf("dsync(%s): ", user->username));
/* update mail_location and create another user for the
@@ -267,18 +267,30 @@
"points to same directory: %s", path1);
}
- worker2 = dsync_worker_init_local(user2, ctx->namespace_prefix,
- *ctx->ctx.set->dsync_alt_char);
+ brain2 = dsync_brain_slave_init(user2, slave2);
+
+ brain1_running = brain2_running = TRUE;
+ changed1 = changed2 = TRUE;
+ while (brain1_running || brain2_running) {
+ if (dsync_brain_has_failed(brain) ||
+ dsync_brain_has_failed(brain2))
+ break;
+
+ i_assert(changed1 || changed2);
+ brain1_running = dsync_brain_run(brain, &changed1);
+ brain2_running = dsync_brain_run(brain2, &changed2);
+ }
mail_user_unref(&user2);
- return worker2;
+ dsync_brain_deinit(&brain2);
+ return slave2;
}
-static struct dsync_worker *
-cmd_dsync_run_remote(struct dsync_cmd_context *ctx, struct mail_user *user)
+static void
+cmd_dsync_run_remote(struct mail_user *user)
{
i_set_failure_prefix(t_strdup_printf("dsync-local(%s): ",
user->username));
- return dsync_worker_init_proxy_client(ctx->fd_in, ctx->fd_out);
+ io_loop_run(current_ioloop);
}
static const char *const *
@@ -297,108 +309,56 @@
return get_ssh_cmd_args(ctx, host, login, username);
}
-static int dsync_lock(struct mail_user *user, unsigned int lock_timeout,
- const char **path_r, struct file_lock **lock_r)
-{
- const char *home, *path;
- int ret, fd;
-
- if ((ret = mail_user_get_home(user, &home)) < 0) {
- i_error("Couldn't look up user's home dir");
- return -1;
- }
- if (ret == 0) {
- i_error("User has no home directory");
- return -1;
- }
-
- path = t_strconcat(home, "/"DSYNC_LOCK_FILENAME, NULL);
- fd = creat(path, 0600);
- if (fd == -1) {
- i_error("Couldn't create lock %s: %m", path);
- return -1;
- }
-
- if (file_wait_lock(fd, path, F_WRLCK, FILE_LOCK_METHOD_FCNTL,
- lock_timeout, lock_r) <= 0) {
- i_error("Couldn't lock %s: %m", path);
- (void)close(fd);
- return -1;
- }
- *path_r = path;
- return fd;
-}
-
-static int
-cmd_dsync_start(struct dsync_cmd_context *ctx, struct dsync_worker *worker1,
- struct dsync_worker *worker2)
-{
- struct dsync_brain *brain;
-
- /* create and run the brain */
- brain = dsync_brain_init(worker1, worker2, ctx->mailbox,
- ctx->brain_flags);
- if (!ctx->remote)
- dsync_brain_sync_all(brain);
- else {
- dsync_brain_sync(brain);
- if (!dsync_brain_has_failed(brain))
- io_loop_run(current_ioloop);
- }
- /* deinit */
- if (dsync_brain_has_unexpected_changes(brain)) {
- i_warning("Mailbox changes caused a desync. "
- "You may want to run dsync again.");
- ctx->ctx.exit_code = 2;
- }
- if (dsync_brain_deinit(&brain) < 0) {
- ctx->ctx.exit_code = EX_TEMPFAIL;
- return -1;
- }
- return 0;
-}
-
static int
cmd_dsync_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user)
{
struct dsync_cmd_context *ctx = (struct dsync_cmd_context *)_ctx;
- struct dsync_worker *worker1, *worker2, *workertmp;
- const char *lock_path;
- struct file_lock *lock;
- int lock_fd, ret = 0;
+ struct dsync_slave *slave, *slave2 = NULL;
+ struct dsync_brain *brain;
+ struct mail_namespace *sync_ns = NULL;
+ int ret = 0;
More information about the dovecot-cvs
mailing list