dovecot-2.0: mdbox: Moving messages to alt storage is done done ...
dovecot at dovecot.org
dovecot at dovecot.org
Tue Apr 20 16:25:39 EEST 2010
details: http://hg.dovecot.org/dovecot-2.0/rev/a92389387cb7
changeset: 11177:a92389387cb7
user: Timo Sirainen <tss at iki.fi>
date: Tue Apr 20 16:25:24 2010 +0300
description:
mdbox: Moving messages to alt storage is done done with doveadm altmove command.
The command can take an arbitrary search query listing what messages should
exist in alt storage. If the message has been copied to multiple mailboxes,
the search query must match all the instances of the message.
Since the search query can now be specified in command line, mdbox_altmove
setting was also removed.
diffstat:
src/config/old-set-parser.c | 6 +-
src/doveadm/Makefile.am | 1 +
src/doveadm/doveadm-mail-altmove.c | 111 +++++
src/doveadm/doveadm-mail.c | 3 +-
src/doveadm/doveadm-mail.h | 3 +
src/lib-storage/index/dbox-multi/Makefile.am | 2 +-
src/lib-storage/index/dbox-multi/mdbox-file-purge.c | 228 ----------
src/lib-storage/index/dbox-multi/mdbox-file.h | 1 -
src/lib-storage/index/dbox-multi/mdbox-mail.c | 17 +-
src/lib-storage/index/dbox-multi/mdbox-map-private.h | 1 -
src/lib-storage/index/dbox-multi/mdbox-map.c | 108 +++-
src/lib-storage/index/dbox-multi/mdbox-map.h | 7 +-
src/lib-storage/index/dbox-multi/mdbox-purge.c | 538 ++++++++++++++++++++++++
src/lib-storage/index/dbox-multi/mdbox-save.c | 4 +-
src/lib-storage/index/dbox-multi/mdbox-settings.c | 4 +-
src/lib-storage/index/dbox-multi/mdbox-settings.h | 1 -
src/lib-storage/index/dbox-multi/mdbox-storage.c | 7 +-
src/lib-storage/index/dbox-multi/mdbox-storage.h | 6 +
src/lib-storage/index/dbox-multi/mdbox-sync.c | 110 -----
src/lib-storage/index/dbox-multi/mdbox-sync.h | 2 -
20 files changed, 765 insertions(+), 395 deletions(-)
diffs (truncated from 1569 to 300 lines):
diff -r 129bc5eab66a -r a92389387cb7 src/config/old-set-parser.c
--- a/src/config/old-set-parser.c Tue Apr 20 16:22:56 2010 +0300
+++ b/src/config/old-set-parser.c Tue Apr 20 16:25:24 2010 +0300
@@ -134,14 +134,10 @@
set_rename(ctx, key, "mdbox_rotate_size", value);
return TRUE;
}
- if (strcmp(key, "dbox_rotate_days") == 0) {
- set_rename(ctx, key, "mdbox_rotate_interval",
- t_strconcat(value, "d", NULL));
- return TRUE;
- }
if (strcmp(key, "login_dir") == 0 ||
strcmp(key, "dbox_rotate_min_size") == 0 ||
+ strcmp(key, "dbox_rotate_days") == 0 ||
strcmp(key, "mail_log_max_lines_per_sec") == 0 ||
strcmp(key, "maildir_copy_preserve_filename") == 0) {
obsolete(ctx, "%s has been removed", key);
diff -r 129bc5eab66a -r a92389387cb7 src/doveadm/Makefile.am
--- a/src/doveadm/Makefile.am Tue Apr 20 16:22:56 2010 +0300
+++ b/src/doveadm/Makefile.am Tue Apr 20 16:25:24 2010 +0300
@@ -52,6 +52,7 @@
doveadm-dump-thread.c \
doveadm-kick.c \
doveadm-mail.c \
+ doveadm-mail-altmove.c \
doveadm-mail-fetch.c \
doveadm-penalty.c \
doveadm-pw.c \
diff -r 129bc5eab66a -r a92389387cb7 src/doveadm/doveadm-mail-altmove.c
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/doveadm/doveadm-mail-altmove.c Tue Apr 20 16:25:24 2010 +0300
@@ -0,0 +1,111 @@
+/* Copyright (c) 2010 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "mail-index.h"
+#include "mail-storage.h"
+#include "mail-namespace.h"
+#include "mail-search-build.h"
+#include "mail-search-parser.h"
+#include "doveadm-mail.h"
+
+static struct mail_search_args *build_search_args(const char *const args[])
+{
+ struct mail_search_parser *parser;
+ struct mail_search_args *sargs;
+ const char *error;
+
+ parser = mail_search_parser_init_cmdline(args);
+ if (mail_search_build(mail_search_register_human, parser, "UTF-8",
+ &sargs, &error) < 0)
+ i_fatal("%s", error);
+ mail_search_parser_deinit(&parser);
+ return sargs;
+}
+
+static int
+cmd_altmove_box(struct mailbox *box, struct mail_search_args *search_args)
+{
+ struct mail_storage *storage;
+ struct mailbox_transaction_context *t;
+ struct mail_search_context *search_ctx;
+ struct mail *mail;
+ const char *box_name;
+ int ret = 0;
+
+ box_name = mailbox_get_vname(box);
+ storage = mailbox_get_storage(box);
+ if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) {
+ i_error("Syncing mailbox %s failed: %s", box_name,
+ mail_storage_get_last_error(storage, NULL));
+ return -1;
+ }
+
+ t = mailbox_transaction_begin(box, 0);
+ search_ctx = mailbox_search_init(t, search_args, NULL);
+ mail = mail_alloc(t, 0, NULL);
+ while (mailbox_search_next(search_ctx, mail)) {
+ if (doveadm_debug)
+ i_debug("altmove: box=%s uid=%u", box_name, mail->uid);
+ mail_update_flags(mail, MODIFY_ADD,
+ MAIL_INDEX_MAIL_FLAG_BACKEND);
+ }
+ mail_free(&mail);
+ if (mailbox_search_deinit(&search_ctx) < 0) {
+ i_error("Searching mailbox %s failed: %s", box_name,
+ mail_storage_get_last_error(storage, NULL));
+ ret = -1;
+ }
+ if (mailbox_transaction_commit(&t) < 0) {
+ i_error("Commiting mailbox %s failed: %s", box_name,
+ mail_storage_get_last_error(storage, NULL));
+ ret = -1;
+ }
+ return ret;
+}
+
+static void
+cmd_altmove_ns(struct mail_namespace *ns, struct mail_search_args *search_args)
+{
+ struct mailbox_list_iterate_context *iter;
+ const struct mailbox_info *info;
+ struct mailbox *box;
+
+ iter = mailbox_list_iter_init(ns->list, "*",
+ MAILBOX_LIST_ITER_RAW_LIST |
+ MAILBOX_LIST_ITER_NO_AUTO_INBOX |
+ MAILBOX_LIST_ITER_RETURN_NO_FLAGS);
+ while ((info = mailbox_list_iter_next(iter)) != NULL) {
+ box = mailbox_alloc(ns->list, info->name,
+ MAILBOX_FLAG_KEEP_RECENT |
+ MAILBOX_FLAG_IGNORE_ACLS);
+ (void)cmd_altmove_box(box, search_args);
+ mailbox_free(&box);
+ }
+ if (mailbox_list_iter_deinit(&iter) < 0) {
+ i_error("Listing namespace '%s' mailboxes failed: %s",
+ ns->prefix,
+ mailbox_list_get_last_error(ns->list, NULL));
+ }
+
+ if (mail_storage_purge(ns->storage) < 0) {
+ i_error("Purging namespace '%s' failed: %s", ns->prefix,
+ mail_storage_get_last_error(ns->storage, NULL));
+ }
+}
+
+void cmd_altmove(struct mail_user *user, const char *const args[])
+{
+ struct mail_search_args *search_args;
+ struct mail_namespace *ns;
+
+ if (args[0] == NULL)
+ doveadm_mail_help_name("altmove");
+ search_args = build_search_args(args);
+
+ for (ns = user->namespaces; ns != NULL; ns = ns->next) {
+ if (ns->type != NAMESPACE_PRIVATE || ns->alias_for != NULL)
+ continue;
+
+ cmd_altmove_ns(ns, search_args);
+ }
+}
diff -r 129bc5eab66a -r a92389387cb7 src/doveadm/doveadm-mail.c
--- a/src/doveadm/doveadm-mail.c Tue Apr 20 16:22:56 2010 +0300
+++ b/src/doveadm/doveadm-mail.c Tue Apr 20 16:25:24 2010 +0300
@@ -312,7 +312,8 @@
static struct doveadm_mail_cmd mail_commands[] = {
{ cmd_purge, "purge", NULL },
{ cmd_force_resync, "force-resync", "<mailbox>" },
- { cmd_fetch, "fetch", "<mailbox> <search query>" }
+ { cmd_fetch, "fetch", "<mailbox> <search query>" },
+ { cmd_altmove, "altmove", "<search query>" }
};
void doveadm_mail_init(void)
diff -r 129bc5eab66a -r a92389387cb7 src/doveadm/doveadm-mail.h
--- a/src/doveadm/doveadm-mail.h Tue Apr 20 16:22:56 2010 +0300
+++ b/src/doveadm/doveadm-mail.h Tue Apr 20 16:25:24 2010 +0300
@@ -1,6 +1,8 @@
#ifndef DOVEADM_MAIL_H
#define DOVEADM_MAIL_H
+#include "doveadm.h"
+
struct mail_user;
typedef void doveadm_mail_command_t(struct mail_user *mail_user,
@@ -29,5 +31,6 @@
doveadm_mailbox_find_and_sync(struct mail_user *user, const char *mailbox);
void cmd_fetch(struct mail_user *user, const char *const args[]);
+void cmd_altmove(struct mail_user *user, const char *const args[]);
#endif
diff -r 129bc5eab66a -r a92389387cb7 src/lib-storage/index/dbox-multi/Makefile.am
--- a/src/lib-storage/index/dbox-multi/Makefile.am Tue Apr 20 16:22:56 2010 +0300
+++ b/src/lib-storage/index/dbox-multi/Makefile.am Tue Apr 20 16:25:24 2010 +0300
@@ -13,9 +13,9 @@
libstorage_dbox_multi_la_SOURCES = \
mdbox-file.c \
- mdbox-file-purge.c \
mdbox-mail.c \
mdbox-map.c \
+ mdbox-purge.c \
mdbox-save.c \
mdbox-settings.c \
mdbox-sync.c \
diff -r 129bc5eab66a -r a92389387cb7 src/lib-storage/index/dbox-multi/mdbox-file-purge.c
--- a/src/lib-storage/index/dbox-multi/mdbox-file-purge.c Tue Apr 20 16:22:56 2010 +0300
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,228 +0,0 @@
-/* Copyright (c) 2007-2010 Dovecot authors, see the included COPYING file */
-
-#include "lib.h"
-#include "array.h"
-#include "istream.h"
-#include "ostream.h"
-#include "str.h"
-#include "hex-binary.h"
-#include "mdbox-storage.h"
-#include "mdbox-file.h"
-#include "mdbox-map.h"
-#include "mdbox-sync.h"
-
-#include <stdlib.h>
-
-struct dbox_mail_move {
- struct dbox_file *file;
- uint32_t offset;
-};
-ARRAY_DEFINE_TYPE(dbox_mail_move, struct dbox_mail_move);
-
-static int mdbox_map_file_msg_offset_cmp(const void *p1, const void *p2)
-{
- const struct dbox_map_file_msg *m1 = p1, *m2 = p2;
-
- if (m1->offset < m2->offset)
- return -1;
- else if (m1->offset > m2->offset)
- return 1;
- else
- return 0;
-}
-
-static int
-mdbox_file_copy_metadata(struct dbox_file *file, struct ostream *output)
-{
- struct dbox_metadata_header meta_hdr;
- const char *line;
- const unsigned char *data;
- size_t size;
- int ret;
-
- ret = i_stream_read_data(file->input, &data, &size,
- sizeof(meta_hdr));
- if (ret <= 0) {
- i_assert(ret == -1);
- if (file->input->stream_errno == 0) {
- dbox_file_set_corrupted(file, "missing metadata");
- return 0;
- }
- mail_storage_set_critical(&file->storage->storage,
- "read(%s) failed: %m", file->cur_path);
- return -1;
- }
-
- memcpy(&meta_hdr, data, sizeof(meta_hdr));
- if (memcmp(meta_hdr.magic_post, DBOX_MAGIC_POST,
- sizeof(meta_hdr.magic_post)) != 0) {
- dbox_file_set_corrupted(file, "invalid metadata magic");
- return 0;
- }
- i_stream_skip(file->input, sizeof(meta_hdr));
- if (output != NULL)
- o_stream_send(output, &meta_hdr, sizeof(meta_hdr));
- while ((line = i_stream_read_next_line(file->input)) != NULL) {
- if (*line == DBOX_METADATA_OLDV1_SPACE || *line == '\0') {
- /* end of metadata */
- break;
- }
- if (output != NULL) {
- o_stream_send_str(output, line);
- o_stream_send(output, "\n", 1);
- }
- }
- if (line == NULL) {
- dbox_file_set_corrupted(file, "missing end-of-metadata line");
- return 0;
- }
- if (output != NULL)
- o_stream_send(output, "\n", 1);
- return 1;
-}
-
-int mdbox_file_purge(struct dbox_file *file)
-{
- struct mdbox_storage *dstorage = (struct mdbox_storage *)file->storage;
- struct dbox_file_append_context *out_file_append;
- struct stat st;
- struct istream *input;
- struct ostream *output = NULL;
- struct dbox_map_append_context *append_ctx;
- ARRAY_TYPE(dbox_map_file_msg) msgs_arr;
- const struct dbox_map_file_msg *msgs;
- ARRAY_TYPE(seq_range) expunged_map_uids;
- ARRAY_TYPE(uint32_t) copied_map_uids;
- unsigned int i, count;
- uoff_t offset, physical_size, msg_size;
- enum dbox_map_append_flags append_flags = 0;
- int ret;
-
- if ((ret = dbox_file_try_lock(file)) <= 0)
- return ret;
-
- /* make sure the file still exists. another process may have already
- deleted it. */
More information about the dovecot-cvs
mailing list