dovecot-2.0: Moved the main functionality from "doveadm index" t...
dovecot at dovecot.org
dovecot at dovecot.org
Tue Jun 14 17:00:07 EEST 2011
details: http://hg.dovecot.org/dovecot-2.0/rev/8c76426a9e53
changeset: 12850:8c76426a9e53
user: Timo Sirainen <tss at iki.fi>
date: Tue Jun 14 16:59:57 2011 +0300
description:
Moved the main functionality from "doveadm index" to MAILBOX_SYNC_FLAG_PRECACHE
This also allows plugins to hook into the sync and implement their own
precaching easily. fts indexing is now done this way rather than kludging.
diffstat:
src/doveadm/doveadm-mail-index.c | 174 +--------------------------------
src/lib-storage/index/index-sync.c | 132 +++++++++++++++++++++++++
src/lib-storage/mail-storage-private.h | 1 +
src/lib-storage/mail-storage.h | 4 +-
src/plugins/fts/fts-storage.c | 90 +++++++++++++---
5 files changed, 210 insertions(+), 191 deletions(-)
diffs (truncated from 523 to 300 lines):
diff -r 09b8701362a4 -r 8c76426a9e53 src/doveadm/doveadm-mail-index.c
--- a/src/doveadm/doveadm-mail-index.c Mon Jun 13 17:17:59 2011 +0300
+++ b/src/doveadm/doveadm-mail-index.c Tue Jun 14 16:59:57 2011 +0300
@@ -6,165 +6,11 @@
#include "mail-search-build.h"
#include "doveadm-mail.h"
-enum cache_mask {
- CACHE_HDR = 0x01,
- CACHE_BODY = 0x02,
- CACHE_RECEIVED_DATE = 0x04,
- CACHE_SAVE_DATE = 0x08,
- CACHE_VIRTUAL_SIZE = 0x10,
- CACHE_PHYSICAL_SIZE = 0x20,
- CACHE_POP3_UIDL = 0x40,
- CACHE_GUID = 0x80
-};
-
-static bool fts_is_enabled = FALSE;
-
-static enum cache_mask cache_fields_get(const struct mailbox_status *status)
-{
- const char *const *cache_fields;
- unsigned int i, count;
- enum cache_mask cache = 0;
-
- cache_fields = array_get(status->cache_fields, &count);
- for (i = 0; i < count; i++) {
- if (strncmp(cache_fields[i], "hdr.", 4) == 0 ||
- strcmp(cache_fields[i], "date.sent") == 0 ||
- strcmp(cache_fields[i], "imap.envelope") == 0)
- cache |= CACHE_HDR;
- else if (strcmp(cache_fields[i], "mime.parts") == 0 ||
- strcmp(cache_fields[i], "imap.body") == 0 ||
- strcmp(cache_fields[i], "imap.bodystructure") == 0)
- cache |= CACHE_BODY;
- else if (strcmp(cache_fields[i], "date.received") == 0)
- cache |= CACHE_RECEIVED_DATE;
- else if (strcmp(cache_fields[i], "date.save") == 0)
- cache |= CACHE_SAVE_DATE;
- else if (strcmp(cache_fields[i], "size.virtual") == 0)
- cache |= CACHE_VIRTUAL_SIZE;
- else if (strcmp(cache_fields[i], "size.physical") == 0)
- cache |= CACHE_PHYSICAL_SIZE;
- else if (strcmp(cache_fields[i], "pop3.uidl") == 0)
- cache |= CACHE_POP3_UIDL;
- else if (strcmp(cache_fields[i], "guid") == 0)
- cache |= CACHE_GUID;
- else if (doveadm_debug) {
- i_debug("Ignoring unknown cache field: %s",
- cache_fields[i]);
- }
- }
- return cache;
-}
-
-static int cache_add(struct mailbox *box, const struct mailbox_status *status,
- enum cache_mask cache)
-{
- struct mailbox_transaction_context *trans;
- struct mail *mail;
- uint32_t seq;
- time_t date;
- uoff_t size;
- const char *str;
-
- if (doveadm_debug) {
- i_debug("%s: Nothing in mailbox cache, skipping",
- mailbox_get_vname(box));
- return 0;
- }
-
- /* find the first message we need to index */
- trans = mailbox_transaction_begin(box, 0);
- mail = mail_alloc(trans, 0, NULL);
- for (seq = status->messages; seq > 0; seq--) {
- mail_set_seq(mail, seq);
- if (mail_is_cached(mail))
- break;
- }
- seq++;
-
- if (doveadm_debug) {
- if (seq > status->messages) {
- i_debug("%s: Cache is already up to date",
- mailbox_get_vname(box));
- } else {
- i_debug("%s: Caching mails seq=%u..%u cache=0x%x",
- mailbox_get_vname(box),
- seq, status->messages, cache);
- }
- }
-
- for (; seq <= status->messages; seq++) {
- mail_set_seq(mail, seq);
-
- if ((cache & (CACHE_HDR | CACHE_BODY)) != 0)
- mail_parse(mail, (cache & CACHE_BODY) != 0);
- if ((cache & CACHE_RECEIVED_DATE) != 0)
- (void)mail_get_received_date(mail, &date);
- if ((cache & CACHE_SAVE_DATE) != 0)
- (void)mail_get_save_date(mail, &date);
- if ((cache & CACHE_VIRTUAL_SIZE) != 0)
- (void)mail_get_virtual_size(mail, &size);
- if ((cache & CACHE_PHYSICAL_SIZE) != 0)
- (void)mail_get_physical_size(mail, &size);
- if ((cache & CACHE_POP3_UIDL) != 0) {
- (void)mail_get_special(mail, MAIL_FETCH_UIDL_BACKEND,
- &str);
- }
- if ((cache & CACHE_GUID) != 0)
- (void)mail_get_special(mail, MAIL_FETCH_GUID, &str);
- }
- mail_free(&mail);
- if (mailbox_transaction_commit(&trans) < 0) {
- i_error("Commiting mailbox %s failed: %s",
- mailbox_get_vname(box),
- mail_storage_get_last_error(mailbox_get_storage(box), NULL));
- return -1;
- }
- return 0;
-}
-
-static int fts_update(struct mailbox *box, const struct mailbox_status *status)
-{
- struct mailbox_transaction_context *t;
- struct mail_search_args *search_args;
- struct mail_search_arg *arg;
- struct mail_search_context *ctx;
- struct mail *mail;
- int ret;
-
- if (!fts_is_enabled)
- return 0;
-
- /* a bit kludgy way to trigger the full text search update:
- search for a string in the last message */
- t = mailbox_transaction_begin(box, 0);
- search_args = mail_search_build_init();
- search_args->charset = "UTF-8";
- mail_search_build_add_seqset(search_args,
- status->messages, status->messages);
- arg = mail_search_build_add(search_args, SEARCH_BODY_FAST);
- arg->value.str = "xyzzy";
-
- ctx = mailbox_search_init(t, search_args, NULL);
- mail_search_args_unref(&search_args);
-
- mail = mail_alloc(t, 0, NULL);
- while (mailbox_search_next(ctx, mail)) {
- }
- mail_free(&mail);
-
- ret = mailbox_search_deinit(&ctx);
- if (mailbox_transaction_commit(&t) < 0)
- ret = -1;
- return ret;
-}
-
static int
cmd_index_box(const struct mailbox_info *info)
{
struct mailbox *box;
- struct mailbox_status status;
const char *storage_name;
- enum cache_mask cache;
int ret = 0;
storage_name = mail_namespace_get_storage_name(info->ns, info->name);
@@ -172,19 +18,13 @@
MAILBOX_FLAG_KEEP_RECENT |
MAILBOX_FLAG_IGNORE_ACLS);
- if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) {
+ if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ |
+ MAILBOX_SYNC_FLAG_PRECACHE) < 0) {
i_error("Syncing mailbox %s failed: %s", info->name,
mail_storage_get_last_error(mailbox_get_storage(box), NULL));
mailbox_free(&box);
return -1;
}
- mailbox_get_status(box, STATUS_MESSAGES | STATUS_CACHE_FIELDS, &status);
-
- cache = cache_fields_get(&status);
- ret = cache_add(box, &status, cache);
-
- if (fts_update(box, &status) < 0)
- ret = -1;
mailbox_free(&box);
return ret;
@@ -203,16 +43,6 @@
struct mailbox_list_iterate_context *iter;
const struct mailbox_info *info;
- if (mail_user_plugin_getenv(user, "fts") != NULL) T_BEGIN {
- const char *const *plugins;
-
- plugins = t_strsplit(user->set->mail_plugins, " ");
- for (; *plugins != NULL; plugins++) {
- if (strncmp(*plugins, "fts", 3) == 0)
- fts_is_enabled = TRUE;
- }
- } T_END;
-
iter = mailbox_list_iter_init_namespaces(user->namespaces, ctx->args,
ns_mask, iter_flags);
while ((info = mailbox_list_iter_next(iter)) != NULL) {
diff -r 09b8701362a4 -r 8c76426a9e53 src/lib-storage/index/index-sync.c
--- a/src/lib-storage/index/index-sync.c Mon Jun 13 17:17:59 2011 +0300
+++ b/src/lib-storage/index/index-sync.c Tue Jun 14 16:59:57 2011 +0300
@@ -6,6 +6,17 @@
#include "array.h"
#include "index-sync-private.h"
+enum cache_mask {
+ CACHE_HDR = 0x01,
+ CACHE_BODY = 0x02,
+ CACHE_RECEIVED_DATE = 0x04,
+ CACHE_SAVE_DATE = 0x08,
+ CACHE_VIRTUAL_SIZE = 0x10,
+ CACHE_PHYSICAL_SIZE = 0x20,
+ CACHE_POP3_UIDL = 0x40,
+ CACHE_GUID = 0x80
+};
+
enum mail_index_sync_flags index_storage_get_sync_flags(struct mailbox *box)
{
enum mail_index_sync_flags sync_flags = 0;
@@ -177,6 +188,7 @@
ctx = i_new(struct index_mailbox_sync_context, 1);
ctx->ctx.box = box;
+ ctx->ctx.flags = flags;
if (failed) {
ctx->failed = TRUE;
@@ -324,6 +336,121 @@
#endif
}
+static enum cache_mask
+cache_fields_get(const struct mailbox_status *status, bool debug)
+{
+ const char *const *cache_fields;
+ unsigned int i, count;
+ enum cache_mask cache = 0;
+
+ cache_fields = array_get(status->cache_fields, &count);
+ for (i = 0; i < count; i++) {
+ if (strncmp(cache_fields[i], "hdr.", 4) == 0 ||
+ strcmp(cache_fields[i], "date.sent") == 0 ||
+ strcmp(cache_fields[i], "imap.envelope") == 0)
+ cache |= CACHE_HDR;
+ else if (strcmp(cache_fields[i], "mime.parts") == 0 ||
+ strcmp(cache_fields[i], "imap.body") == 0 ||
+ strcmp(cache_fields[i], "imap.bodystructure") == 0)
+ cache |= CACHE_BODY;
+ else if (strcmp(cache_fields[i], "date.received") == 0)
+ cache |= CACHE_RECEIVED_DATE;
+ else if (strcmp(cache_fields[i], "date.save") == 0)
+ cache |= CACHE_SAVE_DATE;
+ else if (strcmp(cache_fields[i], "size.virtual") == 0)
+ cache |= CACHE_VIRTUAL_SIZE;
+ else if (strcmp(cache_fields[i], "size.physical") == 0)
+ cache |= CACHE_PHYSICAL_SIZE;
+ else if (strcmp(cache_fields[i], "pop3.uidl") == 0)
+ cache |= CACHE_POP3_UIDL;
+ else if (strcmp(cache_fields[i], "guid") == 0)
+ cache |= CACHE_GUID;
+ else if (debug) {
+ i_debug("Ignoring unknown cache field: %s",
+ cache_fields[i]);
+ }
+ }
+ return cache;
+}
+
+static int cache_add(struct mailbox *box, const struct mailbox_status *status,
+ enum cache_mask cache)
+{
+ struct mailbox_transaction_context *trans;
+ struct mail *mail;
+ uint32_t seq;
+ time_t date;
+ uoff_t size;
+ const char *str;
+
+ if (box->storage->set->mail_debug) {
+ i_debug("%s: Nothing in mailbox cache, skipping",
+ mailbox_get_vname(box));
+ return 0;
+ }
+
+ /* find the first message we need to index */
+ trans = mailbox_transaction_begin(box, 0);
+ mail = mail_alloc(trans, 0, NULL);
+ for (seq = status->messages; seq > 0; seq--) {
+ mail_set_seq(mail, seq);
+ if (mail_is_cached(mail))
More information about the dovecot-cvs
mailing list