dovecot-2.2: lib-storage: Update mailbox vsize header on save/co...
dovecot at dovecot.org
dovecot at dovecot.org
Mon Sep 21 13:37:32 UTC 2015
details: http://hg.dovecot.org/dovecot-2.2/rev/e29d2f7fe53f
changeset: 19168:e29d2f7fe53f
user: Timo Sirainen <tss at iki.fi>
date: Mon Sep 21 16:32:27 2015 +0300
description:
lib-storage: Update mailbox vsize header on save/copy/expunge.
This allows always efficiently looking up maiboxes' vsizes after they're
initially calculated.
The expunge handling is unfortunately done currently in quota handling code,
so it works only if quota is enabled. Ideally this would be solved in v2.3
with some lib-storage core changes.
diffstat:
src/lib-storage/index/Makefile.am | 1 +
src/lib-storage/index/cydir/cydir-sync.c | 10 +-
src/lib-storage/index/dbox-multi/mdbox-sync.c | 19 +-
src/lib-storage/index/dbox-single/sdbox-sync.c | 12 +-
src/lib-storage/index/index-mailbox-size.c | 344 +++++++++++++++++---
src/lib-storage/index/index-storage.c | 69 ++++
src/lib-storage/index/index-storage.h | 9 +
src/lib-storage/index/index-sync.c | 3 +
src/lib-storage/index/maildir/maildir-sync-index.c | 7 +-
src/lib-storage/index/mbox/mbox-sync.c | 11 +-
src/plugins/quota/quota-storage.c | 9 +
11 files changed, 407 insertions(+), 87 deletions(-)
diffs (truncated from 772 to 300 lines):
diff -r 7f26cfb08fcd -r e29d2f7fe53f src/lib-storage/index/Makefile.am
--- a/src/lib-storage/index/Makefile.am Mon Sep 21 16:24:30 2015 +0300
+++ b/src/lib-storage/index/Makefile.am Mon Sep 21 16:32:27 2015 +0300
@@ -40,6 +40,7 @@
istream-mail.h \
index-attachment.h \
index-mail.h \
+ index-mailbox-size.h \
index-rebuild.h \
index-search-private.h \
index-search-result.h \
diff -r 7f26cfb08fcd -r e29d2f7fe53f src/lib-storage/index/cydir/cydir-sync.c
--- a/src/lib-storage/index/cydir/cydir-sync.c Mon Sep 21 16:24:30 2015 +0300
+++ b/src/lib-storage/index/cydir/cydir-sync.c Mon Sep 21 16:32:27 2015 +0300
@@ -112,18 +112,18 @@
if (!force)
sync_flags |= MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES;
- ret = mail_index_sync_begin(mbox->box.index, &ctx->index_sync_ctx,
- &ctx->sync_view, &ctx->trans,
- sync_flags);
+ ret = index_storage_expunged_sync_begin(&mbox->box, &ctx->index_sync_ctx,
+ &ctx->sync_view, &ctx->trans,
+ sync_flags);
if (ret <= 0) {
- if (ret < 0)
- mailbox_set_index_error(&mbox->box);
i_free(ctx);
*ctx_r = NULL;
return ret;
}
cydir_sync_index(ctx);
+ index_storage_expunging_deinit(&mbox->box);
+
*ctx_r = ctx;
return 0;
}
diff -r 7f26cfb08fcd -r e29d2f7fe53f src/lib-storage/index/dbox-multi/mdbox-sync.c
--- a/src/lib-storage/index/dbox-multi/mdbox-sync.c Mon Sep 21 16:24:30 2015 +0300
+++ b/src/lib-storage/index/dbox-multi/mdbox-sync.c Mon Sep 21 16:32:27 2015 +0300
@@ -198,24 +198,20 @@
struct mdbox_mailbox *mbox = ctx->mbox;
int ret;
- ret = mail_index_sync_begin(mbox->box.index, &ctx->index_sync_ctx,
- &ctx->sync_view, &ctx->trans, sync_flags);
+ ret = index_storage_expunged_sync_begin(&mbox->box, &ctx->index_sync_ctx,
+ &ctx->sync_view, &ctx->trans, sync_flags);
if (mail_index_reset_fscked(mbox->box.index))
mdbox_storage_set_corrupted(mbox->storage);
- if (ret < 0) {
- mailbox_set_index_error(&mbox->box);
- return -1;
- }
- if (ret == 0) {
- /* nothing to do */
- return 0;
- }
+ if (ret <= 0)
+ return ret; /* error / nothing to do */
if (!mdbox_map_atomic_is_locked(ctx->atomic) &&
mail_index_sync_has_expunges(ctx->index_sync_ctx)) {
/* we have expunges, so we need to write to map.
it needs to be locked before mailbox index. */
mail_index_sync_rollback(&ctx->index_sync_ctx);
+ index_storage_expunging_deinit(&ctx->mbox->box);
+
if (mdbox_map_atomic_lock(ctx->atomic) < 0)
return -1;
return mdbox_sync_try_begin(ctx, sync_flags);
@@ -262,12 +258,14 @@
ret = mdbox_sync_try_begin(ctx, sync_flags);
if (ret <= 0) {
/* failed / nothing to do */
+ index_storage_expunging_deinit(&mbox->box);
i_free(ctx);
return ret;
}
if ((ret = mdbox_sync_index(ctx)) <= 0) {
mail_index_sync_rollback(&ctx->index_sync_ctx);
+ index_storage_expunging_deinit(&mbox->box);
i_free_and_null(ctx);
if (ret < 0)
@@ -292,6 +290,7 @@
}
return mdbox_sync_begin(mbox, flags, atomic, ctx_r);
}
+ index_storage_expunging_deinit(&mbox->box);
*ctx_r = ctx;
return 0;
diff -r 7f26cfb08fcd -r e29d2f7fe53f src/lib-storage/index/dbox-single/sdbox-sync.c
--- a/src/lib-storage/index/dbox-single/sdbox-sync.c Mon Sep 21 16:24:30 2015 +0300
+++ b/src/lib-storage/index/dbox-single/sdbox-sync.c Mon Sep 21 16:32:27 2015 +0300
@@ -216,15 +216,12 @@
sync_flags |= MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES;
for (i = 0;; i++) {
- ret = mail_index_sync_begin(mbox->box.index,
- &ctx->index_sync_ctx,
- &ctx->sync_view, &ctx->trans,
- sync_flags);
+ ret = index_storage_expunged_sync_begin(&mbox->box,
+ &ctx->index_sync_ctx, &ctx->sync_view,
+ &ctx->trans, sync_flags);
if (mail_index_reset_fscked(mbox->box.index))
sdbox_set_mailbox_corrupted(&mbox->box);
if (ret <= 0) {
- if (ret < 0)
- mailbox_set_index_error(&mbox->box);
array_free(&ctx->expunged_uids);
i_free(ctx);
*ctx_r = NULL;
@@ -255,6 +252,7 @@
}
mail_index_sync_rollback(&ctx->index_sync_ctx);
if (ret < 0) {
+ index_storage_expunging_deinit(&ctx->mbox->box);
array_free(&ctx->expunged_uids);
i_free(ctx);
return -1;
@@ -274,6 +272,7 @@
if (success) {
mail_index_view_ref(ctx->sync_view);
+
if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) {
mailbox_set_index_error(&ctx->mbox->box);
ret = -1;
@@ -285,6 +284,7 @@
mail_index_sync_rollback(&ctx->index_sync_ctx);
}
+ index_storage_expunging_deinit(&ctx->mbox->box);
array_free(&ctx->expunged_uids);
i_free(ctx);
return ret;
diff -r 7f26cfb08fcd -r e29d2f7fe53f src/lib-storage/index/index-mailbox-size.c
--- a/src/lib-storage/index/index-mailbox-size.c Mon Sep 21 16:24:30 2015 +0300
+++ b/src/lib-storage/index/index-mailbox-size.c Mon Sep 21 16:32:27 2015 +0300
@@ -1,53 +1,283 @@
/* Copyright (c) 2002-2015 Dovecot authors, see the included COPYING file */
#include "lib.h"
+#include "file-create-locked.h"
#include "mail-search-build.h"
#include "index-storage.h"
+#include "index-mailbox-size.h"
+
+/*
+ Saving new mails: After transaction is committed and synced, trigger
+ vsize updating. Lock vsize updates. Check if the message count +
+ last-indexed-uid are still valid. If they are, add all the missing new
+ mails. Unlock.
+
+ Fetching vsize: Lock vsize updates. Check if the message count +
+ last-indexed-uid are still valid. If not, set them to zero. Add all
+ the missing mails. Unlock.
+
+ Expunging mails: Check if syncing would expunge any mails. If so, lock the
+ vsize updates before locking syncing (to avoid deadlocks). Check if the
+ message count + last-indexed-uid are still valid. If not, unlock vsize and
+ do nothing else. Otherwise, for each expunged mail whose UID <=
+ last-indexed-uid, decrease the message count and the vsize in memory. After
+ syncing is successfully committed, write the changes to header. Unlock.
+
+ Note that the final expunge handling with some mailbox formats is done while
+ syncing is no longer locked. Because of this we need to have the vsize
+ locking. The final vsize header update requires committing a transaction,
+ which internally is the same as a sync lock. So to avoid deadlocks we always
+ need to lock vsize updates before sync.
+*/
+
+#define VSIZE_LOCK_SUFFIX ".vsize.lock"
+#define VSIZE_UPDATE_MAX_LOCK_SECS 10
+
+struct mailbox_vsize_update {
+ struct mailbox *box;
+ struct mail_index_view *view;
+ struct mailbox_index_vsize vsize_hdr, orig_vsize_hdr;
+
+ char *lock_path;
+ int lock_fd;
+ struct file_lock *lock;
+ bool rebuild;
+ bool written;
+};
+
+static void vsize_header_refresh(struct mailbox_vsize_update *update)
+{
+ const void *data;
+ size_t size;
+
+ if (update->view != NULL)
+ mail_index_view_close(&update->view);
+ (void)mail_index_refresh(update->box->index);
+ update->view = mail_index_view_open(update->box->index);
+
+ mail_index_get_header_ext(update->view, update->box->vsize_hdr_ext_id,
+ &data, &size);
+ memcpy(&update->orig_vsize_hdr, data,
+ I_MIN(size, sizeof(update->orig_vsize_hdr)));
+ if (size == sizeof(update->vsize_hdr))
+ memcpy(&update->vsize_hdr, data, sizeof(update->vsize_hdr));
+ else {
+ if (size != 0) {
+ mail_storage_set_critical(update->box->storage,
+ "vsize-hdr has invalid size: %"PRIuSIZE_T,
+ size);
+ }
+ update->rebuild = TRUE;
+ memset(&update->vsize_hdr, 0, sizeof(update->vsize_hdr));
+ }
+}
+
+static void
+index_mailbox_vsize_check_rebuild(struct mailbox_vsize_update *update)
+{
+ uint32_t seq1, seq2;
+
+ if (update->vsize_hdr.highest_uid == 0)
+ return;
+ if (!mail_index_lookup_seq_range(update->view, 1,
+ update->vsize_hdr.highest_uid,
+ &seq1, &seq2))
+ seq2 = 0;
+
+ if (update->vsize_hdr.message_count != seq2) {
+ if (update->vsize_hdr.message_count < seq2) {
+ mail_storage_set_critical(update->box->storage,
+ "vsize-hdr has invalid message-count (%u < %u)",
+ update->vsize_hdr.message_count, seq2);
+ } else {
+ /* some messages have been expunged, rescan */
+ }
+ memset(&update->vsize_hdr, 0, sizeof(update->vsize_hdr));
+ update->rebuild = TRUE;
+ }
+}
+
+struct mailbox_vsize_update *
+index_mailbox_vsize_update_init(struct mailbox *box)
+{
+ struct mailbox_vsize_update *update;
+
+ update = i_new(struct mailbox_vsize_update, 1);
+ update->box = box;
+ update->lock_fd = -1;
+
+ vsize_header_refresh(update);
+ return update;
+}
+
+static bool vsize_update_lock_full(struct mailbox_vsize_update *update,
+ unsigned int lock_secs)
+{
+ struct mailbox *box = update->box;
+ const struct mailbox_permissions *perm;
+ struct file_create_settings set;
+ const char *error;
+ bool created;
+
+ if (update->lock_path != NULL)
+ return update->lock != NULL;
+ if (MAIL_INDEX_IS_IN_MEMORY(box->index))
+ return FALSE;
+
+ perm = mailbox_get_permissions(box);
+ memset(&set, 0, sizeof(set));
+ set.lock_timeout_secs =
+ mail_storage_get_lock_timeout(box->storage, lock_secs);
+ set.lock_method = box->storage->set->parsed_lock_method;
+ set.mode = perm->file_create_mode;
+ set.gid = perm->file_create_gid;
+ set.gid_origin = perm->file_create_gid_origin;
+
+ update->lock_path = i_strdup_printf("%s/"VSIZE_LOCK_SUFFIX,
+ box->index->dir);
+ update->lock_fd = file_create_locked(update->lock_path, &set,
+ &update->lock, &created, &error);
+ if (update->lock_fd == -1) {
+ if (errno != EAGAIN) {
+ i_error("file_create_locked(%s) failed: %m",
+ update->lock_path);
+ }
+ return FALSE;
+ }
+ update->rebuild = FALSE;
+ vsize_header_refresh(update);
+ index_mailbox_vsize_check_rebuild(update);
+ return TRUE;
+}
+
+bool index_mailbox_vsize_update_try_lock(struct mailbox_vsize_update *update)
+{
+ return vsize_update_lock_full(update, 0);
More information about the dovecot-cvs
mailing list