dovecot-2.0: lib-storage: Added support for quickly getting mail...

dovecot at dovecot.org dovecot at dovecot.org
Wed May 12 14:16:20 EEST 2010


details:   http://hg.dovecot.org/dovecot-2.0/rev/51d0f5d8cc65
changeset: 11280:51d0f5d8cc65
user:      Timo Sirainen <tss at iki.fi>
date:      Wed May 12 13:15:40 2010 +0200
description:
lib-storage: Added support for quickly getting mailbox's virtual size.

diffstat:

 src/doveadm/doveadm-dump-index.c      |  12 +++-
 src/lib-storage/index/index-status.c  |  92 ++++++++++++++++++++++++++++++
 src/lib-storage/index/index-storage.c |   4 +
 src/lib-storage/index/index-storage.h |   6 ++
 src/lib-storage/mail-storage.h        |   5 +-
 5 files changed, 117 insertions(+), 2 deletions(-)

diffs (202 lines):

diff -r 521e36311440 -r 51d0f5d8cc65 src/doveadm/doveadm-dump-index.c
--- a/src/doveadm/doveadm-dump-index.c	Tue May 11 08:49:44 2010 +0300
+++ b/src/doveadm/doveadm-dump-index.c	Wed May 12 13:15:40 2010 +0200
@@ -15,6 +15,10 @@
 #include <stdlib.h>
 #include <time.h>
 
+struct index_vsize_header {
+	uint64_t vsize;
+	uint32_t highest_uid;
+};
 struct maildir_index_header {
 	uint32_t new_check_time, new_mtime, new_mtime_nsecs;
 	uint32_t cur_check_time, cur_mtime, cur_mtime_nsecs;
@@ -105,7 +109,13 @@
 		return;
 
 	data = CONST_PTR_OFFSET(index->map->hdr_base, ext->hdr_offset);
-	if (strcmp(ext->name, "maildir") == 0) {
+	if (strcmp(ext->name, "hdr-vsize") == 0) {
+		const struct index_vsize_header *hdr = data;
+
+		printf("header\n");
+		printf(" - highest uid = %u\n", hdr->highest_uid);
+		printf(" - vsize ..... = %llu\n", (unsigned long long)hdr->vsize);
+	} else if (strcmp(ext->name, "maildir") == 0) {
 		const struct maildir_index_header *hdr = data;
 
 		printf("header\n");
diff -r 521e36311440 -r 51d0f5d8cc65 src/lib-storage/index/index-status.c
--- a/src/lib-storage/index/index-status.c	Tue May 11 08:49:44 2010 +0300
+++ b/src/lib-storage/index/index-status.c	Wed May 12 13:15:40 2010 +0200
@@ -3,6 +3,7 @@
 #include "lib.h"
 #include "array.h"
 #include "mail-cache.h"
+#include "mail-search-build.h"
 #include "index-storage.h"
 #include "mail-index-modseq.h"
 
@@ -30,6 +31,95 @@
 	status_r->cache_fields = cache_fields;
 }
 
+static void
+index_storage_virtual_size_add_new(struct mailbox *box,
+				   struct index_vsize_header *vsize_hdr)
+{
+	struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
+	const struct mail_index_header *hdr;
+	struct mailbox_transaction_context *trans;
+	struct mail_search_context *search_ctx;
+	struct mail_search_args *search_args;
+	struct mail *mail;
+	uint32_t seq1, seq2;
+	uoff_t vsize;
+	int ret = 0;
+
+	hdr = mail_index_get_header(box->view);
+	if (!mail_index_lookup_seq_range(box->view, vsize_hdr->highest_uid+1,
+					 hdr->next_uid, &seq1, &seq2)) {
+		/* the last messages are already expunged,
+		   don't bother updating cache */
+		return;
+	}
+
+	search_args = mail_search_build_init();
+	mail_search_build_add_seqset(search_args, seq1, seq2);
+
+	trans = mailbox_transaction_begin(box, 0);
+	search_ctx = mailbox_search_init(trans, search_args, NULL);
+	mail = mail_alloc(trans, MAIL_FETCH_VIRTUAL_SIZE, NULL);
+	while (mailbox_search_next(search_ctx, mail)) {
+		if (mail_get_virtual_size(mail, &vsize) < 0) {
+			ret = -1;
+			break;
+		}
+		vsize_hdr->vsize += vsize;
+		vsize_hdr->highest_uid = mail->uid;
+	}
+	mail_free(&mail);
+	if (mailbox_search_deinit(&search_ctx) < 0)
+		ret = -1;
+
+	if (ret == 0) {
+		/* success, cache all */
+		vsize_hdr->highest_uid = hdr->next_uid - 1;
+	} else {
+		/* search failed, cache only up to highest seen uid */
+	}
+	mail_index_update_header_ext(trans->itrans, ibox->vsize_hdr_ext_id,
+				     0, vsize_hdr, sizeof(*vsize_hdr));
+	(void)mailbox_transaction_commit(&trans);
+
+}
+
+static void
+index_storage_get_status_virtual_size(struct mailbox *box,
+				      struct mailbox_status *status_r)
+{
+	struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box);
+	struct index_vsize_header vsize_hdr;
+	const void *data;
+	size_t size;
+
+	mail_index_get_header_ext(box->view, ibox->vsize_hdr_ext_id,
+				  &data, &size);
+	if (size == sizeof(vsize_hdr))
+		memcpy(&vsize_hdr, data, sizeof(vsize_hdr));
+	else {
+		if (size != 0) {
+			mail_storage_set_critical(box->storage,
+				"vsize-hdr has invalid size: %"PRIuSIZE_T,
+				size);
+		}
+		memset(&vsize_hdr, 0, sizeof(vsize_hdr));
+	}
+
+	if (vsize_hdr.highest_uid + 1 == status_r->uidnext) {
+		/* up to date */
+		status_r->virtual_size = vsize_hdr.vsize;
+		return;
+	}
+	if (vsize_hdr.highest_uid >= status_r->uidnext) {
+		mail_storage_set_critical(box->storage,
+			"vsize-hdr has invalid highest-uid (%u >= %u)",
+			vsize_hdr.highest_uid, status_r->uidnext);
+		memset(&vsize_hdr, 0, sizeof(vsize_hdr));
+	}
+	index_storage_virtual_size_add_new(box, &vsize_hdr);
+	status_r->virtual_size = vsize_hdr.vsize;
+}
+
 void index_storage_get_status(struct mailbox *box,
 			      enum mailbox_status_items items,
 			      struct mailbox_status *status_r)
@@ -69,4 +159,6 @@
 		status_r->keywords = mail_index_get_keywords(box->index);
 	if ((items & STATUS_CACHE_FIELDS) != 0)
 		index_storage_get_status_cache_fields(box, status_r);
+	if ((items & STATUS_VIRTUAL_SIZE) != 0)
+		index_storage_get_status_virtual_size(box, status_r);
 }
diff -r 521e36311440 -r 51d0f5d8cc65 src/lib-storage/index/index-storage.c
--- a/src/lib-storage/index/index-storage.c	Tue May 11 08:49:44 2010 +0300
+++ b/src/lib-storage/index/index-storage.c	Wed May 12 13:15:40 2010 +0200
@@ -240,6 +240,10 @@
 	index_cache_register_defaults(box);
 	box->view = mail_index_view_open(box->index);
 	ibox->keyword_names = mail_index_get_keywords(box->index);
+	ibox->vsize_hdr_ext_id =
+		mail_index_ext_register(box->index, "hdr-vsize",
+					sizeof(struct index_vsize_header), 0,
+					sizeof(uint64_t));
 
 	box->opened = TRUE;
 
diff -r 521e36311440 -r 51d0f5d8cc65 src/lib-storage/index/index-storage.h
--- a/src/lib-storage/index/index-storage.h	Tue May 11 08:49:44 2010 +0300
+++ b/src/lib-storage/index/index-storage.h	Wed May 12 13:15:40 2010 +0200
@@ -27,6 +27,11 @@
 	struct mail_cache_transaction_ctx *cache_trans;
 };
 
+struct index_vsize_header {
+	uint64_t vsize;
+	uint32_t highest_uid;
+};
+
 struct index_mailbox_context {
 	union mailbox_module_context module_ctx;
 	enum mail_index_open_flags index_flags;
@@ -50,6 +55,7 @@
 	ARRAY_TYPE(seq_range) recent_flags;
 	uint32_t recent_flags_prev_uid;
 	uint32_t recent_flags_count;
+	uint32_t vsize_hdr_ext_id;
 
 	time_t sync_last_check;
 };
diff -r 521e36311440 -r 51d0f5d8cc65 src/lib-storage/mail-storage.h
--- a/src/lib-storage/mail-storage.h	Tue May 11 08:49:44 2010 +0300
+++ b/src/lib-storage/mail-storage.h	Wed May 12 13:15:40 2010 +0200
@@ -62,7 +62,8 @@
 	STATUS_FIRST_UNSEEN_SEQ	= 0x20,
 	STATUS_KEYWORDS		= 0x40,
 	STATUS_HIGHESTMODSEQ	= 0x80,
-	STATUS_CACHE_FIELDS	= 0x100
+	STATUS_CACHE_FIELDS	= 0x100,
+	STATUS_VIRTUAL_SIZE	= 0x200
 };
 
 enum mailbox_search_result_flags {
@@ -181,6 +182,8 @@
 
 	uint32_t first_unseen_seq;
 	uint64_t highest_modseq;
+	/* sum of virtual size of all messages in mailbox */
+	uint64_t virtual_size;
 
 	const ARRAY_TYPE(keywords) *keywords;
 	/* Fields that have "temp" or "yes" caching decision. */


More information about the dovecot-cvs mailing list