[dovecot-cvs] dovecot/src/lib-storage/list Makefile.am, 1.1, 1.2 index-mailbox-list-sync.c, NONE, 1.1 index-mailbox-list.c, NONE, 1.1 index-mailbox-list.h, NONE, 1.1 mailbox-list-fs-iter.c, 1.1, 1.2 mailbox-list-fs.c, 1.1, 1.2 mailbox-list-fs.h, 1.1, 1.2 mailbox-list-maildir-iter.c, 1.1, 1.2 mailbox-list-maildir.c, 1.2, 1.3 mailbox-list-maildir.h, 1.1, 1.2

tss at dovecot.org tss at dovecot.org
Sat Nov 25 22:17:56 UTC 2006


Update of /var/lib/cvs/dovecot/src/lib-storage/list
In directory talvi:/tmp/cvs-serv28938/lib-storage/list

Modified Files:
	Makefile.am mailbox-list-fs-iter.c mailbox-list-fs.c 
	mailbox-list-fs.h mailbox-list-maildir-iter.c 
	mailbox-list-maildir.c mailbox-list-maildir.h 
Added Files:
	index-mailbox-list-sync.c index-mailbox-list.c 
	index-mailbox-list.h 
Log Message:
Mailbox list indexing and related changes. Currently works only with
maildir and mmap_disable=no. This allows doing STATUS to synced mailboxes
without opening their index files at all.



Index: Makefile.am
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/list/Makefile.am,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- Makefile.am	16 Nov 2006 18:57:26 -0000	1.1
+++ Makefile.am	25 Nov 2006 22:17:50 -0000	1.2
@@ -2,10 +2,15 @@
 
 AM_CPPFLAGS = \
 	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-index \
 	-I$(top_srcdir)/src/lib-imap \
-	-I$(top_srcdir)/src/lib-storage
+	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/lib-storage/index
 
 libstorage_list_a_SOURCES = \
+	index-mailbox-list.c \
+	index-mailbox-list-sync.c \
 	mailbox-list-fs.c \
 	mailbox-list-fs-iter.c \
 	mailbox-list-maildir.c \
@@ -13,6 +18,7 @@
 	subscription-file.c
 
 noinst_HEADERS = \
+	index-mailbox-list.h \
 	mailbox-list-fs.h \
 	mailbox-list-maildir.h \
 	subscription-file.h

--- NEW FILE: index-mailbox-list-sync.c ---
/* Copyright (C) 2006 Timo Sirainen */

#include "lib.h"
#include "ioloop.h"
#include "array.h"
#include "index-storage.h"
#include "mailbox-list-index.h"
#include "index-mailbox-list.h"
#include "maildir/maildir-sync.h"

#include <sys/stat.h>

#define INDEX_LIST_STORAGE_CONTEXT(obj) \
	*((void **)array_idx_modifiable(&(obj)->module_contexts, \
					index_list_storage_module_id))

#define CACHED_STATUS_ITEMS \
	(STATUS_MESSAGES | STATUS_UNSEEN | STATUS_RECENT | \
	 STATUS_UIDNEXT | STATUS_UIDVALIDITY)

struct index_list_mailbox {
	struct mailbox_vfuncs super;

	uint32_t log_seq;
	uoff_t log_offset;
};

struct index_list_map {
	const char *name;
	unsigned int eid_offset;
	unsigned int status_offset;
};
#undef DEF
#define DEF(a, b, c) \
	{ a, offsetof(struct index_mailbox_list, b), \
	  offsetof(struct mailbox_status, c) }
static struct index_list_map index_list_map[] = {
	DEF("msgs", eid_messages, messages),
	DEF("unseen", eid_unseen, unseen),
	DEF("recent", eid_recent, recent),
	DEF("uid_validity", eid_uid_validity, uidvalidity),
	DEF("uidnext", eid_uidnext, uidnext),
	{ NULL, 0, 0 }
};

static void (*index_list_next_hook_mailbox_created)(struct mailbox *box);

static unsigned int index_list_storage_module_id = 0;
static bool index_list_storage_module_id_set = FALSE;

static int index_list_box_close(struct mailbox *box)
{
	struct index_list_mailbox *ibox = INDEX_LIST_STORAGE_CONTEXT(box);

	return ibox->super.close(box);
}

static int index_list_update_mail_index(struct index_mailbox_list *ilist,
					struct mailbox *box)
{
	struct index_list_mailbox *ibox = INDEX_LIST_STORAGE_CONTEXT(box);
	struct mail_index_sync_ctx *mail_sync_ctx;
	struct mail_index_view *view;
	struct mail_index_sync_rec sync_rec;

	if (ibox->log_seq == 0)
		return 0;

	if (mail_index_sync_begin(ilist->mail_index, &mail_sync_ctx,
				  &view, ibox->log_seq, ibox->log_offset,
				  FALSE, FALSE) < 0)
		return -1;

	/* we should have only external transactions in here, for which we
	   don't need to do anything but write them to the index */
	while (mail_index_sync_next(mail_sync_ctx, &sync_rec) > 0)
		;

	return mail_index_sync_commit(&mail_sync_ctx);
}

static int
index_list_lookup_stamps(struct index_mailbox_list *ilist,
			 struct mail_index_view *view, uint32_t seq,
			 time_t *new_stamp_r, time_t *cur_stamp_r,
			 uint8_t *dirty_flags_r)
{
	const void *data;

	if (mail_index_lookup_ext(view, seq, ilist->eid_new_sync_stamp,
				  &data) <= 0)
		return -1;
	*new_stamp_r = data == NULL ? 0 : *(const uint32_t *)data;

	if (mail_index_lookup_ext(view, seq, ilist->eid_cur_sync_stamp,
				  &data) <= 0)
		return -1;
	*cur_stamp_r = data == NULL ? 0 : *(const uint32_t *)data;

	if (mail_index_lookup_ext(view, seq, ilist->eid_dirty_flags,
				  &data) <= 0)
		return -1;
	*dirty_flags_r = data == NULL ? 0 : *(const uint8_t *)data;
	return 0;
}

static int
index_list_has_mailbox_changed(struct mailbox *box,
			       struct mail_index_view *view, uint32_t seq)
{
	/* FIXME: this function shouldn't be maildir-specific */
	struct index_mailbox_list *ilist;
	struct mailbox_list *list;
	const char *root_dir, *new_dir, *cur_dir;
	struct stat st;
	time_t idx_new_stamp, idx_cur_stamp, max_stamp;
	uint8_t idx_dirty_flags;

	list = mail_storage_get_list(box->storage);
	ilist = INDEX_LIST_CONTEXT(list);

	if (index_list_lookup_stamps(ilist, view, seq, &idx_new_stamp,
				     &idx_cur_stamp, &idx_dirty_flags) < 0)
		return -1;

	/* if there are dirty flags and the timestamp is old enough,
	   do a resync in any case */
	max_stamp = I_MAX(idx_new_stamp, idx_cur_stamp);
	if (idx_dirty_flags != 0 &&
	    ioloop_time - max_stamp >= MAILDIR_SYNC_SECS)
		return 1;

	/* check if new/ changed */
	root_dir = mailbox_list_get_path(list, box->name,
					 MAILBOX_LIST_PATH_TYPE_MAILBOX);
	new_dir = t_strconcat(root_dir, "/new", NULL);
	if (stat(new_dir, &st) < 0) {
		mail_storage_set_critical(box->storage,
					  "stat(%s) failed: %m", new_dir);
		return -1;
	}
	if (idx_new_stamp != st.st_mtime)
		return 1;

	/* check if cur/ changed */
	cur_dir = t_strconcat(root_dir, "/cur", NULL);
	if (stat(cur_dir, &st) < 0) {
		mail_storage_set_critical(box->storage,
					  "stat(%s) failed: %m", cur_dir);
		return -1;
	}
	if (idx_cur_stamp != st.st_mtime)
		return 1;

	return 0;
}

static int
index_list_mailbox_open_unchanged_view(struct mailbox *box,
				       struct mail_index_view **view_r,
				       uint32_t *seq_r)
{
	struct mailbox_list *list;
	struct index_mailbox_list *ilist;
	struct mail_index_view *view;
	uint32_t uid, seq;
	int ret;

	list = mail_storage_get_list(box->storage);
	ilist = INDEX_LIST_CONTEXT(list);

	ret = mailbox_list_index_lookup(ilist->list_index, box->name, &uid);
	if (ret <= 0)
		return ret;

	/* make sure we're synced */
	if (index_list_update_mail_index(ilist, box) < 0)
		return -1;

	/* found from list index. lookup the mail index record for it */
	view = mail_index_view_open(ilist->mail_index);
	ret = mail_index_lookup_uid_range(view, uid, uid, &seq, &seq);
	if (ret < 0 || seq == 0) {
		mail_index_view_close(&view);
		return ret;
	}

	t_push();
	ret = index_list_has_mailbox_changed(box, view, seq);
	t_pop();
	if (ret != 0) {
		/* error / mailbox has changed. we'll need to sync it. */
		mail_index_view_close(&view);
		return ret < 0 ? -1 : 0;
	}

	*view_r = view;
	*seq_r = seq;
	return 1;
}

static int
index_list_get_cached_status(struct mailbox *box, struct mailbox_status *status)
{
	struct mailbox_list *list;
	struct index_mailbox_list *ilist;
	struct mail_index_view *view;
	const void *data;
	uint32_t seq, *ext_id_p, *counter_p;
	unsigned int i;
	int ret;

	memset(status, 0, sizeof(*status));

	ret = index_list_mailbox_open_unchanged_view(box, &view, &seq);
	if (ret <= 0)
		return ret;

	list = mail_storage_get_list(box->storage);
	ilist = INDEX_LIST_CONTEXT(list);
	for (i = 0; index_list_map[i].name != NULL; i++) {
		ext_id_p = PTR_OFFSET(ilist, index_list_map[i].eid_offset);
		ret = mail_index_lookup_ext(view, seq, *ext_id_p, &data);
		if (ret <= 0 || data == NULL)
			break;

		counter_p = PTR_OFFSET(status, index_list_map[i].status_offset);
		*counter_p = *(const uint32_t *)data;
	}

	mail_index_view_close(&view);
	return 1;
}

static int
index_list_get_status(struct mailbox *box, enum mailbox_status_items items,
		      struct mailbox_status *status)
{
	struct index_list_mailbox *ibox = INDEX_LIST_STORAGE_CONTEXT(box);

	if ((items & ~CACHED_STATUS_ITEMS) == 0) {
		if (index_list_get_cached_status(box, status) > 0)
			return 0;
		/* nonsynced / error, fallback to doing it the slow way */
	}

	return ibox->super.get_status(box, items, status);
}

static int index_list_lookup_or_create(struct index_mailbox_list *ilist,
				       struct mailbox *box, uint32_t *uid_r)
{
	struct mailbox_list_index_sync_ctx *sync_ctx;
	int ret;

	ret = mailbox_list_index_lookup(ilist->list_index, box->name, uid_r);
	if (ret > 0) {
		/* we'll need the mailbox synced since we're updating its
		   contents based on what it already contains */
		if (index_list_update_mail_index(ilist, box) < 0)
			return -1;
		return 1;
	} else if (ret < 0)
		return -1;

	/* create the mailbox by doing a partial sync with the mailbox name
	   as the sync root path */
	if (mailbox_list_index_sync_init(ilist->list_index, box->name,
					 MAILBOX_LIST_SYNC_FLAG_PARTIAL,
					 &sync_ctx) < 0)
		return -1;
	if (mailbox_list_index_sync_commit(&sync_ctx) < 0)
		return -1;

	ret = mailbox_list_index_lookup(ilist->list_index, box->name, uid_r);
	if (ret != 0)
		return ret < 0 ? -1 : 0;

	mail_storage_set_critical(box->storage,
		"mailbox index: Created mailbox %s not found", box->name);
	return -1;
}

static int
index_list_update_sync_stamps(struct index_mailbox_list *ilist,
			      struct mailbox *box,
			      struct mail_index_transaction *trans,
			      struct mail_index_view *view, uint32_t seq)
{
	struct index_mailbox *ibox = (struct index_mailbox *)box;
	const struct mail_index_header *hdr;
	time_t hdr_new_stamp, hdr_cur_stamp;
	time_t idx_new_stamp, idx_cur_stamp;
	uint8_t hdr_dirty_flags, idx_dirty_flags;

	hdr = mail_index_get_header(ibox->view);
	hdr_cur_stamp = hdr->sync_stamp;
	hdr_new_stamp = hdr->sync_size & 0xffffffff;
	hdr_dirty_flags = hdr->sync_size >> 32;

	if (index_list_lookup_stamps(ilist, view, seq, &idx_new_stamp,
				     &idx_cur_stamp, &idx_dirty_flags) < 0)
		return -1;

	if (idx_new_stamp != hdr_new_stamp) {
		mail_index_update_ext(trans, seq, ilist->eid_new_sync_stamp,
				      &hdr_new_stamp, NULL);
	}
	if (idx_cur_stamp != hdr_cur_stamp) {
		mail_index_update_ext(trans, seq, ilist->eid_cur_sync_stamp,
				      &hdr_cur_stamp, NULL);
	}
	if (idx_dirty_flags != hdr_dirty_flags) {
		mail_index_update_ext(trans, seq, ilist->eid_dirty_flags,
				      &hdr_dirty_flags, NULL);
	}
	return 0;
}

static int
index_list_update(struct index_mailbox_list *ilist, struct mailbox *box,
		  struct mail_index_view *view, uint32_t seq,
		  const struct mailbox_status *status)
{
	struct index_list_mailbox *ibox = INDEX_LIST_STORAGE_CONTEXT(box);
	struct mail_index_transaction *trans;
	const void *data;
	const uint32_t *counter_p;
	uint32_t *ext_id_p;
	unsigned int i;
	int ret = 1;

	trans = mail_index_transaction_begin(view, FALSE, TRUE);

	/* update counters */
	for (i = 0; index_list_map[i].name != NULL; i++) {
		ext_id_p = PTR_OFFSET(ilist, index_list_map[i].eid_offset);
		ret = mail_index_lookup_ext(view, seq, *ext_id_p, &data);
		if (ret <= 0)
			break;

		counter_p = CONST_PTR_OFFSET(status,
					     index_list_map[i].status_offset);
		if (data == NULL ||
		    *(const uint32_t *)data != *counter_p) {
			mail_index_update_ext(trans, seq, *ext_id_p,
					      counter_p, NULL);
		}
	}
	if (index_list_update_sync_stamps(ilist, box, trans, view, seq) < 0)
		ret = -1;
	if (ret <= 0) {
		mail_index_transaction_rollback(&trans);
		return -1;
	}

	return mail_index_transaction_commit(&trans, &ibox->log_seq,
					     &ibox->log_offset);
}

static struct mailbox_sync_context *
index_list_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
{
	struct index_list_mailbox *ibox = INDEX_LIST_STORAGE_CONTEXT(box);
	struct mailbox_sync_context *ctx;

	/* clear any cached log seq/offset */
	ibox->log_seq = (uint32_t)-1;
	ibox->log_offset = 0;

	if (!box->opened) {
		/* check using the mailbox list index if the mailbox has
		   changed. if not, we don't need to open the mailbox yet. */
		struct mail_index_view *view;
		uint32_t seq;
		int ret;

		ret = index_list_mailbox_open_unchanged_view(box, &view, &seq);
		if (ret > 0) {
			ctx = i_new(struct mailbox_sync_context, 1);
			ctx->box = box;
			mail_index_view_close(&view);

			/* no changes, so don't bother checking again before
			   next sync */
			ibox->log_seq = 0;
			return ctx;
		}
	}

	return ibox->super.sync_init(box, flags);
}

static int index_list_sync_next(struct mailbox_sync_context *ctx,
				struct mailbox_sync_rec *sync_rec_r)
{
	struct index_list_mailbox *ibox = INDEX_LIST_STORAGE_CONTEXT(ctx->box);

	if (!ctx->box->opened)
		return 0;

	return ibox->super.sync_next(ctx, sync_rec_r);
}

static int index_list_sync_deinit(struct mailbox_sync_context *ctx,
				  enum mailbox_status_items status_items,
				  struct mailbox_status *status_r)
{
	struct mailbox *box = ctx->box;
	struct index_list_mailbox *ibox = INDEX_LIST_STORAGE_CONTEXT(box);
	struct mailbox_list *list;
	struct index_mailbox_list *ilist;
	struct mail_index_view *view;
	struct mailbox_status tmp_status, *status;
	uint32_t uid, seq;

	if (!box->opened) {
		/* nothing synced. just return the status. */
		i_free(ctx);

		return status_items == 0 ? 0 :
			index_list_get_status(box, status_items, status_r);
	}

	/* if status_items == 0, the status_r may be NULL. we really want to
	   know the status anyway, so save it elsewhere then */
	status = status_items == 0 ? &tmp_status : status_r;
	status_items |= CACHED_STATUS_ITEMS;

	if (ibox->super.sync_deinit(ctx, status_items, status) < 0)
		return -1;
	ctx = NULL;

	/* sync mailbox list index */
	list = mail_storage_get_list(box->storage);
	ilist = INDEX_LIST_CONTEXT(list);

	if (index_list_lookup_or_create(ilist, box, &uid) < 0) {
		/* just ignore the error */
		return 0;
	}

	view = mail_index_view_open(ilist->mail_index);
	if (mail_index_lookup_uid_range(view, uid, uid, &seq, &seq) == 0 &&
	    seq > 0)
		(void)index_list_update(ilist, box, view, seq, status);
	mail_index_view_close(&view);
	return 0;
}

static void index_list_mail_mailbox_opened(struct mailbox *box)
{
	struct index_list_mailbox *ibox;

	if (index_list_next_hook_mailbox_created != NULL)
		index_list_next_hook_mailbox_created(box);

	/* FIXME: maildir-only for now */
	if (strcmp(box->storage->name, "maildir") != 0)
		return;

	ibox = p_new(box->pool, struct index_list_mailbox, 1);
	ibox->super = box->v;
	box->v.close = index_list_box_close;
	box->v.get_status = index_list_get_status;
	box->v.sync_init = index_list_sync_init;
	box->v.sync_next = index_list_sync_next;
	box->v.sync_deinit = index_list_sync_deinit;

	if (!index_list_storage_module_id_set) {
		index_list_storage_module_id = mail_storage_module_id++;
		index_list_storage_module_id_set = TRUE;
	}

	array_idx_set(&box->module_contexts,
		      index_list_storage_module_id, &ibox);
}

void index_mailbox_list_sync_init_list(struct mailbox_list *list)
{
	struct index_mailbox_list *ilist = INDEX_LIST_CONTEXT(list);
	unsigned int i;
	uint32_t *ext_id_p;

	for (i = 0; index_list_map[i].name != NULL; i++) {
		ext_id_p = PTR_OFFSET(ilist, index_list_map[i].eid_offset);
		*ext_id_p = mail_index_ext_register(ilist->mail_index,
					index_list_map[i].name, 0,
					sizeof(uint32_t), sizeof(uint32_t));
	}

	/* FIXME: maildir-only: */
	ilist->eid_cur_sync_stamp =
		mail_index_ext_register(ilist->mail_index, "sync-cur", 0,
					sizeof(uint32_t), sizeof(uint32_t));
	ilist->eid_new_sync_stamp =
		mail_index_ext_register(ilist->mail_index, "sync-new", 0,
					sizeof(uint32_t), sizeof(uint32_t));
	ilist->eid_dirty_flags =
		mail_index_ext_register(ilist->mail_index, "sync-dirty", 0,
					sizeof(uint8_t), sizeof(uint8_t));
}

void index_mailbox_list_sync_init(void)
{
	index_list_next_hook_mailbox_created = hook_mailbox_opened;
	hook_mailbox_opened = index_list_mail_mailbox_opened;
}

--- NEW FILE: index-mailbox-list.c ---
/* Copyright (C) 2006 Timo Sirainen */

#include "lib.h"
#include "ioloop.h"
#include "array.h"
#include "imap-match.h"
#include "mail-index.h"
#include "mail-storage.h"
#include "mailbox-list-index.h"
#include "index-mailbox-list.h"

#include <time.h>
#include <sys/stat.h>

/* min 2 seconds */
#define MAILBOX_LIST_SYNC_SECS 2

unsigned int index_mailbox_list_module_id = 0;

static bool index_mailbox_list_module_id_set = FALSE;
static void (*index_next_hook_mailbox_list_created)(struct mailbox_list *list);

static int
index_mailbox_list_is_synced(struct index_mailbox_list_iterate_context *ctx)
{
	const struct mail_index_header *hdr;
	struct stat st;
	const char *path = ctx->ctx.list->set.root_dir;

	/* FIXME: single sync_stamp works only with maildir++ */
	if (stat(path, &st) < 0) {
		mailbox_list_set_critical(ctx->ctx.list,
					  "stat(%s) failed: %m", path);
		return -1;
	}
	/*
	   if mtime is older than 2 secs, we set the first bit on
	   if mtime is 0-2 secs old, we set the first bit off.

	   this way we'll always do a resync later when syncing a recently
	   changed directory. if the directory changes while we're syncing it
	   we'll resync it again later.

	   this would work with 1 second difference if we didn't store the
	   dirtyness flag in the stamp's first bit.
	*/
	if (st.st_mtime < ioloop_time - MAILBOX_LIST_SYNC_SECS)
		st.st_mtime |= 1;
	else
		st.st_mtime &= ~1;

	ctx->sync_stamp = st.st_mtime;

	hdr = mail_index_get_header(ctx->view);
	return hdr->sync_stamp == ctx->sync_stamp;
}

static void mask_parse(struct mailbox_list *list, const char *mask,
		       const char **prefix_r, int *recurse_level_r)
{
	char sep = list->hierarchy_sep;
	const char *prefix_start, *prefix_end;
	bool seen_wildcards = FALSE;
	int recurse_level = 0;

	prefix_start = prefix_end = mask;
	for (; *mask != '\0'; mask++) {
		if (*mask == '%')
			seen_wildcards = TRUE;
		else if (*mask == '*') {
			recurse_level = -1;
			break;
		}

		if (*mask == sep) {
			if (!seen_wildcards)
				prefix_end = mask;
			recurse_level++;
		}
	}

	*prefix_r = prefix_start == prefix_end ? "" :
		t_strdup_until(prefix_start, prefix_end);
	*recurse_level_r = recurse_level;
}

static struct mailbox_list_iterate_context *
index_mailbox_list_iter_init(struct mailbox_list *list, const char *mask,
			     enum mailbox_list_iter_flags flags)
{
	struct index_mailbox_list *ilist = INDEX_LIST_CONTEXT(list);
	struct index_mailbox_list_iterate_context *ctx;
	enum mailbox_list_sync_flags sync_flags;
	const char *prefix;
	int recurse_level;

	ctx = i_new(struct index_mailbox_list_iterate_context, 1);
	ctx->ctx.list = list;
	ctx->ctx.flags = flags;
	ctx->glob = imap_match_init(default_pool, mask, TRUE,
				    list->hierarchy_sep);

	ctx->view = mail_index_view_open(ilist->mail_index);
	if (index_mailbox_list_is_synced(ctx) > 0) {
		/* synced, list from index */
		mask_parse(list, mask, &prefix, &recurse_level);

		ctx->info_pool =
			pool_alloconly_create("mailbox name pool", 128);
		ctx->iter_ctx =
			mailbox_list_index_iterate_init(ilist->list_index,
							prefix, recurse_level);
		ctx->recurse_level = recurse_level;
		ctx->prefix = *prefix == '\0' ? i_strdup("") :
			i_strdup_printf("%s%c", prefix, list->hierarchy_sep);
	} else {
		/* FIXME: this works nicely with maildir++, but not others */
		sync_flags = MAILBOX_LIST_SYNC_FLAG_RECURSIVE;
		mask = "*";
		prefix = "";

		if (mailbox_list_index_sync_init(ilist->list_index, prefix,
						 sync_flags,
						 &ctx->sync_ctx) == 0) {
			ctx->trans =
				mailbox_list_index_sync_get_transaction(ctx->sync_ctx);
		}

		ctx->backend_ctx = ilist->super.iter_init(list, mask, flags);
	}
	return &ctx->ctx;
}

static enum mailbox_info_flags
index_mailbox_list_index_flags_translate(enum mailbox_list_index_flags flags)
{
	enum mailbox_info_flags info_flags = 0;

	if ((flags & MAILBOX_LIST_INDEX_FLAG_CHILDREN) != 0)
		info_flags |= MAILBOX_CHILDREN;
	if ((flags & MAILBOX_LIST_INDEX_FLAG_NOCHILDREN) != 0)
		info_flags |= MAILBOX_NOCHILDREN;

	if ((flags & MAILBOX_LIST_INDEX_FLAG_NONEXISTENT) != 0)
		info_flags |= MAILBOX_NONEXISTENT;
	if ((flags & MAILBOX_LIST_INDEX_FLAG_NOSELECT) != 0)
		info_flags |= MAILBOX_NOSELECT;
	return info_flags;
}

static enum mailbox_list_index_flags
index_mailbox_list_info_flags_translate(enum mailbox_info_flags info_flags)
{
	enum mailbox_list_index_flags flags = 0;

	if ((info_flags & MAILBOX_CHILDREN) != 0)
		flags |= MAILBOX_LIST_INDEX_FLAG_CHILDREN;
	if ((info_flags & MAILBOX_NOCHILDREN) != 0)
		flags |= MAILBOX_LIST_INDEX_FLAG_NOCHILDREN;

	if ((info_flags & MAILBOX_NONEXISTENT) != 0)
		flags |= MAILBOX_LIST_INDEX_FLAG_NONEXISTENT;
	if ((info_flags & MAILBOX_NOSELECT) != 0)
		flags |= MAILBOX_LIST_INDEX_FLAG_NOSELECT;
	return flags;
}

/* skip nonexistent mailboxes when finding with "*" */
#define info_flags_match(ctx, info) \
	(((info)->flags & MAILBOX_NONEXISTENT) == 0 || \
	 (ctx)->recurse_level >= 0)

static int iter_next_nonsync(struct index_mailbox_list_iterate_context *ctx,
			     struct mailbox_info **info_r)
{
	struct index_mailbox_list *ilist = INDEX_LIST_CONTEXT(ctx->ctx.list);
	struct mailbox_list_index_info iinfo;
	const struct mail_index_record *rec;
	uint32_t seq;
	int ret;

	/* find the next matching mailbox */
	do {
		p_clear(ctx->info_pool);
		ret = mailbox_list_index_iterate_next(ctx->iter_ctx, &iinfo);
		if (ret <= 0) {
			*info_r = NULL;
			return ret;
		}

		ctx->info.name = *ctx->prefix == '\0' ? iinfo.name :
			p_strconcat(ctx->info_pool, ctx->prefix,
				    iinfo.name, NULL);
	} while (imap_match(ctx->glob, ctx->info.name) != IMAP_MATCH_YES);

	/* get the mailbox's flags */
	if (mail_index_lookup_uid_range(ctx->view, iinfo.uid, iinfo.uid,
					&seq, &seq) < 0)
		return -1;
	if (seq == 0) {
		mailbox_list_index_set_corrupted(ilist->list_index,
			"Desynced: Record expunged from mail index");
		return -1;
	}

	if (mail_index_lookup(ctx->view, seq, &rec) < 0)
		return -1;
	ctx->info.flags = index_mailbox_list_index_flags_translate(rec->flags);

	/* do some sanity checks to the flags */
	if ((ctx->info.flags & MAILBOX_CHILDREN) != 0 &&
	    (ctx->info.flags & MAILBOX_NOCHILDREN) != 0) {
		mailbox_list_index_set_corrupted(ilist->list_index,
			"Mail index has both children and nochildren flags");
		return -1;
	}
	if ((ctx->info.flags & MAILBOX_NOCHILDREN) != 0 &&
	    iinfo.has_children) {
		mailbox_list_index_set_corrupted(ilist->list_index,
			"Desynced: Children flags wrong in mail index");
	}

	if (!info_flags_match(ctx, &ctx->info))
		return iter_next_nonsync(ctx, info_r);

	*info_r = &ctx->info;
	return 0;
}

static struct mailbox_info *
index_mailbox_list_iter_next(struct mailbox_list_iterate_context *_ctx)
{
	struct index_mailbox_list_iterate_context *ctx =
		(struct index_mailbox_list_iterate_context *)_ctx;
	struct index_mailbox_list *ilist = INDEX_LIST_CONTEXT(_ctx->list);
	struct mailbox_info *info;
	uint32_t seq, flags;

	if (ctx->iter_ctx != NULL) {
		if (iter_next_nonsync(ctx, &info) < 0) {
			ctx->failed = TRUE;
			return NULL;
		}
		return info;
	}

	do {
		info = ilist->super.iter_next(ctx->backend_ctx);
		if (info == NULL || ctx->sync_ctx == NULL)
			return info;

		/* if the sync fails, just ignore it. we don't require synced
		   indexes to return valid output. */
		if (mailbox_list_index_sync_more(ctx->sync_ctx, info->name,
						 &seq) < 0)
			return info;

		flags = index_mailbox_list_info_flags_translate(info->flags);
		mail_index_update_flags(ctx->trans, seq, MODIFY_REPLACE, flags);
	} while (imap_match(ctx->glob, info->name) != IMAP_MATCH_YES ||
		 !info_flags_match(ctx, info));

	return info;
}

static int
index_mailbox_list_iter_deinit(struct mailbox_list_iterate_context *_ctx)
{
	struct index_mailbox_list_iterate_context *ctx =
		(struct index_mailbox_list_iterate_context *)_ctx;
	struct index_mailbox_list *ilist = INDEX_LIST_CONTEXT(_ctx->list);
	int ret = ctx->failed ? -1 : 0;

	if (ctx->iter_ctx != NULL) {
		mailbox_list_index_iterate_deinit(&ctx->iter_ctx);
		pool_unref(ctx->info_pool);
	}

	if (ctx->view != NULL)
		mail_index_view_close(&ctx->view);

	if (ctx->backend_ctx != NULL) {
		/* FIXME: single sync_stamp works only with maildir++ */
		mail_index_update_header(ctx->trans,
			offsetof(struct mail_index_header, sync_stamp),
			&ctx->sync_stamp, sizeof(ctx->sync_stamp), TRUE);

		if ((ret = ilist->super.iter_deinit(ctx->backend_ctx)) < 0)
			mailbox_list_index_sync_rollback(&ctx->sync_ctx);
		else {
			/* index updates aren't that important. if the commit
			   fails, we've still returned full output. */
			(void)mailbox_list_index_sync_commit(&ctx->sync_ctx);
		}
	}

	imap_match_deinit(&ctx->glob);
	i_free(ctx->prefix);
	i_free(ctx);
	return ret;
}

static void index_mailbox_list_deinit(struct mailbox_list *list)
{
	struct index_mailbox_list *ilist = INDEX_LIST_CONTEXT(list);

	mailbox_list_index_free(&ilist->list_index);
	mail_index_free(&ilist->mail_index);

	ilist->super.deinit(list);
}

static void index_mailbox_list_created(struct mailbox_list *list)
{
	struct index_mailbox_list *ilist;
	struct mail_index *mail_index;
	struct mailbox_list_index *list_index;
	enum mail_index_open_flags index_flags;
	enum mail_index_lock_method lock_method;
	enum mail_storage_flags storage_flags;
	const char *dir, *path;
	int ret;

	/* FIXME: for now we only work with maildir++ */
	if (strcmp(list->name, "maildir++") != 0)
		return;

	/* FIXME: a bit ugly way to get the flags, but this will do for now.. */
	index_flags = MAIL_INDEX_OPEN_FLAG_CREATE;
	storage_flags = *list->set.mail_storage_flags;
	if ((storage_flags & MAIL_STORAGE_FLAG_MMAP_DISABLE) != 0) {
		index_flags |= MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE;
		return; /* FIXME: we currently don't support mmap_disable */
	}
#ifndef MMAP_CONFLICTS_WRITE
	if ((storage_flags & MAIL_STORAGE_FLAG_MMAP_NO_WRITE) != 0)
#endif
		index_flags |= MAIL_INDEX_OPEN_FLAG_MMAP_NO_WRITE;

	switch (*list->set.mail_storage_lock_method) {
	case MAIL_STORAGE_LOCK_FCNTL:
		lock_method = MAIL_INDEX_LOCK_FCNTL;
		break;
	case MAIL_STORAGE_LOCK_FLOCK:
		lock_method = MAIL_INDEX_LOCK_FLOCK;
		break;
	case MAIL_STORAGE_LOCK_DOTLOCK:
		lock_method = MAIL_INDEX_LOCK_DOTLOCK;
		break;
	}

	dir = mailbox_list_get_path(list, NULL, MAILBOX_LIST_PATH_TYPE_INDEX);
	path = t_strconcat(dir, "/"MAILBOX_LIST_INDEX_NAME, NULL);

	mail_index = mail_index_alloc(dir, MAIL_INDEX_PREFIX);
	if (mail_index_open(mail_index, index_flags, lock_method) < 0) {
		if (mail_index_move_to_memory(mail_index) < 0) {
			/* try opening once more. it should be created
			   directly into memory now. */
			ret = mail_index_open(mail_index, index_flags,
					      lock_method);
			if (ret <= 0) {
				/* everything failed. there's a bug in the
				   code, but just work around it by disabling
				   the index completely */
				mail_index_free(&mail_index);
				return;
			}
		}
	}

	list_index = mailbox_list_index_alloc(path, list->hierarchy_sep,
					      mail_index);
	if (mailbox_list_index_open_or_create(list_index) < 0) {
		/* skip indexing */
		mailbox_list_index_free(&list_index);
		mail_index_free(&mail_index);
		return;
	}

	ilist = p_new(list->pool, struct index_mailbox_list, 1);
	ilist->super = list->v;
	ilist->mail_index = mail_index;
	ilist->list_index = list_index;

	list->v.deinit = index_mailbox_list_deinit;
	list->v.iter_init = index_mailbox_list_iter_init;
	list->v.iter_deinit = index_mailbox_list_iter_deinit;
	list->v.iter_next = index_mailbox_list_iter_next;

	if (!index_mailbox_list_module_id_set) {
		index_mailbox_list_module_id = mailbox_list_module_id++;
		index_mailbox_list_module_id_set = TRUE;
	}

	array_idx_set(&list->module_contexts,
		      index_mailbox_list_module_id, &ilist);

	index_mailbox_list_sync_init_list(list);
}

void index_mailbox_list_init(void); /* called in mailbox-list-register.c */

void index_mailbox_list_init(void)
{
	index_next_hook_mailbox_list_created = hook_mailbox_list_created;
	hook_mailbox_list_created = index_mailbox_list_created;

	index_mailbox_list_sync_init();
}

--- NEW FILE: index-mailbox-list.h ---
#ifndef __INDEX_MAILBOX_LIST_H
#define __INDEX_MAILBOX_LIST_H

#include "mailbox-list-private.h"

#define MAIL_INDEX_PREFIX "dovecot.list.index"
#define MAILBOX_LIST_INDEX_NAME MAIL_INDEX_PREFIX".uidmap"

#define INDEX_LIST_CONTEXT(obj) \
	*((void **)array_idx_modifiable(&(obj)->module_contexts, \
					index_mailbox_list_module_id))

struct index_mailbox_list {
	struct mailbox_list_vfuncs super;

	struct mail_index *mail_index;
	struct mailbox_list_index *list_index;

	uint32_t eid_messages, eid_unseen, eid_recent;
	uint32_t eid_uid_validity, eid_uidnext;

	uint32_t eid_cur_sync_stamp, eid_new_sync_stamp, eid_dirty_flags;
};

struct index_mailbox_list_iterate_context {
	struct mailbox_list_iterate_context ctx;

	struct mailbox_list_iter_ctx *iter_ctx;
	struct mailbox_list_index_sync_ctx *sync_ctx;
	struct mailbox_list_iterate_context *backend_ctx;

	struct mail_index_view *view;
	struct mail_index_transaction *trans;

	char *prefix;
	int recurse_level;
	struct imap_match_glob *glob;

	pool_t info_pool;
	struct mailbox_info info;
	uint32_t sync_stamp;

	unsigned int failed:1;
};

extern unsigned int index_mailbox_list_module_id;

void index_mailbox_list_sync_init(void);
void index_mailbox_list_sync_init_list(struct mailbox_list *list);

#endif

Index: mailbox-list-fs-iter.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/list/mailbox-list-fs-iter.c,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- mailbox-list-fs-iter.c	16 Nov 2006 00:16:38 -0000	1.1
+++ mailbox-list-fs-iter.c	25 Nov 2006 22:17:50 -0000	1.2
@@ -76,8 +76,7 @@
 }
 
 struct mailbox_list_iterate_context *
-fs_list_iter_init(struct mailbox_list *_list,
-		  const char *ref, const char *mask,
+fs_list_iter_init(struct mailbox_list *_list, const char *mask,
 		  enum mailbox_list_iter_flags flags)
 {
 	struct fs_mailbox_list *list =
@@ -95,20 +94,12 @@
         ctx->next = fs_list_next;
 
 	/* check that we're not trying to do any "../../" lists */
-	if (!mailbox_list_is_valid_mask(_list, ref) ||
-	    !mailbox_list_is_valid_mask(_list, mask)) {
+	if (!mailbox_list_is_valid_mask(_list, mask)) {
 		mailbox_list_set_error(_list, "Invalid mask");
 		ctx->ctx.failed = TRUE;
 		return &ctx->ctx;
 	}
 
-	if (*mask == '/' || *mask == '~') {
-		/* mask overrides reference */
-	} else if (*ref != '\0') {
-		/* merge reference and mask */
-		mask = t_strconcat(ref, mask, NULL);
-	}
-
 	if ((flags & MAILBOX_LIST_ITER_SUBSCRIBED) != 0) {
 		ctx->next = fs_list_subs;
 
@@ -313,7 +304,7 @@
 
 	if (match == IMAP_MATCH_PARENT) {
 		/* placeholder */
-		ctx->info.flags = MAILBOX_PLACEHOLDER;
+		ctx->info.flags = MAILBOX_NONEXISTENT | MAILBOX_CHILDREN;
 		while ((p = strrchr(name, '/')) != NULL) {
 			name = t_strdup_until(name, p);
 			if (imap_match(ctx->glob, name) > 0) {

Index: mailbox-list-fs.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/list/mailbox-list-fs.c,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- mailbox-list-fs.c	16 Nov 2006 00:16:38 -0000	1.1
+++ mailbox-list-fs.c	25 Nov 2006 22:17:50 -0000	1.2
@@ -247,6 +247,19 @@
 	return list->temp_prefix;
 }
 
+static const char *
+fs_list_join_refmask(struct mailbox_list *_list __attr_unused__,
+		     const char *ref, const char *mask)
+{
+	if (*mask == '/' || *mask == '~') {
+		/* mask overrides reference */
+	} else if (*ref != '\0') {
+		/* merge reference and mask */
+		mask = t_strconcat(ref, mask, NULL);
+	}
+	return mask;
+}
+
 static int fs_list_set_subscribed(struct mailbox_list *_list,
 				  const char *name, bool set)
 {
@@ -277,6 +290,7 @@
 		fs_list_get_path,
 		fs_list_get_mailbox_name_status,
 		fs_list_get_temp_prefix,
+		fs_list_join_refmask,
 		fs_list_iter_init,
 		fs_list_iter_next,
 		fs_list_iter_deinit,

Index: mailbox-list-fs.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/list/mailbox-list-fs.h,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- mailbox-list-fs.h	16 Nov 2006 00:16:39 -0000	1.1
+++ mailbox-list-fs.h	25 Nov 2006 22:17:50 -0000	1.2
@@ -14,8 +14,7 @@
 };
 
 struct mailbox_list_iterate_context *
-fs_list_iter_init(struct mailbox_list *_list,
-		  const char *ref, const char *mask,
+fs_list_iter_init(struct mailbox_list *_list, const char *mask,
 		  enum mailbox_list_iter_flags flags);
 int fs_list_iter_deinit(struct mailbox_list_iterate_context *ctx);
 struct mailbox_info *

Index: mailbox-list-maildir-iter.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/list/mailbox-list-maildir-iter.c,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- mailbox-list-maildir-iter.c	16 Nov 2006 00:16:39 -0000	1.1
+++ mailbox-list-maildir-iter.c	25 Nov 2006 22:17:50 -0000	1.2
@@ -33,11 +33,12 @@
 			node->flags |= MAILBOX_CHILDREN;
 			node->flags &= ~MAILBOX_NOCHILDREN;
 			maildir_nodes_fix(node->children, is_subs);
-		} else if ((node->flags & MAILBOX_PLACEHOLDER) != 0) {
+		} else if ((node->flags & MAILBOX_NONEXISTENT) != 0) {
 			if (!is_subs) {
-				node->flags &= ~MAILBOX_PLACEHOLDER;
+				node->flags &= ~MAILBOX_NONEXISTENT;
 				node->flags |= MAILBOX_NOSELECT;
 			}
+			node->flags |= MAILBOX_CHILDREN;
 		}
 		node = node->next;
 	}
@@ -126,7 +127,7 @@
 						 mailbox_c, &created);
 			if (node != NULL) {
 				if (created)
-					node->flags = MAILBOX_PLACEHOLDER;
+					node->flags = MAILBOX_NONEXISTENT;
 
 				node->flags |= MAILBOX_CHILDREN |
 					MAILBOX_FLAG_MATCHED;
@@ -144,8 +145,7 @@
 			if (node != NULL) {
 				if (created)
 					node->flags = MAILBOX_NOCHILDREN;
-				node->flags &= ~(MAILBOX_PLACEHOLDER |
-						 MAILBOX_NONEXISTENT);
+				node->flags &= ~MAILBOX_NONEXISTENT;
 				node->flags |= MAILBOX_FLAG_MATCHED;
 			}
 		}
@@ -165,7 +165,7 @@
 		if (created)
 			node->flags = MAILBOX_NOCHILDREN;
 		else
-			node->flags &= ~MAILBOX_PLACEHOLDER;
+			node->flags &= ~MAILBOX_NONEXISTENT;
 
 		switch (imap_match(glob, "INBOX")) {
 		case IMAP_MATCH_YES:
@@ -218,7 +218,7 @@
 			i_assert(p != NULL);
 
 			node = mailbox_tree_get(ctx->tree_ctx, name, &created);
-			if (created) node->flags = MAILBOX_PLACEHOLDER;
+			if (created) node->flags = MAILBOX_NONEXISTENT;
 			node->flags |= MAILBOX_FLAG_MATCHED | MAILBOX_CHILDREN;
 			node->flags &= ~MAILBOX_NOCHILDREN;
 			break;
@@ -231,8 +231,7 @@
 }
 
 struct mailbox_list_iterate_context *
-maildir_list_iter_init(struct mailbox_list *_list,
-		       const char *ref, const char *mask,
+maildir_list_iter_init(struct mailbox_list *_list, const char *mask,
 		       enum mailbox_list_iter_flags flags)
 {
 	struct maildir_mailbox_list *list =
@@ -251,11 +250,6 @@
 	ctx->pool = pool;
 	ctx->tree_ctx = mailbox_tree_init(_list->hierarchy_sep);
 
-	if (*ref != '\0') {
-		/* join reference + mask */
-		mask = t_strconcat(ref, mask, NULL);
-	}
-
 	glob = imap_match_init(pool, mask, TRUE, _list->hierarchy_sep);
 
 	ctx->dir = _list->set.root_dir;

Index: mailbox-list-maildir.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/list/mailbox-list-maildir.c,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- mailbox-list-maildir.c	18 Nov 2006 14:06:34 -0000	1.2
+++ mailbox-list-maildir.c	25 Nov 2006 22:17:50 -0000	1.3
@@ -239,6 +239,17 @@
 	return list->temp_prefix;
 }
 
+static const char *
+maildir_list_join_refmask(struct mailbox_list *_list __attr_unused__,
+			  const char *ref, const char *mask)
+{
+	if (*ref != '\0') {
+		/* merge reference and mask */
+		mask = t_strconcat(ref, mask, NULL);
+	}
+	return mask;
+}
+
 static int maildir_list_set_subscribed(struct mailbox_list *_list,
 				       const char *name, bool set)
 {
@@ -270,6 +281,7 @@
 		maildir_list_get_path,
 		maildir_list_get_mailbox_name_status,
 		maildir_list_get_temp_prefix,
+		maildir_list_join_refmask,
 		maildir_list_iter_init,
 		maildir_list_iter_next,
 		maildir_list_iter_deinit,

Index: mailbox-list-maildir.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/list/mailbox-list-maildir.h,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- mailbox-list-maildir.h	16 Nov 2006 00:16:39 -0000	1.1
+++ mailbox-list-maildir.h	25 Nov 2006 22:17:50 -0000	1.2
@@ -14,8 +14,7 @@
 };
 
 struct mailbox_list_iterate_context *
-maildir_list_iter_init(struct mailbox_list *_list,
-		       const char *ref, const char *mask,
+maildir_list_iter_init(struct mailbox_list *_list, const char *mask,
 		       enum mailbox_list_iter_flags flags);
 int maildir_list_iter_deinit(struct mailbox_list_iterate_context *ctx);
 struct mailbox_info *



More information about the dovecot-cvs mailing list