[dovecot-cvs] dovecot/src/lib-storage/index/cydir .cvsignore, NONE, 1.1 Makefile.am, NONE, 1.1 cydir-mail.c, NONE, 1.1 cydir-save.c, NONE, 1.1 cydir-storage.c, NONE, 1.1 cydir-storage.h, NONE, 1.1 cydir-sync.c, NONE, 1.1 cydir-sync.h, NONE, 1.1 cydir-transaction.c, NONE, 1.1

[tss at dovecot.org]

Update of /var/lib/cvs/dovecot/src/lib-storage/index/cydir
In directory talvi:/tmp/cvs-serv17119/src/lib-storage/index/cydir

Added Files:
	.cvsignore Makefile.am cydir-mail.c cydir-save.c 
	cydir-storage.c cydir-storage.h cydir-sync.c cydir-sync.h 
	cydir-transaction.c 
Log Message:
Added a simple cydir mail storage backend. It trusts index files completely:
if the indexes are gone, the mailbox starts completely empty and overwriting
the existing mail files. So probably not a good idea to use this in
production yet.



--- NEW FILE: .cvsignore ---
*.la
*.lo
*.o
.deps
.libs
Makefile
Makefile.in
so_locations

--- NEW FILE: Makefile.am ---
noinst_LIBRARIES = libstorage_cydir.a

AM_CPPFLAGS = \
	-I$(top_srcdir)/src/lib \
	-I$(top_srcdir)/src/lib-mail \
	-I$(top_srcdir)/src/lib-imap \
	-I$(top_srcdir)/src/lib-index \
	-I$(top_srcdir)/src/lib-storage \
	-I$(top_srcdir)/src/lib-storage/index

libstorage_cydir_a_SOURCES = \
	cydir-mail.c \
	cydir-save.c \
	cydir-sync.c \
	cydir-storage.c \
	cydir-transaction.c

headers = \
	cydir-storage.h \
	cydir-sync.h

if INSTALL_HEADERS
  pkginc_libdir=$(pkgincludedir)/src/lib-storage/index/cydir
  pkginc_lib_HEADERS = $(headers)
else
  noinst_HEADERS = $(headers)
endif

--- NEW FILE: cydir-mail.c ---
/* Copyright (C) 2007 Timo Sirainen */

#include "lib.h"
#include "istream.h"
#include "index-mail.h"
#include "cydir-storage.h"

#include <fcntl.h>
#include <unistd.h>
#include <sys/stat.h>

static const char *cydir_mail_get_path(struct mail *mail)
{
	const char *dir;

	dir = mailbox_list_get_path(mail->box->storage->list, mail->box->name,
				    MAILBOX_LIST_PATH_TYPE_MAILBOX);
	return t_strdup_printf("%s/%u.", dir, mail->uid);
}

static int cydir_mail_stat(struct mail *mail, struct stat *st_r)
{
	const char *path;

	path = cydir_mail_get_path(mail);
	if (stat(path, st_r) < 0) {
		if (errno == ENOENT)
			mail->expunged = TRUE;
		else {
			mail_storage_set_critical(mail->box->storage,
						  "stat(%s) failed: %m", path);
		}
		return -1;
	}
	return 0;
}

static time_t cydir_mail_get_received_date(struct mail *_mail)
{
	struct index_mail *mail = (struct index_mail *)_mail;
	struct index_mail_data *data = &mail->data;
	struct stat st;
	uint32_t t;

	(void)index_mail_get_received_date(_mail);
	if (data->received_date != (time_t)-1)
		return data->received_date;

	if (cydir_mail_stat(_mail, &st) < 0)
		return (time_t)-1;

	data->received_date = t = st.st_mtime;
	index_mail_cache_add(mail, MAIL_CACHE_RECEIVED_DATE, &t, sizeof(t));
	return data->received_date;
}

static time_t cydir_mail_get_save_date(struct mail *_mail)
{
	struct index_mail *mail = (struct index_mail *)_mail;
	struct index_mail_data *data = &mail->data;
	struct stat st;
	uint32_t t;

	(void)index_mail_get_save_date(_mail);
	if (data->save_date != (time_t)-1)
		return data->save_date;

	if (cydir_mail_stat(_mail, &st) < 0)
		return (time_t)-1;

	data->save_date = t = st.st_ctime;
	index_mail_cache_add(mail, MAIL_CACHE_SAVE_DATE, &t, sizeof(t));
	return data->save_date;
}

static uoff_t cydir_mail_get_physical_size(struct mail *_mail)
{
	struct index_mail *mail = (struct index_mail *)_mail;
	struct index_mail_data *data = &mail->data;
	struct stat st;

	(void)index_mail_get_physical_size(_mail);
	if (data->physical_size != (uoff_t)-1)
		return data->physical_size;

	if (cydir_mail_stat(_mail, &st) < 0)
		return (time_t)-1;

	index_mail_cache_add(mail, MAIL_CACHE_PHYSICAL_FULL_SIZE,
			     &data->physical_size, sizeof(data->physical_size));
	return data->physical_size;
}

static struct istream *
cydir_mail_get_stream(struct mail *_mail, struct message_size *hdr_size,
		      struct message_size *body_size)
{
	struct index_mail *mail = (struct index_mail *)_mail;
	const char *path;
	int fd;

	if (mail->data.stream == NULL) {
		path = cydir_mail_get_path(_mail);
		fd = open(path, O_RDONLY);
		if (fd == -1) {
			if (errno == ENOENT)
				_mail->expunged = TRUE;
			else {
				mail_storage_set_critical(_mail->box->storage,
					"open(%s) failed: %m", path);
			}
			return NULL;
		}
		mail->data.stream =
			i_stream_create_file(fd, default_pool,
					     MAIL_READ_BLOCK_SIZE, TRUE);
	}

	return index_mail_init_stream(mail, hdr_size, body_size);
}

struct mail_vfuncs cydir_mail_vfuncs = {
	index_mail_free,
	index_mail_set_seq,
	index_mail_set_uid,

	index_mail_get_flags,
	index_mail_get_keywords,
	index_mail_get_parts,
	index_mail_get_date,
	cydir_mail_get_received_date,
	cydir_mail_get_save_date,
	cydir_mail_get_physical_size, /* physical = virtual in our case */
	cydir_mail_get_physical_size,
	index_mail_get_first_header,
	index_mail_get_headers,
	index_mail_get_header_stream,
	cydir_mail_get_stream,
	index_mail_get_special,
	index_mail_update_flags,
	index_mail_update_keywords,
	index_mail_expunge
};

--- NEW FILE: cydir-save.c ---
/* Copyright (C) 2007 Timo Sirainen */

#include "lib.h"
#include "hostpid.h"
#include "istream.h"
#include "ostream.h"
#include "ostream-crlf.h"
#include "str.h"
#include "index-mail.h"
#include "cydir-storage.h"
#include "cydir-sync.h"

#include <stdio.h>

struct cydir_save_context {
	struct mail_save_context ctx;

	struct cydir_mailbox *mbox;
	struct mail_index_transaction *trans;

	char *tmp_basename;
	unsigned int mail_count;

	struct cydir_sync_context *sync_ctx;

	/* updated for each appended mail: */
	uint32_t seq;
	struct istream *input;
	struct ostream *output;
	struct mail *mail;

	unsigned int failed:1;
	unsigned int finished:1;
};

static char *cydir_generate_tmp_filename(void)
{
	static unsigned int create_count;

	return i_strdup_printf("%s.P%sQ%uM%s.%s",
			       dec2str(ioloop_timeval.tv_sec), my_pid,
			       create_count++,
			       dec2str(ioloop_timeval.tv_usec), my_hostname);
}

static const char *
cydir_get_save_path(struct cydir_save_context *ctx, unsigned int num)
{
	const char *dir;

	dir = mailbox_list_get_path(STORAGE(ctx->mbox->storage)->list,
				    ctx->mbox->ibox.box.name,
				    MAILBOX_LIST_PATH_TYPE_MAILBOX);
	return t_strdup_printf("%s/%s.%u", dir, ctx->tmp_basename, num);
}

int cydir_save_init(struct mailbox_transaction_context *_t,
		    enum mail_flags flags, struct mail_keywords *keywords,
		    time_t received_date, int timezone_offset __attr_unused__,
		    const char *from_envelope __attr_unused__,
		    struct istream *input, struct mail *dest_mail,
		    struct mail_save_context **ctx_r)
{
	struct cydir_transaction_context *t =
		(struct cydir_transaction_context *)_t;
	struct cydir_mailbox *mbox = (struct cydir_mailbox *)t->ictx.ibox;
	struct cydir_save_context *ctx = t->save_ctx;
	enum mail_flags save_flags;
	struct ostream *output;
	const char *path;
	int fd;

	i_assert((t->ictx.flags & MAILBOX_TRANSACTION_FLAG_EXTERNAL) != 0);

	if (received_date == (time_t)-1)
		received_date = ioloop_time;

	if (ctx == NULL) {
		ctx = t->save_ctx = i_new(struct cydir_save_context, 1);
		ctx->ctx.transaction = &t->ictx.mailbox_ctx;
		ctx->mbox = mbox;
		ctx->trans = t->ictx.trans;
		ctx->tmp_basename = cydir_generate_tmp_filename();
	}
	ctx->input = input;

	t_push();
	path = cydir_get_save_path(ctx, ctx->mail_count);
	fd = open(path, O_WRONLY | O_CREAT | O_EXCL, 0660);
	if (fd != -1) {
		output = o_stream_create_file(fd, default_pool, 0, TRUE);
		ctx->output = o_stream_create_crlf(default_pool, output);
		o_stream_unref(&output);
	} else {
		mail_storage_set_critical(_t->box->storage,
					  "open(%s) failed: %m", path);
		ctx->failed = TRUE;
		t_pop();
		return -1;
	}
	t_pop();

	/* add to index */
	save_flags = (flags & ~MAIL_RECENT) | MAIL_RECENT;
	mail_index_append(ctx->trans, 0, &ctx->seq);
	mail_index_update_flags(ctx->trans, ctx->seq, MODIFY_REPLACE,
				save_flags);
	if (keywords != NULL) {
		mail_index_update_keywords(ctx->trans, ctx->seq,
					   MODIFY_REPLACE, keywords);
	}

	if (dest_mail == NULL) {
		if (ctx->mail == NULL)
			ctx->mail = index_mail_alloc(_t, 0, NULL);
		dest_mail = ctx->mail;
	}
	if (mail_set_seq(dest_mail, ctx->seq) < 0)
		i_unreached();

	if (t->first_saved_mail_seq == 0)
		t->first_saved_mail_seq = ctx->seq;

	*ctx_r = &ctx->ctx;
	return ctx->failed ? -1 : 0;
}

int cydir_save_continue(struct mail_save_context *_ctx)
{
	struct cydir_save_context *ctx = (struct cydir_save_context *)_ctx;

	if (ctx->failed)
		return -1;

	if (o_stream_send_istream(ctx->output, ctx->input) < 0) {
		if (ENOSPACE(ctx->output->stream_errno)) {
			mail_storage_set_error(STORAGE(ctx->mbox->storage),
					       "Not enough disk space");
		} else {
			mail_storage_set_critical(STORAGE(ctx->mbox->storage),
				"o_stream_send_istream(%s) failed: %m",
				cydir_get_save_path(ctx, ctx->mail_count));
		}
		ctx->failed = TRUE;
		return -1;
	}
	return 0;
}

int cydir_save_finish(struct mail_save_context *_ctx)
{
	struct cydir_save_context *ctx = (struct cydir_save_context *)_ctx;

	ctx->finished = TRUE;

	if (!ctx->failed)
		ctx->mail_count++;

	return ctx->failed ? -1 : 0;
}

void cydir_save_cancel(struct mail_save_context *_ctx)
{
	struct cydir_save_context *ctx = (struct cydir_save_context *)_ctx;

	ctx->failed = TRUE;
	(void)cydir_save_finish(_ctx);
}

int cydir_transaction_save_commit_pre(struct cydir_save_context *ctx)
{
	const struct mail_index_header *hdr;
	uint32_t i, uid, next_uid;
	const char *dir;
	string_t *src_path, *dest_path;
	unsigned int src_prefixlen, dest_prefixlen;

	i_assert(ctx->finished);

	if (cydir_sync_begin(ctx->mbox, &ctx->sync_ctx) < 0) {
		ctx->failed = TRUE;
		cydir_transaction_save_rollback(ctx);
		return -1;
	}

	hdr = mail_index_get_header(ctx->sync_ctx->sync_view);
	uid = hdr->next_uid;
	mail_index_append_assign_uids(ctx->trans, uid, &next_uid);

	dir = mailbox_list_get_path(STORAGE(ctx->mbox->storage)->list,
				    ctx->mbox->ibox.box.name,
				    MAILBOX_LIST_PATH_TYPE_MAILBOX);

	src_path = t_str_new(256);
	str_printfa(src_path, "%s/%s.", dir, ctx->tmp_basename);
	src_prefixlen = str_len(src_path);

	dest_path = t_str_new(256);
	str_append(dest_path, dir);
	str_append_c(dest_path, '/');
	dest_prefixlen = str_len(dest_path);

	for (i = 0; i < ctx->mail_count; i++, uid++) {
		str_truncate(src_path, src_prefixlen);
		str_truncate(dest_path, dest_prefixlen);
		str_printfa(src_path, "%u", i);
		str_printfa(dest_path, "%u.", uid);

		if (rename(str_c(src_path), str_c(dest_path)) < 0) {
			mail_storage_set_critical(STORAGE(ctx->mbox->storage),
				"rename(%s, %s) failed: %m",
				str_c(src_path), str_c(dest_path));
			ctx->failed = TRUE;
			cydir_transaction_save_rollback(ctx);
			return -1;
		}
	}

	return 0;
}

void cydir_transaction_save_commit_post(struct cydir_save_context *ctx)
{
	(void)cydir_sync_finish(&ctx->sync_ctx, TRUE);
	cydir_transaction_save_rollback(ctx);
}

void cydir_transaction_save_rollback(struct cydir_save_context *ctx)
{
	if (!ctx->finished)
		cydir_save_cancel(&ctx->ctx);

	if (ctx->sync_ctx != NULL)
		(void)cydir_sync_finish(&ctx->sync_ctx, FALSE);

	if (ctx->mail != NULL)
		index_mail_free(ctx->mail);
	i_free(ctx->tmp_basename);
	i_free(ctx);
}

--- NEW FILE: cydir-storage.c ---
/* Copyright (C) 2007 Timo Sirainen */

#include "lib.h"
#include "array.h"
#include "ioloop.h"
#include "str.h"
#include "mkdir-parents.h"
#include "unlink-directory.h"
#include "index-mail.h"
#include "mail-copy.h"
#include "cydir-sync.h"
#include "cydir-storage.h"

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <dirent.h>
#include <sys/stat.h>

#define CREATE_MODE 0770 /* umask() should limit it more */

#define CYDIR_LIST_CONTEXT(obj) \
	MODULE_CONTEXT(obj, cydir_mailbox_list_module)

extern struct mail_storage cydir_storage;
extern struct mailbox cydir_mailbox;

static MODULE_CONTEXT_DEFINE_INIT(cydir_mailbox_list_module,
				  &mailbox_list_module_register);

static int
cydir_list_delete_mailbox(struct mailbox_list *list, const char *name);
static int cydir_list_iter_is_mailbox(struct mailbox_list_iterate_context *ctx,
				      const char *dir, const char *fname,
				      enum mailbox_list_file_type type,
				      enum mailbox_info_flags *flags);

static int
cydir_get_list_settings(struct mailbox_list_settings *list_set,
			const char *data, enum mail_storage_flags flags)
{
	bool debug = (flags & MAIL_STORAGE_FLAG_DEBUG) != 0;
	const char *p;
	size_t len;

	memset(list_set, 0, sizeof(*list_set));
	list_set->subscription_fname = CYDIR_SUBSCRIPTION_FILE_NAME;
	list_set->maildir_name = "";

	if (data == NULL || *data == '\0') {
		/* we won't do any guessing for this format. */
		if (debug)
			i_info("cydir: mailbox location not given");
		return -1;
	}

	/* <root dir> [:INDEX=<dir>] */
	if (debug)
		i_info("cydir: data=%s", data);
	p = strchr(data, ':');
	if (p == NULL)
		list_set->root_dir = data;
	else {
		list_set->root_dir = t_strdup_until(data, p);

		do {
			p++;
			if (strncmp(p, "INDEX=", 6) == 0)
				list_set->index_dir = t_strcut(p+6, ':');
			p = strchr(p, ':');
		} while (p != NULL);
	}

	/* strip trailing '/' */
	len = strlen(list_set->root_dir);
	if (list_set->root_dir[len-1] == '/')
		list_set->root_dir = t_strndup(list_set->root_dir, len-1);

	if (list_set->index_dir != NULL &&
	    strcmp(list_set->index_dir, "MEMORY") == 0)
		list_set->index_dir = "";
	return 0;
}

static struct mail_storage *
cydir_create(const char *data, const char *user,
	     enum mail_storage_flags flags,
	     enum file_lock_method lock_method)
{
	struct cydir_storage *storage;
	struct index_storage *istorage;
	struct mailbox_list_settings list_set;
	struct mailbox_list *list;
	const char *error;
	struct stat st;
	pool_t pool;

	if (cydir_get_list_settings(&list_set, data, flags) < 0)
		return NULL;
	list_set.mail_storage_flags = &flags;
	list_set.lock_method = &lock_method;

	if ((flags & MAIL_STORAGE_FLAG_NO_AUTOCREATE) != 0) {
		if (stat(list_set.root_dir, &st) < 0) {
			if (errno != ENOENT) {
				i_error("stat(%s) failed: %m",
					list_set.root_dir);
			}
			return NULL;
		}
	}

	if (mkdir_parents(list_set.root_dir, CREATE_MODE) < 0 &&
	    errno != EEXIST) {
		i_error("mkdir_parents(%s) failed: %m", list_set.root_dir);
		return NULL;
	}

	pool = pool_alloconly_create("storage", 512+256);
	storage = p_new(pool, struct cydir_storage, 1);

	if (mailbox_list_init("fs", &list_set,
			      mail_storage_get_list_flags(flags),
			      &list, &error) < 0) {
		i_error("cydir fs: %s", error);
		pool_unref(pool);
		return NULL;
	}
	storage->list_module_ctx.super = list->v;
	list->v.iter_is_mailbox = cydir_list_iter_is_mailbox;
	list->v.delete_mailbox = cydir_list_delete_mailbox;

	MODULE_CONTEXT_SET_FULL(list, cydir_mailbox_list_module,
				storage, &storage->list_module_ctx);

	istorage = INDEX_STORAGE(storage);
	istorage->storage = cydir_storage;
	istorage->storage.pool = pool;

	istorage->user = p_strdup(pool, user);
	index_storage_init(istorage, list, flags, lock_method);

	return STORAGE(storage);
}

static void cydir_free(struct mail_storage *_storage)
{
	struct index_storage *storage = (struct index_storage *) _storage;

	index_storage_deinit(storage);
	pool_unref(storage->storage.pool);
}

static bool cydir_autodetect(const char *data __attr_unused__,
			     enum mail_storage_flags flags __attr_unused__)
{
	return FALSE;
}

static int create_cydir(struct mail_storage *storage, const char *path)
{
	const char *error;

	if (mkdir_parents(path, CREATE_MODE) < 0 && errno != EEXIST) {
		if (mail_storage_errno2str(&error)) {
			mail_storage_set_error(storage, "%s", error);
			return -1;
		}

		mail_storage_set_critical(storage, "mkdir(%s) failed: %m",
					  path);
		return -1;
	}
	return 0;
}

static int create_index_dir(struct mail_storage *storage, const char *name)
{
	const char *root_dir, *index_dir;

	root_dir = mailbox_list_get_path(storage->list, name,
					 MAILBOX_LIST_PATH_TYPE_MAILBOX);
	index_dir = mailbox_list_get_path(storage->list, name,
					  MAILBOX_LIST_PATH_TYPE_INDEX);
	if (strcmp(index_dir, root_dir) == 0)
		return 0;

	if (mkdir_parents(index_dir, CREATE_MODE) < 0 && errno != EEXIST) {
		mail_storage_set_critical(storage, "mkdir(%s) failed: %m",
					  index_dir);
		return -1;
	}

	return 0;
}

static bool cydir_is_recent(struct index_mailbox *ibox __attr_unused__,
			    uint32_t uid __attr_unused__)
{
	return FALSE;
}

static struct mailbox *
cydir_open(struct cydir_storage *storage, const char *name,
	   enum mailbox_open_flags flags)
{
	struct index_storage *istorage = INDEX_STORAGE(storage);
	struct mail_storage *_storage = STORAGE(storage);
	struct cydir_mailbox *mbox;
	struct mail_index *index;
	const char *path, *index_dir;
	pool_t pool;

	path = mailbox_list_get_path(_storage->list, name,
				     MAILBOX_LIST_PATH_TYPE_MAILBOX);
	index_dir = mailbox_list_get_path(_storage->list, name,
					  MAILBOX_LIST_PATH_TYPE_INDEX);

	if (create_cydir(_storage, path) < 0)
		return NULL;
	if (create_index_dir(_storage, name) < 0)
		return NULL;

	index = index_storage_alloc(index_dir, path, CYDIR_INDEX_PREFIX);

	pool = pool_alloconly_create("cydir mailbox", 1024+512);
	mbox = p_new(pool, struct cydir_mailbox, 1);
	mbox->ibox.box = cydir_mailbox;
	mbox->ibox.box.pool = pool;
	mbox->ibox.storage = istorage;
	mbox->ibox.mail_vfuncs = &cydir_mail_vfuncs;
	mbox->ibox.is_recent = cydir_is_recent;
	mbox->ibox.index = index;

	mbox->storage = storage;
	mbox->path = p_strdup(pool, path);

	index_storage_mailbox_init(&mbox->ibox, name, flags, FALSE);
	return &mbox->ibox.box;
}

static struct mailbox *
cydir_mailbox_open(struct mail_storage *_storage, const char *name,
		   struct istream *input, enum mailbox_open_flags flags)
{
	struct cydir_storage *storage = (struct cydir_storage *)_storage;
	const char *path;
	struct stat st;

	mail_storage_clear_error(_storage);

	if (input != NULL) {
		mail_storage_set_critical(_storage,
			"cydir doesn't support streamed mailboxes");
		return NULL;
	}

	if (strcmp(name, "INBOX") == 0)
		return cydir_open(storage, "INBOX", flags);

	if (!mailbox_list_is_valid_existing_name(_storage->list, name)) {
		mail_storage_set_error(_storage, "Invalid mailbox name");
		return NULL;
	}

	path = mailbox_list_get_path(_storage->list, name,
				     MAILBOX_LIST_PATH_TYPE_MAILBOX);
	if (stat(path, &st) == 0) {
		return cydir_open(storage, name, flags);
	} else if (errno == ENOENT) {
		mail_storage_set_error(_storage,
			MAILBOX_LIST_ERR_MAILBOX_NOT_FOUND, name);
		return NULL;
	} else {
		mail_storage_set_critical(_storage, "stat(%s) failed: %m",
					  path);
		return NULL;
	}
}

static int cydir_mailbox_create(struct mail_storage *_storage,
				const char *name,
				bool directory __attr_unused__)
{
	const char *path;
	struct stat st;

	mail_storage_clear_error(_storage);

	if (!mailbox_list_is_valid_create_name(_storage->list, name)) {
		mail_storage_set_error(_storage, "Invalid mailbox name");
		return -1;
	}

	path = mailbox_list_get_path(_storage->list, name,
				     MAILBOX_LIST_PATH_TYPE_MAILBOX);
	if (stat(path, &st) == 0) {
		mail_storage_set_error(_storage, "Mailbox already exists");
		return -1;
	}

	return create_cydir(_storage, path);
}

static int
cydir_delete_nonrecursive(struct mailbox_list *list, const char *path,
			  const char *name)
{
	DIR *dir;
	struct dirent *d;
	string_t *full_path;
	unsigned int dir_len;
	bool unlinked_something = FALSE;

	dir = opendir(path);
	if (dir == NULL) {
		if (errno == ENOENT) {
			mailbox_list_set_error(list, t_strdup_printf(
				MAILBOX_LIST_ERR_MAILBOX_NOT_FOUND, name));
		} else {
			mailbox_list_set_critical(list,
				"opendir(%s) failed: %m", path);
		}
		return -1;
	}

	full_path = t_str_new(256);
	str_append(full_path, path);
	str_append_c(full_path, '/');
	dir_len = str_len(full_path);

	errno = 0;
	while ((d = readdir(dir)) != NULL) {
		if (d->d_name[0] == '.') {
			/* skip . and .. */
			if (d->d_name[1] == '\0')
				continue;
			if (d->d_name[1] == '.' && d->d_name[2] == '\0')
				continue;
		}

		str_truncate(full_path, dir_len);
		str_append(full_path, d->d_name);

		/* trying to unlink() a directory gives either EPERM or EISDIR
		   (non-POSIX). it doesn't really work anywhere in practise,
		   so don't bother stat()ing the file first */
		if (unlink(str_c(full_path)) == 0)
			unlinked_something = TRUE;
		else if (errno != ENOENT && errno != EISDIR && errno != EPERM) {
			mailbox_list_set_critical(list,
				"unlink_directory(%s) failed: %m",
				str_c(full_path));
		}
	}

	if (closedir(dir) < 0) {
		mailbox_list_set_critical(list, "closedir(%s) failed: %m",
					  path);
	}

	if (rmdir(path) == 0)
		unlinked_something = TRUE;
	else if (errno != ENOENT && errno != ENOTEMPTY) {
		mailbox_list_set_critical(list, "rmdir(%s) failed: %m", path);
		return -1;
	}

	if (!unlinked_something) {
		mailbox_list_set_error(list, t_strdup_printf(
			"Directory %s isn't empty, can't delete it.", name));
		return -1;
	}
	return 0;
}

static int
cydir_list_delete_mailbox(struct mailbox_list *list, const char *name)
{
	struct cydir_storage *storage = CYDIR_LIST_CONTEXT(list);
	struct stat st;
	const char *src;

	/* Make sure the indexes are closed before trying to delete the
	   directory that contains them. It can still fail with some NFS
	   implementations if indexes are opened by another session, but
	   that can't really be helped. */
	index_storage_destroy_unrefed();

	/* delete the index and control directories */
	if (storage->list_module_ctx.super.delete_mailbox(list, name) < 0)
		return -1;

	/* check if the mailbox actually exists */
	src = mailbox_list_get_path(list, name, MAILBOX_LIST_PATH_TYPE_MAILBOX);
	if (stat(src, &st) != 0 && errno == ENOENT) {
		mailbox_list_set_error(list, t_strdup_printf(
			MAILBOX_LIST_ERR_MAILBOX_NOT_FOUND, name));
		return -1;
	}

	return cydir_delete_nonrecursive(list, src, name);
}

static int cydir_storage_close(struct mailbox *box)
{
        index_storage_mailbox_free(box);
	return 0;
}

static void
cydir_notify_changes(struct mailbox *box, unsigned int min_interval,
		     mailbox_notify_callback_t *callback, void *context)
{
	struct cydir_mailbox *mbox = (struct cydir_mailbox *)box;

	mbox->ibox.min_notify_interval = min_interval;
	mbox->ibox.notify_callback = callback;
	mbox->ibox.notify_context = context;

	if (callback == NULL) {
		index_mailbox_check_remove_all(&mbox->ibox);
		return;
	}

	index_mailbox_check_add(&mbox->ibox, mbox->path);
}

static int cydir_list_iter_is_mailbox(struct mailbox_list_iterate_context *ctx,
				      const char *dir, const char *fname,
				      enum mailbox_list_file_type type,
				      enum mailbox_info_flags *flags)
{
	const char *mail_path;
	struct stat st;
	int ret = 1;

	if (strchr(fname, '.') != NULL) {
		*flags = MAILBOX_NOSELECT;
		return 0;
	}

	/* try to avoid stat() with these checks */
	if (type != MAILBOX_LIST_FILE_TYPE_DIR &&
	    type != MAILBOX_LIST_FILE_TYPE_SYMLINK &&
	    type != MAILBOX_LIST_FILE_TYPE_UNKNOWN &&
	    (ctx->flags & MAILBOX_LIST_ITER_FAST_FLAGS) != 0) {
		/* it's a file */
		*flags |= MAILBOX_NOSELECT | MAILBOX_NOINFERIORS;
		return 0;
	}

	/* need to stat() then */
	t_push();
	mail_path = t_strconcat(dir, "/", fname, NULL);

	if (stat(mail_path, &st) == 0) {
		if (!S_ISDIR(st.st_mode)) {
			/* non-directory */
			*flags |= MAILBOX_NOSELECT | MAILBOX_NOINFERIORS;
			ret = 0;
		}
	} else {
		/* non-selectable, but may contain subdirs */
		*flags |= MAILBOX_NOSELECT;
	}
	t_pop();

	return ret;
}

static void cydir_class_init(void)
{
	cydir_transaction_class_init();
}

static void cydir_class_deinit(void)
{
	cydir_transaction_class_deinit();
}

struct mail_storage cydir_storage = {
	MEMBER(name) CYDIR_STORAGE_NAME,
	MEMBER(mailbox_is_file) FALSE,

	{
		cydir_class_init,
		cydir_class_deinit,
		cydir_create,
		cydir_free,
		cydir_autodetect,
		index_storage_set_callbacks,
		cydir_mailbox_open,
		cydir_mailbox_create,
		index_storage_get_last_error
	}
};

struct mailbox cydir_mailbox = {
	MEMBER(name) NULL, 
	MEMBER(storage) NULL, 

	{
		index_storage_is_readonly,
		index_storage_allow_new_keywords,
		cydir_storage_close,
		index_storage_get_status,
		cydir_storage_sync_init,
		index_mailbox_sync_next,
		index_mailbox_sync_deinit,
		cydir_notify_changes,
		index_transaction_begin,
		index_transaction_commit,
		index_transaction_rollback,
		index_keywords_create,
		index_keywords_free,
		index_storage_get_uids,
		index_mail_alloc,
		index_header_lookup_init,
		index_header_lookup_deinit,
		index_storage_search_init,
		index_storage_search_deinit,
		index_storage_search_next_nonblock,
		index_storage_search_next_update_seq,
		cydir_save_init,
		cydir_save_continue,
		cydir_save_finish,
		cydir_save_cancel,
		mail_storage_copy,
		index_storage_is_inconsistent
	}
};

--- NEW FILE: cydir-storage.h ---
#ifndef __CYDIR_STORAGE_H
#define __CYDIR_STORAGE_H

#include "index-storage.h"
#include "mailbox-list-private.h"

#define CYDIR_STORAGE_NAME "cydir"
#define CYDIR_SUBSCRIPTION_FILE_NAME "subscriptions."
#define CYDIR_INDEX_PREFIX "dovecot.index"

#define STORAGE(mbox_storage) \
	(&(mbox_storage)->storage.storage)
#define INDEX_STORAGE(mbox_storage) \
	(&(mbox_storage)->storage)

struct cydir_storage {
	struct index_storage storage;
	union mailbox_list_module_context list_module_ctx;
};

struct cydir_mailbox {
	struct index_mailbox ibox;
	struct cydir_storage *storage;

	const char *path;
};

struct cydir_transaction_context {
	struct index_transaction_context ictx;
	union mail_index_transaction_module_context module_ctx;

	uint32_t first_saved_mail_seq;
	struct cydir_save_context *save_ctx;
};

extern struct mail_vfuncs cydir_mail_vfuncs;

void cydir_transaction_created(struct mail_index_transaction *t);
void cydir_transaction_class_init(void);
void cydir_transaction_class_deinit(void);

int cydir_save_init(struct mailbox_transaction_context *_t,
		    enum mail_flags flags, struct mail_keywords *keywords,
		    time_t received_date, int timezone_offset,
		    const char *from_envelope, struct istream *input,
		    struct mail *dest_mail, struct mail_save_context **ctx_r);
int cydir_save_continue(struct mail_save_context *ctx);
int cydir_save_finish(struct mail_save_context *ctx);
void cydir_save_cancel(struct mail_save_context *ctx);

int cydir_transaction_save_commit_pre(struct cydir_save_context *ctx);
void cydir_transaction_save_commit_post(struct cydir_save_context *ctx);
void cydir_transaction_save_rollback(struct cydir_save_context *ctx);

#endif

--- NEW FILE: cydir-sync.c ---
/* Copyright (C) 2007 Timo Sirainen */

#include "lib.h"
#include "ioloop.h"
#include "str.h"
#include "cydir-storage.h"
#include "cydir-sync.h"

static int cydir_sync_set_uidvalidity(struct cydir_sync_context *ctx)
{
	struct mail_index_transaction *trans;
	uint32_t uid_validity = ioloop_time;
	uint32_t seq;
	uoff_t offset;

	trans = mail_index_transaction_begin(ctx->sync_view, FALSE, TRUE);
	mail_index_update_header(trans,
		offsetof(struct mail_index_header, uid_validity),
		&uid_validity, sizeof(uid_validity), TRUE);

	if (mail_index_transaction_commit(&trans, &seq, &offset) < 0) {
		mail_storage_set_index_error(&ctx->mbox->ibox);
		return -1;
	}
	return 0;
}

static string_t *cydir_get_path_prefix(struct cydir_mailbox *mbox)
{
	string_t *path = t_str_new(256);
	const char *dir;

	dir = mailbox_list_get_path(STORAGE(mbox->storage)->list,
				    mbox->ibox.box.name,
				    MAILBOX_LIST_PATH_TYPE_MAILBOX);
	str_append(path, dir);
	str_append_c(path, '/');
	return path;
}

static int cydir_sync_index(struct cydir_sync_context *ctx)
{
	const struct mail_index_header *hdr;
	struct mail_index_sync_rec sync_rec;
	string_t *path = NULL;
	unsigned int prefix_len = 0;
	uint32_t seq1, seq2, uid;
	int ret;

	hdr = mail_index_get_header(ctx->sync_view);
	if (hdr->uid_validity == 0) {
		if (cydir_sync_set_uidvalidity(ctx) < 0)
			return -1;
	}

	/* unlink expunged messages */
	while ((ret = mail_index_sync_next(ctx->index_sync_ctx,
					   &sync_rec)) > 0) {
		if (sync_rec.type != MAIL_INDEX_SYNC_TYPE_EXPUNGE)
			continue;

		if (mail_index_lookup_uid_range(ctx->sync_view,
						sync_rec.uid1, sync_rec.uid2,
						&seq1, &seq2) < 0) {
			mail_storage_set_index_error(&ctx->mbox->ibox);
			return -1;
		}
		if (seq1 == 0) {
			/* already expunged everything. nothing to do. */
			continue;
		}

		if (path == NULL) {
			path = cydir_get_path_prefix(ctx->mbox);
			prefix_len = str_len(path);
		}

		for (; seq1 <= seq2; seq1++) {
			if (mail_index_lookup_uid(ctx->sync_view, seq1,
						  &uid) < 0) {
				mail_storage_set_index_error(&ctx->mbox->ibox);
				return -1;
			}

			str_truncate(path, prefix_len);
			str_printfa(path, "%u.", uid);
			if (unlink(str_c(path)) < 0 && errno != ENOENT) {
				mail_storage_set_critical(
					STORAGE(ctx->mbox->storage),
					"unlink(%s) failed: %m", str_c(path));
				/* continue anyway */
			}
		}
	}
	return 0;
}

int cydir_sync_begin(struct cydir_mailbox *mbox,
		     struct cydir_sync_context **ctx_r)
{
	struct cydir_sync_context *ctx;
	int ret;

	ctx = i_new(struct cydir_sync_context, 1);
	ctx->mbox = mbox;
	ret = mail_index_sync_begin(mbox->ibox.index, &ctx->index_sync_ctx,
				    &ctx->sync_view, (uint32_t)-1, (uoff_t)-1,
				    !mbox->ibox.keep_recent, TRUE);
	if (ret <= 0) {
		if (ret < 0)
			mail_storage_set_index_error(&mbox->ibox);
		i_free(ctx);
		return ret;
	}

	if (cydir_sync_index(ctx) < 0) {
		mail_index_sync_rollback(&ctx->index_sync_ctx);
		i_free(ctx);
		return -1;
	}

	*ctx_r = ctx;
	return 0;
}

int cydir_sync_finish(struct cydir_sync_context **_ctx, bool success)
{
	struct cydir_sync_context *ctx = *_ctx;
	int ret = success ? 0 : -1;

	*_ctx = NULL;
	if (success) {
		if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) {
			mail_storage_set_index_error(&ctx->mbox->ibox);
			ret = -1;
		}
	} else {
		mail_index_sync_rollback(&ctx->index_sync_ctx);
	}
	i_free(ctx);
	return 0;
}

int cydir_sync(struct cydir_mailbox *mbox)
{
	struct cydir_sync_context *sync_ctx;

	if (cydir_sync_begin(mbox, &sync_ctx) < 0)
		return -1;

	return cydir_sync_finish(&sync_ctx, TRUE);
}

struct mailbox_sync_context *
cydir_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
{
	struct cydir_mailbox *mbox = (struct cydir_mailbox *)box;
	int ret = 0;

	if (!box->opened)
		index_storage_mailbox_open(&mbox->ibox);

	if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 ||
	    mbox->ibox.sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <=
	    ioloop_time)
		ret = cydir_sync(mbox);

	return index_mailbox_sync_init(box, flags, ret < 0);
}

--- NEW FILE: cydir-sync.h ---
#ifndef __CYDIR_SYNC_H
#define __CYDIR_SYNC_H

enum mailbox_sync_flags;
struct mailbox;

struct cydir_sync_context {
	struct cydir_mailbox *mbox;
        struct mail_index_sync_ctx *index_sync_ctx;
	struct mail_index_view *sync_view;
};

int cydir_sync_begin(struct cydir_mailbox *mbox,
		     struct cydir_sync_context **ctx_r);
int cydir_sync_finish(struct cydir_sync_context **ctx, bool success);
int cydir_sync(struct cydir_mailbox *mbox);

struct mailbox_sync_context *
cydir_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags);

#endif

--- NEW FILE: cydir-transaction.c ---
/* Copyright (C) 2007 Timo Sirainen */

#include "lib.h"
#include "array.h"
#include "cydir-sync.h"
#include "cydir-storage.h"

static void (*next_hook_mail_index_transaction_created)
	(struct mail_index_transaction *t) = NULL;

static int cydir_transaction_commit(struct mail_index_transaction *t,
				    uint32_t *log_file_seq_r,
				    uoff_t *log_file_offset_r)
{
	struct cydir_transaction_context *dt = MAIL_STORAGE_CONTEXT(t);
	struct cydir_mailbox *mbox = (struct cydir_mailbox *)dt->ictx.ibox;
	struct cydir_save_context *save_ctx;
	bool syncing = t->sync_transaction;
	int ret = 0;

	if (dt->save_ctx != NULL) {
		if (cydir_transaction_save_commit_pre(dt->save_ctx) < 0) {
			dt->save_ctx = NULL;
			ret = -1;
		}
	}

	save_ctx = dt->save_ctx;

	if (ret < 0)
		index_transaction_finish_rollback(&dt->ictx);
	else {
		if (index_transaction_finish_commit(&dt->ictx, log_file_seq_r,
						    log_file_offset_r) < 0)
			ret = -1;
	}

	/* transaction is destroyed now. */
	dt = NULL;

	if (save_ctx != NULL) {
		/* unlock uidlist file after writing to transaction log,
		   to make sure we don't write uids in wrong order. */
		cydir_transaction_save_commit_post(save_ctx);
	}

	if (ret == 0 && !syncing) {
		if (cydir_sync(mbox) < 0)
			ret = -1;
	}

	return ret;
}

static void cydir_transaction_rollback(struct mail_index_transaction *t)
{
	struct cydir_transaction_context *dt = MAIL_STORAGE_CONTEXT(t);

	if (dt->save_ctx != NULL)
		cydir_transaction_save_rollback(dt->save_ctx);

	index_transaction_finish_rollback(&dt->ictx);
}

void cydir_transaction_created(struct mail_index_transaction *t)
{
	struct mailbox *box = MAIL_STORAGE_CONTEXT(t->view->index);

	/* index can be for mailbox list index, in which case box=NULL */
	if (box != NULL &&
	    strcmp(box->storage->name, CYDIR_STORAGE_NAME) == 0) {
		struct cydir_mailbox *cydir = (struct cydir_mailbox *)box;
		struct cydir_transaction_context *mt;

		mt = i_new(struct cydir_transaction_context, 1);
		mt->ictx.trans = t;
		mt->ictx.super = t->v;

		t->v.commit = cydir_transaction_commit;
		t->v.rollback = cydir_transaction_rollback;
		MODULE_CONTEXT_SET(t, mail_storage_mail_index_module, mt);

		index_transaction_init(&mt->ictx, &cydir->ibox);
	}

	if (next_hook_mail_index_transaction_created != NULL)
		next_hook_mail_index_transaction_created(t);
}

void cydir_transaction_class_init(void)
{
	next_hook_mail_index_transaction_created =
		hook_mail_index_transaction_created;
	hook_mail_index_transaction_created = cydir_transaction_created;
}

void cydir_transaction_class_deinit(void)
{
	i_assert(hook_mail_index_transaction_created ==
		 cydir_transaction_created);
	hook_mail_index_transaction_created =
		next_hook_mail_index_transaction_created;
}