[dovecot-cvs] dovecot/src/lib-index Makefile.am, 1.23, 1.24 mail-index-private.h, 1.44, 1.45 mail-index-sync-keywords.c, NONE, 1.1 mail-index-sync-private.h, 1.17, 1.18 mail-index-sync-update.c, 1.77, 1.78 mail-index-sync.c, 1.44, 1.45 mail-index-transaction-private.h, 1.18, 1.19 mail-index-transaction.c, 1.36, 1.37 mail-index-view-sync.c, 1.33, 1.34 mail-index-view.c, 1.31, 1.32 mail-index.c, 1.183, 1.184 mail-index.h, 1.139, 1.140 mail-transaction-log.c, 1.78, 1.79 mail-transaction-log.h, 1.21, 1.22 mail-transaction-util.c, 1.21, 1.22

[cras at dovecot.org]

Update of /var/lib/cvs/dovecot/src/lib-index
In directory talvi:/tmp/cvs-serv26125/lib-index

Modified Files:
	Makefile.am mail-index-private.h mail-index-sync-private.h 
	mail-index-sync-update.c mail-index-sync.c 
	mail-index-transaction-private.h mail-index-transaction.c 
	mail-index-view-sync.c mail-index-view.c mail-index.c 
	mail-index.h mail-transaction-log.c mail-transaction-log.h 
	mail-transaction-util.c 
Added Files:
	mail-index-sync-keywords.c 
Log Message:
Initial support for keywords. Syncing to mbox/maildir doesn't work yet.



Index: Makefile.am
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/Makefile.am,v
retrieving revision 1.23
retrieving revision 1.24
diff -u -d -r1.23 -r1.24
--- Makefile.am	7 Dec 2004 03:59:20 -0000	1.23
+++ Makefile.am	26 Dec 2004 09:12:40 -0000	1.24
@@ -19,6 +19,7 @@
         mail-index-transaction-view.c \
         mail-index-sync.c \
         mail-index-sync-ext.c \
+        mail-index-sync-keywords.c \
         mail-index-sync-update.c \
         mail-index-view.c \
         mail-index-view-sync.c \

Index: mail-index-private.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-private.h,v
retrieving revision 1.44
retrieving revision 1.45
diff -u -d -r1.44 -r1.45
--- mail-index-private.h	5 Dec 2004 04:10:43 -0000	1.44
+++ mail-index-private.h	26 Dec 2004 09:12:40 -0000	1.45
@@ -60,6 +60,23 @@
 	/* unsigned char name[] */
 };
 
+struct mail_keywords {
+	unsigned int count;
+	const char **keywords;
+        struct mail_index_keyword_transaction *transaction;
+};
+
+struct mail_index_keyword_header {
+	uint32_t keywords_count;
+	/* struct mail_index_keyword_header_rec[] */
+	/* char name[][] */
+};
+
+struct mail_index_keyword_header_rec {
+	uint32_t count;
+	uint32_t name_offset; /* relative to beginning of name[] */
+};
+
 struct mail_index_map {
 	int refcount;
 
@@ -78,6 +95,10 @@
 	buffer_t *buffer;
 	buffer_t *hdr_copy_buf;
 
+	pool_t keywords_pool;
+	const char *const *keywords;
+	unsigned int keywords_count;
+
 	unsigned int write_to_disk:1;
 };
 
@@ -108,13 +129,14 @@
 	unsigned int lock_id;
 	char *copy_lock_path;
 	struct dotlock dotlock;
-        enum mail_index_lock_method lock_method;
+	enum mail_index_lock_method lock_method;
 
 	/* These are typically same as map->hdr->log_file_*, but with
 	   mmap_disable we may have synced more than index */
 	uint32_t sync_log_file_seq;
 	uoff_t sync_log_file_offset;
 
+	uint32_t keywords_ext_id;
 	unsigned int last_grow_count;
 
 	char *error;
@@ -196,6 +218,9 @@
 const struct mail_index_ext *
 mail_index_view_get_ext(struct mail_index_view *view, uint32_t ext_id);
 
+int mail_index_map_read_keywords(struct mail_index *index,
+				 struct mail_index_map *map);
+
 int mail_index_fix_header(struct mail_index *index, struct mail_index_map *map,
 			  struct mail_index_header *hdr, const char **error_r);
 

--- NEW FILE: mail-index-sync-keywords.c ---
/* Copyright (C) 2004 Timo Sirainen */

#include "lib.h"
#include "buffer.h"
#include "mail-index-view-private.h"
#include "mail-index-sync-private.h"
#include "mail-transaction-log.h"

static const char *const *
keywords_get_from_header(const struct mail_transaction_keyword_update *rec,
			 size_t size, const uint32_t **uid_r)
{
	buffer_t *buf;
	const unsigned char *p, *end;
	const char *name;
	uint32_t i, diff;

	buf = buffer_create_static_hard(pool_datastack_create(),
					(rec->keywords_count + 1) *
					sizeof(const char *));
	p = (const unsigned char *)(rec->name_size + rec->keywords_count);
	end = CONST_PTR_OFFSET(rec, size);

	if (p >= end)
		return NULL;

	for (i = 0; i < rec->keywords_count; i++) {
		if (p + rec->name_size[i] >= end)
			return NULL;

		name = t_strndup(p, rec->name_size[i]);
		buffer_append(buf, &name, sizeof(name));
		p += rec->name_size[i];
	}

	diff = (p - (const unsigned char *)rec) % 4;
	if (diff != 0)
		p += 4 - diff;

	*uid_r = (const uint32_t *)p;

	name = NULL;
	buffer_append(buf, &name, sizeof(name));
	return buf->data;
}

static int keywords_get_missing(struct mail_index *index,
				const char *const *keywords,
                                const char *const **missing_r)
{
	struct mail_index_map *map = index->map;
	const char *name;
	buffer_t *missing_buf;
	unsigned int i;

	if (mail_index_map_read_keywords(index, map) < 0)
		return -1;

        missing_buf = buffer_create_dynamic(pool_datastack_create(), 64);
	for (; *keywords != NULL; keywords++) {
		for (i = 0; i < map->keywords_count; i++) {
			if (strcmp(map->keywords[i], *keywords) == 0)
				break;
		}
		if (i == map->keywords_count)
			buffer_append(missing_buf, keywords, sizeof(*keywords));
	}

	if (missing_buf->used == 0)
		*missing_r = NULL;
	else {
		name = NULL;
		buffer_append(missing_buf, &name, sizeof(name));
		*missing_r = missing_buf->data;
	}
	return 0;
}

static buffer_t *
keywords_get_header_buf(struct mail_index_map *map,
			const struct mail_index_ext *ext,
			unsigned int new_count, unsigned int *keywords_count_r,
			size_t *rec_offset_r, size_t *name_offset_root_r,
			size_t *name_offset_r)
{
	buffer_t *buf;
	const struct mail_index_keyword_header *kw_hdr;
	const struct mail_index_keyword_header_rec *kw_rec;
	const char *name;
	struct mail_index_keyword_header new_kw_hdr;
	uint32_t offset;

	kw_hdr = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset);
	kw_rec = (const void *)(kw_hdr + 1);
	name = (const char *)(kw_rec + kw_hdr->keywords_count);

	if (kw_hdr->keywords_count == 0)
		return NULL;

	new_kw_hdr = *kw_hdr;
	new_kw_hdr.keywords_count += new_count;
	*keywords_count_r = new_kw_hdr.keywords_count;

	offset = kw_rec[kw_hdr->keywords_count-1].name_offset;
	offset += strlen(name + offset) + 1;

	buf = buffer_create_dynamic(pool_datastack_create(), 512);
	buffer_append(buf, &new_kw_hdr, sizeof(new_kw_hdr));
	buffer_append(buf, kw_rec, sizeof(*kw_rec) * kw_hdr->keywords_count);
	*rec_offset_r = buf->used;
	buffer_write(buf, buf->used + sizeof(*kw_rec) * new_count,
		     name, offset);
	*name_offset_root_r = buf->used;
	*name_offset_r = offset;
	return buf;
}

static int keywords_ext_register(struct mail_index_sync_map_ctx *ctx,
				 uint32_t ext_id, uint32_t reset_id,
				 uint32_t hdr_size, uint32_t keywords_count)
{
	buffer_t *ext_intro_buf;
	struct mail_transaction_ext_intro *u;

	ext_intro_buf =
		buffer_create_static_hard(pool_datastack_create(),
					  sizeof(*u) + sizeof("keywords")-1);

	u = buffer_append_space_unsafe(ext_intro_buf, sizeof(*u));
	u->ext_id = ext_id;
	u->reset_id = reset_id;
	u->hdr_size = hdr_size;
	u->record_size = (keywords_count + CHAR_BIT - 1) / CHAR_BIT;
	if ((u->record_size % 4) != 0) {
		/* since we aren't properly aligned anyway,
		   reserve one extra byte for future */
		u->record_size++;
	}
	u->record_align = 1;

	if (ext_id == (uint32_t)-1) {
		u->name_size = strlen("keywords");
		buffer_append(ext_intro_buf, "keywords", u->name_size);
	}

	return mail_index_sync_ext_intro(ctx, u);
}

static int
keywords_update_header(struct mail_index_sync_map_ctx *ctx,
		       const char *const *keywords)
{
	struct mail_index_map *map = ctx->view->map;
        const struct mail_index_ext *ext = NULL;
	struct mail_index_keyword_header *kw_hdr;
	struct mail_index_keyword_header_rec kw_rec;
	uint32_t ext_id;
	const char *const *missing = keywords;
	buffer_t *buf = NULL;
	size_t rec_offset, name_offset, name_offset_root;
	unsigned int keywords_count;
	int ret = 0;

	ext_id = mail_index_map_lookup_ext(map, "keywords");
	if (ext_id != (uint32_t)-1) {
		/* make sure all keywords exist in the header */
		ext = map->extensions->data;
		ext += ext_id;

		ret = keywords_get_missing(ctx->view->index,
					   keywords, &missing);
		if (ret == 0 && missing != NULL) {
			/* update existing header */
			buf = keywords_get_header_buf(map, ext,
						      strarray_length(missing),
						      &keywords_count,
						      &rec_offset,
						      &name_offset_root,
						      &name_offset);
		}
	}

	if (buf == NULL) {
		/* create new / replace broken header */
		buf = buffer_create_dynamic(pool_datastack_create(), 512);
		kw_hdr = buffer_append_space_unsafe(buf, sizeof(*kw_hdr));
		kw_hdr->keywords_count = strarray_length(missing);

                keywords_count = kw_hdr->keywords_count;
		rec_offset = buf->used;
		name_offset_root = rec_offset +
			kw_hdr->keywords_count * sizeof(kw_rec);
		name_offset = 0;
	}

	if (missing == NULL)
		return 1;

	/* missing some keywords - add them */
	memset(&kw_rec, 0, sizeof(kw_rec));
	kw_rec.name_offset = name_offset;

	for (; *missing != NULL; missing++) {
		size_t len = strlen(*missing) + 1;

		buffer_write(buf, rec_offset, &kw_rec, sizeof(kw_rec));
		buffer_write(buf, name_offset_root, *missing, len);

		rec_offset += sizeof(kw_rec);
		kw_rec.name_offset += len;
		name_offset_root += len;
	}

	if ((buf->used % 4) != 0)
		buffer_append_zero(buf, 4 - (buf->used % 4));

	if (ext == NULL || buf->used > ext->hdr_size ||
	    (uint32_t)ext->record_size * CHAR_BIT < keywords_count) {
		/* if we need to grow the buffer, add some padding */
		buffer_append_zero(buf, 128);

		ret = keywords_ext_register(ctx, ext_id,
					    ext == NULL ? 0 : ext->reset_id,
					    buf->used, keywords_count);
		if (ret <= 0)
			return ret;

		/* map may have changed */
		map = ctx->view->map;

		if (ext == NULL) {
			ext_id = mail_index_map_lookup_ext(map, "keywords");
			i_assert(ext_id != (uint32_t)-1);
		}
		ext = map->extensions->data;
		ext += ext_id;

		i_assert(ext->hdr_size == buf->used);
	}

	buffer_copy(map->hdr_copy_buf, ext->hdr_offset,
		    buf, 0, buf->used);
	map->hdr_base = map->hdr_copy_buf->data;

	return 1;
}

static const unsigned char *
keywords_make_mask(struct mail_index_map *map, const struct mail_index_ext *ext,
		   const char *const *keywords)
{
	const struct mail_index_keyword_header *kw_hdr;
	const struct mail_index_keyword_header_rec *kw_rec;
	const char *name, *const *n;
	unsigned char *mask;
	unsigned int i;

	kw_hdr = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset);
	kw_rec = (const void *)(kw_hdr + 1);
	name = (const char *)(kw_rec + kw_hdr->keywords_count);

	mask = t_malloc0(ext->record_size);

	for (i = 0; i < kw_hdr->keywords_count; i++) {
		for (n = keywords; *n != NULL; n++) {
			if (strcmp(name + kw_rec[i].name_offset, *n) == 0) {
				mask[i / CHAR_BIT] |= 1 << (i % CHAR_BIT);
				break;
			}
		}
	}

	return mask;
}

static int
keywords_update_records(struct mail_index_view *view,
			const struct mail_index_ext *ext,
			const unsigned char *mask,
			enum modify_type type,
			uint32_t uid1, uint32_t uid2)
{
	struct mail_index_record *rec;
	unsigned char *data;
	uint32_t seq1, seq2;
	unsigned int i;

	if (mail_index_lookup_uid_range(view, uid1, uid2, &seq1, &seq2) < 0)
		return -1;

	if (seq1 == 0)
		return 1;

	for (; seq1 <= seq2; seq1++) {
		rec = MAIL_INDEX_MAP_IDX(view->map, seq1-1);
		data = PTR_OFFSET(rec, ext->record_offset);

		switch (type) {
		case MODIFY_ADD:
			for (i = 0; i < ext->record_size; i++)
				data[i] |= mask[i];
			break;
		case MODIFY_REMOVE:
		for (i = 0; i < ext->record_size; i++)
			data[i] &= ~mask[i];
		break;
		case MODIFY_REPLACE:
			memcpy(data, mask, ext->record_size);
			break;
		default:
			i_unreached();
		}
	}
	return 1;
}

int mail_index_sync_keywords(struct mail_index_sync_map_ctx *ctx,
			     const struct mail_transaction_header *hdr,
			     const struct mail_transaction_keyword_update *rec)
{
	const uint32_t *uid, *end;
	const char *const *keywords;
	const struct mail_index_ext *ext;
	const unsigned char *mask;
	uint32_t ext_id;
	int ret;

	keywords = keywords_get_from_header(rec, hdr->size, &uid);
	if (keywords == NULL) {
		mail_transaction_log_view_set_corrupted(ctx->view->log_view,
			"Keyword header ended unexpectedly");
		return -1;
	}
	end = CONST_PTR_OFFSET(rec, hdr->size);

	if (rec->modify_type != MODIFY_REMOVE) {
		ret = keywords_update_header(ctx, keywords);
		if (ret <= 0)
			return ret;
	}

	ext_id = mail_index_map_lookup_ext(ctx->view->map, "keywords");
	if (ext_id == (uint32_t)-1) {
		/* nothing to do */
		i_assert(rec->modify_type == MODIFY_REMOVE);
		return 1;
	}

	ext = ctx->view->map->extensions->data;
	ext += ext_id;

	mask = keywords_make_mask(ctx->view->map, ext, keywords);

	while (uid+2 <= end) {
		ret = keywords_update_records(ctx->view, ext, mask,
					      rec->modify_type,
					      uid[0], uid[1]);
		if (ret <= 0)
			return ret;

		uid += 2;
	}

	return 1;
}

Index: mail-index-sync-private.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-sync-private.h,v
retrieving revision 1.17
retrieving revision 1.18
diff -u -d -r1.17 -r1.18
--- mail-index-sync-private.h	7 Dec 2004 03:59:20 -0000	1.17
+++ mail-index-sync-private.h	26 Dec 2004 09:12:40 -0000	1.18
@@ -90,4 +90,8 @@
 mail_index_sync_ext_rec_update(struct mail_index_sync_map_ctx *ctx,
 			       const struct mail_transaction_ext_rec_update *u);
 
+int mail_index_sync_keywords(struct mail_index_sync_map_ctx *ctx,
+			     const struct mail_transaction_header *hdr,
+			     const struct mail_transaction_keyword_update *rec);
+
 #endif

Index: mail-index-sync-update.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-sync-update.c,v
retrieving revision 1.77
retrieving revision 1.78
diff -u -d -r1.77 -r1.78
--- mail-index-sync-update.c	20 Dec 2004 07:51:40 -0000	1.77
+++ mail-index-sync-update.c	26 Dec 2004 09:12:40 -0000	1.78
@@ -206,9 +206,7 @@
 	struct mail_index_header *hdr;
 	struct mail_index_record *rec;
 	uint8_t flag_mask, old_flags;
-	keywords_mask_t keyword_mask;
-	uint32_t i, idx, seq1, seq2;
-	int update_keywords;
+	uint32_t idx, seq1, seq2;
 
 	if (u->uid1 > u->uid2 || u->uid1 == 0) {
 		mail_transaction_log_view_set_corrupted(ctx->view->log_view,
@@ -228,13 +226,6 @@
 	if ((u->add_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)
 		hdr->flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
 
-	update_keywords = FALSE;
-	for (i = 0; i < INDEX_KEYWORDS_BYTE_COUNT; i++) {
-		if (u->add_keywords[i] != 0 ||
-		    u->remove_keywords[i] != 0)
-			update_keywords = TRUE;
-		keyword_mask[i] = ~u->remove_keywords[i];
-	}
         flag_mask = ~u->remove_flags;
 
 	for (idx = seq1-1; idx < seq2; idx++) {
@@ -242,12 +233,6 @@
 
 		old_flags = rec->flags;
 		rec->flags = (rec->flags & flag_mask) | u->add_flags;
-		if (update_keywords) {
-			for (i = 0; i < INDEX_KEYWORDS_BYTE_COUNT; i++) {
-				rec->keywords[i] = u->add_keywords[i] |
-					(rec->keywords[i] & keyword_mask[i]);
-			}
-		}
 
 		mail_index_header_update_counts(hdr, old_flags, rec->flags);
                 mail_index_header_update_lowwaters(hdr, rec);
@@ -381,6 +366,7 @@
 {
 	int ret = 0;
 
+	t_push();
 	switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
 	case MAIL_TRANSACTION_APPEND: {
 		const struct mail_index_record *rec, *end;
@@ -507,9 +493,16 @@
 		}
 		break;
 	}
+	case MAIL_TRANSACTION_KEYWORD_UPDATE: {
+		const struct mail_transaction_keyword_update *rec = data;
+
+		ret = mail_index_sync_keywords(ctx, hdr, rec);
+		break;
+	}
 	default:
 		i_unreached();
 	}
+	t_pop();
 
 	i_assert(ctx->view->map->records_count ==
 		 ctx->view->map->hdr.messages_count);

Index: mail-index-sync.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-sync.c,v
retrieving revision 1.44
retrieving revision 1.45
diff -u -d -r1.44 -r1.45
--- mail-index-sync.c	13 Dec 2004 00:55:42 -0000	1.44
+++ mail-index-sync.c	26 Dec 2004 09:12:40 -0000	1.45
@@ -19,7 +19,6 @@
 	struct mail_transaction_flag_update *dest;
 	struct mail_transaction_flag_update new_update, tmp_update;
 	size_t i, dest_count;
-	int j;
 
 	dest = buffer_get_modifyable_data(dest_buf, &dest_count);
 	dest_count /= sizeof(*dest);
@@ -91,17 +90,6 @@
 			dest[i].add_flags &= ~new_update.remove_flags;
 			dest[i].remove_flags |= new_update.remove_flags;
 			dest[i].remove_flags &= ~new_update.add_flags;
-
-			for (j = 0; j < INDEX_KEYWORDS_BYTE_COUNT; j++) {
-				dest[i].add_keywords[j] |=
-					new_update.add_keywords[j];
-				dest[i].add_keywords[j] &=
-					~new_update.remove_keywords[j];
-				dest[i].remove_keywords[j] |=
-					new_update.remove_keywords[j];
-				dest[i].remove_keywords[j] &=
-					~new_update.add_keywords[j];
-			}
 		}
 
 		if (new_update.uid1 <= new_update.uid2) {
@@ -147,7 +135,6 @@
 	struct mail_transaction_flag_update update;
 	const struct mail_index_record *rec;
 	uint32_t seq, messages_count;
-	int i;
 
 	memset(&update, 0, sizeof(update));
 
@@ -162,10 +149,6 @@
 		update.uid1 = update.uid2 = rec->uid;
 		update.add_flags = rec->flags;
 		update.remove_flags = ~update.add_flags;
-		memcpy(update.add_keywords, rec->keywords,
-		       INDEX_KEYWORDS_BYTE_COUNT);
-		for (i = 0; i < INDEX_KEYWORDS_BYTE_COUNT; i++)
-			update.remove_keywords[i] = ~update.add_keywords[i];
 
 		mail_index_sync_sort_flags(ctx->updates_buf,
 					   &update, sizeof(update));
@@ -426,11 +409,7 @@
 	rec->uid2 = update->uid2;
 
 	rec->add_flags = update->add_flags;
-	memcpy(rec->add_keywords, update->add_keywords,
-	       sizeof(rec->add_keywords));
 	rec->remove_flags = update->remove_flags;
-	memcpy(rec->remove_keywords, update->remove_keywords,
-	       sizeof(rec->remove_keywords));
 }
 
 static int mail_index_sync_rec_check(struct mail_index_view *view,
@@ -439,6 +418,7 @@
 	switch (rec->type) {
 	case MAIL_INDEX_SYNC_TYPE_EXPUNGE:
 	case MAIL_INDEX_SYNC_TYPE_FLAGS:
+	case MAIL_INDEX_SYNC_TYPE_KEYWORDS:
 		if (rec->uid1 > rec->uid2 || rec->uid1 == 0) {
 			mail_transaction_log_view_set_corrupted(view->log_view,
 				"Broken UID range: %u..%u (type 0x%x)",
@@ -585,15 +565,9 @@
 }
 
 void mail_index_sync_flags_apply(const struct mail_index_sync_rec *sync_rec,
-				 uint8_t *flags, keywords_mask_t keywords)
+				 uint8_t *flags)
 {
-	int i;
-
 	i_assert(sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS);
 
 	*flags = (*flags & ~sync_rec->remove_flags) | sync_rec->add_flags;
-	for (i = 0; i < INDEX_KEYWORDS_BYTE_COUNT; i++) {
-		keywords[i] = (keywords[i] & ~sync_rec->remove_keywords[i]) |
-			sync_rec->add_keywords[i];
-	}
 }

Index: mail-index-transaction-private.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-transaction-private.h,v
retrieving revision 1.18
retrieving revision 1.19
diff -u -d -r1.18 -r1.19
--- mail-index-transaction-private.h	13 Nov 2004 23:08:07 -0000	1.18
+++ mail-index-transaction-private.h	26 Dec 2004 09:12:40 -0000	1.19
@@ -3,6 +3,17 @@
 
 #include "mail-transaction-log.h"
 
+struct mail_index_keyword_transaction {
+	struct mail_keywords keywords;
+	pool_t pool;
+	enum modify_type modify_type;
+	buffer_t *messages;
+
+	/* mail_keywords points to first mail_index_keyword_transaction.
+	   this points to next transaction using the same keywords */
+        struct mail_index_keyword_transaction *next;
+};
+
 struct mail_index_transaction {
 	int refcount;
 	struct mail_index_view *view;
@@ -22,6 +33,7 @@
 	buffer_t *ext_rec_updates; /* buffer[] */
 	buffer_t *ext_resizes; /* struct mail_transaction_ext_intro[] */
 	buffer_t *ext_resets; /* uint32_t[] */
+	buffer_t *keyword_updates; /* struct mail_index_keyword_transaction[] */
 
         struct mail_cache_transaction_ctx *cache_trans_ctx;
 

Index: mail-index-transaction.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-transaction.c,v
retrieving revision 1.36
retrieving revision 1.37
diff -u -d -r1.36 -r1.37
--- mail-index-transaction.c	28 Nov 2004 23:19:53 -0000	1.36
+++ mail-index-transaction.c	26 Dec 2004 09:12:40 -0000	1.37
@@ -12,6 +12,7 @@
 #include "mail-index-transaction-private.h"
 
 #include <stddef.h>
+#include <stdlib.h>
 
 static void mail_index_transaction_add_last(struct mail_index_transaction *t);
 
@@ -58,6 +59,17 @@
 		buffer_free(t->ext_rec_updates);
 	}
 
+	if (t->keyword_updates != NULL) {
+		struct mail_index_keyword_transaction *kt;
+
+		kt = buffer_get_modifyable_data(t->keyword_updates, &size);
+		size /= sizeof(*kt);
+
+		for (i = 0; i < size; i++)
+			pool_unref(kt[i].pool);
+		buffer_free(t->keyword_updates);
+	}
+
 	if (t->appends != NULL)
 		buffer_free(t->appends);
 	if (t->expunges != NULL)
@@ -141,6 +153,21 @@
 		}
 	}
 
+	if (t->keyword_updates != NULL) {
+		struct mail_index_keyword_transaction *kt;
+
+		kt = buffer_get_modifyable_data(t->keyword_updates, &size);
+		size /= sizeof(*kt);
+
+		for (i = 0; i < size; i++) {
+			if (kt[i].messages == NULL)
+				continue;
+
+			mail_index_buffer_convert_to_uids(t, kt[i].messages,
+				sizeof(uint32_t) * 2, TRUE);
+		}
+	}
+
 	mail_index_buffer_convert_to_uids(t, t->expunges,
 		sizeof(struct mail_transaction_expunge), TRUE);
 	mail_index_buffer_convert_to_uids(t, t->updates,
@@ -249,46 +276,39 @@
 	*next_uid_r = first_uid;
 }
 
-void mail_index_expunge(struct mail_index_transaction *t, uint32_t seq)
+struct seq_range {
+	uint32_t seq1, seq2;
+};
+
+static void mail_index_update_seq_range_buffer(buffer_t *buffer, uint32_t seq)
 {
-        struct mail_transaction_expunge exp, *data;
+        struct seq_range *data, value;
 	unsigned int idx, left_idx, right_idx;
 	size_t size;
 
-	i_assert(seq > 0 && seq <= mail_index_view_get_messages_count(t->view));
-
-	t->log_updates = TRUE;
-	exp.uid1 = exp.uid2 = seq;
-
-	/* expunges is a sorted array of {seq1, seq2, ..}, .. */
-
-	if (t->expunges == NULL) {
-		t->expunges = buffer_create_dynamic(default_pool, 1024);
-		buffer_append(t->expunges, &exp, sizeof(exp));
-		return;
-	}
+	value.seq1 = value.seq2 = seq;
 
-	data = buffer_get_modifyable_data(t->expunges, &size);
+	data = buffer_get_modifyable_data(buffer, &size);
 	size /= sizeof(*data);
 	i_assert(size > 0);
 
 	/* quick checks */
-	if (data[size-1].uid2 == seq-1) {
+	if (data[size-1].seq2 == seq-1) {
 		/* grow last range */
-		data[size-1].uid2 = seq;
+		data[size-1].seq2 = seq;
 		return;
 	}
-	if (data[size-1].uid2 < seq) {
-		buffer_append(t->expunges, &exp, sizeof(exp));
+	if (data[size-1].seq2 < seq) {
+		buffer_append(buffer, &value, sizeof(value));
 		return;
 	}
-	if (data[0].uid1 == seq+1) {
+	if (data[0].seq1 == seq+1) {
 		/* grow down first range */
-		data[0].uid1 = seq;
+		data[0].seq1 = seq;
 		return;
 	}
-	if (data[0].uid1 > seq) {
-		buffer_insert(t->expunges, 0, &exp, sizeof(exp));
+	if (data[0].seq1 > seq) {
+		buffer_insert(buffer, 0, &value, sizeof(value));
 		return;
 	}
 
@@ -298,8 +318,8 @@
 	while (left_idx < right_idx) {
 		idx = (left_idx + right_idx) / 2;
 
-		if (data[idx].uid1 <= seq) {
-			if (data[idx].uid2 >= seq) {
+		if (data[idx].seq1 <= seq) {
+			if (data[idx].seq2 >= seq) {
 				/* it's already expunged */
 				return;
 			}
@@ -309,70 +329,77 @@
 		}
 	}
 
-	if (data[idx].uid2 < seq)
+	if (data[idx].seq2 < seq)
 		idx++;
 
         /* idx == size couldn't happen because we already handle it above */
-	i_assert(idx < size && data[idx].uid1 >= seq);
-	i_assert(data[idx].uid1 > seq || data[idx].uid2 < seq);
+	i_assert(idx < size && data[idx].seq1 >= seq);
+	i_assert(data[idx].seq1 > seq || data[idx].seq2 < seq);
 
-	if (data[idx].uid1 == seq+1) {
-		data[idx].uid1 = seq;
-		if (idx > 0 && data[idx-1].uid2 == seq-1) {
+	if (data[idx].seq1 == seq+1) {
+		data[idx].seq1 = seq;
+		if (idx > 0 && data[idx-1].seq2 == seq-1) {
 			/* merge */
-			data[idx-1].uid2 = data[idx].uid2;
-			buffer_delete(t->expunges, idx * sizeof(*data),
+			data[idx-1].seq2 = data[idx].seq2;
+			buffer_delete(buffer, idx * sizeof(*data),
 				      sizeof(*data));
 		}
-	} else if (data[idx].uid2 == seq-1) {
+	} else if (data[idx].seq2 == seq-1) {
 		i_assert(idx+1 < size); /* already handled above */
-		data[idx].uid2 = seq;
-		if (data[idx+1].uid1 == seq+1) {
+		data[idx].seq2 = seq;
+		if (data[idx+1].seq1 == seq+1) {
 			/* merge */
-			data[idx+1].uid1 = data[idx].uid1;
-			buffer_delete(t->expunges, idx * sizeof(*data),
+			data[idx+1].seq1 = data[idx].seq1;
+			buffer_delete(buffer, idx * sizeof(*data),
 				      sizeof(*data));
 		}
 	} else {
-		buffer_insert(t->expunges, idx * sizeof(*data),
-                              &exp, sizeof(exp));
+		buffer_insert(buffer, idx * sizeof(*data),
+                              &value, sizeof(value));
+	}
+}
+
+void mail_index_expunge(struct mail_index_transaction *t, uint32_t seq)
+{
+	i_assert(seq > 0 && seq <= mail_index_view_get_messages_count(t->view));
+
+	t->log_updates = TRUE;
+
+	/* expunges is a sorted array of {seq1, seq2, ..}, .. */
+	if (t->expunges == NULL) {
+		t->expunges = buffer_create_dynamic(default_pool, 1024);
+		buffer_append(t->expunges, &seq, sizeof(seq));
+		buffer_append(t->expunges, &seq, sizeof(seq));
+		return;
 	}
+
+	mail_index_update_seq_range_buffer(t->expunges, seq);
 }
 
 static void mail_index_record_modify_flags(struct mail_index_record *rec,
 					   enum modify_type modify_type,
-					   enum mail_flags flags,
-					   keywords_mask_t keywords)
+					   enum mail_flags flags)
 {
-	int i;
-
 	switch (modify_type) {
 	case MODIFY_REPLACE:
 		rec->flags = flags;
-		memcpy(rec->keywords, keywords, INDEX_KEYWORDS_BYTE_COUNT);
 		break;
 	case MODIFY_ADD:
 		rec->flags |= flags;
-		for (i = 0; i < INDEX_KEYWORDS_BYTE_COUNT; i++)
-			rec->keywords[i] |= keywords[i];
 		break;
 	case MODIFY_REMOVE:
 		rec->flags &= ~flags;
-		for (i = 0; i < INDEX_KEYWORDS_BYTE_COUNT; i++)
-			rec->keywords[i] &= ~keywords[i];
 		break;
 	}
 }
 
-#define IS_COMPATIBLE_UPDATE(t, modify_type, flags, keywords) \
+#define IS_COMPATIBLE_UPDATE(t, modify_type, flags) \
 	((t)->last_update_modify_type == (modify_type) && \
-	 (t)->last_update.add_flags == (flags) && \
-	 memcmp((t)->last_update.add_keywords, keywords, \
-	        INDEX_KEYWORDS_BYTE_COUNT) == 0)
+	 (t)->last_update.add_flags == (flags))
 
 void mail_index_update_flags(struct mail_index_transaction *t, uint32_t seq,
 			     enum modify_type modify_type,
-			     enum mail_flags flags, keywords_mask_t keywords)
+			     enum mail_flags flags)
 {
 	struct mail_index_record *rec;
 
@@ -381,8 +408,7 @@
 	if (seq >= t->first_new_seq) {
 		/* just appended message, modify it directly */
                 rec = mail_index_transaction_lookup(t, seq);
-		mail_index_record_modify_flags(rec, modify_type,
-					       flags, keywords);
+		mail_index_record_modify_flags(rec, modify_type, flags);
 		return;
 	}
 
@@ -393,13 +419,13 @@
 	   transaction (eg. 1:10 +seen, 1:10 +deleted) */
 	if (t->last_update.uid2 == seq-1) {
 		if (t->last_update.uid1 != 0 &&
-		    IS_COMPATIBLE_UPDATE(t, modify_type, flags, keywords)) {
+		    IS_COMPATIBLE_UPDATE(t, modify_type, flags)) {
 			t->last_update.uid2 = seq;
 			return;
 		}
 	} else if (t->last_update.uid1 == seq+1) {
 		if (t->last_update.uid1 != 0 &&
-		    IS_COMPATIBLE_UPDATE(t, modify_type, flags, keywords)) {
+		    IS_COMPATIBLE_UPDATE(t, modify_type, flags)) {
 			t->last_update.uid1 = seq;
 			return;
 		}
@@ -411,24 +437,18 @@
 	t->last_update_modify_type = modify_type;
 	t->last_update.uid1 = t->last_update.uid2 = seq;
 	t->last_update.add_flags = flags;
-	memcpy(t->last_update.add_keywords, keywords,
-	       INDEX_KEYWORDS_BYTE_COUNT);
 }
 
 static void
 mail_index_transaction_get_last(struct mail_index_transaction *t,
 				struct mail_transaction_flag_update *update)
 {
-	int i;
-
 	*update = t->last_update;
 	switch (t->last_update_modify_type) {
 	case MODIFY_REPLACE:
 		/* remove_flags = ~add_flags */
 		update->remove_flags =
 			~update->add_flags & MAIL_INDEX_FLAGS_MASK;
-		for (i = 0; i < INDEX_KEYWORDS_BYTE_COUNT; i++)
-			update->remove_keywords[i] = ~update->add_keywords[i];
 		break;
 	case MODIFY_ADD:
 		/* already in add_flags */
@@ -436,10 +456,7 @@
 	case MODIFY_REMOVE:
 		/* add_flags -> remove_flags */
 		update->remove_flags = update->add_flags;
-		memcpy(&update->remove_keywords, &update->add_keywords,
-		       INDEX_KEYWORDS_BYTE_COUNT);
 		update->add_flags = 0;
-		memset(&update->add_keywords, 0, INDEX_KEYWORDS_BYTE_COUNT);
 		break;
 	}
 }
@@ -698,3 +715,107 @@
 			memset(old_data_r, 0, record_size);
 	}
 }
+
+static int keywords_match(const struct mail_keywords *keywords,
+			  const char *const *list)
+{
+	unsigned int i;
+
+	for (i = 0; list[i] != NULL; i++) {
+		if (strcasecmp(keywords->keywords[i], list[i]) != 0)
+			return FALSE;
+	}
+	i_assert(i == keywords->count);
+	return TRUE;
+}
+
+static struct mail_index_keyword_transaction *
+mail_index_keyword_transaction_new(struct mail_index_transaction *t,
+				   const char *const sorted_keywords[],
+				   unsigned int count)
+{
+	struct mail_index_keyword_transaction *kt;
+	unsigned int i;
+
+	kt = buffer_append_space_unsafe(t->keyword_updates, sizeof(*kt));
+	kt->pool = pool_alloconly_create("keywords", 128);
+	kt->keywords.count = count;
+	kt->keywords.keywords = p_new(kt->pool, const char *, count + 1);
+	kt->keywords.transaction = kt;
+	for (i = 0; i < count; i++) {
+		kt->keywords.keywords[i] =
+			p_strdup(kt->pool, sorted_keywords[i]);
+	}
+
+	return kt;
+}
+
+static struct mail_index_keyword_transaction *
+mail_index_keyword_transaction_clone(struct mail_index_transaction *t,
+				     struct mail_index_keyword_transaction *kt)
+{
+	struct mail_index_keyword_transaction *new_kt;
+
+	new_kt = buffer_append_space_unsafe(t->keyword_updates, sizeof(*kt));
+	new_kt->pool = pool_alloconly_create("keywords", 128);
+	new_kt->keywords = kt->keywords;
+	return new_kt;
+}
+
+struct mail_keywords *
+mail_index_keywords_create(struct mail_index_transaction *t,
+			   const char *const keywords[])
+{
+        struct mail_index_keyword_transaction *kt;
+	const char **sorted_keywords;
+	unsigned int i, count;
+	size_t size;
+
+	count = strarray_length(keywords);
+
+	sorted_keywords = t_new(const char *, count + 1);
+	memcpy(sorted_keywords, keywords, count * sizeof(*sorted_keywords));
+	qsort(sorted_keywords, count, sizeof(*sorted_keywords), strcasecmp_p);
+
+	if (t->keyword_updates == NULL)
+                t->keyword_updates = buffer_create_dynamic(default_pool, 512);
+
+	kt = buffer_get_modifyable_data(t->keyword_updates, &size);
+	size /= sizeof(*kt);
+
+	/* try to use existing ones */
+	for (i = 0; i < size; i++) {
+		if (kt[i].keywords.count == count &&
+		    keywords_match(&kt[i].keywords, sorted_keywords))
+			return &kt[i].keywords;
+	}
+
+	kt = mail_index_keyword_transaction_new(t, sorted_keywords, count);
+	return &kt->keywords;
+}
+
+void mail_index_update_keywords(struct mail_index_transaction *t, uint32_t seq,
+				enum modify_type modify_type,
+				const struct mail_keywords *keywords)
+{
+	struct mail_index_keyword_transaction *kt;
+
+	kt = keywords->transaction;
+	while (kt->modify_type != modify_type && kt->messages != NULL) {
+		if (kt->next == NULL) {
+			kt = mail_index_keyword_transaction_clone(t, kt);
+			break;
+		}
+		kt = kt->next;
+	}
+
+	if (kt->messages == NULL) {
+		kt->messages = buffer_create_dynamic(kt->pool, 32);
+		kt->modify_type = modify_type;
+		buffer_append(kt->messages, &seq, sizeof(seq));
+		buffer_append(kt->messages, &seq, sizeof(seq));
+	} else {
+		mail_index_update_seq_range_buffer(kt->messages, seq);
+	}
+	t->log_updates = TRUE;
+}

Index: mail-index-view-sync.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-view-sync.c,v
retrieving revision 1.33
retrieving revision 1.34
diff -u -d -r1.33 -r1.34
--- mail-index-view-sync.c	5 Dec 2004 01:47:03 -0000	1.33
+++ mail-index-view-sync.c	26 Dec 2004 09:12:40 -0000	1.34
@@ -74,7 +74,7 @@
 
 #define MAIL_INDEX_VIEW_VISIBLE_SYNC_MASK \
 	(MAIL_TRANSACTION_EXPUNGE | MAIL_TRANSACTION_APPEND | \
-	 MAIL_TRANSACTION_FLAG_UPDATE)
+	 MAIL_TRANSACTION_FLAG_UPDATE | MAIL_TRANSACTION_KEYWORD_UPDATE)
 
 int mail_index_view_sync_begin(struct mail_index_view *view,
                                enum mail_index_sync_type sync_mask,
@@ -240,17 +240,14 @@
 	return 1;
 }
 
-#define FLAG_UPDATE_IS_INTERNAL(u, empty) \
+#define FLAG_UPDATE_IS_INTERNAL(u) \
 	((((u)->add_flags | (u)->remove_flags) & \
-	  ~(MAIL_INDEX_MAIL_FLAG_DIRTY | MAIL_RECENT)) == 0 && \
-	 memcmp((u)->add_keywords, empty, INDEX_KEYWORDS_BYTE_COUNT) == 0 && \
-	 memcmp((u)->add_keywords, empty, INDEX_KEYWORDS_BYTE_COUNT) == 0)
+	  ~(MAIL_INDEX_MAIL_FLAG_DIRTY | MAIL_RECENT)) == 0)
 
 static int
 mail_index_view_sync_get_rec(struct mail_index_view_sync_ctx *ctx,
 			     struct mail_index_sync_rec *rec)
 {
-	static keywords_mask_t empty_keywords = { 0, };
 	const struct mail_transaction_header *hdr = ctx->hdr;
 	const void *data = ctx->data;
 
@@ -275,7 +272,7 @@
 
 		for (;;) {
 			ctx->data_offset += sizeof(*update);
-			if (!FLAG_UPDATE_IS_INTERNAL(update, empty_keywords))
+			if (!FLAG_UPDATE_IS_INTERNAL(update))
 				break;
 
 			if (ctx->data_offset == ctx->hdr->size)
@@ -284,6 +281,31 @@
                 mail_index_sync_get_update(rec, update);
 		break;
 	}
+	case MAIL_TRANSACTION_KEYWORD_UPDATE: {
+		const struct mail_transaction_keyword_update *update = data;
+		const unsigned char *p;
+		const uint32_t *uids;
+		uint32_t i;
+
+		if (ctx->data_offset == 0) {
+			p = (const unsigned char *)
+				(update->name_size + update->keywords_count);
+
+			for (i = 0; i < update->keywords_count; i++)
+				p += update->name_size[i];
+
+			ctx->data_offset = p - (const unsigned char *)update;
+			if ((ctx->data_offset % 4) != 0)
+				ctx->data_offset += 4 - (ctx->data_offset % 4);
+		}
+
+		uids = CONST_PTR_OFFSET(data, ctx->data_offset);
+		rec->type = MAIL_INDEX_SYNC_TYPE_KEYWORDS;
+		rec->uid1 = uids[0];
+		rec->uid2 = uids[1];
+		ctx->data_offset += sizeof(uint32_t) * 2;
+		break;
+	}
 	default:
 		i_unreached();
 	}

Index: mail-index-view.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-view.c,v
retrieving revision 1.31
retrieving revision 1.32
diff -u -d -r1.31 -r1.32
--- mail-index-view.c	5 Dec 2004 01:45:53 -0000	1.31
+++ mail-index-view.c	26 Dec 2004 09:12:40 -0000	1.32
@@ -400,6 +400,7 @@
 				const void **data_r, size_t *data_size_r)
 {
 	const struct mail_index_ext *ext;
+	uint32_t idx;
 
 	if (map != NULL) {
 		if (mail_index_view_lock(view) < 0)
@@ -411,8 +412,14 @@
 		map = view->index->map;
 	}
 
+	if (!mail_index_map_get_ext_idx(map, ext_id, &idx)) {
+		*data_r = NULL;
+		*data_size_r = 0;
+		return 0;
+	}
+
 	ext = map->extensions->data;
-	ext += ext_id;
+	ext += idx;
 
 	*data_r = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset);
 	*data_size_r = ext->hdr_size;
@@ -453,6 +460,64 @@
 	return view->methods.lookup_full(view, seq, map_r, rec_r);
 }
 
+int mail_index_lookup_keywords(struct mail_index_view *view, uint32_t seq,
+			       buffer_t *buf, const char *const **keywords_r)
+{
+	struct mail_index_map *map;
+	const struct mail_index_ext *ext;
+	const void *data;
+	unsigned int i, j;
+	uint32_t ext_id, idx;
+	int ret;
+
+	*keywords_r = NULL;
+	buffer_set_used_size(buf, 0);
+
+	ext_id = view->index->keywords_ext_id;
+	ret = mail_index_lookup_ext_full(view, seq, ext_id, &map, &data);
+	if (ret < 0)
+		return -1;
+
+	if (!mail_index_map_get_ext_idx(map, ext_id, &idx)) {
+		buffer_append_zero(buf, sizeof(const char *));
+		*keywords_r = buf->data;
+		return ret;
+	}
+
+	ext = map->extensions->data;
+	ext += idx;
+
+	for (i = 0, idx = 0; i < ext->record_size; i++) {
+		if (((const char *)data)[i] == 0)
+			continue;
+
+		for (j = 0; j < CHAR_BIT; j++, idx++) {
+			if ((((const char *)data)[i] & (1 << j)) == 0)
+				continue;
+
+			if (idx >= map->keywords_count) {
+				/* keyword header is updated, re-read
+				   it so we know what this one is
+				   called */
+				if (mail_index_map_read_keywords(view->index,
+								 map) < 0)
+					return -1;
+				if (idx >= map->keywords_count) {
+					/* extra bits set in keyword bytes.
+					   shouldn't happen, but just ignore. */
+					break;
+				}
+			}
+			buffer_append(buf, &map->keywords[idx],
+				      sizeof(const char *));
+		}
+	}
+	buffer_append_zero(buf, sizeof(const char *));
+	*keywords_r = buf->data;
+
+	return ret;
+}
+
 int mail_index_lookup_uid(struct mail_index_view *view, uint32_t seq,
 			  uint32_t *uid_r)
 {

Index: mail-index.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index.c,v
retrieving revision 1.183
retrieving revision 1.184
diff -u -d -r1.183 -r1.184
--- mail-index.c	7 Dec 2004 21:32:09 -0000	1.183
+++ mail-index.c	26 Dec 2004 09:12:40 -0000	1.184
@@ -34,6 +34,9 @@
 
 	index->mode = 0600;
 	index->gid = (gid_t)-1;
+
+	index->keywords_ext_id =
+		mail_index_ext_register(index, "keywords", 128, 2, 1);
 	return index;
 }
 
@@ -270,6 +273,92 @@
 	return 1;
 }
 
+int mail_index_map_read_keywords(struct mail_index *index,
+				 struct mail_index_map *map)
+{
+	const struct mail_index_ext *ext;
+	const struct mail_index_keyword_header *kw_hdr;
+	const struct mail_index_keyword_header_rec *kw_rec;
+	const char *name, **keywords_list;
+	unsigned int i, name_len;
+	uint32_t ext_id;
+
+	ext_id = mail_index_map_lookup_ext(map, "keywords");
+	if (ext_id == (uint32_t)-1) {
+		map->keywords = NULL;
+		map->keywords_count = 0;
+		return 0;
+	}
+
+	ext = map->extensions->data;
+	ext += ext_id;
+
+	kw_hdr = CONST_PTR_OFFSET(map->hdr_base, ext->hdr_offset);
+	kw_rec = (const void *)(kw_hdr + 1);
+	name = (const char *)(kw_rec + kw_hdr->keywords_count);
+
+	if ((size_t)(name - (const char *)kw_hdr) > ext->hdr_size) {
+		mail_index_set_error(index, "Corrupted index file %s: "
+				     "keywords_count larger than header size",
+				     index->filepath);
+		return -1;
+	}
+
+	/* make sure the header is valid */
+	name_len = (const char *)kw_hdr + ext->hdr_size - name;
+	for (i = 0; i < kw_hdr->keywords_count; i++) {
+		if (kw_rec[i].name_offset > name_len) {
+			mail_index_set_error(index, "Corrupted index file %s: "
+				"name_offset points outside allocated header",
+				index->filepath);
+			return -1;
+		}
+	}
+	if (name[name_len-1] != '\0') {
+		mail_index_set_error(index, "Corrupted index file %s: "
+				     "header doesn't end with NUL",
+				     index->filepath);
+		return -1;
+	}
+
+	if (map->keywords_pool == NULL)
+		map->keywords_pool = pool_alloconly_create("keywords", 1024);
+
+	/* Save keywords in memory. Only new keywords should come into the
+	   mapping, so keep the existing keyword strings in memory to allow
+	   mail_index_lookup_keywords() to safely return direct pointers
+	   into them. */
+	if (kw_hdr->keywords_count < map->keywords_count) {
+		mail_index_set_error(index, "Corrupted index file %s: "
+				     "Keywords removed unexpectedly",
+				     index->filepath);
+		return -1;
+	}
+	if (kw_hdr->keywords_count == map->keywords_count) {
+		/* nothing changed */
+		return 0;
+	}
+
+	/* @UNSAFE */
+	keywords_list = p_new(map->keywords_pool,
+			      const char *, kw_hdr->keywords_count + 1);
+	for (i = 0; i < map->keywords_count; i++)
+		keywords_list[i] = map->keywords[i];
+	for (; i < kw_hdr->keywords_count; i++) {
+		keywords_list[i] = p_strdup(map->keywords_pool,
+					    name + kw_rec[i].name_offset);
+	}
+	map->keywords = keywords_list;
+	map->keywords_count = kw_hdr->keywords_count;
+	return 0;
+}
+
+const char *const *mail_index_get_keywords(struct mail_index *index)
+{
+	(void)mail_index_map_read_keywords(index, index->map);
+	return index->map->keywords;
+}
+
 static int mail_index_check_header(struct mail_index *index,
 				   struct mail_index_map *map)
 {
@@ -305,14 +394,6 @@
 		return -1;
 	}
 
-	if (hdr->keywords_mask_size != sizeof(keywords_mask_t)) {
-		mail_index_set_error(index, "Corrupted index file %s: "
-				     "keywords_mask_size mismatch: %d != %d",
-				     index->filepath, hdr->keywords_mask_size,
-				     (int)sizeof(keywords_mask_t));
-		return -1;
-	}
-
 	if (hdr->record_size < sizeof(struct mail_index_record)) {
 		mail_index_set_error(index, "Corrupted index file %s: "
 				     "record_size too small: %u < %"PRIuSIZE_T,
@@ -368,6 +449,8 @@
 	mail_index_map_clear(index, map);
 	if (map->extension_pool != NULL)
 		pool_unref(map->extension_pool);
+	if (map->keywords_pool != NULL)
+		pool_unref(map->keywords_pool);
 	buffer_free(map->hdr_copy_buf);
 	i_free(map);
 }
@@ -1080,7 +1163,6 @@
 	hdr->base_header_size = sizeof(*hdr);
 	hdr->header_size = sizeof(*hdr);
 	hdr->record_size = sizeof(struct mail_index_record);
-	hdr->keywords_mask_size = sizeof(keywords_mask_t);
 
 #ifndef WORDS_BIGENDIAN
 	hdr->compat_data[0] = MAIL_INDEX_COMPAT_LITTLE_ENDIAN;

Index: mail-index.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index.h,v
retrieving revision 1.139
retrieving revision 1.140
diff -u -d -r1.139 -r1.140
--- mail-index.h	5 Dec 2004 01:45:53 -0000	1.139
+++ mail-index.h	26 Dec 2004 09:12:40 -0000	1.140
@@ -3,15 +3,11 @@
 
 #include "mail-types.h"
 
-#define MAIL_INDEX_MAJOR_VERSION 6
+#define MAIL_INDEX_MAJOR_VERSION 7
 #define MAIL_INDEX_MINOR_VERSION 0
 
 #define MAIL_INDEX_HEADER_MIN_SIZE 120
 
-/* Number of keywords in mail_index_record. */
-#define INDEX_KEYWORDS_COUNT (3*8)
-#define INDEX_KEYWORDS_BYTE_COUNT ((INDEX_KEYWORDS_COUNT+CHAR_BIT-1) / CHAR_BIT)
-
 enum mail_index_open_flags {
 	/* Create index if it doesn't exist */
 	MAIL_INDEX_OPEN_FLAG_CREATE		= 0x01,
@@ -58,8 +54,6 @@
 #define MAIL_INDEX_FLAGS_MASK \
 	(MAIL_ANSWERED | MAIL_FLAGGED | MAIL_DELETED | MAIL_SEEN | MAIL_DRAFT)
 
-typedef unsigned char keywords_mask_t[INDEX_KEYWORDS_BYTE_COUNT];
-
 struct mail_index_header {
 	/* major version is increased only when you can't have backwards
 	   compatibility. minor version is increased when header size is
@@ -69,8 +63,7 @@
 
 	uint16_t base_header_size;
 	uint32_t header_size; /* base + extended header size */
-	uint16_t record_size;
-	uint16_t keywords_mask_size;
+	uint32_t record_size;
 
 	/* 0 = flags
 	   1 = sizeof(uoff_t)
@@ -110,14 +103,14 @@
 
 struct mail_index_record {
 	uint32_t uid;
-	uint8_t flags; /* mail_flags | mail_index_mail_flags */
-	keywords_mask_t keywords;
+	uint8_t flags; /* enum mail_flags | enum mail_index_mail_flags */
 };
 
 enum mail_index_sync_type {
 	MAIL_INDEX_SYNC_TYPE_APPEND	= 0x01,
 	MAIL_INDEX_SYNC_TYPE_EXPUNGE	= 0x02,
-	MAIL_INDEX_SYNC_TYPE_FLAGS	= 0x04
+	MAIL_INDEX_SYNC_TYPE_FLAGS	= 0x04,
+	MAIL_INDEX_SYNC_TYPE_KEYWORDS	= 0x08
 };
 #define MAIL_INDEX_SYNC_MASK_ALL 0xff
 
@@ -127,11 +120,10 @@
 
 	/* MAIL_INDEX_SYNC_TYPE_FLAGS: */
 	uint8_t add_flags;
-	keywords_mask_t add_keywords;
 	uint8_t remove_flags;
-	keywords_mask_t remove_keywords;
 };
 
+struct mail_keywords;
 struct mail_index;
 struct mail_index_map;
 struct mail_index_view;
@@ -258,6 +250,8 @@
 int mail_index_lookup_full(struct mail_index_view *view, uint32_t seq,
 			   struct mail_index_map **map_r,
 			   const struct mail_index_record **rec_r);
+int mail_index_lookup_keywords(struct mail_index_view *view, uint32_t seq,
+			       buffer_t *buf, const char *const **keywords_r);
 /* Returns the UID for given message. May be slightly faster than
    mail_index_lookup()->uid. */
 int mail_index_lookup_uid(struct mail_index_view *view, uint32_t seq,
@@ -286,7 +280,20 @@
 /* Update flags in index. */
 void mail_index_update_flags(struct mail_index_transaction *t, uint32_t seq,
 			     enum modify_type modify_type,
-			     enum mail_flags flags, keywords_mask_t keywords);
+			     enum mail_flags flags);
+
+/* Return a list of all existing keywords, or NULL if there is none. */
+const char *const *mail_index_get_keywords(struct mail_index *index);
+/* Create a keyword list structure. It's freed automatically at the end of
+   the transaction. */
+struct mail_keywords *
+mail_index_keywords_create(struct mail_index_transaction *t,
+			   const char *const keywords[]);
+/* Update keywords for given message. */
+void mail_index_update_keywords(struct mail_index_transaction *t, uint32_t seq,
+				enum modify_type modify_type,
+				const struct mail_keywords *keywords);
+
 /* Update field in header. */
 void mail_index_update_header(struct mail_index_transaction *t,
 			      size_t offset, const void *data, size_t size);
@@ -302,7 +309,7 @@
 /* Apply changes in MAIL_INDEX_SYNC_TYPE_FLAGS typed sync records to given
    flags variables. */
 void mail_index_sync_flags_apply(const struct mail_index_sync_rec *sync_rec,
-				 uint8_t *flags, keywords_mask_t keywords);
+				 uint8_t *flags);
 
 /* register index extension. name is a unique identifier for the extension.
    returns unique identifier for the name. */

Index: mail-transaction-log.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-transaction-log.c,v
retrieving revision 1.78
retrieving revision 1.79
diff -u -d -r1.78 -r1.79
--- mail-transaction-log.c	7 Dec 2004 01:02:04 -0000	1.78
+++ mail-transaction-log.c	26 Dec 2004 09:12:40 -0000	1.79
@@ -1188,6 +1188,50 @@
 	return 0;
 }
 
+static int log_append_keyword_updates(struct mail_transaction_log_file *file,
+				      struct mail_index_transaction *t)
+{
+	struct mail_index_keyword_transaction *kt;
+	struct mail_transaction_keyword_update *kt_hdr;
+	buffer_t *buf;
+	size_t i, size;
+	unsigned int j;
+
+	buf = buffer_create_dynamic(pool_datastack_create(), 128);
+
+	kt = buffer_get_modifyable_data(t->keyword_updates, &size);
+	size /= sizeof(*kt);
+	for (i = 0; i < size; i++) {
+		if (kt[i].messages == NULL)
+			continue;
+
+		buffer_set_used_size(buf, 0);
+		kt_hdr = buffer_append_space_unsafe(buf, sizeof(*kt_hdr));
+		kt_hdr->keywords_count = kt[i].keywords.count;
+		kt_hdr->modify_type = kt[i].modify_type;
+		kt_hdr->name_size[0] = strlen(kt[i].keywords.keywords[0]);
+
+		for (j = 1; j < kt[i].keywords.count; j++) {
+			uint16_t name_size = strlen(kt[i].keywords.keywords[j]);
+			buffer_append(buf, &name_size, sizeof(name_size));
+		}
+		for (j = 0; j < kt[i].keywords.count; j++) {
+			const char *name = kt[i].keywords.keywords[j];
+			buffer_append(buf, name, strlen(name));
+		}
+		if ((buf->used % 4) != 0)
+			buffer_append_zero(buf, 4 - (buf->used % 4));
+		buffer_append_buf(buf, kt[i].messages, 0, (size_t)-1);
+
+		if (log_append_buffer(file, buf, NULL,
+				      MAIL_TRANSACTION_KEYWORD_UPDATE,
+				      t->external) < 0)
+			return -1;
+	}
+
+	return 0;
+}
+
 int mail_transaction_log_append(struct mail_index_transaction *t,
 				uint32_t *log_file_seq_r,
 				uoff_t *log_file_offset_r)
@@ -1272,6 +1316,9 @@
 	if (t->ext_rec_updates != NULL && ret == 0)
 		ret = log_append_ext_rec_updates(file, t);
 
+	if (t->keyword_updates != NULL && ret == 0)
+		ret = log_append_keyword_updates(file, t);
+
 	if (t->expunges != NULL && ret == 0) {
 		ret = log_append_buffer(file, t->expunges, NULL,
 					MAIL_TRANSACTION_EXPUNGE, t->external);

Index: mail-transaction-log.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-transaction-log.h,v
retrieving revision 1.21
retrieving revision 1.22
diff -u -d -r1.21 -r1.22
--- mail-transaction-log.h	6 Nov 2004 17:20:19 -0000	1.21
+++ mail-transaction-log.h	26 Dec 2004 09:12:40 -0000	1.22
@@ -21,6 +21,7 @@
 	MAIL_TRANSACTION_EXT_RESET		= 0x00000080,
 	MAIL_TRANSACTION_EXT_HDR_UPDATE		= 0x00000100,
 	MAIL_TRANSACTION_EXT_REC_UPDATE		= 0x00000200,
+	MAIL_TRANSACTION_KEYWORD_UPDATE		= 0x00000400,
 
 	MAIL_TRANSACTION_TYPE_MASK		= 0x0000ffff,
 
@@ -46,9 +47,17 @@
 struct mail_transaction_flag_update {
 	uint32_t uid1, uid2;
 	uint8_t add_flags;
-	keywords_mask_t add_keywords;
 	uint8_t remove_flags;
-	keywords_mask_t remove_keywords;
+};
+
+struct mail_transaction_keyword_update {
+	uint32_t keywords_count;
+	uint8_t modify_type; /* enum modify_type */
+	uint8_t padding;
+	uint16_t name_size[1]; /* [keywords_count] */
+	/* unsigned char name[keywords_count][name_size];
+	   array of { uint32_t uid1, uid2; }
+	*/
 };
 
 struct mail_transaction_header_update {

Index: mail-transaction-util.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-transaction-util.c,v
retrieving revision 1.21
retrieving revision 1.22
diff -u -d -r1.21 -r1.22
--- mail-transaction-util.c	6 Nov 2004 17:20:19 -0000	1.21
+++ mail-transaction-util.c	26 Dec 2004 09:12:40 -0000	1.22
@@ -19,6 +19,7 @@
 	  sizeof(struct mail_transaction_ext_reset) },
 	{ MAIL_TRANSACTION_EXT_HDR_UPDATE, 0, 1 },
 	{ MAIL_TRANSACTION_EXT_REC_UPDATE, 0, 1 },
+	{ MAIL_TRANSACTION_KEYWORD_UPDATE, MAIL_INDEX_SYNC_TYPE_KEYWORDS, 1 },
 	{ 0, 0, 0 }
 };