[dovecot-cvs] dovecot/src/lib-storage/index/dbox Makefile.am, 1.2, 1.3 dbox-file.c, 1.4, 1.5 dbox-file.h, 1.1, 1.2 dbox-keywords.c, NONE, 1.1 dbox-keywords.h, NONE, 1.1 dbox-mail.c, 1.4, 1.5 dbox-storage.h, 1.5, 1.6 dbox-sync-full.c, 1.2, 1.3 dbox-sync.c, 1.5, 1.6 dbox-uidlist.c, 1.13, 1.14

[cras at dovecot.org]

Update of /var/lib/cvs/dovecot/src/lib-storage/index/dbox
In directory talvi:/tmp/cvs-serv12050

Modified Files:
	Makefile.am dbox-file.c dbox-file.h dbox-mail.c dbox-storage.h 
	dbox-sync-full.c dbox-sync.c dbox-uidlist.c 
Added Files:
	dbox-keywords.c dbox-keywords.h 
Log Message:
Support storing keywords in dbox files. Doesn't yet work while saving.



Index: Makefile.am
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/dbox/Makefile.am,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- Makefile.am	21 Dec 2005 18:43:16 -0000	1.2
+++ Makefile.am	28 Jan 2006 22:05:48 -0000	1.3
@@ -10,6 +10,7 @@
 
 libstorage_dbox_a_SOURCES = \
 	dbox-file.c \
+	dbox-keywords.c \
 	dbox-list.c \
 	dbox-mail.c \
 	dbox-save.c \
@@ -22,6 +23,7 @@
 
 noinst_HEADERS = \
 	dbox-file.h \
+	dbox-keywords.h \
 	dbox-storage.h \
 	dbox-sync.h \
 	dbox-uidlist.h

Index: dbox-file.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/dbox/dbox-file.c,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -d -r1.4 -r1.5
--- dbox-file.c	14 Jan 2006 18:47:46 -0000	1.4
+++ dbox-file.c	28 Jan 2006 22:05:48 -0000	1.5
@@ -1,6 +1,8 @@
-/* Copyright (C) 2005 Timo Sirainen */
+/* Copyright (C) 2005-2006 Timo Sirainen */
 
 #include "lib.h"
+#include "array.h"
+#include "bsearch-insert-pos.h"
 #include "hex-dec.h"
 #include "istream.h"
 #include "ostream.h"
@@ -38,12 +40,18 @@
 
 void dbox_file_close(struct dbox_file *file)
 {
+	if (array_is_created(&file->file_idx_keywords)) {
+		array_free(&file->idx_file_keywords);
+		array_free(&file->file_idx_keywords);
+	}
+
 	if (file->input != NULL)
 		i_stream_unref(&file->input);
 	if (file->fd != -1) {
 		if (close(file->fd) < 0)
 			i_error("close(dbox) failed: %m");
 	}
+	i_free(file->seeked_keywords);
 	i_free(file->path);
 	i_free(file);
 }
@@ -56,6 +64,7 @@
 	const unsigned char *data;
 	size_t size;
 
+	/* read the header */
 	i_stream_seek(file->input, offset);
 	(void)i_stream_read_data(file->input, &data, &size,
 				 file->mail_header_size-1);
@@ -68,10 +77,15 @@
 					  "read(%s) failed: %m", file->path);
 		return -1;
 	}
+
 	memcpy(&file->seeked_mail_header, data,
 	       sizeof(file->seeked_mail_header));
+	/* @UNSAFE */
+	memcpy(file->seeked_keywords, data + sizeof(file->seeked_mail_header),
+	       file->keyword_count);
 	file->seeked_offset = offset;
 
+	/* parse the header */
 	hdr = &file->seeked_mail_header;
 	file->seeked_mail_size =
 		hex2dec(hdr->mail_size_hex, sizeof(hdr->mail_size_hex));
@@ -83,15 +97,13 @@
 			"Corrupted mail header in dbox file %s", file->path);
 		return -1;
 	}
-	if (file->seeked_mail_size == 0 || file->seeked_uid == 0) {
-		/* could be legitimately just not written yet. we're at EOF. */
-		return 0;
-	}
 	return 1;
 }
 
 int dbox_file_seek(struct dbox_mailbox *mbox, uint32_t file_seq, uoff_t offset)
 {
+	int ret;
+
 	if (mbox->file != NULL && mbox->file->file_seq != file_seq) {
 		dbox_file_close(mbox->file);
 		mbox->file = NULL;
@@ -129,39 +141,99 @@
 	if (offset == 0)
 		offset = mbox->file->header_size;
 
-	return dbox_file_read_mail_header(mbox, mbox->file, offset);
+	if ((ret = dbox_file_read_mail_header(mbox, mbox->file, offset)) <= 0)
+		return ret;
+
+	if (mbox->file->seeked_mail_size == 0 || mbox->file->seeked_uid == 0) {
+		/* could be legitimately just not written yet. we're at EOF. */
+		return 0;
+	}
+	return 1;
 }
 
 int dbox_file_seek_next_nonexpunged(struct dbox_mailbox *mbox)
 {
+	const struct dbox_mail_header *hdr;
 	uoff_t offset;
 	int ret;
 
-	offset = mbox->file->seeked_offset +
-		mbox->file->mail_header_size + mbox->file->seeked_mail_size;
+	for (;;) {
+		offset = mbox->file->seeked_offset +
+			mbox->file->mail_header_size +
+			mbox->file->seeked_mail_size;
 
-	while ((ret = dbox_file_seek(mbox, mbox->file->file_seq, offset)) > 0) {
-		if (mbox->file->seeked_mail_header.expunged != '1')
-			break;
+		ret = dbox_file_seek(mbox, mbox->file->file_seq, offset);
+		if (ret <= 0)
+			return ret;
 
-		/* marked expunged, get to next mail. */
+		hdr = &mbox->file->seeked_mail_header;
+		if (hdr->expunged != '1') {
+			/* non-expunged mail found */
+			break;
+		}
 	}
-	return ret;
+
+	return 1;
 }
 
 void dbox_file_header_init(struct dbox_file_header *hdr)
 {
-	uint16_t header_size = sizeof(*hdr);
+	uint16_t base_header_size = sizeof(*hdr);
+	uint32_t header_size =
+		base_header_size + DBOX_KEYWORD_NAMES_RESERVED_SPACE;
 	uint32_t append_offset = header_size;
-	uint16_t mail_header_size = sizeof(struct dbox_mail_header);
+	uint16_t keyword_count = DBOX_KEYWORD_COUNT;
+	uint16_t mail_header_size =
+		sizeof(struct dbox_mail_header) + keyword_count;
 	uint32_t create_time = ioloop_time;
 
 	memset(hdr, '0', sizeof(*hdr));
+	DEC2HEX(hdr->base_header_size_hex, base_header_size);
 	DEC2HEX(hdr->header_size_hex, header_size);
 	DEC2HEX(hdr->append_offset_hex, append_offset);
 	DEC2HEX(hdr->create_time_hex, create_time);
 	DEC2HEX(hdr->mail_header_size_hex, mail_header_size);
-	// FIXME: set keyword_count
+	DEC2HEX(hdr->keyword_list_offset_hex, base_header_size);
+	DEC2HEX(hdr->keyword_count_hex, keyword_count);
+}
+
+int dbox_file_header_parse(struct dbox_mailbox *mbox, struct dbox_file *file,
+			   const struct dbox_file_header *hdr)
+{
+	file->base_header_size = hex2dec(hdr->base_header_size_hex,
+					 sizeof(hdr->base_header_size_hex));
+	file->header_size = hex2dec(hdr->header_size_hex,
+				    sizeof(hdr->header_size_hex));
+	file->append_offset = hex2dec(hdr->append_offset_hex,
+				      sizeof(hdr->append_offset_hex));
+	file->create_time = hex2dec(hdr->create_time_hex,
+				    sizeof(hdr->create_time_hex));
+	file->mail_header_size = hex2dec(hdr->mail_header_size_hex,
+					 sizeof(hdr->mail_header_size_hex));
+	file->mail_header_align =
+		hex2dec(hdr->mail_header_align_hex,
+			sizeof(hdr->mail_header_align_hex));
+	file->keyword_count = hex2dec(hdr->keyword_count_hex,
+				      sizeof(hdr->keyword_count_hex));
+	file->keyword_list_offset =
+		hex2dec(hdr->keyword_list_offset_hex,
+			sizeof(hdr->keyword_list_offset_hex));
+
+	if (file->base_header_size == 0 ||
+	    file->header_size < file->base_header_size ||
+	    file->append_offset < file->header_size ||
+            file->keyword_list_offset < file->base_header_size ||
+	    file->mail_header_size < sizeof(struct dbox_mail_header) ||
+	    file->keyword_count > file->mail_header_size -
+	    sizeof(struct dbox_mail_header)) {
+		mail_storage_set_critical(STORAGE(mbox->storage),
+			"dbox %s: broken file header", file->path);
+		return -1;
+	}
+
+	i_free(file->seeked_keywords);
+	file->seeked_keywords = i_malloc(file->keyword_count);
+	return 0;
 }
 
 int dbox_file_read_header(struct dbox_mailbox *mbox, struct dbox_file *file)
@@ -170,6 +242,7 @@
 	const unsigned char *data;
 	size_t size;
 
+	/* read the file header */
 	i_stream_seek(file->input, 0);
 	(void)i_stream_read_data(file->input, &data, &size, sizeof(hdr)-1);
 	if (size < sizeof(hdr)) {
@@ -187,25 +260,46 @@
 	memcpy(&hdr, data, sizeof(hdr));
 
 	/* parse the header */
-	file->header_size = hex2dec(hdr.header_size_hex,
-				    sizeof(hdr.header_size_hex));
-	file->append_offset = hex2dec(hdr.append_offset_hex,
-				      sizeof(hdr.append_offset_hex));
-	file->create_time = hex2dec(hdr.create_time_hex,
-				    sizeof(hdr.create_time_hex));
-	file->mail_header_size = hex2dec(hdr.mail_header_size_hex,
-					 sizeof(hdr.mail_header_size_hex));
-	file->mail_header_padding =
-		hex2dec(hdr.mail_header_padding_hex,
-			sizeof(hdr.mail_header_padding_hex));
-	file->keyword_count = hex2dec(hdr.keyword_count_hex,
-				      sizeof(hdr.keyword_count_hex));
+	if (dbox_file_header_parse(mbox, file, &hdr) < 0)
+		return -1;
 
-	if (file->header_size == 0 || file->append_offset < sizeof(hdr) ||
-	    file->mail_header_size < sizeof(struct dbox_mail_header)) {
+	/* keywords may not be up to date anymore */
+	if (array_is_created(&file->idx_file_keywords)) {
+		array_free(&file->idx_file_keywords);
+		array_free(&file->file_idx_keywords);
+	}
+	return 0;
+}
+
+int dbox_file_write_header(struct dbox_mailbox *mbox, struct dbox_file *file)
+{
+	struct dbox_file_header hdr;
+	char buf[1024];
+	int ret;
+
+	dbox_file_header_init(&hdr);
+	ret = dbox_file_header_parse(mbox, file, &hdr);
+	i_assert(ret == 0);
+
+	/* write header + LF to mark end-of-keywords list */
+	if (o_stream_send(file->output, &hdr, sizeof(hdr)) < 0 ||
+	    o_stream_send_str(file->output, "\n") < 0) {
 		mail_storage_set_critical(STORAGE(mbox->storage),
-			"dbox %s: broken file header", file->path);
+			"write(%s) failed: %m", file->path);
 		return -1;
 	}
+
+	/* fill the rest of the header with spaces */
+	memset(buf, ' ', sizeof(buf));
+	while (file->output->offset < file->header_size) {
+		unsigned int size = I_MIN(sizeof(buf), file->header_size -
+					  file->output->offset);
+
+		if (o_stream_send(file->output, buf, size) < 0) {
+			mail_storage_set_critical(STORAGE(mbox->storage),
+				"write(%s) failed: %m", file->path);
+			return -1;
+		}
+	}
 	return 0;
 }

Index: dbox-file.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/dbox/dbox-file.h,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- dbox-file.h	27 Nov 2005 23:05:29 -0000	1.1
+++ dbox-file.h	28 Jan 2006 22:05:48 -0000	1.2
@@ -13,10 +13,14 @@
 
 void dbox_file_close(struct dbox_file *file);
 /* Returns -1 = error, 0 = EOF (mail was just moved / file broken), 1 = ok */
-int dbox_file_seek(struct dbox_mailbox *mbox, uint32_t file_seq, uoff_t offset);
+int dbox_file_seek(struct dbox_mailbox *mbox,
+		   uint32_t file_seq, uoff_t offset);
 int dbox_file_seek_next_nonexpunged(struct dbox_mailbox *mbox);
 
 void dbox_file_header_init(struct dbox_file_header *hdr);
+int dbox_file_header_parse(struct dbox_mailbox *mbox, struct dbox_file *file,
+			   const struct dbox_file_header *hdr);
 int dbox_file_read_header(struct dbox_mailbox *mbox, struct dbox_file *file);
+int dbox_file_write_header(struct dbox_mailbox *mbox, struct dbox_file *file);
 
 #endif

--- NEW FILE: dbox-keywords.c ---
/* Copyright (C) 2006 Timo Sirainen */

#include "lib.h"
#include "array.h"
#include "str.h"
#include "istream.h"
#include "write-full.h"
#include "seq-range-array.h"
#include "bsearch-insert-pos.h"
#include "dbox-file.h"
#include "dbox-storage.h"
#include "dbox-keywords.h"

#include <stdlib.h>

static int dbox_keyword_map_compare(const void *p1, const void *p2)
{
	const struct keyword_map *map1 = p1, *map2 = p2;

	return map1->index_idx < map2->index_idx ? -1 :
		map1->index_idx > map2->index_idx ? 1 : 0;
}

int dbox_file_read_keywords(struct dbox_mailbox *mbox, struct dbox_file *file)
{
	struct keyword_map *map, *pos, kw;
	const char *line;
	unsigned int idx, count;
	uoff_t last_offset;

	if (array_is_created(&file->idx_file_keywords)) {
		array_clear(&file->idx_file_keywords);
		array_clear(&file->file_idx_keywords);
	} else {
		ARRAY_CREATE(&file->idx_file_keywords, default_pool,
			     struct keyword_map, file->keyword_count);
		ARRAY_CREATE(&file->file_idx_keywords, default_pool,
			     unsigned int, file->keyword_count);
	}

	/* currently we assume that all extra space at the end of header
	   belongs to keyword list. */
	file->keyword_list_size_alloc =
		file->header_size - file->keyword_list_offset;

	i_stream_seek(file->input, file->keyword_list_offset);
	idx = 0;
	last_offset = file->input->v_offset;
	while ((line = i_stream_read_next_line(file->input)) != NULL) {
		if (*line == '\0') {
			/* end of list */
			break;
		}
		last_offset = file->input->v_offset;

		/* set up map record for the keyword */
		(void)mail_index_keyword_lookup(mbox->ibox.index, line, TRUE,
						&kw.index_idx);
		kw.file_idx = idx;

		/* look up the position where to insert it */
		map = array_get_modifyable(&file->idx_file_keywords, &count);
		pos = idx == 0 ? map :
			bsearch_insert_pos(line, map, count, sizeof(*map),
					   dbox_keyword_map_compare);
		array_insert(&file->idx_file_keywords, pos - map, &kw, 1);
		array_append(&file->file_idx_keywords, &kw.index_idx, 1);

		if (++idx == file->keyword_count)
			break;
	}

	if (line == NULL || file->input->v_offset > file->header_size) {
		/* unexpected end of list, or list continues outside its
		   allocated area */
		mail_storage_set_critical(STORAGE(mbox->storage),
			"Corrupted keyword list offset in dbox file %s",
			file->path);
		array_clear(&file->idx_file_keywords);
		return 0;
	}

	file->keyword_list_size_used =
		last_offset - file->keyword_list_offset;
	return 1;
}

static int keyword_lookup_cmp(const void *key, const void *obj)
{
	const unsigned int *index_idx = key;
	const struct keyword_map *map = obj;

	return *index_idx < map->index_idx ? -1 :
		*index_idx > map->index_idx ? 1 : 0;
}

bool dbox_file_lookup_keyword(struct dbox_mailbox *mbox, struct dbox_file *file,
			      unsigned int index_idx, unsigned int *idx_r)
{
	const struct keyword_map *map, *pos;
	unsigned int count;

	if (!array_is_created(&file->idx_file_keywords)) {
		/* Read the keywords, if there are any */
		if (dbox_file_read_keywords(mbox, file) <= 0)
			return FALSE;
	}

	map = array_get(&file->idx_file_keywords, &count);
	pos = bsearch(&index_idx, map, count, sizeof(*map),
		      keyword_lookup_cmp);
	if (pos != NULL && idx_r != NULL)
		*idx_r = pos->file_idx;
	return pos != NULL;
}

int dbox_file_append_keywords(struct dbox_mailbox *mbox, struct dbox_file *file,
			      const struct seq_range *idx_range,
			      unsigned int count)
{
	const array_t *idx_keywords;
	ARRAY_SET_TYPE(idx_keywords, const char *);
	string_t *keyword_str;
	const char *const *idx_keyword_names;
	unsigned int i, idx_keyword_count, new_pos;
	int ret;

	t_push();
	keyword_str = t_str_new(2048);
	idx_keywords = mail_index_get_keywords(mbox->ibox.index);
	idx_keyword_names = array_get(idx_keywords, &idx_keyword_count);

	/* make sure we've read the existing keywords */
	if (!array_is_created(&file->idx_file_keywords)) {
		ret = dbox_file_read_keywords(mbox, file);
		if (ret < 0)
			return -1;

		if (ret == 0) {
			/* broken keywords list. */
			file->keyword_list_size_used = 0;
		}
	}

	/* append existing keywords */
	if (array_count(&file->idx_file_keywords) > 0) {
		const unsigned int *file_idx;
		unsigned int file_count;

		file_idx = array_get(&file->file_idx_keywords, &file_count);
		for (i = 0; i < file_count; i++) {
			i_assert(file_idx[i] < idx_keyword_count);

			str_append(keyword_str, idx_keyword_names[file_idx[i]]);
			str_append_c(keyword_str, '\n');
		}
	}

	/* append new keywords */
	if (file->keyword_list_size_used == 0)
		new_pos = 0;
	else {
		new_pos = str_len(keyword_str);
		i_assert(new_pos == file->keyword_list_size_used);
	}
	for (i = 0; i < count; i++) {
		unsigned int idx;

		for (idx = idx_range[i].seq1; idx <= idx_range[i].seq2; idx++) {
			size_t prev_len;

			i_assert(idx < idx_keyword_count);
			i_assert(!dbox_file_lookup_keyword(mbox, file,
							   idx, NULL));

			prev_len = str_len(keyword_str);
			str_append(keyword_str, idx_keyword_names[idx]);
			str_append_c(keyword_str, '\n');

			if (str_len(keyword_str) >=
			    file->keyword_list_size_alloc) {
				/* FIXME: keyword list doesn't fit to the
				   space allocated for it. create a new file
				   where there's more space for keywords and
				   move the mails there.

				   for now we'll just ignore the problem. */
				str_truncate(keyword_str, prev_len);
				break;
			}
		}
	}

	str_append_c(keyword_str, '\n');
	i_assert(str_len(keyword_str) <= file->keyword_list_size_alloc);

	/* we can reuse the existing keyword list position */
	if (pwrite_full(file->fd, str_data(keyword_str) + new_pos,
			str_len(keyword_str) - new_pos,
			file->keyword_list_offset + new_pos) < 0) {
		mail_storage_set_critical(STORAGE(mbox->storage),
			"pwrite_full(%s) failed: %m", file->path);
	}

	/* FIXME: we could do this faster than by reading them.. */
	ret = 0;
	if (dbox_file_read_keywords(mbox, file) <= 0)
		ret = -1;

	t_pop();
	return ret;
}

--- NEW FILE: dbox-keywords.h ---
#ifndef __DBOX_KEYWORDS_H
#define __DBOX_KEYWORDS_H

struct seq_range;

/* Read keywords from file into memory. Returns 1 if ok, 0 if the list is
   broken or -1 if I/O error. */
int dbox_file_read_keywords(struct dbox_mailbox *mbox, struct dbox_file *file);
/* Index file -> dbox file keyword index lookup. Returns TRUE if found. */
bool dbox_file_lookup_keyword(struct dbox_mailbox *mbox, struct dbox_file *file,
			      unsigned int index_idx, unsigned int *idx_r);
/* Save keywords to dbox file. Returns -1 if error, 0 if ok. */
int dbox_file_append_keywords(struct dbox_mailbox *mbox, struct dbox_file *file,
			      const struct seq_range *idx_range,
			      unsigned int count);

#endif

Index: dbox-mail.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/dbox/dbox-mail.c,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -d -r1.4 -r1.5
--- dbox-mail.c	13 Jan 2006 20:26:29 -0000	1.4
+++ dbox-mail.c	28 Jan 2006 22:05:48 -0000	1.5
@@ -21,9 +21,7 @@
 	const struct dbox_mail_header *hdr = &file->seeked_mail_header;
 	uint32_t hdr_uid = hex2dec(hdr->uid_hex, sizeof(hdr->uid_hex));
 
-	if (hdr_uid != mail->mail.mail.uid ||
-	    memcmp(hdr->magic, DBOX_MAIL_HEADER_MAGIC,
-		   sizeof(hdr->magic)) != 0) {
+	if (hdr_uid != mail->mail.mail.uid) {
 		mail_storage_set_critical(STORAGE(mbox->storage),
 			"dbox %s: Cached file offset broken",
 			mbox->file->path);
@@ -144,9 +142,8 @@
 		data->received_date = 0;
 	}
 
-	mail_cache_add(mail->trans->cache_trans, mail->data.seq,
-		       MAIL_CACHE_RECEIVED_DATE,
-		       &data->received_date, sizeof(data->received_date));
+	index_mail_cache_add(mail, MAIL_CACHE_RECEIVED_DATE,
+			     &data->received_date, sizeof(data->received_date));
 	return data->received_date;
 }
 
@@ -163,9 +160,8 @@
 	if (dbox_mail_open(mail, &offset) <= 0)
 		return (uoff_t)-1;
 
-	mail_cache_add(mail->trans->cache_trans, mail->data.seq,
-		       MAIL_CACHE_PHYSICAL_FULL_SIZE,
-		       &data->physical_size, sizeof(data->physical_size));
+	index_mail_cache_add(mail, MAIL_CACHE_PHYSICAL_FULL_SIZE,
+			     &data->physical_size, sizeof(data->physical_size));
 	return data->physical_size;
 
 }

Index: dbox-storage.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/dbox/dbox-storage.h,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -d -r1.5 -r1.6
--- dbox-storage.h	13 Jan 2006 20:26:29 -0000	1.5
+++ dbox-storage.h	28 Jan 2006 22:05:48 -0000	1.6
@@ -6,6 +6,9 @@
 #define DBOX_MAILDIR_NAME "Mails"
 #define DBOX_MAIL_FILE_PREFIX "msg."
 
+#define DBOX_KEYWORD_COUNT 64
+#define DBOX_KEYWORD_NAMES_RESERVED_SPACE (2048-sizeof(struct dbox_file_header))
+
 #include "index-storage.h"
 
 #define STORAGE(mbox_storage) \
@@ -21,17 +24,37 @@
 struct dbox_uidlist;
 
 struct dbox_file_header {
+	/* Size of the base header. sizeof(struct dbox_file_header) */
+	unsigned char base_header_size_hex[4];
+	/* Size of the full header, including keywords list and padding */
 	unsigned char header_size_hex[8];
+	/* Offset where to store the next mail. note that a mail may already
+	   have been fully written here and added to uidlist, but this offset
+	   just wasn't updated. In that case the append_offset should be
+	   updated instead of overwriting the mail. */
 	unsigned char append_offset_hex[16];
+	/* Initial file creation time as UNIX timestamp. */
 	unsigned char create_time_hex[8];
+	/* Size of each message's header. */
 	unsigned char mail_header_size_hex[4];
-	unsigned char mail_header_padding_hex[4];
+	/* If set, mail headers start always at given alignmentation.
+	   Currently not supported. */
+	unsigned char mail_header_align_hex[4];
+	/* Number of keywords allocated for each mail (not necessarily used) */
 	unsigned char keyword_count_hex[4];
-	/* possible padding to fill header_size */
+	/* Offset for the keyword list inside the file header. */
+	unsigned char keyword_list_offset_hex[8];
+
+	/* space reserved for keyword list and possible other future
+	   extensions. */
+	/* unsigned char [header_size - header_base_size]; */
 };
 
 #define DBOX_MAIL_HEADER_MAGIC "\001\003"
 struct dbox_mail_header {
+	/* This field acts as kind of a verification marker to make sure that
+	   seeked offset is valid. So the magic value should be something that
+	   normally doesn't occur in mails. */
 	unsigned char magic[2];
 	unsigned char uid_hex[8];
 	unsigned char mail_size_hex[16];
@@ -42,13 +65,18 @@
 	unsigned char seen;
 	unsigned char draft;
 	unsigned char expunged;
-	/* unsigned char keywords[]; */
+	/* unsigned char keywords[keywords_count]; */
 };
 
 struct dbox_storage {
 	struct index_storage storage;
 };
 
+struct keyword_map {
+	unsigned int index_idx;
+	unsigned int file_idx;
+};
+
 struct dbox_file {
 	uint32_t file_seq;
 	char *path;
@@ -57,17 +85,27 @@
 	struct istream *input;
 	struct ostream *output; /* while appending mails */
 
+	uint16_t base_header_size;
 	uint32_t header_size;
 	time_t create_time;
 	uint64_t append_offset;
 	uint16_t mail_header_size;
-	uint16_t mail_header_padding;
+	uint16_t mail_header_align;
 	uint16_t keyword_count;
+	uint64_t keyword_list_offset;
+	uint32_t keyword_list_size_alloc;
+	uint32_t keyword_list_size_used;
 
 	uoff_t seeked_offset;
 	uoff_t seeked_mail_size;
 	uint32_t seeked_uid;
-        struct dbox_mail_header seeked_mail_header;
+	struct dbox_mail_header seeked_mail_header;
+	unsigned char *seeked_keywords;
+
+	/* Keywords list, sorted by index_idx. */
+	array_t ARRAY_DEFINE(idx_file_keywords, struct keyword_map);
+	/* idx -> index_idx array */
+	array_t ARRAY_DEFINE(file_idx_keywords, unsigned int);
 };
 
 struct dbox_mailbox {

Index: dbox-sync-full.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/dbox/dbox-sync-full.c,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- dbox-sync-full.c	5 Jan 2006 01:27:51 -0000	1.2
+++ dbox-sync-full.c	28 Jan 2006 22:05:48 -0000	1.3
@@ -6,6 +6,7 @@
 #include "dbox-storage.h"
 #include "dbox-uidlist.h"
 #include "dbox-file.h"
+#include "dbox-keywords.h"
 #include "dbox-sync.h"
 
 #include <stdlib.h>
@@ -13,11 +14,36 @@
 #include <fcntl.h>
 #include <dirent.h>
 
+static int
+dbox_mail_get_keywords(struct dbox_mailbox *mbox, struct dbox_file *file,
+		       array_t *keywords)
+{
+	const unsigned int *map;
+	unsigned int i, count;
+
+	ARRAY_SET_TYPE(keywords, unsigned int);
+
+	if (!array_is_created(&file->file_idx_keywords)) {
+		if (dbox_file_read_keywords(mbox, file) < 0)
+			return -1;
+	}
+
+	map = array_get(&file->file_idx_keywords, &count);
+	for (i = 0; i < count; i++) {
+		if (file->seeked_keywords[i] != '0')
+			array_append(keywords, &map[i], 1);
+	}
+
+	return 0;
+}
+
 static int dbox_sync_full_mail(struct dbox_sync_context *ctx, uint32_t *seq_r)
 {
 	struct dbox_mailbox *mbox = ctx->mbox;
 	const struct dbox_mail_header *hdr = &mbox->file->seeked_mail_header;
 	enum mail_flags flags;
+        struct mail_keywords *keywords;
+	array_t ARRAY_DEFINE(keywords_arr, unsigned int);
 	uint32_t seq;
 	uint64_t hdr_offset = mbox->file->seeked_offset;
 
@@ -60,7 +86,20 @@
 	if (hdr->draft == '1')
 		flags |= MAIL_DRAFT;
 	mail_index_update_flags(ctx->trans, seq, MODIFY_REPLACE, flags);
-	// FIXME: keywords
+
+	t_push();
+	ARRAY_CREATE(&keywords_arr, pool_datastack_create(),
+		     unsigned int, mbox->file->keyword_count);
+	if (dbox_mail_get_keywords(mbox, mbox->file, &keywords_arr) < 0) {
+		t_pop();
+		return -1;
+	}
+	keywords = mail_index_keywords_create_from_indexes(ctx->trans,
+							   &keywords_arr);
+	mail_index_update_keywords(ctx->trans, seq, MODIFY_REPLACE, keywords);
+	mail_index_keywords_free(&keywords);
+	t_pop();
+
 	mail_index_update_ext(ctx->trans, seq, mbox->dbox_file_ext_idx,
 			      &mbox->file->file_seq, NULL);
 	mail_index_update_ext(ctx->trans, seq, mbox->dbox_offset_ext_idx,

Index: dbox-sync.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/dbox/dbox-sync.c,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -d -r1.5 -r1.6
--- dbox-sync.c	14 Jan 2006 18:47:46 -0000	1.5
+++ dbox-sync.c	28 Jan 2006 22:05:48 -0000	1.6
@@ -4,8 +4,10 @@
 #include "ioloop.h"
 #include "array.h"
 #include "hash.h"
+#include "seq-range-array.h"
 #include "write-full.h"
 #include "dbox-file.h"
+#include "dbox-keywords.h"
 #include "dbox-sync.h"
 #include "dbox-uidlist.h"
 #include "dbox-storage.h"
@@ -120,45 +122,18 @@
 	return 0;
 }
 
-int dbox_sync_update_flags(struct dbox_sync_context *ctx,
-			   const struct dbox_sync_rec *sync_rec)
+static int
+dbox_sync_write_mask(struct dbox_sync_context *ctx,
+		     const struct dbox_sync_rec *sync_rec,
+                     unsigned int first_flag_offset, unsigned int flag_count,
+		     const unsigned char *array, const unsigned char *mask)
 {
-	static enum mail_flags dbox_flag_list[] = {
-		MAIL_ANSWERED,
-		MAIL_FLAGGED,
-		MAIL_DELETED,
-		MAIL_SEEN,
-		MAIL_DRAFT,
-		0 /* expunged */
-	};
-#define DBOX_FLAG_COUNT (sizeof(dbox_flag_list)/sizeof(dbox_flag_list[0]))
 	struct dbox_mailbox *mbox = ctx->mbox;
-	unsigned char dbox_flag_array[DBOX_FLAG_COUNT];
-	unsigned char dbox_flag_mask[DBOX_FLAG_COUNT];
 	uint32_t file_seq, uid2;
 	uoff_t offset;
-	unsigned int i, start, first_flag_offset;
+	unsigned int i, start;
 	int ret;
 
-	/* first build flag array and mask */
-	if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE) {
-		memset(dbox_flag_array, '0', sizeof(dbox_flag_array));
-		memset(dbox_flag_mask, 0, sizeof(dbox_flag_mask));
-		dbox_flag_mask[5] = 1;
-		dbox_flag_array[5] = '1';
-	} else {
-		i_assert(sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS);
-		for (i = 0; i < DBOX_FLAG_COUNT; i++) {
-			dbox_flag_array[i] =
-				(sync_rec->value.flags.add &
-				 dbox_flag_list[i]) != 0 ? '1' : '0';
-			dbox_flag_mask[i] = dbox_flag_array[i] ||
-				(sync_rec->value.flags.remove &
-				 dbox_flag_list[i]) != 0;
-		}
-	}
-	first_flag_offset = offsetof(struct dbox_mail_header, answered);
-
 	if (dbox_sync_get_file_offset(ctx, sync_rec->seq1,
 				      &file_seq, &offset) < 0)
 		return -1;
@@ -172,18 +147,18 @@
 		return ret;
 
 	while (mbox->file->seeked_uid <= uid2) {
-		for (i = 0; i < DBOX_FLAG_COUNT; ) {
-			if (!dbox_flag_mask[i])
+		for (i = 0; i < flag_count; ) {
+			if (!mask[i])
 				continue;
 
 			start = i;
-			while (i < DBOX_FLAG_COUNT) {
-				if (!dbox_flag_mask[i])
+			while (i < flag_count) {
+				if (!mask[i])
 					break;
 				i++;
 			}
 			ret = pwrite_full(ctx->mbox->file->fd,
-					  dbox_flag_array+start, i - start,
+					  array + start, i - start,
 					  offset + first_flag_offset + start);
 			if (ret < 0) {
 				mail_storage_set_critical(
@@ -205,11 +180,143 @@
 	return 0;
 }
 
+int dbox_sync_update_flags(struct dbox_sync_context *ctx,
+			   const struct dbox_sync_rec *sync_rec)
+{
+	static enum mail_flags dbox_flag_list[] = {
+		MAIL_ANSWERED,
+		MAIL_FLAGGED,
+		MAIL_DELETED,
+		MAIL_SEEN,
+		MAIL_DRAFT,
+		0 /* expunged */
+	};
+#define DBOX_FLAG_COUNT (sizeof(dbox_flag_list)/sizeof(dbox_flag_list[0]))
+	unsigned char dbox_flag_array[DBOX_FLAG_COUNT];
+	unsigned char dbox_flag_mask[DBOX_FLAG_COUNT];
+	unsigned int i, first_flag_offset;
+
+	/* first build flag array and mask */
+	if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE) {
+		memset(dbox_flag_array, '0', sizeof(dbox_flag_array));
+		memset(dbox_flag_mask, 0, sizeof(dbox_flag_mask));
+		dbox_flag_mask[5] = 1;
+		dbox_flag_array[5] = '1';
+	} else {
+		i_assert(sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS);
+		for (i = 0; i < DBOX_FLAG_COUNT; i++) {
+			dbox_flag_array[i] =
+				(sync_rec->value.flags.add &
+				 dbox_flag_list[i]) != 0 ? '1' : '0';
+			dbox_flag_mask[i] = dbox_flag_array[i] ||
+				(sync_rec->value.flags.remove &
+				 dbox_flag_list[i]) != 0;
+		}
+	}
+	first_flag_offset = offsetof(struct dbox_mail_header, answered);
+
+	return dbox_sync_write_mask(ctx, sync_rec,
+				    first_flag_offset, DBOX_FLAG_COUNT,
+				    dbox_flag_array, dbox_flag_mask);
+}
+
+static int
+dbox_sync_update_keyword(struct dbox_sync_context *ctx,
+			 const struct dbox_sync_rec *sync_rec, bool set)
+{
+	unsigned char keyword_array, keyword_mask = 1;
+	unsigned int file_idx, first_flag_offset;
+
+	keyword_array = set ? '1' : '0';
+
+	if (!dbox_file_lookup_keyword(ctx->mbox, ctx->mbox->file,
+				      sync_rec->value.keyword_idx, &file_idx)) {
+		/* not found. if removing, just ignore.
+
+		   if adding, it currently happens only if the maximum keyword
+		   count was reached. once we support moving mails to new file
+		   to grow keywords count, this should never happen.
+		   for now, just ignore this. */
+		return 0;
+	}
+
+	first_flag_offset = sizeof(struct dbox_mail_header) + file_idx;
+	return dbox_sync_write_mask(ctx, sync_rec, first_flag_offset, 1,
+				    &keyword_array, &keyword_mask);
+}
+
+static int
+dbox_sync_reset_keyword(struct dbox_sync_context *ctx,
+			 const struct dbox_sync_rec *sync_rec)
+{
+	unsigned char *keyword_array, *keyword_mask;
+	unsigned int first_flag_offset;
+	int ret;
+
+	t_push();
+	keyword_array = t_malloc(ctx->mbox->file->keyword_count);
+	keyword_mask = t_malloc(ctx->mbox->file->keyword_count);
+	memset(keyword_array, '0', ctx->mbox->file->keyword_count);
+	memset(keyword_mask, 1, ctx->mbox->file->keyword_count);
+
+	first_flag_offset = sizeof(struct dbox_mail_header);
+	ret = dbox_sync_write_mask(ctx, sync_rec, first_flag_offset,
+				   ctx->mbox->file->keyword_count,
+				   keyword_array, keyword_mask);
+	t_pop();
+	return ret;
+}
+
+static int
+dbox_sync_file_add_keywords(struct dbox_sync_context *ctx,
+			    const struct dbox_sync_file_entry *entry,
+			    unsigned int i)
+{
+	array_t ARRAY_DEFINE(keywords, struct seq_range);
+	const struct dbox_sync_rec *sync_recs;
+	const struct seq_range *range;
+	unsigned int count, file_idx, keyword_idx;
+	int ret = 0;
+
+	if (dbox_file_seek(ctx->mbox, entry->file_seq, 0) <= 0)
+		return -1;
+
+	/* Get a list of all new keywords. Using seq_range is the easiest
+	   way to do this and should be pretty fast too. */
+	t_push();
+	ARRAY_CREATE(&keywords, pool_datastack_create(), struct seq_range, 16);
+	sync_recs = array_get(&entry->sync_recs, &count);
+	for (; i < count; i++) {
+		if (sync_recs[i].type != MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD)
+			continue;
+
+		/* check if it's already in the file */
+                keyword_idx = sync_recs[i].value.keyword_idx;
+		if (dbox_file_lookup_keyword(ctx->mbox, ctx->mbox->file,
+					     keyword_idx, &file_idx))
+			continue;
+
+		/* add it. if it already exists, it's handled internally. */
+		seq_range_array_add(&keywords, 0, keyword_idx);
+	}
+
+	/* now, write them to file */
+	range = array_get(&keywords, &count);
+	if (count > 0) {
+		ret = dbox_file_append_keywords(ctx->mbox, ctx->mbox->file,
+						range, count);
+	}
+
+	t_pop();
+	return ret;
+}
+
 static int dbox_sync_file(struct dbox_sync_context *ctx,
                           const struct dbox_sync_file_entry *entry)
 {
 	const struct dbox_sync_rec *sync_recs;
 	unsigned int i, count;
+	bool first_keyword = TRUE;
 	int ret;
 
 	sync_recs = array_get(&entry->sync_recs, &count);
@@ -232,9 +339,25 @@
 				return -1;
 			break;
 		case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD:
+			if (first_keyword) {
+				/* add all new keywords in one go */
+				first_keyword = FALSE;
+				if (dbox_sync_file_add_keywords(ctx, entry,
+								i) < 0)
+					return -1;
+			}
+			if (dbox_sync_update_keyword(ctx, &sync_recs[i],
+						     TRUE) < 0)
+				return -1;
+			break;
 		case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE:
+			if (dbox_sync_update_keyword(ctx, &sync_recs[i],
+						     FALSE) < 0)
+				return -1;
+			break;
 		case MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET:
-			/* FIXME */
+			if (dbox_sync_reset_keyword(ctx, &sync_recs[i]) < 0)
+				return -1;
 			break;
 		case MAIL_INDEX_SYNC_TYPE_APPEND:
 			i_unreached();

Index: dbox-uidlist.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/dbox/dbox-uidlist.c,v
retrieving revision 1.13
retrieving revision 1.14
diff -u -d -r1.13 -r1.14
--- dbox-uidlist.c	14 Jan 2006 18:47:46 -0000	1.13
+++ dbox-uidlist.c	28 Jan 2006 22:05:48 -0000	1.14
@@ -794,26 +794,6 @@
 	return 0;
 }
 
-static int dbox_file_write_header(struct dbox_mailbox *mbox,
-				  struct dbox_file *file)
-{
-	struct dbox_file_header hdr;
-
-	// FIXME: code duplication
-	file->header_size = sizeof(hdr);
-	file->append_offset = file->header_size;
-	file->create_time = ioloop_time;
-	file->mail_header_size = sizeof(struct dbox_mail_header);
-
-	dbox_file_header_init(&hdr);
-	if (o_stream_send(file->output, &hdr, sizeof(hdr)) < 0) {
-		mail_storage_set_critical(STORAGE(mbox->storage),
-			"write(%s) failed: %m", file->path);
-		return -1;
-	}
-	return 0;
-}
-
 static int dbox_uidlist_files_lookup(struct dbox_uidlist_append_ctx *ctx,
 				     uint32_t file_seq)
 {
@@ -951,12 +931,7 @@
 
 	file->input = i_stream_create_file(file->fd, default_pool,
 					   65536, FALSE);
-
-	/* we'll be using CRLF linefeeds always */
-	output = o_stream_create_file(file->fd, default_pool, 0, FALSE);
-	file->output = o_stream_create_crlf(default_pool, output);
-	o_stream_unref(&output);
-
+	file->output = o_stream_create_file(file->fd, default_pool, 0, FALSE);
 	if ((uoff_t)st.st_size < sizeof(struct dbox_file_header)) {
 		if (dbox_file_write_header(mbox, file) < 0) {
 			dbox_file_close(file);
@@ -967,9 +942,16 @@
 			dbox_file_close(file);
 			return -1;
 		}
-		o_stream_seek(file->output, file->append_offset);
 	}
 
+	/* we'll always use CRLF linefeeds for mails (but not the header,
+	   so don't do this before dbox_file_write_header()) */
+	output = o_stream_create_crlf(default_pool, file->output);
+	o_stream_unref(&file->output);
+	file->output = output;
+
+	o_stream_seek(file->output, file->append_offset);
+
 	save_file->dev = st.st_dev;
 	save_file->ino = st.st_ino;
 	ARRAY_CREATE(&save_file->seqs, ctx->pool, unsigned int, 8);