[dovecot-cvs] dovecot/src/lib-index Makefile.am, 1.19, 1.20 mail-cache-compress.c, 1.11, 1.12 mail-cache-decisions.c, 1.4, 1.5 mail-cache-fields.c, NONE, 1.1 mail-cache-lookup.c, 1.13, 1.14 mail-cache-private.h, 1.9, 1.10 mail-cache-transaction.c, 1.14, 1.15 mail-cache.c, 1.40, 1.41 mail-cache.h, 1.18, 1.19

[cras at dovecot.org]

Update of /home/cvs/dovecot/src/lib-index
In directory talvi:/tmp/cvs-serv3398/lib-index

Modified Files:
	Makefile.am mail-cache-compress.c mail-cache-decisions.c 
	mail-cache-lookup.c mail-cache-private.h 
	mail-cache-transaction.c mail-cache.c mail-cache.h 
Added Files:
	mail-cache-fields.c 
Log Message:
Header caching redesigned. New design allows caching decisions per field, so
they can be divided to temporary/permanent. Cached headers are now always
returned in original order, old code didn't guarantee it. Some other caching
changes. (still missing code to store changes in caching decisions)



Index: Makefile.am
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/Makefile.am,v
retrieving revision 1.19
retrieving revision 1.20
diff -u -d -r1.19 -r1.20
--- Makefile.am	28 Jun 2004 17:35:27 -0000	1.19
+++ Makefile.am	18 Jul 2004 02:25:07 -0000	1.20
@@ -8,6 +8,7 @@
 	mail-cache.c \
 	mail-cache-compress.c \
 	mail-cache-decisions.c \
+	mail-cache-fields.c \
 	mail-cache-lookup.c \
 	mail-cache-transaction.c \
         mail-index.c \

Index: mail-cache-compress.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache-compress.c,v
retrieving revision 1.11
retrieving revision 1.12
diff -u -d -r1.11 -r1.12
--- mail-cache-compress.c	8 Jul 2004 21:27:22 -0000	1.11
+++ mail-cache-compress.c	18 Jul 2004 02:25:07 -0000	1.12
@@ -10,49 +10,40 @@
 
 struct mail_cache_copy_context {
 	int new_msg;
-	char field_seen[32], keep_fields[32], temp_fields[32];
-	buffer_t *buffer, *header;
+	buffer_t *buffer, *field_seen;
+	uint8_t field_seen_value;
 };
 
 static int
-mail_cache_compress_callback(struct mail_cache_view *view __attr_unused__,
-			     enum mail_cache_field field,
+mail_cache_compress_callback(struct mail_cache_view *view, uint32_t file_field,
 			     const void *data, size_t data_size, void *context)
 {
-        struct mail_cache_copy_context *ctx = context;
+	struct mail_cache_copy_context *ctx = context;
+	enum mail_cache_decision_type dec;
+	unsigned int field;
+	uint8_t *field_seen;
 	uint32_t size32;
-	int i;
-
-	if (ctx->new_msg) {
-		if (!ctx->temp_fields[field])
-			return 1;
-	} else {
-		if (!ctx->keep_fields[field])
-			return 1;
-	}
 
-	if (ctx->field_seen[field]) {
-		/* drop duplicates */
+	field_seen = buffer_get_space_unsafe(ctx->field_seen, file_field, 1);
+	if (*field_seen == ctx->field_seen_value) {
+		/* duplicate */
 		return 1;
 	}
-	ctx->field_seen[field] = TRUE;
+	*field_seen = ctx->field_seen_value;
 
-	for (i = 0; i < MAIL_CACHE_HEADERS_COUNT; i++) {
-		if (mail_cache_header_fields[i] == field) {
-			/* it's header - save it into header field */
-			size32 = buffer_get_used_size(ctx->header);
-			if (size32 > 0) {
-				/* remove old terminating \0 */
-				buffer_set_used_size(ctx->header, size32-1);
-			}
-			buffer_append(ctx->header, data, data_size);
+	field = view->cache->file_field_map[file_field];
+	dec = view->cache->fields[field].decision & ~MAIL_CACHE_DECISION_FORCED;
+	if (ctx->new_msg) {
+		if (dec == MAIL_CACHE_DECISION_NO)
+			return 1;
+	} else {
+		if (dec != MAIL_CACHE_DECISION_YES)
 			return 1;
-		}
 	}
 
-	buffer_append(ctx->buffer, &field, sizeof(field));
+	buffer_append(ctx->buffer, &file_field, sizeof(file_field));
 
-	if (mail_cache_field_sizes[field] == (unsigned int)-1) {
+	if (view->cache->fields[field].field_size == (unsigned int)-1) {
 		size32 = (uint32_t)data_size;
 		buffer_append(ctx->buffer, &size32, sizeof(size32));
 	}
@@ -72,12 +63,11 @@
 	const struct mail_index_header *idx_hdr;
 	struct mail_cache_header hdr;
 	struct mail_cache_record cache_rec;
-        enum mail_cache_field field;
 	struct ostream *output;
-	const char *str;
-	uint32_t size32, message_count, seq, first_new_seq, old_offset;
+	buffer_t *buffer;
+	size_t size;
+	uint32_t message_count, seq, first_new_seq, old_offset;
 	uoff_t offset;
-	int i, ret, header_idx;
 
 	/* get sequence of first message which doesn't need it's temp fields
 	   removed. */
@@ -104,85 +94,44 @@
 	hdr.indexid = idx_hdr->indexid;
 	hdr.file_seq = idx_hdr->cache_file_seq + 1;
 
-	if (cache->hdr != NULL) {
-		memcpy(hdr.field_usage_decision_type,
-		       cache->hdr->field_usage_decision_type,
-		       sizeof(hdr.field_usage_decision_type));
-		memcpy(hdr.field_usage_last_used,
-		       cache->hdr->field_usage_last_used,
-		       sizeof(hdr.field_usage_last_used));
-	} else {
-		memcpy(hdr.field_usage_decision_type,
-		       cache->default_field_usage_decision_type,
-		       sizeof(hdr.field_usage_decision_type));
-	}
-
-	memset(&ctx, 0, sizeof(ctx));
-	ctx.buffer = buffer_create_dynamic(default_pool, 4096, (size_t)-1);
-	ctx.header = buffer_create_dynamic(default_pool, 4096, (size_t)-1);
-
-	for (i = 0; i < 32; i++) {
-		if (hdr.field_usage_decision_type[i] & MAIL_CACHE_DECISION_YES)
-			ctx.keep_fields[i] = TRUE;
-		else if (hdr.field_usage_decision_type[i] &
-			 MAIL_CACHE_DECISION_TEMP) {
-			ctx.temp_fields[i] = TRUE;
-			ctx.keep_fields[i] = TRUE;
-		}
+	if (cache->fields_count != 0) {
+		hdr.field_header_offset =
+			mail_cache_uint32_to_offset(sizeof(hdr));
 	}
-
 	o_stream_send(output, &hdr, sizeof(hdr));
 
-	/* merge all the header pieces into one. if some message doesn't have
-	   all the required pieces, we'll just have to drop them all. */
-	for (i = MAIL_CACHE_HEADERS_COUNT-1; i >= 0; i--) {
-		str = mail_cache_get_header_fields_str(cache, i);
-		if (str != NULL)
-			break;
+	if (cache->fields_count != 0) {
+		t_push();
+		buffer = buffer_create_dynamic(pool_datastack_create(),
+					       256, (size_t)-1);
+		mail_cache_header_fields_get(cache, buffer);
+		o_stream_send(output, buffer_get_data(buffer, NULL),
+			      buffer_get_used_size(buffer));
+		t_pop();
 	}
 
-	if (str == NULL)
-		header_idx = -1;
-	else {
-		hdr.header_offsets[0] =
-			mail_cache_uint32_to_offset(output->offset);
-		header_idx = i;
-
-		size32 = strlen(str) + 1;
-		o_stream_send(output, &size32, sizeof(size32));
-		o_stream_send(output, str, size32);
-		if ((size32 & 3) != 0)
-			o_stream_send(output, null4, 4 - (size32 & 3));
-	}
+	memset(&ctx, 0, sizeof(ctx));
+	ctx.buffer = buffer_create_dynamic(default_pool, 4096, (size_t)-1);
+	ctx.field_seen = buffer_create_dynamic(default_pool, 64, (size_t)-1);
+	ctx.field_seen_value = 0;
 
 	mail_index_reset_cache(t, hdr.file_seq);
 
-	ret = 0;
 	for (seq = 1; seq <= message_count; seq++) {
 		ctx.new_msg = seq >= first_new_seq;
 		buffer_set_used_size(ctx.buffer, 0);
-		buffer_set_used_size(ctx.header, 0);
-		memset(ctx.field_seen, 0, sizeof(ctx.field_seen));
+
+		if (++ctx.field_seen_value == 0) {
+			memset(buffer_get_modifyable_data(ctx.field_seen, NULL),
+			       0, buffer_get_size(ctx.field_seen));
+			ctx.field_seen_value++;
+		}
 
 		memset(&cache_rec, 0, sizeof(cache_rec));
 		buffer_append(ctx.buffer, &cache_rec, sizeof(cache_rec));
 
-		mail_cache_foreach(cache_view, seq,
-				   mail_cache_compress_callback, &ctx);
-
-		size32 = buffer_get_used_size(ctx.header);
-		if (size32 > 0 && ctx.field_seen[header_idx]) {
-			field = MAIL_CACHE_HEADERS1;
-			buffer_append(ctx.buffer, &field, sizeof(field));
-			buffer_append(ctx.buffer, &size32, sizeof(size32));
-			buffer_append(ctx.buffer,
-				      buffer_get_data(ctx.header, NULL),
-				      size32);
-			if ((size32 & 3) != 0) {
-				buffer_append(ctx.buffer, null4,
-					      4 - (size32 & 3));
-			}
-		}
+		(void)mail_cache_foreach(cache_view, seq,
+					 mail_cache_compress_callback, &ctx);
 
 		cache_rec.size = buffer_get_used_size(ctx.buffer);
 		if (cache_rec.size == sizeof(cache_rec))
@@ -197,7 +146,6 @@
 	}
 	hdr.used_file_size = output->offset;
 	buffer_free(ctx.buffer);
-	buffer_free(ctx.header);
 
 	o_stream_seek(output, 0);
 	o_stream_send(output, &hdr, sizeof(hdr));
@@ -236,6 +184,12 @@
 		return -1;
 	locked = ret > 0;
 
+	/* get the latest info on fields */
+	if (mail_cache_header_fields_read(cache) < 0) {
+		if (locked) mail_cache_unlock(cache);
+		return -1;
+	}
+
 #ifdef DEBUG
 	i_warning("Compressing cache file %s", cache->filepath);
 #endif
@@ -246,6 +200,7 @@
 			       MAIL_CACHE_LOCK_IMMEDIATE_TIMEOUT, NULL, NULL);
 	if (fd == -1) {
 		mail_cache_set_syscall_error(cache, "file_dotlock_open()");
+		if (locked) mail_cache_unlock(cache);
 		return -1;
 	}
 
@@ -268,13 +223,11 @@
 
 			if (mail_cache_map(cache, 0, 0) < 0)
 				ret = -1;
+			else if (mail_cache_header_fields_read(cache) < 0)
+				ret = -1;
 		}
 	}
 
-	/* headers could have changed, reread them */
-	memset(cache->split_offsets, 0, sizeof(cache->split_offsets));
-	memset(cache->split_headers, 0, sizeof(cache->split_headers));
-
 	if (locked)
 		mail_cache_unlock(cache);
 

Index: mail-cache-decisions.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache-decisions.c,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -d -r1.4 -r1.5
--- mail-cache-decisions.c	8 Jul 2004 20:26:15 -0000	1.4
+++ mail-cache-decisions.c	18 Jul 2004 02:25:07 -0000	1.5
@@ -67,39 +67,21 @@
 */
 
 #include "lib.h"
-#include "write-full.h"
+#include "ioloop.h"
 #include "mail-cache-private.h"
 
-#include <stddef.h>
-
-static void
-mail_cache_set_decision_type(struct mail_cache *cache,
-			     enum mail_cache_field field,
-			     enum mail_cache_decision_type type)
-{
-	uint8_t value = type;
-
-	/* update the header without locking, we'll just write one byte and
-	   it's very unlikely someone else tries to write different value for
-	   it at the same time. even then it's just a wrong decision which
-	   will be corrected sometimes later, not too bad.. */
-	if (pwrite_full(cache->fd, &value, 1,
-			offsetof(struct mail_cache_header,
-				 field_usage_decision_type) + field) < 0) {
-		mail_cache_set_syscall_error(cache, "pwrite_full()");
-	}
-}
-
 void mail_cache_decision_lookup(struct mail_cache_view *view, uint32_t seq,
-				enum mail_cache_field field)
+				unsigned int field)
 {
+	struct mail_cache *cache = view->cache;
 	const struct mail_index_header *hdr;
 	uint32_t uid;
 
-	if (view->cache->hdr->field_usage_decision_type[field] !=
-	    MAIL_CACHE_DECISION_TEMP) {
+	i_assert(field < cache->fields_count);
+
+	if (cache->fields[field].decision != MAIL_CACHE_DECISION_TEMP) {
 		/* a) forced decision
-		   b) not cached, mail_cache_mark_missing() will handle this
+		   b) not cached, mail_cache_decision_add() will handle this
 		   c) permanently cached already, okay. */
 		return;
 	}
@@ -109,7 +91,13 @@
 	    mail_index_get_header(view->view, &hdr) < 0)
 		return;
 
-	if (uid < view->cache->field_usage_uid_highwater[field] ||
+	if (ioloop_time - cache->fields[field].last_used > 3600*24) {
+		/* update last_used about once a day */
+		cache->fields[field].last_used = ioloop_time;
+		cache->field_header_write_pending = TRUE;
+	}
+
+	if (uid < cache->fields[field].uid_highwater ||
 	    uid < hdr->day_first_uid[7]) {
 		/* a) nonordered access within this session. if client doesn't
 		      request messages in growing order, we assume it doesn't
@@ -118,32 +106,34 @@
 		      client with no local cache. if it was just a new client
 		      generating the local cache for the first time, we'll
 		      drop back to TEMP within few months. */
-		mail_cache_set_decision_type(view->cache, field,
-					     MAIL_CACHE_DECISION_YES);
+		cache->fields[field].decision = MAIL_CACHE_DECISION_YES;
+		cache->field_header_write_pending = TRUE;
 	} else {
-		view->cache->field_usage_uid_highwater[field] = uid;
+		cache->fields[field].uid_highwater = uid;
 	}
 }
 
 void mail_cache_decision_add(struct mail_cache_view *view, uint32_t seq,
-			     enum mail_cache_field field)
+			     unsigned int field)
 {
+	struct mail_cache *cache = view->cache;
 	uint32_t uid;
 
-	if (MAIL_CACHE_IS_UNUSABLE(view->cache))
+	i_assert(field < cache->fields_count);
+
+	if (MAIL_CACHE_IS_UNUSABLE(cache))
 		return;
 
-	if (view->cache->hdr->field_usage_decision_type[field] !=
-	    MAIL_CACHE_DECISION_NO) {
+	if (cache->fields[field].decision != MAIL_CACHE_DECISION_NO) {
 		/* a) forced decision
 		   b) we're already caching it, so it just wasn't in cache */
 		return;
 	}
 
 	/* field used the first time */
-	mail_cache_set_decision_type(view->cache, field,
-				     MAIL_CACHE_DECISION_TEMP);
+	cache->fields[field].decision = MAIL_CACHE_DECISION_TEMP;
+	cache->field_header_write_pending = TRUE;
 
 	if (mail_index_lookup_uid(view->view, seq, &uid) == 0)
-		view->cache->field_usage_uid_highwater[field] = uid;
+		cache->fields[field].uid_highwater = uid;
 }

--- NEW FILE: mail-cache-fields.c ---
/* Copyright (C) 2004 Timo Sirainen */

#include "lib.h"
#include "buffer.h"
#include "hash.h"
#include "mail-cache-private.h"

#include <stddef.h>

#define CACHE_HDR_PREFETCH 1024

static const unsigned char *null4[] = { 0, 0, 0, 0 };

void mail_cache_register_fields(struct mail_cache *cache,
				struct mail_cache_field *fields,
				size_t fields_count)
{
	void *orig_key, *orig_value;
	unsigned int new_idx;
	size_t i;

	new_idx = cache->fields_count;
	for (i = 0; i < fields_count; i++) {
		if (hash_lookup_full(cache->field_name_hash, fields[i].name,
				     &orig_key, &orig_value)) {
			fields[i].idx =
				POINTER_CAST_TO(orig_value, unsigned int);
			continue;
		}

		fields[i].idx = new_idx++;
	}

	if (new_idx == cache->fields_count)
		return;

	/* @UNSAFE */
	cache->fields = p_realloc(cache->field_pool, cache->fields,
				  cache->fields_count * sizeof(*cache->fields),
				  new_idx * sizeof(*cache->fields));
	cache->field_file_map =
		p_realloc(cache->field_pool, cache->field_file_map,
			  cache->fields_count * sizeof(*cache->field_file_map),
			  new_idx * sizeof(*cache->field_file_map));

	for (i = 0; i < fields_count; i++) {
		unsigned int idx = fields[i].idx;

		if (idx < cache->fields_count)
			continue;

		/* new index - save it */
		cache->fields[idx] = fields[i];
		cache->fields[idx].name =
			p_strdup(cache->field_pool, fields[i].name);
		cache->field_file_map[idx] = (uint32_t)-1;

		switch (cache->fields[idx].type) {
		case MAIL_CACHE_FIELD_FIXED_SIZE:
		case MAIL_CACHE_FIELD_BITMASK:
			break;
		case MAIL_CACHE_FIELD_VARIABLE_SIZE:
		case MAIL_CACHE_FIELD_STRING:
		case MAIL_CACHE_FIELD_HEADER:
			cache->fields[idx].field_size = (unsigned int)-1;
			break;
		}

		hash_insert(cache->field_name_hash,
			    (char *)cache->fields[idx].name,
			    POINTER_CAST(idx));
	}
	cache->fields_count = new_idx;
}

unsigned int
mail_cache_register_lookup(struct mail_cache *cache, const char *name)
{
	void *orig_key, *orig_value;

	if (hash_lookup_full(cache->field_name_hash, name,
			     &orig_key, &orig_value))
		return POINTER_CAST_TO(orig_value, unsigned int);
	else
		return (unsigned int)-1;
}

static int mail_cache_header_fields_get_offset(struct mail_cache *cache,
					       uint32_t *offset_r)
{
	const struct mail_cache_header_fields *field_hdr;
	uint32_t offset, next_offset;

	if (MAIL_CACHE_IS_UNUSABLE(cache)) {
		*offset_r = 0;
		return 0;
	}

	/* find the latest header */
	offset = 0;
	next_offset =
		mail_cache_offset_to_uint32(cache->hdr->field_header_offset);
	while (next_offset != 0) {
		offset = next_offset;

		if (mail_cache_map(cache, offset,
				   sizeof(*field_hdr) + CACHE_HDR_PREFETCH) < 0)
			return -1;

		field_hdr = CONST_PTR_OFFSET(cache->mmap_base, offset);
		next_offset =
			mail_cache_offset_to_uint32(field_hdr->next_offset);
	}

	*offset_r = offset;
	return 0;
}

int mail_cache_header_fields_read(struct mail_cache *cache)
{
	const struct mail_cache_header_fields *field_hdr = NULL;
	struct mail_cache_field field;
	const uint32_t *last_used, *sizes;
	const uint8_t *types, *decisions;
	const char *p, *names, *end;
	uint32_t offset, i;

	if (mail_cache_header_fields_get_offset(cache, &offset) < 0)
		return -1;

	if (offset == 0) {
		/* no fields - the file is empty */
		return 0;
	}

	field_hdr = CONST_PTR_OFFSET(cache->mmap_base, offset);
	if (offset + field_hdr->size > cache->mmap_length) {
		mail_cache_set_corrupted(cache,
					 "field header points outside file");
		return -1;
	}

	/* check the fixed size of the header. name[] has to be checked
	   separately */
	if (field_hdr->size < sizeof(*field_hdr) +
	    field_hdr->fields_count * (sizeof(uint32_t)*2 + 1 + 2)) {
		mail_cache_set_corrupted(cache, "invalid field header size");
		return -1;
	}

	if (field_hdr->size > sizeof(*field_hdr) + CACHE_HDR_PREFETCH) {
		if (mail_cache_map(cache, offset, field_hdr->size) < 0)
			return -1;
	}
	field_hdr = CONST_PTR_OFFSET(cache->mmap_base, offset);

	cache->file_field_map =
		i_realloc(cache->file_field_map,
			  cache->file_fields_count * sizeof(unsigned int),
			  field_hdr->fields_count * sizeof(unsigned int));
	cache->file_fields_count = field_hdr->fields_count;

        last_used = MAIL_CACHE_FIELD_LAST_USED(field_hdr);
	sizes = MAIL_CACHE_FIELD_SIZE(field_hdr);
	types = MAIL_CACHE_FIELD_TYPE(field_hdr);
	decisions = MAIL_CACHE_FIELD_DECISION(field_hdr);
	names = MAIL_CACHE_FIELD_NAMES(field_hdr);
	end = CONST_PTR_OFFSET(field_hdr, field_hdr->size);

	/* clear the old mapping */
	for (i = 0; i < cache->fields_count; i++)
		cache->field_file_map[i] = (uint32_t)-1;

	memset(&field, 0, sizeof(field));
	for (i = 0; i < field_hdr->fields_count; i++) {
		for (p = names; p != end && *p != '\0'; p++) ;
		if (p == end) {
			mail_cache_set_corrupted(cache,
				"field header names corrupted");
			return -1;
		}

		field.name = names;
		field.type = types[i];
		field.field_size = sizes[i];
		field.decision = decisions[i];
		field.last_used = (time_t)last_used[i];
		mail_cache_register_fields(cache, &field, 1);
		cache->field_file_map[field.idx] = i;
		cache->file_field_map[i] = field.idx;

		names = p + 1;
	}
	return 0;
}

int mail_cache_header_fields_update(struct mail_cache *cache)
{
	int locked = cache->locked;

	if (!locked) {
		if (mail_cache_lock(cache) <= 0)
			return -1;
	}

	// FIXME

	if (!locked)
		mail_cache_unlock(cache);
}

#define UGLY_COPY_MACRO(field_name, type) \
	for (i = 0; i < cache->file_fields_count; i++) {                \
		field = cache->file_field_map[i];                       \
		field_name = (type)cache->fields[field].field_name;     \
		buffer_append(dest, &field_name, sizeof(field_name));   \
	}                                                               \
	for (i = 0; i < cache->fields_count; i++) {                     \
		if (cache->field_file_map[i] != (uint32_t)-1)           \
			continue;                                       \
		field_name = (type)cache->fields[i].field_name;         \
		buffer_append(dest, &field_name, sizeof(field_name));   \
	}

void mail_cache_header_fields_get(struct mail_cache *cache, buffer_t *dest)
{
	struct mail_cache_header_fields hdr;
	unsigned int field;
	const char *name;
	uint32_t i, last_used, field_size;
	uint8_t type, decision;

	memset(&hdr, 0, sizeof(hdr));
	hdr.fields_count = cache->fields_count;
	buffer_append(dest, &hdr, sizeof(hdr));

	/* we have to keep the field order for the existing fields. */
        UGLY_COPY_MACRO(last_used, uint32_t);
        UGLY_COPY_MACRO(field_size, uint32_t);
        UGLY_COPY_MACRO(type, uint8_t);
        UGLY_COPY_MACRO(decision, uint8_t);

	for (i = 0; i < cache->file_fields_count; i++) {
		field = cache->file_field_map[i];
		name = cache->fields[field].name;
		buffer_append(dest, name, strlen(name)+1);
	}
	for (i = 0; i < cache->fields_count; i++) {
		if (cache->field_file_map[i] != (uint32_t)-1)
			continue;
		name = cache->fields[i].name;
		buffer_append(dest, name, strlen(name)+1);
	}

	hdr.size = buffer_get_used_size(dest);
	buffer_write(dest, 0, &hdr, sizeof(hdr));

	if ((hdr.size & 3) != 0)
		buffer_append(dest, null4, 4 - (hdr.size & 3));
}

int mail_cache_header_fields_get_next_offset(struct mail_cache *cache,
					     uint32_t *offset_r)
{
	if (mail_cache_header_fields_get_offset(cache, offset_r) < 0)
		return -1;

	if (*offset_r == 0) {
		*offset_r = offsetof(struct mail_cache_header,
				     field_header_offset);
	} else {
		*offset_r += offsetof(struct mail_cache_header_fields,
				      next_offset);
	}
	return 0;
}

Index: mail-cache-lookup.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache-lookup.c,v
retrieving revision 1.13
retrieving revision 1.14
diff -u -d -r1.13 -r1.14
--- mail-cache-lookup.c	10 Jul 2004 11:16:05 -0000	1.13
+++ mail-cache-lookup.c	18 Jul 2004 02:25:07 -0000	1.14
@@ -5,114 +5,9 @@
 #include "str.h"
 #include "mail-cache-private.h"
 
-#define CACHE_PREFETCH 1024
-
-const char *
-mail_cache_get_header_fields_str(struct mail_cache *cache, unsigned int idx)
-{
-	uint32_t offset, data_size;
-	const unsigned char *buf;
-
-	if (MAIL_CACHE_IS_UNUSABLE(cache))
-		return NULL;
-
-	offset = mail_cache_offset_to_uint32(cache->hdr->header_offsets[idx]);
-
-	if (offset == 0)
-		return NULL;
-
-	if (mail_cache_map(cache, offset, CACHE_PREFETCH) < 0)
-		return NULL;
-
-	if (offset + sizeof(data_size) > cache->mmap_length) {
-		mail_cache_set_corrupted(cache, "Header %u points outside file",
-					 idx);
-		return NULL;
-	}
-
-	buf = cache->mmap_base;
-	memcpy(&data_size, buf + offset, sizeof(data_size));
-	offset += sizeof(data_size);
-
-	if (data_size == 0) {
-		mail_cache_set_corrupted(cache,
-			"Header %u points to empty string", idx);
-		return NULL;
-	}
-
-	if (data_size + sizeof(data_size) > CACHE_PREFETCH) {
-		if (mail_cache_map(cache, offset, data_size) < 0)
-			return NULL;
-	}
-
-	if (offset + data_size > cache->mmap_length) {
-		mail_cache_set_corrupted(cache, "Header %u points outside file",
-					 idx);
-		return NULL;
-	}
-
-	buf = cache->mmap_base;
-	if (buf[offset + data_size - 1] != '\0') {
-		mail_cache_set_corrupted(cache,
-			"Header %u points to invalid string", idx);
-		return NULL;
-	}
-
-	return buf + offset;
-}
-
-const char *const *
-mail_cache_split_header(struct mail_cache *cache, const char *header)
-{
-	const char *const *arr, *const *tmp;
-	const char *null = NULL;
-	char *str;
-	buffer_t *buf;
-
-	if (header == NULL)
-		return NULL;
-
-	arr = t_strsplit(header, "\n");
-	buf = buffer_create_dynamic(cache->split_header_pool, 32, (size_t)-1);
-	for (tmp = arr; *tmp != NULL; tmp++) {
-		str = p_strdup(cache->split_header_pool, *tmp);
-		buffer_append(buf, &str, sizeof(str));
-	}
-	buffer_append(buf, &null, sizeof(null));
-
-	return buffer_get_data(buf, NULL);
-}
-
-const char *const *mail_cache_get_header_fields(struct mail_cache_view *view,
-						unsigned int idx)
-{
-	struct mail_cache *cache = view->cache;
-	const char *str;
-	int i;
-
-	i_assert(idx < MAIL_CACHE_HEADERS_COUNT);
-
-	if (MAIL_CACHE_IS_UNUSABLE(view->cache))
-		return NULL;
-
-	/* t_strsplit() is a bit slow, so we cache it */
-	if (cache->hdr->header_offsets[idx] != cache->split_offsets[idx]) {
-		p_clear(cache->split_header_pool);
-
-		t_push();
-		for (i = 0; i < MAIL_CACHE_HEADERS_COUNT; i++) {
-			cache->split_offsets[i] =
-				cache->hdr->header_offsets[i];
-
-			str = mail_cache_get_header_fields_str(cache, i);
-			cache->split_headers[i] =
-				mail_cache_split_header(cache, str);
-		}
-		t_pop();
-	}
+#include <stdlib.h>
 
-	return cache->split_headers[idx];
-}
+#define CACHE_PREFETCH 1024
 
 struct mail_cache_record *
 mail_cache_get_record(struct mail_cache *cache, uint32_t offset)
@@ -177,23 +72,38 @@
 		       const struct mail_cache_record *cache_rec,
 		       mail_cache_foreach_callback_t *callback, void *context)
 {
+	struct mail_cache *cache = view->cache;
 	size_t pos, next_pos, max_size, data_size;
-	uint32_t field;
+	uint32_t file_field;
+	unsigned int field;
 	int ret;
 
 	max_size = cache_rec->size;
 	if (max_size < sizeof(*cache_rec) + sizeof(uint32_t)*2) {
-		mail_cache_set_corrupted(view->cache,
-					 "record has invalid size");
+		mail_cache_set_corrupted(cache, "record has invalid size");
 		return -1;
 	}
 	max_size -= sizeof(uint32_t);
 
 	for (pos = sizeof(*cache_rec); pos < max_size; ) {
-		field = *((const uint32_t *)CONST_PTR_OFFSET(cache_rec, pos));
+		file_field =
+			*((const uint32_t *)CONST_PTR_OFFSET(cache_rec, pos));
 		pos += sizeof(uint32_t);
 
-		data_size = mail_cache_field_sizes[field];
+		if (file_field >= cache->file_fields_count) {
+			/* new field, have to re-read fields header to figure
+			   out it's size */
+			if (mail_cache_header_fields_read(cache) < 0)
+				return -1;
+			if (file_field >= cache->file_fields_count) {
+				mail_cache_set_corrupted(cache,
+					"field index too large");
+				return -1;
+			}
+		}
+
+		field = cache->file_field_map[file_field];
+		data_size = cache->fields[field].field_size;
 		if (data_size == (unsigned int)-1) {
 			data_size = *((const uint32_t *)
 				      CONST_PTR_OFFSET(cache_rec, pos));
@@ -202,14 +112,15 @@
 
 		next_pos = pos + ((data_size + 3) & ~3);
 		if (next_pos > cache_rec->size) {
-			mail_cache_set_corrupted(view->cache,
+			mail_cache_set_corrupted(cache,
 				"Record continues outside it's allocated size");
 			return -1;
 		}
 
-		ret = callback(view, field, CONST_PTR_OFFSET(cache_rec, pos),
+		ret = callback(view, file_field,
+			       CONST_PTR_OFFSET(cache_rec, pos),
 			       data_size, context);
-		if (ret <= 0)
+		if (ret != 1)
 			return ret;
 
 		pos = next_pos;
@@ -227,8 +138,15 @@
         if (MAIL_CACHE_IS_UNUSABLE(view->cache))
 		return 0;
 
-	if ((ret = mail_cache_lookup_offset(view, seq, &offset)) <= 0)
-		return ret;
+	if (view->cached_offset_seq == seq)
+		offset = view->cached_offset;
+	else {
+		if ((ret = mail_cache_lookup_offset(view, seq, &offset)) <= 0)
+			return ret;
+
+		view->cached_offset_seq = seq;
+		view->cached_offset = offset;
+	}
 
 	cache_rec = mail_cache_get_record(view->cache, offset);
 	while (cache_rec != NULL) {
@@ -251,13 +169,14 @@
 	return 1;
 }
 
-static int mail_cache_seq_callback(struct mail_cache_view *view,
-				   enum mail_cache_field field,
-				   const void *data __attr_unused__,
-				   size_t data_size __attr_unused__,
-				   void *context __attr_unused__)
+static int
+mail_cache_seq_callback(struct mail_cache_view *view, uint32_t file_field,
+			const void *data __attr_unused__,
+			size_t data_size __attr_unused__,
+			void *context __attr_unused__)
 {
-	view->cached_exists[field] = TRUE;
+	buffer_write(view->cached_exists_buf, file_field,
+		     &view->cached_exists_value, 1);
 	return 1;
 }
 
@@ -265,53 +184,63 @@
 {
 	int ret;
 
-	view->cached_exists_seq = seq;
-	memset(view->cached_exists, 0, sizeof(view->cached_exists));
+	if (++view->cached_exists_value == 0) {
+		/* wrapped, we'll have to clear the buffer */
+		memset(buffer_get_modifyable_data(view->cached_exists_buf,
+						  NULL), 0,
+		       buffer_get_size(view->cached_exists_buf));
+		view->cached_exists_value++;
+	}
 
+	view->cached_exists_seq = seq;
 	ret = mail_cache_foreach(view, seq, mail_cache_seq_callback, NULL);
 	return ret < 0 ? -1 : 0;
 }
 
 int mail_cache_field_exists(struct mail_cache_view *view, uint32_t seq,
-			    enum mail_cache_field field)
+			    unsigned int field)
 {
-	i_assert(field < MAIL_CACHE_FIELD_COUNT);
+	const uint8_t *data;
+	uint32_t file_field;
+	size_t size;
 
-        if (MAIL_CACHE_IS_UNUSABLE(view->cache))
+	i_assert(field < view->cache->fields_count);
+
+	file_field = view->cache->field_file_map[field];
+	if (file_field == (uint32_t)-1)
 		return 0;
 
 	if (view->cached_exists_seq != seq) {
 		if (mail_cache_seq(view, seq) < 0)
 			return -1;
 	}
-	return view->cached_exists[field];
+
+	data = buffer_get_data(view->cached_exists_buf, &size);
+	return size <= file_field ? FALSE :
+		data[file_field] == view->cached_exists_value;
 }
 
 enum mail_cache_decision_type
-mail_cache_field_get_decision(struct mail_cache *cache,
-			      enum mail_cache_field field)
+mail_cache_field_get_decision(struct mail_cache *cache, unsigned int field)
 {
-	i_assert(field < MAIL_CACHE_FIELD_COUNT);
-
-        if (MAIL_CACHE_IS_UNUSABLE(cache))
-		return cache->default_field_usage_decision_type[field];
+	i_assert(field < cache->fields_count);
 
-	return cache->hdr->field_usage_decision_type[field];
+	return cache->fields[field].decision;
 }
 
 struct mail_cache_lookup_context {
 	buffer_t *dest_buf;
-	enum mail_cache_field field;
+	uint32_t file_field;
 };
 
 static int
 mail_cache_lookup_callback(struct mail_cache_view *view __attr_unused__,
-			   enum mail_cache_field field,
-			   const void *data, size_t data_size, void *context)
+			   uint32_t file_field, const void *data,
+			   size_t data_size, void *context)
 {
         struct mail_cache_lookup_context *ctx = context;
 
-	if (ctx->field != field)
+	if (ctx->file_field != file_field)
 		return 1;
 
 	buffer_append(ctx->dest_buf, data, data_size);
@@ -319,60 +248,174 @@
 }
 
 int mail_cache_lookup_field(struct mail_cache_view *view, buffer_t *dest_buf,
-			    uint32_t seq, enum mail_cache_field field)
+			    uint32_t seq, unsigned int field)
 {
-        struct mail_cache_lookup_context ctx;
-
-	i_assert(field < MAIL_CACHE_FIELD_COUNT);
+	struct mail_cache_lookup_context ctx;
+	int ret;
 
-        if (MAIL_CACHE_IS_UNUSABLE(view->cache))
-		return 0;
+	if ((ret = mail_cache_field_exists(view, seq, field)) <= 0)
+		return ret;
 
 	mail_cache_decision_lookup(view, seq, field);
 
-	if (view->cached_exists_seq != seq) {
-		if (mail_cache_seq(view, seq) < 0)
-			return -1;
-	}
-
-	if (!view->cached_exists[field])
-		return 0;
-
 	/* should exist. find it. */
-	ctx.field = field;
+	ctx.file_field = view->cache->field_file_map[field];
 	ctx.dest_buf = dest_buf;
 	return mail_cache_foreach(view, seq, mail_cache_lookup_callback,
 				  &ctx) == 0;
 }
 
-int mail_cache_lookup_string_field(struct mail_cache_view *view, string_t *dest,
-				   uint32_t seq, enum mail_cache_field field)
+struct header_lookup_data_rec {
+	uint32_t offset;
+	uint32_t data_size;
+};
+
+struct header_lookup_data {
+	uint32_t line_num;
+        struct header_lookup_data_rec *data;
+};
+
+struct header_lookup_context {
+	unsigned int *fields;
+	size_t fields_count;
+	buffer_t *data;
+
+	unsigned int max_field;
+	uint8_t *fields_found;
+};
+
+static int
+headers_find_callback(struct mail_cache_view *view, uint32_t file_field,
+		      const void *data, size_t data_size, void *context)
 {
-	size_t old_size, new_size;
+	struct header_lookup_context *ctx = context;
+	const uint32_t *lines = data;
+	struct header_lookup_data hdr_data;
+        struct header_lookup_data_rec *hdr_data_rec;
+	unsigned int i, lines_count;
 
-	i_assert(field < MAIL_CACHE_FIELD_COUNT);
+	if (file_field > ctx->max_field || ctx->fields_found[file_field] != 1) {
+		/* a) don't want it, b) duplicate */
+		return 1;
+	}
+	ctx->fields_found[file_field]++;
 
-        if (MAIL_CACHE_IS_UNUSABLE(view->cache))
-		return 0;
+	/* data = { line_nums[], 0, "headers" } */
+	for (i = 0; data_size >= sizeof(uint32_t); i++) {
+		data_size -= sizeof(uint32_t);
+		if (lines[i] == 0)
+			break;
+	}
+	lines_count = i;
 
-	old_size = str_len(dest);
-	if (!mail_cache_lookup_field(view, dest, seq, field))
-		return 0;
+	/* FIXME: this relies on mmap() too heavily */
+	hdr_data_rec = t_new(struct header_lookup_data_rec, 1);
+	hdr_data_rec->offset = (const char *)&lines[lines_count+1] -
+		(const char *)view->cache->mmap_base;
+	hdr_data_rec->data_size = (uint32_t)data_size;
 
-	new_size = str_len(dest);
-	if (old_size == new_size ||
-	    str_data(dest)[new_size-1] != '\0') {
-		mail_cache_set_corrupted(view->cache,
-			"String field %x doesn't end with NUL", field);
-		return -1;
+	for (i = 0; i < lines_count; i++) {
+		hdr_data.line_num = lines[i];
+		hdr_data.data = hdr_data_rec;
+		buffer_append(ctx->data, &hdr_data, sizeof(hdr_data));
 	}
-	str_truncate(dest, new_size-1);
 	return 1;
 }
 
-enum mail_cache_record_flag
-mail_cache_get_record_flags(struct mail_cache_view *view, uint32_t seq)
+static int header_lookup_data_cmp(const void *p1, const void *p2)
 {
-	// FIXME:
-	return 0;
+	const struct header_lookup_data *d1 = p1, *d2 = p2;
+
+	return (int)d1->line_num - (int)d2->line_num;
+}
+
+int mail_cache_lookup_headers(struct mail_cache_view *view, string_t *dest,
+			      uint32_t seq, unsigned int fields[],
+			      size_t fields_count)
+{
+	struct mail_cache *cache = view->cache;
+	struct header_lookup_context ctx;
+	struct header_lookup_data *data;
+	const unsigned char *p, *start, *end;
+	size_t i, size, hdr_size;
+	unsigned int field_idx;
+	int ret;
+
+	if (fields_count == 0)
+		return 1;
+
+	t_push();
+
+	/* @UNSAFE */
+	memset(&ctx, 0, sizeof(ctx));
+	ctx.fields = t_new(unsigned int, fields_count);
+	ctx.fields_count = fields_count;
+
+	ctx.max_field = 1;
+	ctx.fields_found = t_buffer_get(ctx.max_field);
+	for (i = 0; i < fields_count; i++) {
+		i_assert(fields[i] < cache->fields_count);
+		field_idx = cache->field_file_map[fields[i]];
+		if (field_idx == (unsigned int)-1) {
+			/* not cached at all */
+			t_pop();
+			return 0;
+		}
+
+		if (field_idx > ctx.max_field) {
+			ctx.fields_found = t_buffer_reget(ctx.fields_found,
+							  field_idx + 1);
+			memset(ctx.fields_found + ctx.max_field + 1, 0,
+			       field_idx - ctx.max_field - 1);
+			ctx.max_field = field_idx;
+		}
+		ctx.fields_found[field_idx] = 1;
+                ctx.fields[i] = field_idx;
+	}
+        t_buffer_alloc(ctx.max_field + 1);
+
+	ctx.data = buffer_create_dynamic(pool_datastack_create(),
+					 256, (size_t)-1);
+
+	/* we need to return them in sorted order. create array:
+	   { line number -> cache file offset } */
+	ret = mail_cache_foreach(view, seq, headers_find_callback, &ctx);
+	if (ret <= 0) {
+		t_pop();
+		return ret;
+	}
+
+	/* check that all fields were found */
+	for (i = 0; i < ctx.max_field; i++) {
+		if (ctx.fields_found[i] == 1) {
+			t_pop();
+			return 0;
+		}
+	}
+
+	data = buffer_get_modifyable_data(ctx.data, &size);
+	size /= sizeof(*data);
+	qsort(data, size, sizeof(*data), header_lookup_data_cmp);
+
+	/* then start filling dest buffer from the headers */
+	for (i = 0; i < size; i++) {
+		start = CONST_PTR_OFFSET(cache->mmap_base,
+					 data[i].data->offset);
+		end = start + data[i].data->data_size;
+
+		for (p = start; p != end; p++) {
+			if (*p == '\n' &&
+			    (p+1 == end || (p[1] != ' ' && p[1] != '\t'))) {
+				p++;
+				break;
+			}
+		}
+		hdr_size = (size_t)(p - start);
+		data[i].data->offset += hdr_size;
+		data[i].data->data_size += hdr_size;
+		buffer_append(dest, start, hdr_size);
+	}
+
+	t_pop();
+	return 1;
 }

Index: mail-cache-private.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache-private.h,v
retrieving revision 1.9
retrieving revision 1.10
diff -u -d -r1.9 -r1.10
--- mail-cache-private.h	10 Jul 2004 11:16:05 -0000	1.9
+++ mail-cache-private.h	18 Jul 2004 02:25:07 -0000	1.10
@@ -50,12 +50,44 @@
 	uint32_t used_file_size;
 	uint32_t deleted_space;
 
-	uint32_t field_usage_last_used[32]; /* time_t */
-	uint8_t field_usage_decision_type[32];
+	uint32_t field_header_offset;
+};
 
-	uint32_t header_offsets[MAIL_CACHE_HEADERS_COUNT];
+struct mail_cache_header_fields {
+	uint32_t next_offset;
+	uint32_t size;
+	uint32_t fields_count;
+
+#if 0
+	/* last time the field was accessed. not updated more often than
+	   once a day. */
+	uint32_t last_used[fields_count];
+	/* (uint32_t)-1 for variable sized fields */
+	uint32_t size[fields_count];
+	/* enum mail_cache_field_type */
+	uint8_t type[fields_count];
+	/* enum mail_cache_decision_type */
+	uint8_t decision[fields_count];
+	/* NUL-separated list of field names */
+	char name[fields_count][];
+#endif
 };
 
+#define MAIL_CACHE_FIELD_LAST_USED(field_hdr) \
+	CONST_PTR_OFFSET(field_hdr, sizeof(uint32_t) * 3)
+#define MAIL_CACHE_FIELD_SIZE(field_hdr) \
+	CONST_PTR_OFFSET(MAIL_CACHE_FIELD_LAST_USED(field_hdr), \
+			 sizeof(uint32_t) * (field_hdr)->fields_count)
+#define MAIL_CACHE_FIELD_TYPE(field_hdr) \
+	CONST_PTR_OFFSET(MAIL_CACHE_FIELD_SIZE(field_hdr), \
+			 sizeof(uint32_t) * (field_hdr)->fields_count)
+#define MAIL_CACHE_FIELD_DECISION(field_hdr) \
+	CONST_PTR_OFFSET(MAIL_CACHE_FIELD_TYPE(field_hdr), \
+			 sizeof(uint8_t) * (field_hdr)->fields_count)
+#define MAIL_CACHE_FIELD_NAMES(field_hdr) \
+	CONST_PTR_OFFSET(MAIL_CACHE_FIELD_DECISION(field_hdr), \
+			 sizeof(uint8_t) * (field_hdr)->fields_count)
+
 struct mail_cache_record {
 	uint32_t prev_offset;
 	uint32_t size; /* full record size, including this header */
@@ -85,16 +117,19 @@
 	const struct mail_cache_header *hdr;
 	struct mail_cache_header hdr_copy;
 
-	pool_t split_header_pool;
-	uint32_t split_offsets[MAIL_CACHE_HEADERS_COUNT];
-	const char *const *split_headers[MAIL_CACHE_HEADERS_COUNT];
+	pool_t field_pool;
+	struct mail_cache_field *fields;
+	uint32_t *field_file_map;
+	unsigned int fields_count;
+	struct hash_table *field_name_hash; /* name -> idx */
 
-	uint8_t default_field_usage_decision_type[32];
-	uint32_t field_usage_uid_highwater[32];
+	unsigned int *file_field_map;
+	unsigned int file_fields_count;
 
 	unsigned int locked:1;
 	unsigned int need_compress:1;
 	unsigned int hdr_modified:1;
+	unsigned int field_header_write_pending:1;
 };
 
 struct mail_cache_view {
@@ -104,18 +139,20 @@
 	struct mail_cache_transaction_ctx *transaction;
 	uint32_t trans_seq1, trans_seq2;
 
-	char cached_exists[32];
+	/* if cached_exists_buf[field] == cached_exists_value, it's cached.
+	   this allows us to avoid constantly clearing the whole buffer.
+	   it needs to be cleared only when cached_exists_value is wrapped. */
+	buffer_t *cached_exists_buf;
+	uint8_t cached_exists_value;
 	uint32_t cached_exists_seq;
+	uint32_t cached_offset, cached_offset_seq;
 };
 
 typedef int mail_cache_foreach_callback_t(struct mail_cache_view *view,
-					  enum mail_cache_field field,
+					  uint32_t file_field,
 					  const void *data, size_t data_size,
 					  void *context);
 
-extern unsigned int mail_cache_field_sizes[32];
-extern enum mail_cache_field mail_cache_header_fields[MAIL_CACHE_HEADERS_COUNT];
-
 uint32_t mail_cache_uint32_to_offset(uint32_t offset);
 uint32_t mail_cache_offset_to_uint32(uint32_t offset);
 
@@ -124,10 +161,11 @@
 int mail_cache_lock(struct mail_cache *cache);
 void mail_cache_unlock(struct mail_cache *cache);
 
-const char *
-mail_cache_get_header_fields_str(struct mail_cache *cache, unsigned int idx);
-const char *const *
-mail_cache_split_header(struct mail_cache *cache, const char *header);
+int mail_cache_header_fields_read(struct mail_cache *cache);
+int mail_cache_header_fields_update(struct mail_cache *cache);
+void mail_cache_header_fields_get(struct mail_cache *cache, buffer_t *dest);
+int mail_cache_header_fields_get_next_offset(struct mail_cache *cache,
+					     uint32_t *offset_r);
 
 struct mail_cache_record *
 mail_cache_get_record(struct mail_cache *cache, uint32_t offset);
@@ -152,9 +190,9 @@
 int mail_cache_delete(struct mail_cache *cache, uint32_t offset);
 
 void mail_cache_decision_lookup(struct mail_cache_view *view, uint32_t seq,
-				enum mail_cache_field field);
+				uint32_t field);
 void mail_cache_decision_add(struct mail_cache_view *view, uint32_t seq,
-			     enum mail_cache_field field);
+			     uint32_t field);
 
 void mail_cache_set_syscall_error(struct mail_cache *cache,
 				  const char *function);

Index: mail-cache-transaction.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache-transaction.c,v
retrieving revision 1.14
retrieving revision 1.15
diff -u -d -r1.14 -r1.15
--- mail-cache-transaction.c	15 Jul 2004 12:26:31 -0000	1.14
+++ mail-cache-transaction.c	18 Jul 2004 02:25:07 -0000	1.15
@@ -16,9 +16,6 @@
 	struct mail_cache_view *view;
 	struct mail_index_transaction *trans;
 
-	uint32_t update_header_offsets[MAIL_CACHE_HEADERS_COUNT];
-	unsigned int next_unused_header_lowwater;
-
 	buffer_t *cache_data, *cache_data_seq;
 	uint32_t prev_seq;
 	size_t prev_pos;
@@ -279,6 +276,9 @@
 	size_t size;
 	int ret;
 
+	i_assert((min_size & 3) == 0);
+	i_assert((max_size & 3) == 0);
+
 	if (min_size > ctx->reserved_space) {
 		if (!locked) {
 			if (mail_cache_lock(ctx->cache) <= 0)
@@ -302,6 +302,7 @@
 	ctx->reserved_space -= size;
 	if (available_space_r != NULL)
 		*available_space_r = size;
+	i_assert((size & 3) == 0);
 
 	if (size == max_size && commit) {
 		/* final commit - see if we can free the rest of the
@@ -436,8 +437,7 @@
 int mail_cache_transaction_commit(struct mail_cache_transaction_ctx *ctx)
 {
 	struct mail_cache *cache = ctx->cache;
-	uint32_t offset;
-	int i, ret = 0;
+	int ret = 0;
 
 	if (!ctx->changes) {
 		mail_cache_transaction_free(ctx);
@@ -461,24 +461,7 @@
 		ret = -1;
 	}
 
-	if (ret == 0) {
-		for (i = 0; i < MAIL_CACHE_HEADERS_COUNT; i++) {
-			offset = ctx->update_header_offsets[i];
-			if (offset != 0) {
-				cache->hdr_copy.header_offsets[i] =
-					mail_cache_uint32_to_offset(offset);
-				cache->hdr_modified = TRUE;
-			}
-		}
-	}
-
 	mail_cache_unlock(cache);
-
-	if (ctx->next_unused_header_lowwater == MAIL_CACHE_HEADERS_COUNT) {
-		/* they're all used - compress the cache to get more */
-		cache->need_compress = TRUE;
-	}
-
 	mail_cache_transaction_free(ctx);
 	return ret;
 }
@@ -488,7 +471,6 @@
 	struct mail_cache *cache = ctx->cache;
 	const uint32_t *buf;
 	size_t size;
-	unsigned int i;
 
 	if (mail_cache_lock(cache) > 0) {
 		mail_cache_transaction_free_space(ctx);
@@ -510,112 +492,86 @@
 		mail_cache_unlock(cache);
 	}
 
-	/* make sure we don't cache the headers */
-	for (i = 0; i < ctx->next_unused_header_lowwater; i++) {
-		uint32_t offset = cache->hdr->header_offsets[i];
-		if (mail_cache_offset_to_uint32(offset) == 0)
-			cache->split_offsets[i] = 1;
-	}
-
 	mail_cache_transaction_free(ctx);
 }
 
-static const char *write_header_string(const char *const headers[],
-				       uint32_t *size_r)
+static int
+mail_cache_header_write_fields(struct mail_cache_transaction_ctx *ctx)
 {
+	struct mail_cache *cache = ctx->cache;
 	buffer_t *buffer;
+	const void *data;
 	size_t size;
+	uint32_t offset, hdr_offset;
+	int ret = 0;
 
-	buffer = buffer_create_dynamic(pool_datastack_create(),
-				       512, (size_t)-1);
-
-	while (*headers != NULL) {
-		if (buffer_get_used_size(buffer) != 0)
-			buffer_append(buffer, "\n", 1);
-		buffer_append(buffer, *headers, strlen(*headers));
-		headers++;
-	}
-	buffer_append(buffer, null4, 1);
-
-	size = buffer_get_used_size(buffer);
-	if ((size & 3) != 0) {
-		buffer_append(buffer, null4, 4 - (size & 3));
-		size += 4 - (size & 3);
-	}
-	*size_r = size;
-	return buffer_get_data(buffer, NULL);
-}
-
-int mail_cache_set_header_fields(struct mail_cache_transaction_ctx *ctx,
-				 unsigned int idx, const char *const headers[])
-{
-	struct mail_cache *cache = ctx->cache;
-	uint32_t offset, size, total_size;
-	const char *header_str, *prev_str;
-
-	i_assert(*headers != NULL);
-	i_assert(idx < MAIL_CACHE_HEADERS_COUNT);
-	i_assert(idx >= ctx->next_unused_header_lowwater);
-	i_assert(mail_cache_offset_to_uint32(cache->hdr->
-					     header_offsets[idx]) == 0);
+	if (mail_cache_lock(cache) <= 0)
+		return -1;
 
 	t_push();
+	buffer = buffer_create_dynamic(pool_datastack_create(),
+				       256, (size_t)-1);
+	mail_cache_header_fields_get(cache, buffer);
+	data = buffer_get_data(buffer, &size);
 
-	header_str = write_header_string(headers, &size);
-	if (idx != 0) {
-		prev_str = mail_cache_get_header_fields_str(cache, idx-1);
-		if (prev_str == NULL) {
-			t_pop();
-			return FALSE;
-		}
-
-		i_assert(strcmp(header_str, prev_str) != 0);
-	}
-
-	total_size = size + sizeof(uint32_t);
-	offset = mail_cache_transaction_get_space(ctx, total_size, total_size,
-						  NULL, FALSE);
-	if (offset != 0) {
-		if (pwrite_full(cache->fd, &size, sizeof(size), offset) < 0 ||
-		    pwrite_full(cache->fd, header_str, size,
-				offset + sizeof(uint32_t)) < 0) {
+	offset = mail_cache_transaction_get_space(ctx, size, size, &size, TRUE);
+	if (offset == 0)
+		ret = -1;
+	else if (pwrite_full(cache->fd, data, size, offset) < 0) {
+		mail_cache_set_syscall_error(cache, "pwrite_full()");
+		ret = -1;
+	} else if (fdatasync(cache->fd) < 0) {
+		mail_cache_set_syscall_error(cache, "fdatasync()");
+		ret = -1;
+	} else if (mail_cache_header_fields_get_next_offset(cache,
+							    &hdr_offset) < 0)
+		ret = -1;
+	else {
+		/* after it's guaranteed to be in disk, update header offset */
+		offset = mail_cache_uint32_to_offset(offset);
+		if (pwrite_full(cache->fd, &offset, sizeof(offset),
+				hdr_offset) < 0) {
 			mail_cache_set_syscall_error(cache, "pwrite_full()");
-			offset = 0;
+			ret = -1;
+		} else {
+			/* we'll need to fix mappings. */
+			if (mail_cache_header_fields_read(cache) < 0)
+				ret = -1;
 		}
 	}
-
-	if (offset != 0) {
-		ctx->update_header_offsets[idx] = offset;
-		ctx->changes = TRUE;
-
-		/* update cached headers */
-		cache->split_offsets[idx] = cache->hdr->header_offsets[idx];
-		cache->split_headers[idx] =
-			mail_cache_split_header(cache, header_str);
-
-		/* make sure get_header_fields() still works for this header
-		   while the transaction isn't yet committed. */
-		ctx->next_unused_header_lowwater = idx + 1;
-	}
-
 	t_pop();
-	return offset > 0;
+
+	mail_cache_unlock(cache);
+	return ret;
 }
 
 void mail_cache_add(struct mail_cache_transaction_ctx *ctx, uint32_t seq,
-		    enum mail_cache_field field,
-		    const void *data, size_t data_size)
+		    unsigned int field, const void *data, size_t data_size)
 {
-	uint32_t fixed_size, data_size32;
+	uint32_t file_field, data_size32;
+	unsigned int fixed_size;
 	size_t full_size;
 
-	i_assert(field < MAIL_CACHE_FIELD_COUNT);
-	i_assert(data_size > 0);
+	i_assert(field < ctx->cache->fields_count);
 	i_assert(data_size < (uint32_t)-1);
 
+	if (ctx->cache->fields[field].decision ==
+	    (MAIL_CACHE_DECISION_NO | MAIL_CACHE_DECISION_FORCED))
+		return;
+
+	file_field = ctx->cache->field_file_map[field];
+	if (file_field == (uint32_t)-1) {
+		/* we'll have to add this field to headers */
+		if (mail_cache_header_write_fields(ctx) < 0)
+			return;
+
+		file_field = ctx->cache->field_file_map[field];
+		i_assert(file_field != (uint32_t)-1);
+	}
+
 	mail_cache_decision_add(ctx->view, seq, field);
 
-	fixed_size = mail_cache_field_sizes[field];
+	fixed_size = ctx->cache->fields[field].field_size;
 	i_assert(fixed_size == (unsigned int)-1 || fixed_size == data_size);
 
 	data_size32 = (uint32_t)data_size;
@@ -643,7 +599,7 @@
 			return;
 	}
 
-	buffer_append(ctx->cache_data, &field, sizeof(field));
+	buffer_append(ctx->cache_data, &file_field, sizeof(file_field));
 	if (fixed_size == (unsigned int)-1) {
 		buffer_append(ctx->cache_data, &data_size32,
 			      sizeof(data_size32));
@@ -654,12 +610,6 @@
                 buffer_append(ctx->cache_data, null4, 4 - (data_size & 3));
 }
 
-int mail_cache_update_record_flags(struct mail_cache_view *view, uint32_t seq,
-				   enum mail_cache_record_flag flags)
-{
-	return -1;
-}
-
 int mail_cache_transaction_lookup(struct mail_cache_transaction_ctx *ctx,
 				  uint32_t seq, uint32_t *offset_r)
 {

Index: mail-cache.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache.c,v
retrieving revision 1.40
retrieving revision 1.41
diff -u -d -r1.40 -r1.41
--- mail-cache.c	9 Jul 2004 19:12:32 -0000	1.40
+++ mail-cache.c	18 Jul 2004 02:25:07 -0000	1.41
@@ -1,37 +1,15 @@
 /* Copyright (C) 2003-2004 Timo Sirainen */
 
 #include "lib.h"
+#include "buffer.h"
+#include "hash.h"
 #include "file-lock.h"
-#include "file-set-size.h"
 #include "mmap-util.h"
 #include "write-full.h"
 #include "mail-cache-private.h"
 
 #include <unistd.h>
 
-unsigned int mail_cache_field_sizes[32] = {
-	sizeof(enum mail_cache_record_flag),
-	sizeof(struct mail_sent_date),
-	sizeof(time_t),
-	sizeof(uoff_t),
-
-	/* variable sized */
-	(unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1,
-	(unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1,
-	(unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1,
-	(unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1,
-	(unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1,
-	(unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1,
-	(unsigned int)-1, (unsigned int)-1, (unsigned int)-1, (unsigned int)-1
-};
-
-enum mail_cache_field mail_cache_header_fields[MAIL_CACHE_HEADERS_COUNT] = {
-	MAIL_CACHE_HEADERS1,
-	MAIL_CACHE_HEADERS2,
-	MAIL_CACHE_HEADERS3,
-	MAIL_CACHE_HEADERS4
-};
-
 uint32_t mail_cache_uint32_to_offset(uint32_t offset)
 {
 	unsigned char buf[4];
@@ -129,6 +107,9 @@
 	if (mail_cache_map(cache, 0, 0) < 0)
 		return -1;
 
+	if (mail_cache_header_fields_read(cache) < 0)
+		return -1;
+
 	if (cache->hdr->file_seq != cache->index->hdr->cache_file_seq) {
 		/* still different - maybe a race condition or maybe the
 		   file_seq really is corrupted. either way, this shouldn't
@@ -240,7 +221,10 @@
 		return -1;
 	}
 
-	return mail_cache_map(cache, 0, sizeof(struct mail_cache_header));
+	if (mail_cache_map(cache, 0, sizeof(struct mail_cache_header)) < 0)
+		return -1;
+
+	return mail_cache_header_fields_read(cache);
 }
 
 struct mail_cache *mail_cache_open_or_create(struct mail_index *index)
@@ -250,7 +234,10 @@
 	cache = i_new(struct mail_cache, 1);
 	cache->index = index;
 	cache->fd = -1;
-        cache->split_header_pool = pool_alloconly_create("Headers", 512);
+        cache->field_pool = pool_alloconly_create("Cache fields", 512);
+	cache->field_name_hash =
+		hash_create(default_pool, cache->field_pool, 0,
+			    strcase_hash, (hash_cmp_callback_t *)strcasecmp);
 
 	if (!index->mmap_disable && !index->mmap_no_write) {
 		if (mail_cache_open_and_verify(cache) < 0) {
@@ -267,18 +254,13 @@
 {
 	mail_cache_file_close(cache);
 
-	pool_unref(cache->split_header_pool);
+	hash_destroy(cache->field_name_hash);
+	pool_unref(cache->field_pool);
+	i_free(cache->file_field_map);
 	i_free(cache->filepath);
 	i_free(cache);
 }
 
-void mail_cache_set_defaults(struct mail_cache *cache,
-			     const enum mail_cache_decision_type dec[32])
-{
-	memcpy(cache->default_field_usage_decision_type, dec,
-	       sizeof(cache->default_field_usage_decision_type));
-}
-
 int mail_cache_lock(struct mail_cache *cache)
 {
 	int i, ret;
@@ -367,10 +349,15 @@
 	view = i_new(struct mail_cache_view, 1);
 	view->cache = cache;
 	view->view = iview;
+	view->cached_exists_buf =
+		buffer_create_dynamic(default_pool,
+				      cache->file_fields_count + 10,
+				      (size_t)-1);
 	return view;
 }
 
 void mail_cache_view_close(struct mail_cache_view *view)
 {
+	buffer_free(view->cached_exists_buf);
 	i_free(view);
 }

Index: mail-cache.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache.h,v
retrieving revision 1.18
retrieving revision 1.19
diff -u -d -r1.18 -r1.19
--- mail-cache.h	8 Jul 2004 20:26:15 -0000	1.18
+++ mail-cache.h	18 Jul 2004 02:25:07 -0000	1.19
@@ -5,8 +5,6 @@
 
 #define MAIL_CACHE_FILE_PREFIX ".cache"
 
-#define MAIL_CACHE_HEADERS_COUNT 4
-
 struct mail_cache;
 struct mail_cache_view;
 struct mail_cache_transaction_ctx;
@@ -23,54 +21,37 @@
 	MAIL_CACHE_DECISION_FORCED	= 0x80
 };
 
-enum mail_cache_record_flag {
-	/* If binary flags are set, it's not checked whether mail is
-	   missing CRs. So this flag may be set as an optimization for
-	   regular non-binary mails as well if it's known that it contains
-	   valid CR+LF line breaks. */
-	MAIL_INDEX_FLAG_BINARY_HEADER		= 0x0001,
-	MAIL_INDEX_FLAG_BINARY_BODY		= 0x0002,
-
-	/* Mail header or body is known to contain NUL characters. */
-	MAIL_INDEX_FLAG_HAS_NULS		= 0x0004,
-	/* Mail header or body is known to not contain NUL characters. */
-	MAIL_INDEX_FLAG_HAS_NO_NULS		= 0x0008
+enum mail_cache_field_type {
+	MAIL_CACHE_FIELD_FIXED_SIZE,
+	MAIL_CACHE_FIELD_VARIABLE_SIZE,
+	MAIL_CACHE_FIELD_STRING,
+	MAIL_CACHE_FIELD_BITMASK,
+	MAIL_CACHE_FIELD_HEADER
 };
 
-/* when modifying, remember to update mail_cache_field_sizes[] too */
-enum mail_cache_field {
-	/* fixed size fields */
-	MAIL_CACHE_INDEX_FLAGS = 0,
-	MAIL_CACHE_SENT_DATE,
-	MAIL_CACHE_RECEIVED_DATE,
-	MAIL_CACHE_VIRTUAL_FULL_SIZE,
-
-	/* variable sized field */
-	MAIL_CACHE_HEADERS1,
-	MAIL_CACHE_HEADERS2,
-	MAIL_CACHE_HEADERS3,
-	MAIL_CACHE_HEADERS4,
-	MAIL_CACHE_BODY,
-	MAIL_CACHE_BODYSTRUCTURE,
-	MAIL_CACHE_ENVELOPE,
-	MAIL_CACHE_MESSAGEPART,
-	MAIL_CACHE_UID_STRING,
+struct mail_cache_field {
+	const char *name;
+	unsigned int idx;
 
-	MAIL_CACHE_FIELD_COUNT
-};
+	enum mail_cache_field_type type;
+	unsigned int field_size;
+	enum mail_cache_decision_type decision;
 
-struct mail_sent_date {
-	time_t time;
-	int32_t timezone;
+	/* internal: */
+	uint32_t uid_highwater;
+	time_t last_used;
 };
 
-extern enum mail_cache_field mail_cache_header_fields[MAIL_CACHE_HEADERS_COUNT];
-
 struct mail_cache *mail_cache_open_or_create(struct mail_index *index);
 void mail_cache_free(struct mail_cache *cache);
 
-void mail_cache_set_defaults(struct mail_cache *cache,
-			     const enum mail_cache_decision_type dec[32]);
+/* Register fields. fields[].idx is updated to contain field index. */
+void mail_cache_register_fields(struct mail_cache *cache,
+				struct mail_cache_field *fields,
+				size_t fields_count);
+/* Returns registered field index, or (unsigned int)-1 if not found. */
+unsigned int
+mail_cache_register_lookup(struct mail_cache *cache, const char *name);
 
 /* Returns TRUE if cache should be compressed. */
 int mail_cache_need_compress(struct mail_cache *cache);
@@ -86,46 +67,28 @@
 mail_cache_get_transaction(struct mail_cache_view *view,
 			   struct mail_index_transaction *t);
 
-/* Return NULL-terminated list of headers for given index, or NULL if
-   header index isn't used. */
-const char *const *mail_cache_get_header_fields(struct mail_cache_view *view,
-						unsigned int idx);
-/* Set list of headers for given index. */
-int mail_cache_set_header_fields(struct mail_cache_transaction_ctx *ctx,
-				 unsigned int idx, const char *const headers[]);
-
 /* Add new field to given record. Updates are not allowed. Fixed size fields
    must be exactly the expected size. */
 void mail_cache_add(struct mail_cache_transaction_ctx *ctx, uint32_t seq,
-		    enum mail_cache_field field,
-		    const void *data, size_t data_size);
+		    unsigned int field, const void *data, size_t data_size);
 
-/* Retursn TRUE if field exists. */
+/* Retursn 1 if field exists, 0 if not, -1 if error. */
 int mail_cache_field_exists(struct mail_cache_view *view, uint32_t seq,
-			    enum mail_cache_field field);
+			    unsigned int field);
 /* Returns current caching decision for given field. */
 enum mail_cache_decision_type
-mail_cache_field_get_decision(struct mail_cache *cache,
-			      enum mail_cache_field field);
+mail_cache_field_get_decision(struct mail_cache *cache, unsigned int field);
 
 /* Set data_r and size_r to point to wanted field in cache file.
-   Returns TRUE if field was found. If field contains multiple fields,
-   first one found is returned. This is mostly useful for finding headers. */
+   Returns 1 if field was found, 0 if not, -1 if error. */
 int mail_cache_lookup_field(struct mail_cache_view *view, buffer_t *dest_buf,
-			    uint32_t seq, enum mail_cache_field field);
-
-/* Return string field. */
-int mail_cache_lookup_string_field(struct mail_cache_view *view, string_t *dest,
-				   uint32_t seq, enum mail_cache_field field);
-
-/* Return record flags. */
-enum mail_cache_record_flag
-mail_cache_get_record_flags(struct mail_cache_view *view, uint32_t seq);
+			    uint32_t seq, unsigned int field);
 
-/* Update record flags. The cache file must be locked and the flags must be
-   already inserted to the record. */
-int mail_cache_update_record_flags(struct mail_cache_view *view, uint32_t seq,
-				   enum mail_cache_record_flag flags);
+/* Return specified cached headers. Returns 1 if all fields were found,
+   0 if not, -1 if error. dest is updated only if all fields were found. */
+int mail_cache_lookup_headers(struct mail_cache_view *view, string_t *dest,
+			      uint32_t seq, unsigned int fields[],
+			      size_t fields_count);
 
 /* "Error in index cache file %s: ...". */
 void mail_cache_set_corrupted(struct mail_cache *cache, const char *fmt, ...)