[dovecot-cvs] dovecot/src/lib-index mail-cache-compress.c, 1.5, 1.6 mail-cache-lookup.c, 1.8, 1.9 mail-cache-private.h, 1.5, 1.6 mail-cache-transaction.c, 1.8, 1.9 mail-cache.c, 1.35, 1.36 mail-cache.h, 1.14, 1.15 mail-index-sync-private.h, 1.9, 1.10 mail-index-sync-update.c, 1.32, 1.33

[cras at procontrol.fi]

Update of /home/cvs/dovecot/src/lib-index
In directory talvi:/tmp/cvs-serv24885/lib-index

Modified Files:
	mail-cache-compress.c mail-cache-lookup.c mail-cache-private.h 
	mail-cache-transaction.c mail-cache.c mail-cache.h 
	mail-index-sync-private.h mail-index-sync-update.c 
Log Message:
Caching fixes and optimizations. Removed all network byte ordering code -
it's not worth the trouble and would require massive changes in indexing
code as well to be useful. Changed next_offset to prev_offset which is
updated while syncing index.



Index: mail-cache-compress.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache-compress.c,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -d -r1.5 -r1.6
--- mail-cache-compress.c	4 Jul 2004 11:50:48 -0000	1.5
+++ mail-cache-compress.c	4 Jul 2004 14:26:22 -0000	1.6
@@ -2,7 +2,6 @@
 
 #include "lib.h"
 #include "buffer.h"
-#include "byteorder.h"
 #include "ostream.h"
 #include "file-set-size.h"
 #include "mail-cache-private.h"
@@ -19,7 +18,7 @@
 	buffer_t *buffer;
 	const void *data;
 	size_t size, pos;
-	uint32_t nb_size;
+	uint32_t size32;
 	int i;
 
 	memset(&cache_rec, 0, sizeof(cache_rec));
@@ -37,20 +36,20 @@
 			continue;
 		}
 
-		nb_size = uint32_to_nbo((uint32_t)size);
+		size32 = (uint32_t)size;
 
 		if ((field & MAIL_CACHE_FIXED_MASK) == 0)
-			buffer_append(buffer, &nb_size, sizeof(nb_size));
+			buffer_append(buffer, &size32, sizeof(size32));
 		buffer_append(buffer, data, size);
-		if ((size & 3) != 0)
-			buffer_append(buffer, null4, 4 - (size & 3));
+		if ((size32 & 3) != 0)
+			buffer_append(buffer, null4, 4 - (size32 & 3));
 	}
 
 	/* now merge all the headers if we have them all */
 	if ((orig_cached_fields & mail_cache_header_fields[header_idx]) != 0) {
-		nb_size = 0;
+		size32 = 0;
 		pos = buffer_get_used_size(buffer);
-		buffer_append(buffer, &nb_size, sizeof(nb_size));
+		buffer_append(buffer, &size32, sizeof(size32));
 
 		for (i = 0; i <= header_idx; i++) {
 			field = mail_cache_header_fields[i];
@@ -58,22 +57,20 @@
 						    &data, &size) && size > 1) {
 				size--; /* terminating \0 */
 				buffer_append(buffer, data, size);
-				nb_size += size;
+				size32 += size;
 			}
 		}
 		buffer_append(buffer, null4, 1);
-		nb_size++;
-		if ((nb_size & 3) != 0)
-			buffer_append(buffer, null4, 4 - (nb_size & 3));
-
-		nb_size = uint32_to_nbo(nb_size);
-		buffer_write(buffer, pos, &nb_size, sizeof(nb_size));
+		size32++;
+		if ((size32 & 3) != 0)
+			buffer_append(buffer, null4, 4 - (size32 & 3));
+		buffer_write(buffer, pos, &size32, sizeof(size32));
 
 		cached_fields |= MAIL_CACHE_HEADERS1;
 	}
 
 	cache_rec.fields = cached_fields;
-	cache_rec.size = uint32_to_nbo(buffer_get_used_size(buffer));
+	cache_rec.size = buffer_get_used_size(buffer);
 	buffer_write(buffer, 0, &cache_rec, sizeof(cache_rec));
 
 	data = buffer_get_data(buffer, &size);
@@ -93,7 +90,7 @@
 	enum mail_cache_field keep_fields, temp_fields;
 	enum mail_cache_field cached_fields, new_fields;
 	const char *str;
-	uint32_t size, nb_size, message_count, seq, first_new_seq;
+	uint32_t size, size32, message_count, seq, first_new_seq;
 	uoff_t offset;
 	int i, header_idx, ret;
 
@@ -118,6 +115,7 @@
 	output = o_stream_create_file(fd, default_pool, 0, FALSE);
 
 	memset(&hdr, 0, sizeof(hdr));
+	hdr.version = MAIL_CACHE_VERSION;
 	hdr.indexid = idx_hdr->indexid;
 	hdr.file_seq = idx_hdr->cache_file_seq + 1;
 
@@ -157,13 +155,11 @@
 			mail_cache_uint32_to_offset(output->offset);
 		header_idx = i;
 
-		size = strlen(str) + 1;
-		nb_size = uint32_to_nbo(size);
-
-		o_stream_send(output, &nb_size, sizeof(nb_size));
-		o_stream_send(output, str, size);
-		if ((size & 3) != 0)
-			o_stream_send(output, null4, 4 - (size & 3));
+		size32 = strlen(str) + 1;
+		o_stream_send(output, &size32, sizeof(size32));
+		o_stream_send(output, str, size32);
+		if ((size32 & 3) != 0)
+			o_stream_send(output, null4, 4 - (size32 & 3));
 	}
 
 	mail_index_reset_cache(t, hdr.file_seq);
@@ -183,14 +179,15 @@
 		}
 
 		if (keep_fields == cached_fields &&
-		    mail_cache_offset_to_uint32(cache_rec->next_offset) == 0) {
+		    cache_rec->prev_offset == 0) {
 			/* just one unmodified block, save it */
-			size = nbo_to_uint32(cache_rec->size);
                         mail_index_update_cache(t, seq, output->offset);
-			o_stream_send(output, cache_rec, size);
+			o_stream_send(output, cache_rec, cache_rec->size);
 
-			if ((size & 3) != 0)
-				o_stream_send(output, null4, 4 - (size & 3));
+			if ((cache_rec->size & 3) != 0) {
+				o_stream_send(output, null4,
+					      4 - (cache_rec->size & 3));
+			}
 		} else {
 			/* a) dropping fields
 			   b) multiple blocks, sort them into buffer */
@@ -205,7 +202,7 @@
 			t_pop();
 		}
 	}
-	hdr.used_file_size = uint32_to_nbo(output->offset);
+	hdr.used_file_size = output->offset;
 
 	o_stream_seek(output, 0);
 	o_stream_send(output, &hdr, sizeof(hdr));
@@ -259,6 +256,8 @@
 		return -1;
 	}
 
+	// FIXME: check that cache file was just recreated
+
 	ret = 0;
 	if (mail_cache_copy(cache, view, fd) < 0) {
 		(void)file_dotlock_delete(cache->filepath, NULL, fd);
@@ -283,10 +282,8 @@
 	memset(cache->split_offsets, 0, sizeof(cache->split_offsets));
 	memset(cache->split_headers, 0, sizeof(cache->split_headers));
 
-	if (locked) {
-		if (mail_cache_unlock(cache) < 0)
-			return -1;
-	}
+	if (locked)
+		mail_cache_unlock(cache);
 
 	if (ret == 0)
                 cache->need_compress = FALSE;

Index: mail-cache-lookup.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache-lookup.c,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -d -r1.8 -r1.9
--- mail-cache-lookup.c	4 Jul 2004 11:50:49 -0000	1.8
+++ mail-cache-lookup.c	4 Jul 2004 14:26:22 -0000	1.9
@@ -2,7 +2,6 @@
 
 #include "lib.h"
 #include "buffer.h"
-#include "byteorder.h"
 #include "mail-cache-private.h"
 
 #define CACHE_PREFETCH 1024
@@ -32,7 +31,6 @@
 
 	buf = cache->mmap_base;
 	memcpy(&data_size, buf + offset, sizeof(data_size));
-	data_size = nbo_to_uint32(data_size);
 	offset += sizeof(data_size);
 
 	if (data_size == 0) {
@@ -116,14 +114,10 @@
 }
 
 struct mail_cache_record *
-mail_cache_get_record(struct mail_cache *cache, uint32_t offset,
-		      int index_offset)
+mail_cache_get_record(struct mail_cache *cache, uint32_t offset)
 {
 	struct mail_cache_record *cache_rec;
-	size_t size;
 
-	if (!index_offset)
-		offset = mail_cache_offset_to_uint32(offset);
 	if (offset == 0)
 		return NULL;
 
@@ -137,17 +131,16 @@
 	}
 	cache_rec = CACHE_RECORD(cache, offset);
 
-	size = nbo_to_uint32(cache_rec->size);
-	if (size < sizeof(*cache_rec)) {
+	if (cache_rec->size < sizeof(*cache_rec)) {
 		mail_cache_set_corrupted(cache, "invalid record size");
 		return NULL;
 	}
-	if (size > CACHE_PREFETCH) {
-		if (mail_cache_map(cache, offset, size) < 0)
+	if (cache_rec->size > CACHE_PREFETCH) {
+		if (mail_cache_map(cache, offset, cache_rec->size) < 0)
 			return NULL;
 	}
 
-	if (offset + size > cache->mmap_length) {
+	if (offset + cache_rec->size > cache->mmap_length) {
 		mail_cache_set_corrupted(cache, "record points outside file");
 		return NULL;
 	}
@@ -195,7 +188,7 @@
 	if (mail_cache_lookup_offset(view, seq, &offset, FALSE) <= 0)
 		return NULL;
 
-	return mail_cache_get_record(view->cache, offset, TRUE);
+	return mail_cache_get_record(view->cache, offset);
 }
 
 enum mail_cache_field
@@ -208,26 +201,22 @@
 	while (cache_rec != NULL) {
 		fields |= cache_rec->fields;
 		cache_rec = mail_cache_get_record(view->cache,
-						  cache_rec->next_offset,
-						  FALSE);
+						  cache_rec->prev_offset);
 	}
 
 	return fields;
 }
 
 static int cache_get_field(struct mail_cache *cache,
-			   struct mail_cache_record *cache_rec,
+			   const struct mail_cache_record *cache_rec,
 			   enum mail_cache_field field,
 			   const void **data_r, size_t *size_r)
 {
-	unsigned char *buf;
 	unsigned int mask;
-	uint32_t rec_size, data_size;
-	size_t offset, next_offset;
+	uint32_t data_size;
+	size_t offset, prev_offset;
 	int i;
 
-	rec_size = nbo_to_uint32(cache_rec->size);
-	buf = (unsigned char *) cache_rec;
 	offset = sizeof(*cache_rec);
 
 	for (i = 0, mask = 1; i < 31; i++, mask <<= 1) {
@@ -236,7 +225,7 @@
 
 		/* all records are at least 32bit. we have to check this
 		   before getting data_size. */
-		if (offset + sizeof(uint32_t) > rec_size) {
+		if (offset + sizeof(uint32_t) > cache_rec->size) {
 			mail_cache_set_corrupted(cache,
 				"Record continues outside it's allocated size");
 			return FALSE;
@@ -245,13 +234,13 @@
 		if ((mask & MAIL_CACHE_FIXED_MASK) != 0)
 			data_size = mail_cache_field_sizes[i];
 		else {
-			memcpy(&data_size, buf + offset, sizeof(data_size));
-			data_size = nbo_to_uint32(data_size);
+			memcpy(&data_size, CONST_PTR_OFFSET(cache_rec, offset),
+			       sizeof(data_size));
 			offset += sizeof(data_size);
 		}
 
-		next_offset = offset + ((data_size + 3) & ~3);
-		if (next_offset > rec_size) {
+		prev_offset = offset + ((data_size + 3) & ~3);
+		if (prev_offset > cache_rec->size) {
 			mail_cache_set_corrupted(cache,
 				"Record continues outside it's allocated size");
 			return FALSE;
@@ -263,11 +252,11 @@
 							 "Field size is 0");
 				return FALSE;
 			}
-			*data_r = buf + offset;
+			*data_r = CONST_PTR_OFFSET(cache_rec, offset);
 			*size_r = data_size;
 			return TRUE;
 		}
-		offset = next_offset;
+		offset = prev_offset;
 	}
 
 	i_unreached();
@@ -289,8 +278,7 @@
 					       data_r, size_r);
 		}
 		cache_rec = mail_cache_get_record(view->cache,
-						  cache_rec->next_offset,
-						  FALSE);
+						  cache_rec->prev_offset);
 	}
 
 	return FALSE;
@@ -308,7 +296,7 @@
 	if (!mail_cache_lookup_field(view, seq, field, &data, &size))
 		return NULL;
 
-	if (((const char *) data)[size-1] != '\0') {
+	if (((const char *)data)[size-1] != '\0') {
 		mail_cache_set_corrupted(view->cache,
 			"String field %x doesn't end with NUL", field);
 		return NULL;

Index: mail-cache-private.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache-private.h,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -d -r1.5 -r1.6
--- mail-cache-private.h	4 Jul 2004 11:50:49 -0000	1.5
+++ mail-cache-private.h	4 Jul 2004 14:26:22 -0000	1.6
@@ -4,6 +4,8 @@
 #include "mail-index-private.h"
 #include "mail-cache.h"
 
+#define MAIL_CACHE_VERSION 1
+
 /* Never compress the file if it's smaller than this */
 #define COMPRESS_MIN_SIZE (1024*50)
 
@@ -43,6 +45,11 @@
 };
 
 struct mail_cache_header {
+	/* version is increased only when you can't have backwards
+	   compatibility. */
+	uint8_t version;
+	uint8_t unused[3];
+
 	uint32_t indexid;
 	uint32_t file_seq;
 
@@ -59,7 +66,7 @@
 
 struct mail_cache_record {
 	uint32_t fields; /* enum mail_cache_field */
-	uint32_t next_offset;
+	uint32_t prev_offset;
 	uint32_t size; /* full record size, including this header */
 };
 
@@ -109,8 +116,7 @@
 mail_cache_split_header(struct mail_cache *cache, const char *header);
 
 struct mail_cache_record *
-mail_cache_get_record(struct mail_cache *cache, uint32_t offset,
-		      int index_offset);
+mail_cache_get_record(struct mail_cache *cache, uint32_t offset);
 
 int mail_cache_lookup_offset(struct mail_cache_view *view, uint32_t seq,
 			     uint32_t *offset, int skip_expunged);
@@ -126,6 +132,9 @@
 void mail_cache_file_close(struct mail_cache *cache);
 int mail_cache_reopen(struct mail_cache *cache);
 
+int mail_cache_link(struct mail_cache *cache, uint32_t old_offset,
+		    uint32_t new_offset);
+
 void mail_cache_handle_decisions(struct mail_cache_view *view, uint32_t seq,
 				 enum mail_cache_field field);
 

Index: mail-cache-transaction.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache-transaction.c,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -d -r1.8 -r1.9
--- mail-cache-transaction.c	4 Jul 2004 11:50:49 -0000	1.8
+++ mail-cache-transaction.c	4 Jul 2004 14:26:22 -0000	1.9
@@ -2,7 +2,6 @@
 
 #include "lib.h"
 #include "buffer.h"
-#include "byteorder.h"
 #include "file-set-size.h"
 #include "mmap-util.h"
 #include "mail-cache-private.h"
@@ -46,23 +45,19 @@
 	ctx->view = view;
 	ctx->trans = t;
 	ctx->cache_data = buffer_create_dynamic(system_pool, 8192, (size_t)-1);
-	ctx->used_file_size = nbo_to_uint32(ctx->cache->hdr->used_file_size);
+	ctx->used_file_size = ctx->cache->hdr->used_file_size;
 
 	view->cache->trans_ctx = ctx;
 	*ctx_r = ctx;
 	return 1;
 }
 
-int mail_cache_transaction_end(struct mail_cache_transaction_ctx *ctx)
+void mail_cache_transaction_end(struct mail_cache_transaction_ctx *ctx)
 {
-	int ret = 0;
-
 	i_assert(ctx->cache->trans_ctx != NULL);
 
 	(void)mail_cache_transaction_rollback(ctx);
-
-	if (mail_cache_unlock(ctx->cache) < 0)
-		ret = -1;
+	mail_cache_unlock(ctx->cache);
 
 	ctx->cache->trans_ctx = NULL;
 
@@ -70,7 +65,6 @@
 		buffer_free(ctx->cache_marks);
 	buffer_free(ctx->cache_data);
 	i_free(ctx);
-	return ret;
 }
 
 static void mail_cache_transaction_flush(struct mail_cache_transaction_ctx *ctx)
@@ -116,7 +110,6 @@
 static int commit_all_changes(struct mail_cache_transaction_ctx *ctx)
 {
 	struct mail_cache *cache = ctx->cache;
-	uint32_t cont;
 
 	/* write everything to disk */
 	if (msync(cache->mmap_base, cache->mmap_length, MS_SYNC) < 0) {
@@ -138,18 +131,17 @@
 		return -1;
 
 	/* update continued records count */
-	cont = nbo_to_uint32(cache->hdr->continued_record_count);
-	cont += buffer_get_used_size(ctx->cache_marks) /
+        cache->hdr->continued_record_count +=
+		buffer_get_used_size(ctx->cache_marks) /
 		(sizeof(uint32_t) * 2);
 
-	if (cont * 100 / cache->index->hdr->messages_count >=
+	if (cache->hdr->continued_record_count * 100 /
+	    cache->index->hdr->messages_count >=
 	    COMPRESS_CONTINUED_PERCENTAGE &&
 	    ctx->used_file_size >= COMPRESS_MIN_SIZE) {
 		/* too many continued rows, compress */
 		cache->need_compress = TRUE;
 	}
-
-	cache->hdr->continued_record_count = uint32_to_nbo(cont);
 	return 0;
 }
 
@@ -218,7 +210,6 @@
 static int mail_cache_write(struct mail_cache_transaction_ctx *ctx)
 {
 	struct mail_cache *cache = ctx->cache;
-	struct mail_cache_record *cache_rec;
 	uint32_t offset, write_offset;
 	const void *buf;
 	size_t size, buf_size;
@@ -226,8 +217,8 @@
 
 	buf = buffer_get_data(ctx->cache_data, &buf_size);
 
-	size = sizeof(*cache_rec) + buf_size;
-	ctx->cache_rec.size = uint32_to_nbo(size);
+	size = sizeof(ctx->cache_rec) + buf_size;
+	ctx->cache_rec.size = size;
 
         ret = mail_cache_lookup_offset(ctx->view, ctx->prev_seq, &offset, TRUE);
 	if (ret < 0)
@@ -240,27 +231,10 @@
 		if (write_offset == 0)
 			return -1;
 
-		cache_rec = mail_cache_get_record(cache, offset, TRUE);
-		if (cache_rec == NULL) {
-			/* first cache record - update offset in index file */
-			mail_index_update_cache(ctx->trans, ctx->prev_seq,
-						write_offset);
-		} else {
-			/* find offset to last cache record */
-			for (;;) {
-				cache_rec = mail_cache_get_record(cache, offset,
-								  FALSE);
-				if (cache_rec == NULL)
-					break;
-				offset = cache_rec->next_offset;
-			}
-
-			/* mark next_offset to be updated later */
-			offset = mail_cache_offset_to_uint32(offset) +
-				offsetof(struct mail_cache_record, next_offset);
-			mark_update(&ctx->cache_marks, offset,
-				    mail_cache_uint32_to_offset(write_offset));
-		}
+		/* write the offset to index file. this record's prev_offset
+		   is updated to point to old cache record when index is
+		   being synced. */
+		mail_index_update_cache(ctx->trans, ctx->prev_seq, write_offset);
 
 		memcpy((char *) cache->mmap_base + write_offset,
 		       &ctx->cache_rec, sizeof(ctx->cache_rec));
@@ -291,7 +265,7 @@
 			return -1;
 	}
 
-	ctx->cache->hdr->used_file_size = uint32_to_nbo(ctx->used_file_size);
+	ctx->cache->hdr->used_file_size = ctx->used_file_size;
 
 	if (commit_all_changes(ctx) < 0)
 		ret = -1;
@@ -312,7 +286,7 @@
 
 	/* no need to actually modify the file - we just didn't update
 	   used_file_size */
-	ctx->used_file_size = nbo_to_uint32(cache->hdr->used_file_size);
+	ctx->used_file_size = cache->hdr->used_file_size;
 
 	/* make sure we don't cache the headers */
 	for (i = 0; i < ctx->next_unused_header_lowwater; i++) {
@@ -378,11 +352,10 @@
 
 	offset = mail_cache_append_space(ctx, size + sizeof(uint32_t));
 	if (offset != 0) {
-		memcpy((char *) cache->mmap_base + offset + sizeof(uint32_t),
+		memcpy(PTR_OFFSET(cache->mmap_base, offset + sizeof(uint32_t)),
 		       header_str, size);
 
-		size = uint32_to_nbo(size);
-		memcpy((char *) cache->mmap_base + offset,
+		memcpy(PTR_OFFSET(cache->mmap_base, offset),
 		       &size, sizeof(uint32_t));
 
 		/* update cached headers */
@@ -428,7 +401,6 @@
 			else {
 				memcpy(&data_size, buf + offset,
 				       sizeof(data_size));
-				data_size = nbo_to_uint32(data_size);
 				offset += sizeof(data_size);
 			}
 			offset += (data_size + 3) & ~3;
@@ -456,7 +428,7 @@
 		   enum mail_cache_field field,
 		   const void *data, size_t data_size)
 {
-	uint32_t nb_data_size;
+	uint32_t data_size32;
 	size_t full_size, offset;
 	unsigned char *buf;
 	int field_num;
@@ -464,7 +436,7 @@
 	i_assert(data_size > 0);
 	i_assert(data_size < (uint32_t)-1);
 
-	nb_data_size = uint32_to_nbo((uint32_t)data_size);
+	data_size32 = (uint32_t)data_size;
 
 	if ((field & MAIL_CACHE_FIXED_MASK) != 0) {
 		field_num = get_field_num(field);
@@ -484,7 +456,7 @@
 
 	full_size = (data_size + 3) & ~3;
 	if ((field & MAIL_CACHE_FIXED_MASK) == 0)
-		full_size += sizeof(nb_data_size);
+		full_size += sizeof(data_size32);
 
 	/* fields must be ordered. find where to insert it. */
 	if (field > ctx->cache_rec.fields)
@@ -500,8 +472,8 @@
 
 	/* @UNSAFE */
 	if ((field & MAIL_CACHE_FIXED_MASK) == 0) {
-		memcpy(buf, &nb_data_size, sizeof(nb_data_size));
-		buf += sizeof(nb_data_size);
+		memcpy(buf, &data_size32, sizeof(data_size32));
+		buf += sizeof(data_size32);
 	}
 	memcpy(buf, data, data_size); buf += data_size;
 	if ((data_size & 3) != 0)
@@ -533,12 +505,12 @@
 	   the data. also it's actually useful as some index views are still
 	   able to ask cached data from messages that have already been
 	   expunged. */
-	deleted_space = nbo_to_uint32(cache->hdr->deleted_space);
+	deleted_space = cache->hdr->deleted_space;
 
 	do {
-		deleted_space -= nbo_to_uint32(cache_rec->size);
-		cache_rec = mail_cache_get_record(cache, cache_rec->next_offset,
-						  FALSE);
+		deleted_space += cache_rec->size;
+		cache_rec =
+			mail_cache_get_record(cache, cache_rec->prev_offset);
 	} while (cache_rec != NULL);
 
 	/* see if we've reached the max. deleted space in file */
@@ -547,7 +519,7 @@
 	    ctx->used_file_size >= COMPRESS_MIN_SIZE)
 		cache->need_compress = TRUE;
 
-	cache->hdr->deleted_space = uint32_to_nbo(deleted_space);
+	cache->hdr->deleted_space = deleted_space;
 	return 0;
 }
 
@@ -580,3 +552,22 @@
 {
 	return -1;
 }
+
+int mail_cache_link(struct mail_cache *cache, uint32_t old_offset,
+		    uint32_t new_offset)
+{
+	struct mail_cache_record *cache_rec;
+
+	i_assert(cache->locks > 0);
+
+	if (mail_cache_map(cache, new_offset, sizeof(*cache_rec)) < 0)
+		return -1;
+
+	if (new_offset + sizeof(*cache_rec) > cache->mmap_length) {
+		mail_cache_set_corrupted(cache, "record points outside file");
+		return -1;
+	}
+	cache_rec = CACHE_RECORD(cache, new_offset);
+	cache_rec->prev_offset = old_offset;
+	return 0;
+}

Index: mail-cache.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache.c,v
retrieving revision 1.35
retrieving revision 1.36
diff -u -d -r1.35 -r1.36
--- mail-cache.c	4 Jul 2004 11:50:49 -0000	1.35
+++ mail-cache.c	4 Jul 2004 14:26:22 -0000	1.36
@@ -1,7 +1,6 @@
 /* Copyright (C) 2003-2004 Timo Sirainen */
 
 #include "lib.h"
-#include "byteorder.h"
 #include "file-lock.h"
 #include "file-set-size.h"
 #include "mmap-util.h"
@@ -156,7 +155,6 @@
 static int mmap_verify_header(struct mail_cache *cache)
 {
 	struct mail_cache_header *hdr;
-	uint32_t used_file_size;
 
 	/* check that the header is still ok */
 	if (cache->mmap_length < sizeof(struct mail_cache_header)) {
@@ -165,6 +163,11 @@
 	}
 	cache->hdr = hdr = cache->mmap_base;
 
+	if (cache->hdr->version != MAIL_CACHE_VERSION) {
+		/* version changed - upgrade silently */
+		return FALSE;
+	}
+
 	if (cache->hdr->indexid != cache->index->indexid) {
 		/* index id changed */
 		mail_cache_set_corrupted(cache, "indexid changed");
@@ -180,17 +183,16 @@
 	if (cache->locks == 0)
 		return TRUE;
 
-	used_file_size = nbo_to_uint32(hdr->used_file_size);
-	if (used_file_size < sizeof(struct mail_cache_header)) {
+	if (hdr->used_file_size < sizeof(struct mail_cache_header)) {
 		mail_cache_set_corrupted(cache, "used_file_size too small");
 		return FALSE;
 	}
-	if ((used_file_size % sizeof(uint32_t)) != 0) {
+	if ((hdr->used_file_size % sizeof(uint32_t)) != 0) {
 		mail_cache_set_corrupted(cache, "used_file_size not aligned");
 		return FALSE;
 	}
 
-	if (used_file_size > cache->mmap_length) {
+	if (hdr->used_file_size > cache->mmap_length) {
 		mail_cache_set_corrupted(cache, "used_file_size too large");
 		return FALSE;
 	}
@@ -354,17 +356,13 @@
 	return ret;
 }
 
-int mail_cache_unlock(struct mail_cache *cache)
+void mail_cache_unlock(struct mail_cache *cache)
 {
 	if (--cache->locks > 0)
-		return 0;
+		return;
 
-	if (file_wait_lock(cache->fd, F_UNLCK) <= 0) {
+	if (file_wait_lock(cache->fd, F_UNLCK) <= 0)
 		mail_cache_set_syscall_error(cache, "file_wait_lock(F_UNLCK)");
-		return -1;
-	}
-
-	return 0;
 }
 
 int mail_cache_is_locked(struct mail_cache *cache)

Index: mail-cache.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache.h,v
retrieving revision 1.14
retrieving revision 1.15
diff -u -d -r1.14 -r1.15
--- mail-cache.h	4 Jul 2004 11:50:49 -0000	1.14
+++ mail-cache.h	4 Jul 2004 14:26:22 -0000	1.15
@@ -86,7 +86,7 @@
 /* Explicitly lock the cache file. Returns -1 if error, 1 if ok, 0 if we
    couldn't lock */
 int mail_cache_lock(struct mail_cache *cache, int nonblock);
-int mail_cache_unlock(struct mail_cache *cache);
+void mail_cache_unlock(struct mail_cache *cache);
 
 /* Returns TRUE if cache file is locked. */
 int mail_cache_is_locked(struct mail_cache *cache);
@@ -106,8 +106,7 @@
 int mail_cache_transaction_commit(struct mail_cache_transaction_ctx *ctx);
 void mail_cache_transaction_rollback(struct mail_cache_transaction_ctx *ctx);
 
-/* Should be called only by mail_transaction_commit/rollback: */
-int mail_cache_transaction_end(struct mail_cache_transaction_ctx *ctx);
+void mail_cache_transaction_end(struct mail_cache_transaction_ctx *ctx);
 
 /* Return NULL-terminated list of headers for given index, or NULL if
    header index isn't used. */

Index: mail-index-sync-private.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-sync-private.h,v
retrieving revision 1.9
retrieving revision 1.10
diff -u -d -r1.9 -r1.10
--- mail-index-sync-private.h	24 Jun 2004 11:10:41 -0000	1.9
+++ mail-index-sync-private.h	4 Jul 2004 14:26:22 -0000	1.10
@@ -22,6 +22,7 @@
 	unsigned int lock_id;
 
 	unsigned int sync_appends:1;
+	unsigned int cache_locked:1;
 };
 
 extern struct mail_transaction_map_functions mail_index_map_sync_funcs;

Index: mail-index-sync-update.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-sync-update.c,v
retrieving revision 1.32
retrieving revision 1.33
diff -u -d -r1.32 -r1.33
--- mail-index-sync-update.c	2 Jul 2004 10:33:48 -0000	1.32
+++ mail-index-sync-update.c	4 Jul 2004 14:26:22 -0000	1.33
@@ -9,6 +9,7 @@
 #include "mail-index-sync-private.h"
 #include "mail-transaction-log.h"
 #include "mail-transaction-util.h"
+#include "mail-cache-private.h"
 
 #include <time.h>
 
@@ -58,7 +59,8 @@
 
 static int sync_expunge(const struct mail_transaction_expunge *e, void *context)
 {
-	struct mail_index_view *view = context;
+        struct mail_index_sync_ctx *sync_ctx = context;
+	struct mail_index_view *view = sync_ctx->view;
 	struct mail_index_map *map = view->map;
 	struct mail_index_header *hdr = &map->hdr_copy;
 	struct mail_index_record *rec;
@@ -98,7 +100,8 @@
 static int sync_append(const struct mail_transaction_append_header *hdr,
 		       const struct mail_index_record *rec, void *context)
 {
-	struct mail_index_view *view = context;
+        struct mail_index_sync_ctx *sync_ctx = context;
+	struct mail_index_view *view = sync_ctx->view;
 	struct mail_index_map *map = view->map;
 	void *dest;
 
@@ -139,7 +142,8 @@
 static int sync_flag_update(const struct mail_transaction_flag_update *u,
 			    void *context)
 {
-        struct mail_index_view *view = context;
+        struct mail_index_sync_ctx *sync_ctx = context;
+	struct mail_index_view *view = sync_ctx->view;
 	struct mail_index_record *rec;
 	struct mail_index_header *hdr;
 	uint8_t flag_mask, old_flags;
@@ -188,7 +192,8 @@
 static int sync_cache_reset(const struct mail_transaction_cache_reset *u,
 			    void *context)
 {
-	struct mail_index_view *view = context;
+        struct mail_index_sync_ctx *sync_ctx = context;
+	struct mail_index_view *view = sync_ctx->view;
 	uint32_t i;
 
 	view->map->hdr_copy.cache_file_seq = u->new_file_seq;
@@ -201,7 +206,9 @@
 static int sync_cache_update(const struct mail_transaction_cache_update *u,
 			     void *context)
 {
-	struct mail_index_view *view = context;
+        struct mail_index_sync_ctx *sync_ctx = context;
+	struct mail_index_view *view = sync_ctx->view;
+	struct mail_index_record *rec;
 	uint32_t seq;
 	int ret;
 
@@ -209,10 +216,25 @@
 					  &seq, &seq);
 	i_assert(ret == 0);
 
-	if (seq != 0) {
-		MAIL_INDEX_MAP_IDX(view->map, seq-1)->cache_offset =
-			u->cache_offset;
+	if (seq == 0) {
+		/* already expunged */
+		return 1;
 	}
+
+	rec = MAIL_INDEX_MAP_IDX(view->map, seq-1);
+	if (rec->cache_offset != 0) {
+		/* we'll need to link the old and new cache records */
+		if (!sync_ctx->cache_locked) {
+			if (mail_cache_lock(view->index->cache, FALSE) <= 0)
+				return -1;
+			sync_ctx->cache_locked = TRUE;
+		}
+
+		if (mail_cache_link(view->index->cache,
+				    rec->cache_offset, u->cache_offset) < 0)
+			return -1;
+	}
+	rec->cache_offset = u->cache_offset;
 	return 1;
 }
 
@@ -423,12 +445,17 @@
 
 		if (mail_transaction_map(index, hdr, data,
 					 &mail_index_map_sync_funcs,
-					 view) < 0) {
+					 sync_ctx) < 0) {
 			ret = -1;
 			break;
 		}
 	}
 
+	if (sync_ctx->cache_locked) {
+		mail_cache_unlock(index->cache);
+		sync_ctx->cache_locked = FALSE;
+	}
+
 	if (ret < 0) {
 		mail_index_view_unlock(view);
 		return -1;