Thu Jun 24 14:10:44 EEST 2004

Update of /home/cvs/dovecot/src/lib-index
In directory talvi:/tmp/cvs-serv3603/lib-index

Modified Files:
	mail-index-fsck.c mail-index-lock.c mail-index-private.h 
	mail-index-sync-private.h mail-index-sync-update.c 
	mail-index-sync.c mail-index-transaction-private.h 
	mail-index-transaction-view.c mail-index-transaction.c 
	mail-index-view-sync.c mail-index-view.c mail-index.c 
	mail-index.h mail-transaction-log.c mail-transaction-log.h 
	mail-transaction-util.c mail-transaction-util.h 
Log Message:
Record size is allowed to change between index files. This will allow adding
extensions dynamically for existing indexes.



Index: mail-index-fsck.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-fsck.c,v
retrieving revision 1.22
retrieving revision 1.23
diff -u -d -r1.22 -r1.23

--- mail-index-fsck.c	13 Jun 2004 23:38:47 -0000	1.22
+++ mail-index-fsck.c	24 Jun 2004 11:10:41 -0000	1.23
@@ -75,7 +75,7 @@
 			hdr.first_deleted_uid_lowwater = rec->uid;
 
 		last_uid = rec->uid;
-		rec = CONST_PTR_OFFSET(rec, index->record_size);
+		rec = CONST_PTR_OFFSET(rec, hdr.record_size);
 	}
 
 	if (hdr.next_uid <= last_uid) {
@@ -100,7 +100,7 @@
         CHECK(first_unseen_uid_lowwater, <);
         CHECK(first_deleted_uid_lowwater, <);
 
-	if (mail_index_write_header(index, &hdr) < 0)
+	if (mail_index_write_base_header(index, &hdr) < 0)
 		return -1;
 
 	return 1;

Index: mail-index-lock.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-lock.c,v
retrieving revision 1.22
retrieving revision 1.23
diff -u -d -r1.22 -r1.23
--- mail-index-lock.c	20 Jun 2004 13:14:17 -0000	1.22
+++ mail-index-lock.c	24 Jun 2004 11:10:41 -0000	1.23
@@ -187,7 +187,7 @@
 	ret = write_full(fd, index->map->hdr, sizeof(*index->map->hdr));
 	if (ret < 0 || write_full(fd, index->map->records,
 				  index->map->records_count *
-				  index->record_size) < 0) {
+				  index->map->hdr->record_size) < 0) {
 		mail_index_file_set_syscall_error(index, path, "write_full()");
 		(void)close(fd);
 		(void)unlink(path);

Index: mail-index-private.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-private.h,v
retrieving revision 1.21
retrieving revision 1.22
diff -u -d -r1.21 -r1.22
--- mail-index-private.h	20 Jun 2004 09:13:14 -0000	1.21
+++ mail-index-private.h	24 Jun 2004 11:10:41 -0000	1.22
@@ -22,9 +22,15 @@
 #define MAIL_INDEX_MAP_IS_IN_MEMORY(map) \
 	((map)->buffer != NULL)
 
-#define MAIL_INDEX_MAP_IDX(index, map, idx) \
+#define MAIL_INDEX_MAP_IDX(map, idx) \
 	((struct mail_index_record *) \
-		PTR_OFFSET((map)->records, (idx) * (index)->record_size))
+		PTR_OFFSET((map)->records, (idx) * (map)->hdr->record_size))
+
+struct mail_index_extra_record_info {
+	const char *name;
+	uint16_t offset;
+	uint16_t size;
+};
 
 struct mail_index_map {
 	int refcount;
@@ -33,6 +39,9 @@
 	void *records; /* struct mail_index_record[] */
 	unsigned int records_count;
 
+	struct mail_index_extra_record_info *
+		extra_record_map[MAIL_INDEX_MAX_EXTRA_RECORDS];
+
 	void *mmap_base;
 	size_t mmap_size, mmap_used_size;
 
@@ -46,12 +55,6 @@
 	unsigned int write_to_disk:1;
 };
 
-struct mail_index_extra_record_info {
-	const char *name;
-	uint16_t offset;
-	uint16_t size;
-};
-
 struct mail_index {
 	char *dir, *prefix;
 
@@ -65,8 +68,7 @@
 	buffer_t *extra_records_buf;
 	const struct mail_index_extra_record_info *extra_records;
 	unsigned int extra_records_count;
-
-	unsigned int record_size;
+	unsigned int max_record_size;
 
 	char *filepath;
 	int fd;
@@ -95,8 +97,8 @@
 	unsigned int fsck:1;
 };
 
-int mail_index_write_header(struct mail_index *index,
-			    const struct mail_index_header *hdr);
+int mail_index_write_base_header(struct mail_index *index,
+				 const struct mail_index_header *hdr);
 
 int mail_index_reopen(struct mail_index *index, int fd);
 int mail_index_create_tmp_file(struct mail_index *index, const char **path_r);
@@ -122,7 +124,7 @@
 /* Unreference given mapping and unmap it if it's dropped to zero. */
 void mail_index_unmap(struct mail_index *index, struct mail_index_map *map);
 struct mail_index_map *
-mail_index_map_to_memory(struct mail_index *index, struct mail_index_map *map);
+mail_index_map_to_memory(struct mail_index_map *map, uint32_t new_record_size);
 
 int mail_index_lookup_full(struct mail_index_view *view, uint32_t seq,
 			   struct mail_index_map **map_r,

Index: mail-index-sync-private.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-sync-private.h,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -d -r1.8 -r1.9
--- mail-index-sync-private.h	30 May 2004 04:32:36 -0000	1.8
+++ mail-index-sync-private.h	24 Jun 2004 11:10:41 -0000	1.9
@@ -5,12 +5,14 @@
 	struct mail_index *index;
 	struct mail_index_view *view;
 
-	buffer_t *expunges_buf, *updates_buf, *appends_buf;
+	buffer_t *expunges_buf, *updates_buf;
 
 	const struct mail_transaction_expunge *expunges;
 	const struct mail_transaction_flag_update *updates;
 	size_t expunges_count, updates_count;
 
+	uint32_t append_uid_first, append_uid_last;
+
 	const struct mail_transaction_header *hdr;
 	const void *data;
 

Index: mail-index-sync-update.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-sync-update.c,v
retrieving revision 1.29
retrieving revision 1.30
diff -u -d -r1.29 -r1.30
--- mail-index-sync-update.c	20 Jun 2004 13:02:08 -0000	1.29
+++ mail-index-sync-update.c	24 Jun 2004 11:10:41 -0000	1.30
@@ -56,7 +56,6 @@
 static int sync_expunge(const struct mail_transaction_expunge *e, void *context)
 {
 	struct mail_index_view *view = context;
-	struct mail_index *index = view->index;
 	struct mail_index_map *map = view->map;
 	struct mail_index_header *hdr = &map->hdr_copy;
 	struct mail_index_record *rec;
@@ -72,15 +71,14 @@
 		return 1;
 
 	for (seq = seq1; seq <= seq2; seq++) {
-                rec = MAIL_INDEX_MAP_IDX(index, map, seq-1);
+                rec = MAIL_INDEX_MAP_IDX(map, seq-1);
 		mail_index_header_update_counts(hdr, rec->flags, 0);
 	}
 
 	/* @UNSAFE */
 	count = seq2 - seq1 + 1;
-	memmove(MAIL_INDEX_MAP_IDX(index, map, seq1-1),
-		MAIL_INDEX_MAP_IDX(index, map, seq2),
-		(map->records_count - seq2) * view->index->record_size);
+	memmove(MAIL_INDEX_MAP_IDX(map, seq1-1), MAIL_INDEX_MAP_IDX(map, seq2),
+		(map->records_count - seq2) * map->hdr->record_size);
 
 	map->records_count -= count;
 	hdr->messages_count -= count;
@@ -88,17 +86,20 @@
 
 	if (map->buffer != NULL) {
 		buffer_set_used_size(map->buffer, map->records_count *
-				     view->index->record_size);
+				     map->hdr->record_size);
 		map->records = buffer_get_modifyable_data(map->buffer, NULL);
 	}
 	return 1;
 }
 
-static int sync_append(const struct mail_index_record *rec, void *context)
+static int sync_append(const struct mail_transaction_append_header *hdr,
+		       const struct mail_index_record *rec, void *context)
 {
 	struct mail_index_view *view = context;
-	struct mail_index *index = view->index;
 	struct mail_index_map *map = view->map;
+	void *dest;
+
+	i_assert(hdr->record_size <= map->hdr->record_size);
 
 	if (rec->uid < map->hdr_copy.next_uid) {
 		mail_transaction_log_view_set_corrupted(view->log_view,
@@ -108,16 +109,19 @@
 	}
 
 	if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
-		i_assert(map->records_count * index->record_size ==
+		i_assert(map->records_count * map->hdr->record_size ==
 			 buffer_get_used_size(map->buffer));
-		buffer_append(map->buffer, rec, index->record_size);
+		dest = buffer_append_space_unsafe(map->buffer,
+						  map->hdr->record_size);
 		map->records = buffer_get_modifyable_data(map->buffer, NULL);
 	} else {
-		i_assert((map->records_count+1) * index->record_size <=
+		i_assert((map->records_count+1) * map->hdr->record_size <=
 			 map->mmap_size);
-		memcpy(MAIL_INDEX_MAP_IDX(index, map, map->records_count),
-		       rec, index->record_size);
+		dest = MAIL_INDEX_MAP_IDX(map, map->records_count);
 	}
+	memcpy(dest, rec, hdr->record_size);
+	memset(PTR_OFFSET(dest, hdr->record_size), 0,
+	       map->hdr->record_size - hdr->record_size);
 
 	map->hdr_copy.messages_count++;
 	map->hdr_copy.next_uid = rec->uid+1;
@@ -161,7 +165,7 @@
         flag_mask = ~u->remove_flags;
 
 	for (idx = seq1-1; idx < seq2; idx++) {
-                rec = MAIL_INDEX_MAP_IDX(view->index, view->map, idx);
+                rec = MAIL_INDEX_MAP_IDX(view->map, idx);
 
 		old_flags = rec->flags;
 		rec->flags = (rec->flags & flag_mask) | u->add_flags;
@@ -187,7 +191,7 @@
 	view->map->hdr_copy.cache_file_seq = u->new_file_seq;
 
 	for (i = 0; i < view->messages_count; i++)
-		MAIL_INDEX_MAP_IDX(view->index, view->map, i)->cache_offset = 0;
+		MAIL_INDEX_MAP_IDX(view->map, i)->cache_offset = 0;
 	return 1;
 }
 
@@ -203,8 +207,8 @@
 	i_assert(ret == 0);
 
 	if (seq != 0) {
-		MAIL_INDEX_MAP_IDX(view->index, view->map, seq-1)->
-			cache_offset = u->cache_offset;
+		MAIL_INDEX_MAP_IDX(view->map, seq-1)->cache_offset =
+			u->cache_offset;
 	}
 	return 1;
 }
@@ -241,7 +245,7 @@
 		offset = view->index->extra_records[hdr->idx].offset;
 		size = view->index->extra_records[hdr->idx].size;
 
-		rec = MAIL_INDEX_MAP_IDX(view->index, view->map, seq-1);
+		rec = MAIL_INDEX_MAP_IDX(view->map, seq-1);
 		memcpy(PTR_OFFSET(rec, offset), u->data, size);
 	}
 	return 1;
@@ -259,7 +263,7 @@
 	i_assert(map == index->map);
 
 	size = map->hdr->header_size +
-		(map->records_count + count) * index->record_size;
+		(map->records_count + count) * map->hdr->record_size;
 	if (size <= map->mmap_size)
 		return 0;
 
@@ -271,7 +275,7 @@
 	index->last_grow_count = count;
 
 	size = map->hdr->header_size +
-		(map->records_count + count) * index->record_size;
+		(map->records_count + count) * map->hdr->record_size;
 	if (file_set_size(index->fd, (off_t)size) < 0)
 		return mail_index_set_syscall_error(index, "file_set_size()");
 
@@ -290,6 +294,18 @@
 	return 0;
 }
 
+static void mail_index_sync_replace_map(struct mail_index_view *view,
+					struct mail_index_map *map)
+{
+	mail_index_unmap(view->index, view->map);
+	view->map = map;
+	view->map->refcount++;
+	mail_index_unmap(view->index, view->index->map);
+	view->index->map = map;
+	view->index->hdr = map->hdr;
+	map->write_to_disk = TRUE;
+}
+
 int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx)
 {
 	struct mail_index *index = sync_ctx->index;
@@ -335,18 +351,23 @@
 			   replace the whole index file. to avoid extra disk
 			   I/O we copy the index into memory rather than to
 			   temporary file */
-			map = mail_index_map_to_memory(index, map);
-			mail_index_unmap(index, view->map);
-			view->map = map;
-			view->map->refcount++;
-			mail_index_unmap(index, index->map);
-			index->map = map;
-			index->hdr = map->hdr;
-			map->write_to_disk = TRUE;
+			map = mail_index_map_to_memory(map,
+						       map->hdr->record_size);
+			mail_index_sync_replace_map(view, map);
 		}
 
 		if ((hdr->type & MAIL_TRANSACTION_APPEND) != 0) {
-			count = hdr->size / index->record_size;
+                        const struct mail_transaction_append_header *append_hdr;
+
+			append_hdr = data;
+			if (append_hdr->record_size > map->hdr->record_size) {
+				/* we have to grow our record size */
+				map = mail_index_map_to_memory(map,
+					append_hdr->record_size);
+				mail_index_sync_replace_map(view, map);
+			}
+			count = (hdr->size - sizeof(*append_hdr)) /
+				 append_hdr->record_size;
 			if (mail_index_grow(index, view->map, count) < 0) {
 				ret = -1;
 				break;
@@ -381,7 +402,7 @@
 		const struct mail_index_record *rec;
 
 		for (i = 0; i < map->records_count; i++) {
-			rec = MAIL_INDEX_MAP_IDX(index, map, i);
+			rec = MAIL_INDEX_MAP_IDX(map, i);
 			if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
 				map->hdr_copy.flags |=
 					MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
@@ -392,7 +413,7 @@
 
 	if (!MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
 		map->mmap_used_size = index->hdr->header_size +
-			map->records_count * index->record_size;
+			map->records_count * map->hdr->record_size;
 
 		memcpy(map->mmap_base, &map->hdr_copy, sizeof(map->hdr_copy));
 		if (msync(map->mmap_base, map->mmap_used_size, MS_SYNC) < 0) {

Index: mail-index-sync.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-sync.c,v
retrieving revision 1.23
retrieving revision 1.24
diff -u -d -r1.23 -r1.24
--- mail-index-sync.c	21 Jun 2004 14:44:47 -0000	1.23
+++ mail-index-sync.c	24 Jun 2004 11:10:41 -0000	1.24
@@ -80,11 +80,23 @@
 			mail_index_sync_sort_flags(ctx);
 		}
 		break;
-	case MAIL_TRANSACTION_APPEND:
-		buffer_append(ctx->appends_buf, ctx->data, ctx->hdr->size);
+	case MAIL_TRANSACTION_APPEND: {
+		const struct mail_transaction_append_header *hdr = ctx->data;
+		const struct mail_index_record *rec = ctx->data;
+
+		if (ctx->append_uid_first == 0 ||
+		    rec->uid < ctx->append_uid_first)
+			ctx->append_uid_first = rec->uid;
+
+		rec = CONST_PTR_OFFSET(ctx->data,
+				       ctx->hdr->size - hdr->record_size);
+		if (rec->uid > ctx->append_uid_last)
+			ctx->append_uid_last = rec->uid;
+
                 ctx->sync_appends = TRUE;
 		break;
 	}
+	}
 }
 
 static int mail_index_sync_read_and_sort(struct mail_index_sync_ctx *ctx)
@@ -172,8 +184,6 @@
 						  1024, (size_t)-1);
 	ctx->updates_buf = buffer_create_dynamic(default_pool,
 						 1024, (size_t)-1);
-	ctx->appends_buf = buffer_create_dynamic(default_pool,
-						 1024, (size_t)-1);
 	if (mail_index_sync_read_and_sort(ctx) < 0) {
                 mail_index_sync_end(ctx);
 		return -1;
@@ -304,12 +314,8 @@
 	if (ctx->sync_appends) {
 		ctx->sync_appends = FALSE;
 		sync_rec->type = MAIL_INDEX_SYNC_TYPE_APPEND;
-		sync_rec->appends = buffer_get_data(ctx->appends_buf,
-						    &sync_rec->appends_count);
-		sync_rec->appends_count /= ctx->index->record_size;
-		sync_rec->uid1 = sync_rec->appends[0].uid;
-		sync_rec->uid2 =
-			sync_rec->appends[sync_rec->appends_count-1].uid;
+		sync_rec->uid1 = ctx->append_uid_first;
+		sync_rec->uid2 = ctx->append_uid_last;
 		return 1;
 	}
 
@@ -355,8 +361,6 @@
 		buffer_free(ctx->expunges_buf);
 	if (ctx->updates_buf != NULL)
 		buffer_free(ctx->updates_buf);
-	if (ctx->appends_buf != NULL)
-		buffer_free(ctx->appends_buf);
 	i_free(ctx);
 	return ret;
 }

Index: mail-index-transaction-private.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-transaction-private.h,v
retrieving revision 1.6
retrieving revision 1.7
diff -u -d -r1.6 -r1.7
--- mail-index-transaction-private.h	22 Jun 2004 07:36:33 -0000	1.6
+++ mail-index-transaction-private.h	24 Jun 2004 11:10:41 -0000	1.7
@@ -8,6 +8,7 @@
 
         buffer_t *appends;
 	uint32_t first_new_seq, last_new_seq;
+	unsigned int append_record_size;
 
 	buffer_t *expunges;
 

Index: mail-index-transaction-view.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-transaction-view.c,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- mail-index-transaction-view.c	22 Jun 2004 07:36:33 -0000	1.1
+++ mail-index-transaction-view.c	24 Jun 2004 11:10:41 -0000	1.2
@@ -130,7 +130,7 @@
 			*seq_r = seq;
 			break;
 		}
-		rec = CONST_PTR_OFFSET(rec, view->index->record_size);
+		rec = CONST_PTR_OFFSET(rec, view->index->max_record_size);
 	}
 
 	return 0;

Index: mail-index-transaction.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-transaction.c,v
retrieving revision 1.14
retrieving revision 1.15
diff -u -d -r1.14 -r1.15
--- mail-index-transaction.c	22 Jun 2004 07:36:33 -0000	1.14
+++ mail-index-transaction.c	24 Jun 2004 11:10:41 -0000	1.15
@@ -70,12 +70,10 @@
 		seq = (uint32_t *)&data[i];
 		i_assert(seq[0] <= view->map->records_count);
 
-		seq[0] = MAIL_INDEX_MAP_IDX(view->index, view->map,
-					    seq[0]-1)->uid;
+		seq[0] = MAIL_INDEX_MAP_IDX(view->map, seq[0]-1)->uid;
 		if (range) {
 			i_assert(seq[1] <= view->map->records_count);
-			seq[1] = MAIL_INDEX_MAP_IDX(view->index, view->map,
-						    seq[1]-1)->uid;
+			seq[1] = MAIL_INDEX_MAP_IDX(view->map, seq[1]-1)->uid;
 		}
 	}
 }
@@ -139,6 +137,40 @@
         mail_index_transaction_free(t);
 }
 
+static void
+mail_index_transaction_update_append_size(struct mail_index_transaction *t)
+{
+	buffer_t *new_buf;
+	unsigned int new_record_size;
+	const void *src;
+	void *dest;
+	size_t i, size;
+
+	new_record_size = t->view->index->max_record_size;
+	if (t->append_record_size == new_record_size)
+		return;
+
+	i_assert(t->append_record_size < new_record_size);
+
+	if (t->append_record_size != 0) {
+		/* resize the records in buffer */
+		src = buffer_get_data(t->appends, &size);
+		size /= t->append_record_size;
+
+		new_buf = buffer_create_dynamic(default_pool,
+						size * new_record_size,
+						(size_t)-1);
+		for (i = 0; i < size; i++) {
+			dest = buffer_append_space_unsafe(new_buf,
+							  new_record_size);
+			memcpy(dest, src, t->append_record_size);
+			src = CONST_PTR_OFFSET(src, t->append_record_size);
+		}
+	}
+
+	t->append_record_size = new_record_size;
+}
+
 struct mail_index_record *
 mail_index_transaction_lookup(struct mail_index_transaction *t, uint32_t seq)
 {
@@ -146,9 +178,10 @@
 
 	i_assert(seq >= t->first_new_seq && seq <= t->last_new_seq);
 
-	pos = (seq - t->first_new_seq) * t->view->index->record_size;
-	return buffer_get_space_unsafe(t->appends, pos,
-				       t->view->index->record_size);
+	mail_index_transaction_update_append_size(t);
+
+	pos = (seq - t->first_new_seq) * t->append_record_size;
+	return buffer_get_space_unsafe(t->appends, pos, t->append_record_size);
 }
 
 void mail_index_append(struct mail_index_transaction *t, uint32_t uid,
@@ -159,7 +192,9 @@
 	if (t->appends == NULL) {
 		t->appends = buffer_create_dynamic(default_pool,
 						   4096, (size_t)-1);
+		t->append_record_size = t->view->index->max_record_size;
 	}
+	mail_index_transaction_update_append_size(t);
 
 	/* sequence number is visible only inside given view,
 	   so let it generate it */
@@ -168,9 +203,8 @@
 	else
 		*seq_r = t->last_new_seq = t->first_new_seq;
 
-	rec = buffer_append_space_unsafe(t->appends,
-					 t->view->index->record_size);
-	memset(rec, 0, t->view->index->record_size);
+	rec = buffer_append_space_unsafe(t->appends, t->append_record_size);
+	memset(rec, 0, t->append_record_size);
 	rec->uid = uid;
 }
 

Index: mail-index-view-sync.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-view-sync.c,v
retrieving revision 1.13
retrieving revision 1.14
diff -u -d -r1.13 -r1.14
--- mail-index-view-sync.c	13 Jun 2004 23:38:47 -0000	1.13
+++ mail-index-view-sync.c	24 Jun 2004 11:10:41 -0000	1.14
@@ -125,7 +125,8 @@
 		if (view->map != view->index->map)
 			ctx->sync_map_update = TRUE;
 
-		map = mail_index_map_to_memory(view->index, view->map);
+		map = mail_index_map_to_memory(view->map,
+					       view->map->hdr->record_size);
 		mail_index_unmap(view->index, view->map);
 		view->map = map;
 	}

Index: mail-index-view.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-view.c,v
retrieving revision 1.17
retrieving revision 1.18
diff -u -d -r1.17 -r1.18
--- mail-index-view.c	22 Jun 2004 07:36:33 -0000	1.17
+++ mail-index-view.c	24 Jun 2004 11:10:41 -0000	1.18
@@ -178,7 +178,7 @@
 	if (mail_index_view_lock(view) < 0)
 		return -1;
 
-	rec = MAIL_INDEX_MAP_IDX(view->index, view->map, seq-1);
+	rec = MAIL_INDEX_MAP_IDX(view->map, seq-1);
 	if (view->map == view->index->map) {
 		*map_r = view->map;
 		*rec_r = rec;
@@ -203,7 +203,7 @@
 	do {
 		// FIXME: we could be skipping more by uid diff
 		seq--;
-		n_rec = MAIL_INDEX_MAP_IDX(view->index, map, seq);
+		n_rec = MAIL_INDEX_MAP_IDX(map, seq);
 		if (n_rec->uid <= uid)
 			break;
 	} while (seq > 0);
@@ -225,7 +225,7 @@
 	if (mail_index_view_lock(view) < 0)
 		return -1;
 
-	*uid_r = MAIL_INDEX_MAP_IDX(view->index, view->map, seq-1)->uid;
+	*uid_r = MAIL_INDEX_MAP_IDX(view->map, seq-1)->uid;
 	return 0;
 }
 
@@ -239,7 +239,7 @@
 	i_assert(view->messages_count <= view->map->records_count);
 
 	rec_base = view->map->records;
-	record_size = view->index->record_size;
+	record_size = view->map->hdr->record_size;
 
 	idx = left_idx = *left_idx_p;
 	right_idx = view->messages_count;
@@ -301,8 +301,7 @@
 	left_idx = 0;
 	*first_seq_r = mail_index_bsearch_uid(view, first_uid, &left_idx, 1);
 	if (*first_seq_r == 0 ||
-	    MAIL_INDEX_MAP_IDX(view->index, view->map, *first_seq_r-1)->uid >
-	    last_uid) {
+	    MAIL_INDEX_MAP_IDX(view->map, *first_seq_r-1)->uid > last_uid) {
 		*first_seq_r = 0;
 		*last_seq_r = 0;
 		return 0;
@@ -351,7 +350,7 @@
 	}
 
 	for (; seq <= view->messages_count; seq++) {
-		rec = MAIL_INDEX_MAP_IDX(view->index, view->map, seq-1);
+		rec = MAIL_INDEX_MAP_IDX(view->map, seq-1);
 		if ((rec->flags & flags_mask) == (uint8_t)flags) {
 			*seq_r = seq;
 			break;

Index: mail-index.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index.c,v
retrieving revision 1.136
retrieving revision 1.137
diff -u -d -r1.136 -r1.137
--- mail-index.c	20 Jun 2004 12:23:27 -0000	1.136
+++ mail-index.c	24 Jun 2004 11:10:41 -0000	1.137
@@ -31,7 +31,7 @@
 	index->extra_records_buf =
 		buffer_create_dynamic(index->extra_records_pool,
 				      64, (size_t)-1);
-	index->record_size = sizeof(struct mail_index_record);
+	index->max_record_size = sizeof(struct mail_index_record);
 
 	index->mode = 0600;
 	index->gid = (gid_t)-1;
@@ -66,7 +66,7 @@
 
 	i_assert(size % 4 == 0);
 	i_assert(!index->opened);
-	i_assert(index->record_size + size <= 65535);
+	i_assert(index->max_record_size + size <= 65535);
 
 	if (index->extra_records_count >= MAIL_INDEX_MAX_EXTRA_RECORDS) {
 		i_panic("Maximum extra record count reached, "
@@ -78,14 +78,14 @@
 	memset(&info, 0, sizeof(info));
 	info.name = p_strdup(index->extra_records_pool, name);
 	info.size = size;
-	info.offset = index->record_size;
+	info.offset = index->max_record_size;
 
 	buffer_append(index->extra_records_buf, &info, sizeof(info));
 	index->extra_records =
 		buffer_get_data(index->extra_records_buf, &buf_size);
 	index->extra_records_count = buf_size / sizeof(info);
 
-	index->record_size += size;
+	index->max_record_size += size;
 	return index->extra_records_count-1;
 }
 
@@ -132,11 +132,11 @@
 		return -1;
 	}
 
-	if (hdr->record_size != index->record_size) {
+	if (hdr->record_size < sizeof(struct mail_index_record)) {
 		mail_index_set_error(index, "Corrupted index file %s: "
-				     "record_size mismatch: %d != %d",
+				     "record_size too small: %u < %"PRIuSIZE_T,
 				     index->filepath, hdr->record_size,
-				     (int)index->record_size);
+				     sizeof(struct mail_index_record));
 		return -1;
 	}
 
@@ -227,11 +227,11 @@
 	hdr = map->mmap_base;
 	map->hdr = hdr;
 	map->mmap_used_size = hdr->header_size +
-		hdr->messages_count * index->record_size;
+		hdr->messages_count * hdr->record_size;
 
 	if (map->mmap_used_size > map->mmap_size) {
 		records_count = (map->mmap_size - hdr->header_size) /
-			index->record_size;
+			hdr->record_size;
 		mail_index_set_error(index, "Corrupted index file %s: "
 				     "messages_count too large (%u > %u)",
 				     index->filepath, hdr->messages_count,
@@ -271,7 +271,7 @@
 			pos += ret;
 	}
 	if (ret >= 0 && pos >= MAIL_INDEX_HEADER_MIN_SIZE) {
-		records_size = hdr.messages_count * index->record_size;
+		records_size = hdr.messages_count * hdr.record_size;
 
 		if (map->buffer == NULL) {
 			map->buffer = buffer_create_dynamic(default_pool,
@@ -367,7 +367,7 @@
 			return -1;
 
 		used_size = hdr->header_size +
-			hdr->messages_count * index->record_size;
+			hdr->messages_count * hdr->record_size;
 		if (map->mmap_size >= used_size && !force) {
 			map->records_count = hdr->messages_count;
 			return 1;
@@ -410,27 +410,41 @@
 }
 
 struct mail_index_map *
-mail_index_map_to_memory(struct mail_index *index, struct mail_index_map *map)
+mail_index_map_to_memory(struct mail_index_map *map, uint32_t new_record_size)
 {
 	struct mail_index_map *mem_map;
-	size_t size;
+	void *src, *dest;
+	size_t size, copy_size;
+	unsigned int i;
 
 	if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
 		map->refcount++;
 		return map;
 	}
 
-        size = map->records_count * index->record_size;
+        size = map->records_count * new_record_size;
 
 	mem_map = i_new(struct mail_index_map, 1);
 	mem_map->refcount = 1;
 	mem_map->buffer = buffer_create_dynamic(default_pool, size, (size_t)-1);
-	buffer_append(mem_map->buffer, map->records, size);
+	if (map->hdr->record_size == new_record_size)
+		buffer_append(mem_map->buffer, map->records, size);
+	else {
+		copy_size = I_MIN(map->hdr->record_size, new_record_size);
+		src = map->records;
+		for (i = 0; i < map->records_count; i++) {
+			dest = buffer_append_space_unsafe(mem_map->buffer,
+							  new_record_size);
+			memcpy(dest, src, copy_size);
+			src = PTR_OFFSET(src, map->hdr->record_size);
+		}
+	}
 
 	mem_map->records = buffer_get_modifyable_data(mem_map->buffer, NULL);
 	mem_map->records_count = map->records_count;
 
 	mem_map->hdr_copy = *map->hdr;
+	mem_map->hdr_copy.record_size = new_record_size;
 	mem_map->hdr = &mem_map->hdr_copy;
 	return mem_map;
 }
@@ -497,15 +511,19 @@
 	return ret;
 }
 
-int mail_index_write_header(struct mail_index *index,
-			    const struct mail_index_header *hdr)
+int mail_index_write_base_header(struct mail_index *index,
+				 const struct mail_index_header *hdr)
 {
+	size_t hdr_size;
+
+	hdr_size = I_MIN(sizeof(*hdr), hdr->base_header_size);
+
 	if (!MAIL_INDEX_MAP_IS_IN_MEMORY(index->map)) {
-		memcpy(index->map->mmap_base, hdr, sizeof(*hdr));
-		if (msync(index->map->mmap_base, sizeof(*hdr), MS_SYNC) < 0)
+		memcpy(index->map->mmap_base, hdr, hdr_size);
+		if (msync(index->map->mmap_base, hdr_size, MS_SYNC) < 0)
 			return mail_index_set_syscall_error(index, "msync()");
 	} else {
-		if (pwrite_full(index->fd, hdr, sizeof(*hdr), 0) < 0) {
+		if (pwrite_full(index->fd, hdr, hdr_size, 0) < 0) {
 			mail_index_set_syscall_error(index, "pwrite_full()");
 			return -1;
 		}
@@ -602,7 +620,7 @@
 	hdr->minor_version = MAIL_INDEX_MINOR_VERSION;
 	hdr->base_header_size = sizeof(*hdr);
 	hdr->header_size = sizeof(*hdr);
-	hdr->record_size = index->record_size;
+	hdr->record_size = index->max_record_size;
 	hdr->keywords_mask_size = sizeof(keywords_mask_t);
 
 #ifndef WORDS_BIGENDIAN
@@ -890,7 +908,7 @@
 
 	hdr = *index->hdr;
 	hdr.flags |= MAIL_INDEX_HDR_FLAG_CORRUPTED;
-	if (mail_index_write_header(index, &hdr) == 0) {
+	if (mail_index_write_base_header(index, &hdr) == 0) {
 		if (fsync(index->fd) < 0)
 			mail_index_set_syscall_error(index, "fsync()");
 	}

Index: mail-index.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index.h,v
retrieving revision 1.119
retrieving revision 1.120
diff -u -d -r1.119 -r1.120
--- mail-index.h	22 Jun 2004 07:36:33 -0000	1.119
+++ mail-index.h	24 Jun 2004 11:10:41 -0000	1.120
@@ -123,10 +123,6 @@
 	keywords_mask_t add_keywords;
 	uint8_t remove_flags;
 	keywords_mask_t remove_keywords;
-
-	/* MAIL_INDEX_SYNC_TYPE_APPEND: */
-	const struct mail_index_record *appends;
-	size_t appends_count;
 };
 
 struct mail_index;

Index: mail-transaction-log.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-transaction-log.c,v
retrieving revision 1.41
retrieving revision 1.42
diff -u -d -r1.41 -r1.42
--- mail-transaction-log.c	20 Jun 2004 13:02:08 -0000	1.41
+++ mail-transaction-log.c	24 Jun 2004 11:10:41 -0000	1.42
@@ -874,6 +874,7 @@
 	struct mail_transaction_log_view *sync_view;
 	const struct mail_index_record *old, *old_end;
 	struct mail_index_record *appends, *end, *rec, *dest;
+        const struct mail_transaction_append_header *append_hdr;
 	const struct mail_transaction_header *hdr;
 	const void *data;
 	size_t size;
@@ -883,7 +884,7 @@
 	if (t->appends == NULL)
 		return 0;
 
-	record_size = log->index->record_size;
+	record_size = t->append_record_size;
 	appends = buffer_get_modifyable_data(t->appends, &size);
 	end = PTR_OFFSET(appends, size);
 
@@ -905,8 +906,10 @@
 		    MAIL_TRANSACTION_APPEND)
 			continue;
 
-		old = data;
-		old_end = CONST_PTR_OFFSET(old, hdr->size);
+                append_hdr = data;
+
+		old = CONST_PTR_OFFSET(data, sizeof(*append_hdr));
+		old_end = CONST_PTR_OFFSET(data, hdr->size);
 		while (old != old_end) {
 			/* appends are sorted */
 			for (rec = appends; rec != end; ) {
@@ -919,7 +922,7 @@
 				}
 				rec = PTR_OFFSET(rec, record_size);
 			}
-                        old = CONST_PTR_OFFSET(old, record_size);
+                        old = CONST_PTR_OFFSET(old, append_hdr->record_size);
 		}
 	}
 
@@ -1097,7 +1100,15 @@
 
 	ret = 0;
 	if (t->appends != NULL) {
-		ret = log_append_buffer(file, t->appends, NULL,
+		struct mail_transaction_append_header hdr;
+
+		memset(&hdr, 0, sizeof(hdr));
+		hdr.record_size = t->append_record_size;
+
+		hdr_buf = buffer_create_data(pool_datastack_create(),
+					     &hdr, sizeof(hdr));
+		buffer_set_used_size(hdr_buf, sizeof(hdr));
+		ret = log_append_buffer(file, t->appends, hdr_buf,
 					MAIL_TRANSACTION_APPEND,
 					view->external);
 	}

Index: mail-transaction-log.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-transaction-log.h,v
retrieving revision 1.13
retrieving revision 1.14
diff -u -d -r1.13 -r1.14
--- mail-transaction-log.h	20 Jun 2004 09:13:14 -0000	1.13
+++ mail-transaction-log.h	24 Jun 2004 11:10:41 -0000	1.14
@@ -75,6 +75,10 @@
 	unsigned char data[1]; /* variable size */
 };
 
+struct mail_transaction_append_header {
+	uint32_t record_size;
+};
+
 struct mail_transaction_log *
 mail_transaction_log_open_or_create(struct mail_index *index);
 void mail_transaction_log_close(struct mail_transaction_log *log);

Index: mail-transaction-util.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-transaction-util.c,v
retrieving revision 1.13
retrieving revision 1.14
diff -u -d -r1.13 -r1.14
--- mail-transaction-util.c	20 Jun 2004 09:13:14 -0000	1.13
+++ mail-transaction-util.c	24 Jun 2004 11:10:41 -0000	1.14
@@ -60,23 +60,23 @@
 			 struct mail_transaction_map_functions *map,
 			 void *context)
 {
-
 	int ret = 0;
 
 	switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
 	case MAIL_TRANSACTION_APPEND: {
+                const struct mail_transaction_append_header *append_hdr = data;
 		const struct mail_index_record *rec, *end;
-		uint32_t record_size = index->record_size;
 
 		if (map->append == NULL)
 			break;
 
+		rec = CONST_PTR_OFFSET(data, sizeof(*append_hdr));
 		end = CONST_PTR_OFFSET(data, hdr->size);
-		for (rec = data; rec != end; ) {
-			ret = map->append(rec, context);
+		while (rec != end) {
+			ret = map->append(append_hdr, rec, context);
 			if (ret <= 0)
 				break;
-			rec = CONST_PTR_OFFSET(rec, record_size);
+			rec = CONST_PTR_OFFSET(rec, append_hdr->record_size);
 		}
 		break;
 	}

Index: mail-transaction-util.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-transaction-util.h,v
retrieving revision 1.7
retrieving revision 1.8
diff -u -d -r1.7 -r1.8
--- mail-transaction-util.h	20 Jun 2004 09:13:14 -0000	1.7
+++ mail-transaction-util.h	24 Jun 2004 11:10:41 -0000	1.8
@@ -10,7 +10,8 @@
 
 struct mail_transaction_map_functions {
 	int (*expunge)(const struct mail_transaction_expunge *e, void *context);
-	int (*append)(const struct mail_index_record *rec, void *context);
+	int (*append)(const struct mail_transaction_append_header *hdr,
+		      const struct mail_index_record *rec, void *context);
 	int (*flag_update)(const struct mail_transaction_flag_update *u,
 			   void *context);
 	int (*cache_reset)(const struct mail_transaction_cache_reset *u,

