[dovecot-cvs] dovecot/src/lib-index mail-index-fsck.c, 1.21, 1.22 mail-index-lock.c, 1.19, 1.20 mail-index-private.h, 1.17, 1.18 mail-index-sync-update.c, 1.21, 1.22 mail-index-sync.c, 1.18, 1.19 mail-index-transaction.c, 1.7, 1.8 mail-index-view-sync.c, 1.12, 1.13 mail-index-view.c, 1.10, 1.11 mail-index.c, 1.128, 1.129 mail-index.h, 1.112, 1.113 mail-transaction-log.c, 1.34, 1.35 mail-transaction-util.c, 1.8, 1.9 mail-transaction-util.h, 1.4, 1.5

Mon Jun 14 02:38:49 EEST 2004

Update of /home/cvs/dovecot/src/lib-index
In directory talvi:/tmp/cvs-serv4162/lib-index

Modified Files:
	mail-index-fsck.c mail-index-lock.c mail-index-private.h 
	mail-index-sync-update.c mail-index-sync.c 
	mail-index-transaction.c mail-index-view-sync.c 
	mail-index-view.c mail-index.c mail-index.h 
	mail-transaction-log.c mail-transaction-util.c 
	mail-transaction-util.h 
Log Message:
Added support for per-index sized mail_index_record.



Index: mail-index-fsck.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-fsck.c,v
retrieving revision 1.21
retrieving revision 1.22
diff -u -d -r1.21 -r1.22

--- mail-index-fsck.c	24 May 2004 02:29:43 -0000	1.21
+++ mail-index-fsck.c	13 Jun 2004 23:38:47 -0000	1.22
@@ -27,8 +27,8 @@
 				  const char **error_r)
 {
 	struct mail_index_header hdr;
-	const struct mail_index_record *rec, *end;
-	uint32_t last_uid;
+	const struct mail_index_record *rec;
+	uint32_t i, last_uid;
 
 	*error_r = NULL;
 
@@ -49,8 +49,8 @@
 	hdr.first_unseen_uid_lowwater = 0;
 	hdr.first_deleted_uid_lowwater = 0;
 
-	end = index->map->records + index->map->records_count; last_uid = 0;
-	for (rec = index->map->records; rec != end; rec++) {
+	rec = index->map->records; last_uid = 0;
+	for (i = 0; i < index->map->records_count; i++) {
 		if (rec->uid <= last_uid) {
 			*error_r = "Record UIDs are not ordered";
 			return 0;
@@ -75,6 +75,7 @@
 			hdr.first_deleted_uid_lowwater = rec->uid;
 
 		last_uid = rec->uid;
+		rec = CONST_PTR_OFFSET(rec, index->record_size);
 	}
 
 	if (hdr.next_uid <= last_uid) {

Index: mail-index-lock.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-lock.c,v
retrieving revision 1.19
retrieving revision 1.20
diff -u -d -r1.19 -r1.20
--- mail-index-lock.c	25 May 2004 23:13:57 -0000	1.19
+++ mail-index-lock.c	13 Jun 2004 23:38:47 -0000	1.20
@@ -187,7 +187,7 @@
 	ret = write_full(fd, index->map->hdr, sizeof(*index->map->hdr));
 	if (ret < 0 || write_full(fd, index->map->records,
 				  index->map->records_count *
-				  sizeof(struct mail_index_record)) < 0) {
+				  index->record_size) < 0) {
 		mail_index_file_set_syscall_error(index, path, "write_full()");
 		(void)close(fd);
 		(void)unlink(path);

Index: mail-index-private.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-private.h,v
retrieving revision 1.17
retrieving revision 1.18
diff -u -d -r1.17 -r1.18
--- mail-index-private.h	28 May 2004 01:04:01 -0000	1.17
+++ mail-index-private.h	13 Jun 2004 23:38:47 -0000	1.18
@@ -17,11 +17,15 @@
 #define MAIL_INDEX_MAP_IS_IN_MEMORY(map) \
 	((map)->buffer != NULL)
 
+#define MAIL_INDEX_MAP_IDX(index, map, idx) \
+	((struct mail_index_record *) \
+		PTR_OFFSET((map)->records, (idx) * (index)->record_size))
+
 struct mail_index_map {
 	int refcount;
 
 	const struct mail_index_header *hdr;
-	struct mail_index_record *records;
+	void *records; /* struct mail_index_record[] */
 	unsigned int records_count;
 
 	void *mmap_base;
@@ -45,6 +49,7 @@
 
 	mode_t mode;
 	gid_t gid;
+	unsigned int record_size;
 
 	char *filepath;
 	int fd;
@@ -99,7 +104,8 @@
 int mail_index_map(struct mail_index *index, int force);
 /* Unreference given mapping and unmap it if it's dropped to zero. */
 void mail_index_unmap(struct mail_index *index, struct mail_index_map *map);
-struct mail_index_map *mail_index_map_to_memory(struct mail_index_map *map);
+struct mail_index_map *
+mail_index_map_to_memory(struct mail_index *index, struct mail_index_map *map);
 
 void mail_index_update_cache(struct mail_index_transaction *t,
 			     uint32_t seq, uint32_t offset);

Index: mail-index-sync-update.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-sync-update.c,v
retrieving revision 1.21
retrieving revision 1.22
diff -u -d -r1.21 -r1.22
--- mail-index-sync-update.c	30 May 2004 04:32:36 -0000	1.21
+++ mail-index-sync-update.c	13 Jun 2004 23:38:47 -0000	1.22
@@ -56,6 +56,7 @@
 static void mail_index_sync_expunge(struct mail_index_view *view,
 				    const struct mail_transaction_expunge *e)
 {
+	struct mail_index *index = view->index;
 	struct mail_index_map *map = view->map;
 	struct mail_index_header *hdr = &map->hdr_copy;
 	struct mail_index_record *rec;
@@ -70,22 +71,24 @@
 	if (seq1 == 0)
 		return;
 
-	rec = &map->records[seq1-1];
-	for (seq = seq1; seq <= seq2; seq++, rec++)
+	for (seq = seq1; seq <= seq2; seq++) {
+                rec = MAIL_INDEX_MAP_IDX(index, map, seq-1);
 		mail_index_header_update_counts(hdr, rec->flags, 0);
+	}
 
 	/* @UNSAFE */
 	count = seq2 - seq1 + 1;
-	memcpy(map->records + (seq1-1), map->records + seq2,
-	       (map->records_count - seq2) * sizeof(*map->records));
+	memcpy(MAIL_INDEX_MAP_IDX(index, map, seq-1),
+               MAIL_INDEX_MAP_IDX(index, map, seq2),
+	       (map->records_count - seq2) * view->index->record_size);
 
 	map->records_count -= count;
 	hdr->messages_count -= count;
 	view->messages_count -= count;
 
 	if (map->buffer != NULL) {
-		buffer_set_used_size(map->buffer,
-				     map->records_count * sizeof(*rec));
+		buffer_set_used_size(map->buffer, map->records_count *
+				     view->index->record_size);
 		map->records = buffer_get_modifyable_data(map->buffer, NULL);
 	}
 }
@@ -101,6 +104,7 @@
 static int sync_append(const struct mail_index_record *rec, void *context)
 {
 	struct mail_index_view *view = context;
+	struct mail_index *index = view->index;
 	struct mail_index_map *map = view->map;
 
 	if (rec->uid < map->hdr_copy.next_uid) {
@@ -111,14 +115,15 @@
 	}
 
 	if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
-		i_assert(map->records_count * sizeof(*rec) ==
+		i_assert(map->records_count * index->record_size ==
 			 buffer_get_used_size(map->buffer));
-		buffer_append(map->buffer, rec, sizeof(*rec));
+		buffer_append(map->buffer, rec, index->record_size);
 		map->records = buffer_get_modifyable_data(map->buffer, NULL);
 	} else {
-		i_assert((map->records_count+1) * sizeof(*rec) <=
+		i_assert((map->records_count+1) * index->record_size <=
 			 map->mmap_size);
-		map->records[map->records_count] = *rec;
+		memcpy(MAIL_INDEX_MAP_IDX(index, map, map->records_count),
+		       rec, index->record_size);
 	}
 
 	map->hdr_copy.messages_count++;
@@ -135,12 +140,12 @@
 			    void *context)
 {
         struct mail_index_view *view = context;
-	struct mail_index_record *rec, *end;
+	struct mail_index_record *rec;
 	struct mail_index_header *hdr;
 	uint8_t flag_mask, old_flags;
 	keywords_mask_t keyword_mask;
-	uint32_t seq1, seq2;
-	int i, update_keywords, ret;
+	uint32_t i, seq1, seq2;
+	int update_keywords, ret;
 
 	ret = mail_index_lookup_uid_range(view, u->uid1, u->uid2, &seq1, &seq2);
 	i_assert(ret == 0);
@@ -162,9 +167,9 @@
 	}
         flag_mask = ~u->remove_flags;
 
-	rec = &view->map->records[seq1-1];
-	end = rec + (seq2 - seq1) + 1;
-	for (; rec != end; rec++) {
+	for (i = seq1-1; i < seq2; i++) {
+                rec = MAIL_INDEX_MAP_IDX(view->index, view->map, i);
+
 		old_flags = rec->flags;
 		rec->flags = (rec->flags & flag_mask) | u->add_flags;
 		if (update_keywords) {
@@ -191,8 +196,10 @@
 					  &seq, &seq);
 	i_assert(ret == 0);
 
-	if (seq != 0)
-		view->map->records[seq-1].cache_offset = u->cache_offset;
+	if (seq != 0) {
+		MAIL_INDEX_MAP_IDX(view->index, view->map, seq-1)->
+			cache_offset = u->cache_offset;
+	}
 	return 1;
 }
 
@@ -218,7 +225,7 @@
 	i_assert(map == index->map);
 
 	size = map->hdr->header_size +
-		(map->records_count + count) * sizeof(struct mail_index_record);
+		(map->records_count + count) * index->record_size;
 	if (size <= map->mmap_size)
 		return 0;
 
@@ -230,7 +237,7 @@
 	index->last_grow_count = count;
 
 	size = map->hdr->header_size +
-		(map->records_count + count) * sizeof(struct mail_index_record);
+		(map->records_count + count) * index->record_size;
 	if (file_set_size(index->fd, (off_t)size) < 0)
 		return mail_index_set_syscall_error(index, "file_set_size()");
 
@@ -282,7 +289,7 @@
 			   replace the whole index file. to avoid extra disk
 			   I/O we copy the index into memory rather than to
 			   temporary file */
-			map = mail_index_map_to_memory(map);
+			map = mail_index_map_to_memory(index, map);
 			mail_index_unmap(index, view->map);
 			view->map = map;
 			view->map->refcount++;
@@ -293,12 +300,13 @@
 		}
 
 		if ((hdr->type & MAIL_TRANSACTION_APPEND) != 0) {
-			count = hdr->size / sizeof(struct mail_index_record);
+			count = hdr->size / index->record_size;
 			if (mail_index_grow(index, view->map, count) < 0)
 				return -1;
 		}
 
-		if (mail_transaction_map(hdr, data, &mail_index_map_sync_funcs,
+		if (mail_transaction_map(index, hdr, data,
+					 &mail_index_map_sync_funcs,
 					 view) < 0) {
 			ret = -1;
 			break;
@@ -316,9 +324,11 @@
 	if ((map->hdr_copy.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) == 0 &&
 	    had_dirty) {
 		/* do we have dirty flags anymore? */
+		const struct mail_index_record *rec;
+
 		for (i = 0; i < map->records_count; i++) {
-			if ((map->records[i].flags &
-			     MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
+			rec = MAIL_INDEX_MAP_IDX(index, map, i);
+			if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
 				map->hdr_copy.flags |=
 					MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
 				break;
@@ -328,7 +338,7 @@
 
 	if (!MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
 		map->mmap_used_size = index->hdr->header_size +
-			map->records_count * sizeof(struct mail_index_record);
+			map->records_count * index->record_size;
 
 		memcpy(map->mmap_base, &map->hdr_copy, sizeof(map->hdr_copy));
 		if (msync(map->mmap_base, map->mmap_used_size, MS_SYNC) < 0) {

Index: mail-index-sync.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-sync.c,v
retrieving revision 1.18
retrieving revision 1.19
diff -u -d -r1.18 -r1.19
--- mail-index-sync.c	28 May 2004 01:33:11 -0000	1.18
+++ mail-index-sync.c	13 Jun 2004 23:38:47 -0000	1.19
@@ -297,7 +297,7 @@
 		sync_rec->type = MAIL_INDEX_SYNC_TYPE_APPEND;
 		sync_rec->appends = buffer_get_data(ctx->appends_buf,
 						    &sync_rec->appends_count);
-		sync_rec->appends_count /= sizeof(*sync_rec->appends);
+		sync_rec->appends_count /= ctx->index->record_size;
 		sync_rec->uid1 = sync_rec->uid2 = 0;
 		return 1;
 	}

Index: mail-index-transaction.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-transaction.c,v
retrieving revision 1.7
retrieving revision 1.8
diff -u -d -r1.7 -r1.8
--- mail-index-transaction.c	24 May 2004 01:50:16 -0000	1.7
+++ mail-index-transaction.c	13 Jun 2004 23:38:47 -0000	1.8
@@ -54,9 +54,12 @@
 	for (i = 0; i < size; i += record_size) {
 		seq = (uint32_t *)&data[i];
 
-		seq[0] = view->map->records[seq[0]-1].uid;
-		if (range)
-			seq[1] = view->map->records[seq[1]-1].uid;
+		seq[0] = MAIL_INDEX_MAP_IDX(view->index, view->map,
+					    seq[0]-1)->uid;
+		if (range) {
+			seq[1] = MAIL_INDEX_MAP_IDX(view->index, view->map,
+						    seq[1]-1)->uid;
+		}
 	}
 }
 
@@ -124,8 +127,9 @@
 			mail_index_view_get_message_count(t->view)+1;
 	}
 
-	rec = buffer_append_space_unsafe(t->appends, sizeof(*rec));
-	memset(rec, 0, sizeof(*rec));
+	rec = buffer_append_space_unsafe(t->appends,
+					 t->view->index->record_size);
+	memset(rec, 0, t->view->index->record_size);
 	rec->uid = uid;
 }
 
@@ -262,8 +266,9 @@
 		/* just appended message, modify it directly */
 		i_assert(seq > 0 && seq <= t->last_new_seq);
 
-		pos = (seq - t->first_new_seq) * sizeof(*rec);
-		rec = buffer_get_space_unsafe(t->appends, pos, sizeof(*rec));
+		pos = (seq - t->first_new_seq) * t->view->index->record_size;
+		rec = buffer_get_space_unsafe(t->appends, pos,
+					      t->view->index->record_size);
 		mail_index_record_modify_flags(rec, modify_type,
 					       flags, keywords);
 		return;

Index: mail-index-view-sync.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-view-sync.c,v
retrieving revision 1.12
retrieving revision 1.13
diff -u -d -r1.12 -r1.13
--- mail-index-view-sync.c	28 May 2004 01:33:11 -0000	1.12
+++ mail-index-view-sync.c	13 Jun 2004 23:38:47 -0000	1.13
@@ -125,7 +125,7 @@
 		if (view->map != view->index->map)
 			ctx->sync_map_update = TRUE;
 
-		map = mail_index_map_to_memory(view->map);
+		map = mail_index_map_to_memory(view->index, view->map);
 		mail_index_unmap(view->index, view->map);
 		view->map = map;
 	}
@@ -189,7 +189,7 @@
 	   see only updated information. */
 	if (ctx->sync_map_update &&
 	    (ctx->hdr->type & MAIL_TRANSACTION_EXPUNGE) == 0) {
-		if (mail_transaction_map(ctx->hdr, ctx->data,
+		if (mail_transaction_map(view->index, ctx->hdr, ctx->data,
 					 &mail_index_map_sync_funcs, view) < 0)
 			return -1;
 	}

Index: mail-index-view.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-view.c,v
retrieving revision 1.10
retrieving revision 1.11
diff -u -d -r1.10 -r1.11
--- mail-index-view.c	25 May 2004 23:21:56 -0000	1.10
+++ mail-index-view.c	13 Jun 2004 23:38:47 -0000	1.11
@@ -173,7 +173,7 @@
 	if (mail_index_view_lock(view) < 0)
 		return -1;
 
-	rec = &view->map->records[seq-1];
+	rec = MAIL_INDEX_MAP_IDX(view->index, view->map, seq-1);
 	if (view->map == view->index->map) {
 		*rec_r = rec;
 		return 0;
@@ -190,12 +190,13 @@
 	map = view->index->map;
 	while (seq > 0) {
 		// FIXME: we could be skipping more by uid diff
-		if (map->records[--seq].uid <= uid)
+		seq--;
+		if (MAIL_INDEX_MAP_IDX(view->index, map, seq-1)->uid <= uid)
 			break;
 	}
 
-	*rec_r = map->records[seq].uid == uid ?
-		&map->records[seq] : rec;
+	*rec_r = MAIL_INDEX_MAP_IDX(view->index, map, seq)->uid == uid ?
+		MAIL_INDEX_MAP_IDX(view->index, map, seq) : rec;
 	return 0;
 }
 
@@ -208,7 +209,7 @@
 	if (mail_index_view_lock(view) < 0)
 		return -1;
 
-	*uid_r = view->map->records[seq-1].uid;
+	*uid_r = MAIL_INDEX_MAP_IDX(view->index, view->map, seq-1)->uid;
 	return 0;
 }
 
@@ -216,10 +217,11 @@
 				       uint32_t uid, uint32_t *left_idx_p,
 				       int nearest_side)
 {
-	const struct mail_index_record *rec;
-	uint32_t idx, left_idx, right_idx;
+	const struct mail_index_record *rec_base, *rec;
+	uint32_t idx, left_idx, right_idx, record_size;
 
-	rec = view->map->records;
+	rec_base = view->map->records;
+	record_size = view->index->record_size;
 
 	idx = left_idx = *left_idx_p;
 	right_idx = view->messages_count;
@@ -227,9 +229,10 @@
 	while (left_idx < right_idx) {
 		idx = (left_idx + right_idx) / 2;
 
-		if (rec[idx].uid < uid)
+                rec = CONST_PTR_OFFSET(rec_base, idx * record_size);
+		if (rec->uid < uid)
 			left_idx = idx+1;
-		else if (rec[idx].uid > uid)
+		else if (rec->uid > uid)
 			right_idx = idx;
 		else
 			break;
@@ -241,14 +244,15 @@
 	}
 
         *left_idx_p = left_idx;
-	if (rec[idx].uid != uid) {
+	rec = CONST_PTR_OFFSET(rec_base, idx * record_size);
+	if (rec->uid != uid) {
 		if (nearest_side > 0) {
 			/* we want uid or larger */
-			return rec[idx].uid > uid ? idx+1 :
+			return rec->uid > uid ? idx+1 :
 				idx == view->messages_count-1 ? 0 : idx+2;
 		} else {
 			/* we want uid or smaller */
-			return rec[idx].uid < uid ? idx + 1 : idx;
+			return rec->uid < uid ? idx + 1 : idx;
 		}
 	}
 
@@ -279,7 +283,8 @@
 	left_idx = 0;
 	*first_seq_r = mail_index_bsearch_uid(view, first_uid, &left_idx, 1);
 	if (*first_seq_r == 0 ||
-	    view->map->records[*first_seq_r-1].uid > last_uid) {
+	    MAIL_INDEX_MAP_IDX(view->index, view->map, *first_seq_r-1)->uid >
+	    last_uid) {
 		*first_seq_r = 0;
 		*last_seq_r = 0;
 		return 0;
@@ -326,8 +331,8 @@
 			return 0;
 	}
 
-	rec = &view->map->records[seq-1];
 	for (; seq <= view->messages_count; seq++, rec++) {
+		rec = MAIL_INDEX_MAP_IDX(view->index, view->map, seq-1);
 		if ((rec->flags & flags_mask) == (uint8_t)flags) {
 			*seq_r = seq;
 			break;

Index: mail-index.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index.c,v
retrieving revision 1.128
retrieving revision 1.129
diff -u -d -r1.128 -r1.129
--- mail-index.c	30 May 2004 16:35:18 -0000	1.128
+++ mail-index.c	13 Jun 2004 23:38:47 -0000	1.129
@@ -24,6 +24,7 @@
 	index->dir = i_strdup(dir);
 	index->prefix = i_strdup(prefix);
 	index->fd = -1;
+	index->record_size = sizeof(struct mail_index_record);
 
 	index->mode = 0600;
 	index->gid = (gid_t)-1;
@@ -40,20 +41,28 @@
 	i_free(index);
 }
 
+uint16_t mail_index_register_record_extra(struct mail_index *index,
+					  uint16_t size)
+{
+	i_assert(!index->opened);
+	i_assert(index->record_size + size <= 65535);
+
+	index->record_size += size;
+	return index->record_size - size;
+}
+
 static int mail_index_check_header(struct mail_index *index,
 				   struct mail_index_map *map)
 {
 	const struct mail_index_header *hdr = map->hdr;
-	unsigned char compat_data[3];
+	unsigned char compat_data[sizeof(hdr->compat_data)];
 
+	memset(compat_data, 0, sizeof(compat_data));
 #ifndef WORDS_BIGENDIAN
 	compat_data[0] = MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
-#else
-	compat_data[0] = 0;
 #endif
 	compat_data[1] = sizeof(uoff_t);
 	compat_data[2] = sizeof(time_t);
-	compat_data[3] = sizeof(keywords_mask_t);
 
 	if (hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
 		/* major version change - handle silently(?) */
@@ -77,6 +86,22 @@
 		return -1;
 	}
 
+	if (hdr->keywords_mask_size != sizeof(keywords_mask_t)) {
+		mail_index_set_error(index, "Corrupted index file %s: "
+				     "keywords_mask_size mismatch: %d != %d",
+				     index->filepath, hdr->keywords_mask_size,
+				     (int)sizeof(keywords_mask_t));
+		return -1;
+	}
+
+	if (hdr->record_size != index->record_size) {
+		mail_index_set_error(index, "Corrupted index file %s: "
+				     "record_size mismatch: %d != %d",
+				     index->filepath, hdr->record_size,
+				     (int)index->record_size);
+		return -1;
+	}
+
 	if (hdr->next_uid == 0)
 		return 0;
 
@@ -157,11 +182,11 @@
 
 	hdr = map->mmap_base;
 	map->mmap_used_size = hdr->header_size +
-		hdr->messages_count * sizeof(struct mail_index_record);
+		hdr->messages_count * index->record_size;
 
 	if (map->mmap_used_size > map->mmap_size) {
 		records_count = (map->mmap_size - hdr->header_size) /
-			sizeof(struct mail_index_record);
+			index->record_size;
 		mail_index_set_error(index, "Corrupted index file %s: "
 				     "messages_count too large (%u > %u)",
 				     index->filepath, map->hdr->messages_count,
@@ -202,8 +227,7 @@
 			pos += ret;
 	}
 	if (ret >= 0 && pos >= MAIL_INDEX_HEADER_MIN_SIZE) {
-		records_size = hdr.messages_count *
-			sizeof(struct mail_index_record);
+		records_size = hdr.messages_count * index->record_size;
 
 		if (map->buffer == NULL) {
 			map->buffer = buffer_create_dynamic(default_pool,
@@ -298,7 +322,7 @@
 			return -1;
 
 		used_size = hdr->header_size +
-			hdr->messages_count * sizeof(struct mail_index_record);
+			hdr->messages_count * index->record_size;
 		if (map->mmap_size >= used_size && !force) {
 			map->records_count = hdr->messages_count;
 			return 1;
@@ -340,7 +364,8 @@
 	return 1;
 }
 
-struct mail_index_map *mail_index_map_to_memory(struct mail_index_map *map)
+struct mail_index_map *
+mail_index_map_to_memory(struct mail_index *index, struct mail_index_map *map)
 {
 	const struct mail_index_header *hdr;
 	struct mail_index_map *mem_map;
@@ -351,7 +376,7 @@
 		return map;
 	}
 
-        size = map->records_count * sizeof(struct mail_index_record);
+        size = map->records_count * index->record_size;
 
 	mem_map = i_new(struct mail_index_map, 1);
 	mem_map->refcount = 1;
@@ -525,7 +550,8 @@
 	return 1;
 }
 
-static void mail_index_header_init(struct mail_index_header *hdr)
+static void mail_index_header_init(struct mail_index *index,
+				   struct mail_index_header *hdr)
 {
 	time_t now = time(NULL);
 
@@ -534,13 +560,14 @@
 	hdr->major_version = MAIL_INDEX_MAJOR_VERSION;
 	hdr->minor_version = MAIL_INDEX_MINOR_VERSION;
 	hdr->header_size = sizeof(*hdr);
+	hdr->record_size = index->record_size;
+	hdr->keywords_mask_size = sizeof(keywords_mask_t);
 
 #ifndef WORDS_BIGENDIAN
 	hdr->compat_data[0] = MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
 #endif
 	hdr->compat_data[1] = sizeof(uoff_t);
 	hdr->compat_data[2] = sizeof(time_t);
-	hdr->compat_data[3] = sizeof(keywords_mask_t);
 
 	hdr->indexid = now;
 
@@ -561,7 +588,7 @@
 		/* doesn't exist, or corrupted */
 		if ((flags & MAIL_INDEX_OPEN_FLAG_CREATE) == 0)
 			return 0;
-		mail_index_header_init(&hdr);
+		mail_index_header_init(index, &hdr);
 		index->hdr = &hdr;
 	} else if (ret < 0)
 		return -1;

Index: mail-index.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index.h,v
retrieving revision 1.112
retrieving revision 1.113
diff -u -d -r1.112 -r1.113
--- mail-index.h	25 May 2004 23:13:57 -0000	1.112
+++ mail-index.h	13 Jun 2004 23:38:47 -0000	1.113
@@ -6,7 +6,7 @@
 #define MAIL_INDEX_MAJOR_VERSION 4
 #define MAIL_INDEX_MINOR_VERSION 0
 
-#define MAIL_INDEX_HEADER_MIN_SIZE 72
+#define MAIL_INDEX_HEADER_MIN_SIZE 80
 
 /* Number of keywords in mail_index_record. */
 #define INDEX_KEYWORDS_COUNT (3*8)
@@ -62,12 +62,15 @@
 	   increased to contain new non-critical fields. */
 	uint8_t major_version;
 	uint8_t minor_version;
+
 	uint16_t header_size;
+	uint16_t record_size;
+	uint16_t keywords_mask_size;
 
 	/* 0 = flags
 	   1 = sizeof(uoff_t)
 	   2 = sizeof(time_t)
-	   3 = sizeof(keywords_mask_t) */
+	   3 = unused */
 	uint8_t compat_data[4];
 
 	uint32_t indexid;
@@ -133,6 +136,12 @@
 struct mail_index *mail_index_alloc(const char *dir, const char *prefix);
 void mail_index_free(struct mail_index *index);
 
+/* register extra data to be used in mail_index_record. calls to this function
+   must remain in same order as long as the index exists, or it breaks.
+   returns the relative offset in mail_index_record for the data. */
+uint16_t mail_index_register_record_extra(struct mail_index *index,
+					  uint16_t size);
+
 int mail_index_open(struct mail_index *index, enum mail_index_open_flags flags);
 void mail_index_close(struct mail_index *index);
 

Index: mail-transaction-log.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-transaction-log.c,v
retrieving revision 1.34
retrieving revision 1.35
diff -u -d -r1.34 -r1.35
--- mail-transaction-log.c	31 May 2004 19:08:57 -0000	1.34
+++ mail-transaction-log.c	13 Jun 2004 23:38:47 -0000	1.35
@@ -874,11 +874,13 @@
 	const struct mail_transaction_header *hdr;
 	const void *data;
 	size_t size;
+	uint32_t record_size;
 	int ret, deleted = FALSE;
 
 	if (t->appends == NULL)
 		return 0;
 
+	record_size = log->index->record_size;
 	appends = buffer_get_modifyable_data(t->appends, &size);
 	end = PTR_OFFSET(appends, size);
 
@@ -902,9 +904,9 @@
 
 		old = data;
 		old_end = CONST_PTR_OFFSET(old, hdr->size);
-		for (; old != old_end; old++) {
+		while (old != old_end) {
 			/* appends are sorted */
-			for (rec = appends; rec != end; rec++) {
+			for (rec = appends; rec != end; ) {
 				if (rec->uid >= old->uid) {
 					if (rec->uid == old->uid) {
 						rec->uid = 0;
@@ -912,7 +914,9 @@
 					}
 					break;
 				}
+				rec = PTR_OFFSET(rec, record_size);
 			}
+                        old = CONST_PTR_OFFSET(old, record_size);
 		}
 	}
 

Index: mail-transaction-util.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-transaction-util.c,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -d -r1.8 -r1.9
--- mail-transaction-util.c	24 May 2004 01:50:16 -0000	1.8
+++ mail-transaction-util.c	13 Jun 2004 23:38:47 -0000	1.9
@@ -15,7 +15,7 @@
 
 const struct mail_transaction_type_map mail_transaction_type_map[] = {
 	{ MAIL_TRANSACTION_APPEND, MAIL_INDEX_SYNC_TYPE_APPEND,
-	  sizeof(struct mail_index_record) },
+	  1 }, /* index-specific size, use 1 */
 	{ MAIL_TRANSACTION_EXPUNGE, MAIL_INDEX_SYNC_TYPE_EXPUNGE,
 	  sizeof(struct mail_transaction_expunge) },
 	{ MAIL_TRANSACTION_FLAG_UPDATE, MAIL_INDEX_SYNC_TYPE_FLAGS,
@@ -51,7 +51,8 @@
 	return type;
 }
 
-int mail_transaction_map(const struct mail_transaction_header *hdr,
+int mail_transaction_map(struct mail_index *index,
+			 const struct mail_transaction_header *hdr,
 			 const void *data,
 			 struct mail_transaction_map_functions *map,
 			 void *context)
@@ -61,15 +62,17 @@
 	switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
 	case MAIL_TRANSACTION_APPEND: {
 		const struct mail_index_record *rec, *end;
+		uint32_t record_size = index->record_size;
 
 		if (map->append == NULL)
 			break;
 
 		end = CONST_PTR_OFFSET(data, hdr->size);
-		for (rec = data; rec != end; rec++) {
+		for (rec = data; rec != end; ) {
 			ret = map->append(rec, context);
 			if (ret <= 0)
 				break;
+			rec = CONST_PTR_OFFSET(rec, record_size);
 		}
 		break;
 	}

Index: mail-transaction-util.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-transaction-util.h,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -d -r1.4 -r1.5
--- mail-transaction-util.h	24 May 2004 01:50:16 -0000	1.4
+++ mail-transaction-util.h	13 Jun 2004 23:38:47 -0000	1.5
@@ -24,7 +24,8 @@
 enum mail_transaction_type
 mail_transaction_type_mask_get(enum mail_index_sync_type sync_type);
 
-int mail_transaction_map(const struct mail_transaction_header *hdr,
+int mail_transaction_map(struct mail_index *index,
+			 const struct mail_transaction_header *hdr,
 			 const void *data,
 			 struct mail_transaction_map_functions *map,
 			 void *context);

