Mon May 24 04:50:19 EEST 2004

Update of /home/cvs/dovecot/src/lib-index
In directory talvi:/tmp/cvs-serv13926/lib-index

Modified Files:
	mail-index-lock.c mail-index-sync-private.h 
	mail-index-sync-update.c mail-index-sync.c 
	mail-index-transaction-private.h mail-index-transaction.c 
	mail-index-view-private.h mail-index-view-sync.c mail-index.c 
	mail-index.h mail-transaction-log-view.c 
	mail-transaction-log.c mail-transaction-log.h 
	mail-transaction-util.c mail-transaction-util.h 
Log Message:
Index header changes now go through transaction log. Removed the kludgy
parameters for mail_index_sync_end(). Removed code duplication of syncing
index root mapping and view mapping. Some fixes to handling uidvalidity and
nextuid in syncing.



Index: mail-index-lock.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-lock.c,v
retrieving revision 1.16
retrieving revision 1.17
diff -u -d -r1.16 -r1.17

--- a/mail-index-lock.c	24 May 2004 01:01:42 -0000	1.16
+++ b/mail-index-lock.c	24 May 2004 01:50:16 -0000	1.17
@@ -126,6 +126,7 @@
 	int prot;
 
 	if (!MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
+		i_assert(map->mmap_size != 0);
 		prot = lock_type == F_UNLCK ? PROT_NONE :
 			lock_type == F_WRLCK ? (PROT_READ|PROT_WRITE) :
 			PROT_READ;

Index: mail-index-sync-private.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-sync-private.h,v
retrieving revision 1.6
retrieving revision 1.7
diff -u -d -r1.6 -r1.7
--- a/mail-index-sync-private.h	22 May 2004 21:30:42 -0000	1.6
+++ b/mail-index-sync-private.h	24 May 2004 01:50:16 -0000	1.7
@@ -22,13 +22,12 @@
 	unsigned int sync_appends:1;
 };
 
-int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx,
-				 uint32_t sync_stamp, uint64_t sync_size);
+extern struct mail_transaction_map_functions mail_index_map_sync_funcs;
 
-void mail_index_header_update_counts(struct mail_index_header *hdr,
-				     uint8_t old_flags, uint8_t new_flags);
-void mail_index_header_update_lowwaters(struct mail_index_header *hdr,
-					const struct mail_index_record *rec);
+int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx);
+
+void mail_index_sync_expunge(struct mail_index_view *view,
+			     const struct mail_transaction_expunge *e);
 
 void
 mail_index_sync_get_expunge(struct mail_index_sync_rec *rec,

Index: mail-index-sync-update.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-sync-update.c,v
retrieving revision 1.16
retrieving revision 1.17
diff -u -d -r1.16 -r1.17
--- a/mail-index-sync-update.c	22 May 2004 22:23:29 -0000	1.16
+++ b/mail-index-sync-update.c	24 May 2004 01:50:16 -0000	1.17
@@ -7,18 +7,11 @@
 #include "mail-index-view-private.h"
 #include "mail-index-sync-private.h"
 #include "mail-transaction-log.h"
+#include "mail-transaction-util.h"
 
-struct mail_index_update_ctx {
-	struct mail_index *index;
-	struct mail_index_view *view;
-	struct mail_index_header hdr;
-	struct mail_transaction_log_view *log_view;
-
-	unsigned int have_dirty:1;
-};
-
-void mail_index_header_update_counts(struct mail_index_header *hdr,
-				     uint8_t old_flags, uint8_t new_flags)
+static void
+mail_index_header_update_counts(struct mail_index_header *hdr,
+				uint8_t old_flags, uint8_t new_flags)
 {
 	if (((old_flags ^ new_flags) & MAIL_RECENT) != 0) {
 		/* different recent-flag */
@@ -45,8 +38,9 @@
 	}
 }
 
-void mail_index_header_update_lowwaters(struct mail_index_header *hdr,
-					const struct mail_index_record *rec)
+static void
+mail_index_header_update_lowwaters(struct mail_index_header *hdr,
+				   const struct mail_index_record *rec)
 {
 	if ((rec->flags & MAIL_RECENT) != 0 &&
 	    rec->uid < hdr->first_recent_uid_lowwater)
@@ -59,70 +53,161 @@
 		hdr->first_deleted_uid_lowwater = rec->uid;
 }
 
-static void mail_index_sync_update_expunges(struct mail_index_update_ctx *ctx,
-					    uint32_t seq1, uint32_t seq2)
+void mail_index_sync_expunge(struct mail_index_view *view,
+			     const struct mail_transaction_expunge *e)
 {
+	struct mail_index_map *map = view->map;
+	struct mail_index_header *hdr = &map->hdr_copy;
 	struct mail_index_record *rec;
+	uint32_t count, seq, seq1, seq2;
+	int ret;
 
-	rec = &ctx->index->map->records[seq1-1];
-	for (; seq1 <= seq2; seq1++, rec++)
-		mail_index_header_update_counts(&ctx->hdr, rec->flags, 0);
+	i_assert(MAIL_INDEX_MAP_IS_IN_MEMORY(map));
+
+	ret = mail_index_lookup_uid_range(view, e->uid1, e->uid2, &seq1, &seq2);
+	i_assert(ret == 0);
+
+	if (seq1 == 0)
+		return;
+
+	rec = &map->records[seq1-1];
+	for (seq = seq1; seq <= seq2; seq++, rec++)
+		mail_index_header_update_counts(hdr, rec->flags, 0);
+
+	/* @UNSAFE */
+	count = seq2 - seq1 + 1;
+	memcpy(map->records + (seq1-1), map->records + seq2,
+	       (map->records_count - seq2) * sizeof(*map->records));
+
+	map->records_count -= count;
+	hdr->messages_count -= count;
+
+	if (map->buffer != NULL) {
+		buffer_set_used_size(map->buffer, map->records_count);
+		map->records = buffer_get_modifyable_data(map->buffer, NULL);
+	}
 }
 
-static void mail_index_sync_update_flags(struct mail_index_update_ctx *ctx,
-					 struct mail_index_sync_rec *syncrec)
+static int sync_expunge(const struct mail_transaction_expunge *e, void *context)
+{
+	struct mail_index_view *view = context;
+
+	mail_index_sync_expunge(view, e);
+	return 1;
+}
+
+static int sync_append(const struct mail_index_record *rec, void *context)
 {
+	struct mail_index_view *view = context;
+	struct mail_index_map *map = view->map;
+
+	if (rec->uid < map->hdr_copy.next_uid) {
+		mail_transaction_log_view_set_corrupted(view->log_view,
+			"Append with UID %u, but next_uid = %u",
+			rec->uid, map->hdr_copy.next_uid);
+		return -1;
+	}
+
+	if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
+		if (map->records_count * sizeof(*rec) >
+		    buffer_get_used_size(map->buffer)) {
+			(void)buffer_append_space_unsafe(map->buffer,
+							 sizeof(*rec));
+			map->records =
+				buffer_get_modifyable_data(map->buffer, NULL);
+		}
+	} else {
+		i_assert(map->records_count * sizeof(*rec) <= map->mmap_size);
+	}
+
+	map->records[map->records_count++] = *rec;
+	map->hdr_copy.messages_count++;
+	map->hdr_copy.next_uid = rec->uid+1;
+
+	mail_index_header_update_counts(&map->hdr_copy, 0, rec->flags);
+	mail_index_header_update_lowwaters(&map->hdr_copy, rec);
+	return 1;
+}
+
+static int sync_flag_update(const struct mail_transaction_flag_update *u,
+			    void *context)
+{
+        struct mail_index_view *view = context;
 	struct mail_index_record *rec, *end;
+	struct mail_index_header *hdr;
 	uint8_t flag_mask, old_flags;
 	keywords_mask_t keyword_mask;
 	uint32_t seq1, seq2;
 	int i, update_keywords, ret;
 
-	ret = mail_index_lookup_uid_range(ctx->view, syncrec->uid1,
-					  syncrec->uid2, &seq1, &seq2);
+	ret = mail_index_lookup_uid_range(view, u->uid1, u->uid2, &seq1, &seq2);
 	i_assert(ret == 0);
 
 	if (seq1 == 0)
-		return;
+		return 1;
 
-	if ((syncrec->add_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
-		ctx->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
-		ctx->have_dirty = TRUE;
-	}
+	hdr = &view->map->hdr_copy;
+
+	if ((u->add_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0)
+		hdr->flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
 
 	update_keywords = FALSE;
 	for (i = 0; i < INDEX_KEYWORDS_BYTE_COUNT; i++) {
-		if (syncrec->add_keywords[i] != 0)
-			update_keywords = TRUE;
-		if (syncrec->remove_keywords[i] != 0)
+		if (u->add_keywords[i] != 0 ||
+		    u->remove_keywords[i] != 0)
 			update_keywords = TRUE;
-		keyword_mask[i] = ~syncrec->remove_keywords[i];
+		keyword_mask[i] = ~u->remove_keywords[i];
 	}
+        flag_mask = ~u->remove_flags;
 
-	flag_mask = ~syncrec->remove_flags;
-	rec = &ctx->index->map->records[seq1-1];
+	rec = &view->map->records[seq1-1];
 	end = rec + (seq2 - seq1) + 1;
 	for (; rec != end; rec++) {
 		old_flags = rec->flags;
-		rec->flags = (rec->flags & flag_mask) | syncrec->add_flags;
+		rec->flags = (rec->flags & flag_mask) | u->add_flags;
 		if (update_keywords) {
 			for (i = 0; i < INDEX_KEYWORDS_BYTE_COUNT; i++) {
-				rec->keywords[i] =
-					(rec->keywords[i] & keyword_mask[i]) |
-					syncrec->add_keywords[i];
+				rec->keywords[i] = u->add_keywords[i] |
+					(rec->keywords[i] & keyword_mask[i]);
 			}
 		}
 
-		mail_index_header_update_counts(&ctx->hdr,
-						old_flags, rec->flags);
-                mail_index_header_update_lowwaters(&ctx->hdr, rec);
+		mail_index_header_update_counts(hdr, old_flags, rec->flags);
+                mail_index_header_update_lowwaters(hdr, rec);
 	}
+	return 1;
 }
 
-static int mail_index_grow(struct mail_index *index, unsigned int count)
+static int sync_cache_update(const struct mail_transaction_cache_update *u,
+			     void *context)
+{
+	struct mail_index_view *view = context;
+	uint32_t seq;
+	int ret;
+
+	ret = mail_index_lookup_uid_range(view, u->uid, u->uid,
+					  &seq, &seq);
+	i_assert(ret == 0);
+
+	if (seq != 0)
+		view->map->records[seq-1].cache_offset = u->cache_offset;
+	return 1;
+}
+
+static int sync_header_update(const struct mail_transaction_header_update *u,
+			      void *context)
+{
+	struct mail_index_view *view = context;
+	void *data;
+
+	data = PTR_OFFSET(&view->map->hdr_copy, u->offset);
+	memcpy(data, u->data, u->size);
+	return 1;
+}
+
+static int mail_index_grow(struct mail_index *index, struct mail_index_map *map,
+			   unsigned int count)
 {
-	struct mail_index_map *map = index->map;
-	unsigned int records_count;
 	size_t size;
 
 	if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
@@ -132,6 +217,8 @@
 		return 0;
 	}
 
+	i_assert(map == index->map);
+
 	size = map->hdr->header_size +
 		(map->records_count + count) * sizeof(struct mail_index_record);
 	if (size <= map->mmap_size)
@@ -149,192 +236,115 @@
 	if (file_set_size(index->fd, (off_t)size) < 0)
 		return mail_index_set_syscall_error(index, "file_set_size()");
 
-	records_count = map->records_count;
-
 	if (mail_index_map(index, TRUE) <= 0)
 		return -1;
 
 	i_assert(map->mmap_size >= size);
-	map->records_count = records_count;
-	return 0;
-}
-
-static int mail_index_sync_appends(struct mail_index_update_ctx *ctx,
-				   const struct mail_index_record *appends,
-				   unsigned int count)
-{
-	struct mail_index_map *map = ctx->index->map;
-	unsigned int i;
-	uint32_t next_uid;
-
-	if (mail_index_grow(ctx->index, count) < 0)
-		return -1;
-
-	next_uid = ctx->hdr.next_uid;
-	for (i = 0; i < count; i++) {
-		mail_index_header_update_counts(&ctx->hdr, 0, appends[i].flags);
-                mail_index_header_update_lowwaters(&ctx->hdr, &appends[i]);
-
-		if (appends[i].uid < next_uid) {
-			mail_transaction_log_view_set_corrupted(ctx->log_view,
-				"Append with UID %u, but next_uid = %u",
-				appends[i].uid, next_uid);
-			return -1;
-		}
-		next_uid = appends[i].uid+1;
-	}
-	ctx->hdr.next_uid = next_uid;
-
-	memcpy(map->records + map->records_count, appends,
-	       count * sizeof(*appends));
-	map->records_count += count;
 	return 0;
 }
 
-int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx,
-				 uint32_t sync_stamp, uint64_t sync_size)
+int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx)
 {
 	struct mail_index *index = sync_ctx->index;
+	struct mail_index_view *view = sync_ctx->view;
 	struct mail_index_map *map;
-        struct mail_index_update_ctx ctx;
-	struct mail_index_sync_rec rec;
-	const struct mail_index_record *appends;
-	unsigned int append_count;
-	uint32_t count, file_seq, src_idx, dest_idx, i;
-	uint32_t seq1, seq2;
-	uoff_t file_offset;
-	unsigned int lock_id;
-	int ret, changed;
-
-	/* rewind */
-	sync_ctx->update_idx = sync_ctx->expunge_idx = 0;
-	sync_ctx->sync_appends =
-		buffer_get_used_size(sync_ctx->appends_buf) != 0;
-
-	changed = mail_index_sync_have_more(sync_ctx);
-
-	memset(&ctx, 0, sizeof(ctx));
-	ctx.index = index;
-	ctx.view = sync_ctx->view;
-	ctx.hdr = *index->hdr;
-	ctx.log_view = sync_ctx->view->log_view;
-
-	/* see if we need to update sync headers */
-	if (ctx.hdr.sync_stamp != sync_stamp && sync_stamp != 0) {
-		ctx.hdr.sync_stamp = sync_stamp;
-		changed = TRUE;
-	}
-	if (ctx.hdr.sync_size != sync_size && sync_size != 0) {
-		ctx.hdr.sync_size = sync_size;
-		changed = TRUE;
-	}
-
-	if (!changed) {
-		/* nothing to sync */
-		return 0;
-	}
+	const struct mail_transaction_header *hdr;
+	const void *data;
+	unsigned int lock_id, count;
+	uint32_t seq, i;
+	uoff_t offset;
+	int ret, had_dirty, skipped;
 
 	if (mail_index_lock_exclusive(index, &lock_id) < 0)
 		return -1;
 
+	/* NOTE: locking may change index->map so make sure assignment
+	   after locking */
 	map = index->map;
 	if (MAIL_INDEX_MAP_IS_IN_MEMORY(map))
 		map->write_to_disk = TRUE;
 
-	src_idx = dest_idx = 0;
-	append_count = 0; appends = NULL;
-	while (mail_index_sync_next(sync_ctx, &rec) > 0) {
-		switch (rec.type) {
-		case MAIL_INDEX_SYNC_TYPE_APPEND:
-			i_assert(appends == NULL);
-			appends = rec.appends;
-			append_count = rec.appends_count;
-			break;
-		case MAIL_INDEX_SYNC_TYPE_EXPUNGE:
-			ret = mail_index_lookup_uid_range(sync_ctx->view,
-							  rec.uid1, rec.uid2,
-							  &seq1, &seq2);
-			i_assert(ret == 0);
-
-			if (seq1 == 0)
-				break;
+	map->hdr_copy = *map->hdr;
+	map->hdr = &map->hdr_copy;
 
-			if (src_idx == 0) {
-				/* expunges have to be atomic. so we'll have
-				   to copy the mapping, do the changes there
-				   and then finally replace the whole index
-				   file. to avoid extra disk I/O we copy the
-				   index into memory rather than to temporary
-				   file */
-				map = mail_index_map_to_memory(map);
-				mail_index_unmap(index, index->map);
-				index->map = map;
-				index->hdr = map->hdr;
-				map->write_to_disk = TRUE;
+	mail_index_unmap(index, view->map);
+	view->map = map;
+	view->map->refcount++;
 
-				dest_idx = seq1-1;
-			} else {
-				count = (seq1-1) - src_idx;
-				memmove(map->records + dest_idx,
-					map->records + src_idx,
-					count * sizeof(*map->records));
-				dest_idx += count;
-			}
+	had_dirty = (map->hdr_copy.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0;
+	if (had_dirty)
+		map->hdr_copy.flags &= ~MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
 
-			mail_index_sync_update_expunges(&ctx, seq1, seq2);
-			src_idx = seq2;
-			break;
-		case MAIL_INDEX_SYNC_TYPE_FLAGS:
-			mail_index_sync_update_flags(&ctx, &rec);
-			break;
+	while ((ret = mail_transaction_log_view_next(view->log_view, &hdr,
+						     &data, &skipped)) > 0) {
+		if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0 &&
+		    !map->write_to_disk) {
+			/* expunges have to be atomic. so we'll have to copy
+			   the mapping, do the changes there and then finally
+			   replace the whole index file. to avoid extra disk
+			   I/O we copy the index into memory rather than to
+			   temporary file */
+			map = mail_index_map_to_memory(map);
+			mail_index_unmap(index, view->map);
+			view->map = map;
+			view->map->refcount++;
+			mail_index_unmap(index, index->map);
+			index->map = map;
+			index->hdr = map->hdr;
+			map->write_to_disk = TRUE;
 		}
-	}
 
-	if (src_idx != 0) {
-		count = map->records_count - src_idx;
-		memmove(map->records + dest_idx,
-			map->records + src_idx,
-			count * sizeof(*map->records));
-		dest_idx += count;
+		if ((hdr->type & MAIL_TRANSACTION_APPEND) != 0) {
+			count = hdr->size / sizeof(struct mail_index_record);
+			if (mail_index_grow(index, view->map, count) < 0)
+				return -1;
+		}
 
-		map->records_count = dest_idx;
+		if (mail_transaction_map(hdr, data, &mail_index_map_sync_funcs,
+					 view) < 0) {
+			ret = -1;
+			break;
+		}
 	}
 
-	ret = 0;
-	if (append_count > 0)
-		ret = mail_index_sync_appends(&ctx, appends, append_count);
+	if (ret < 0)
+		return -1;
 
-	mail_transaction_log_get_head(index->log, &file_seq, &file_offset);
+	mail_transaction_log_get_head(index->log, &seq, &offset);
 
-	ctx.hdr.messages_count = map->records_count;
-	ctx.hdr.log_file_seq = file_seq;
-	ctx.hdr.log_file_offset = file_offset;
+	map->hdr_copy.log_file_seq = seq;
+	map->hdr_copy.log_file_offset = offset;
 
-	if ((ctx.hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) &&
-	    !ctx.have_dirty) {
+	if ((map->hdr_copy.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) == 0 &&
+	    had_dirty) {
 		/* do we have dirty flags anymore? */
 		for (i = 0; i < map->records_count; i++) {
-			if (map->records[i].flags & MAIL_INDEX_MAIL_FLAG_DIRTY)
+			if ((map->records[i].flags &
+			     MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
+				map->hdr_copy.flags |=
+					MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
 				break;
+			}
 		}
-		if (i == map->records_count)
-			ctx.hdr.flags &= ~MAIL_INDEX_HDR_FLAG_HAVE_DIRTY;
 	}
 
 	if (!MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
 		map->mmap_used_size = index->hdr->header_size +
 			map->records_count * sizeof(struct mail_index_record);
 
-		memcpy(map->mmap_base, &ctx.hdr, sizeof(ctx.hdr));
+		memcpy(map->mmap_base, &map->hdr_copy, sizeof(map->hdr_copy));
 		if (msync(map->mmap_base, map->mmap_used_size, MS_SYNC) < 0) {
 			mail_index_set_syscall_error(index, "msync()");
 			ret = -1;
 		}
-	} else {
-		map->hdr_copy = ctx.hdr;
-		map->hdr = &map->hdr_copy;
+		map->hdr = map->mmap_base;
 	}
 
 	mail_index_unlock(index, lock_id);
 	return ret;
 }
+
+struct mail_transaction_map_functions mail_index_map_sync_funcs = {
+	sync_expunge, sync_append, sync_flag_update,
+	sync_cache_update, sync_header_update
+};

Index: mail-index-sync.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-sync.c,v
retrieving revision 1.16
retrieving revision 1.17
diff -u -d -r1.16 -r1.17
--- a/mail-index-sync.c	22 May 2004 21:30:42 -0000	1.16
+++ b/mail-index-sync.c	24 May 2004 01:50:16 -0000	1.17
@@ -164,7 +164,7 @@
 					  index->hdr->log_file_offset,
 					  seq, offset,
 					  MAIL_TRANSACTION_TYPE_MASK) < 0) {
-                mail_index_sync_end(ctx, 0, 0);
+                mail_index_sync_end(ctx);
 		return -1;
 	}
 
@@ -177,7 +177,7 @@
 	ctx->appends_buf = buffer_create_dynamic(default_pool,
 						 1024, (size_t)-1);
 	if (mail_index_sync_read_and_sort(ctx, FALSE) < 0) {
-                mail_index_sync_end(ctx, 0, 0);
+                mail_index_sync_end(ctx);
 		return -1;
 	}
 
@@ -312,8 +312,7 @@
 		ctx->sync_appends;
 }
 
-int mail_index_sync_end(struct mail_index_sync_ctx *ctx,
-			uint32_t sync_stamp, uint64_t sync_size)
+int mail_index_sync_end(struct mail_index_sync_ctx *ctx)
 {
 	const struct mail_index_header *hdr;
 	uint32_t seq;
@@ -337,8 +336,12 @@
 	if (ret == 0) {
 		mail_index_sync_read_and_sort(ctx, TRUE);
 
-		if (mail_index_sync_update_index(ctx, sync_stamp,
-						 sync_size) < 0)
+		mail_transaction_log_view_unset(ctx->view->log_view);
+		if (mail_transaction_log_view_set(ctx->view->log_view,
+				hdr->log_file_seq, hdr->log_file_offset,
+				seq, offset, MAIL_TRANSACTION_TYPE_MASK) < 0)
+			ret = -1;
+		if (mail_index_sync_update_index(ctx) < 0)
 			ret = -1;
 	}
 

Index: mail-index-transaction-private.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-transaction-private.h,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- a/mail-index-transaction-private.h	28 Apr 2004 00:21:00 -0000	1.2
+++ b/mail-index-transaction-private.h	24 May 2004 01:50:16 -0000	1.3
@@ -13,8 +13,12 @@
         struct mail_transaction_flag_update last_update;
 	enum modify_type last_update_modify_type;
 
+	unsigned char hdr_change[sizeof(struct mail_index_header)];
+	unsigned char hdr_mask[sizeof(struct mail_index_header)];
+
 	buffer_t *cache_updates;
 	unsigned int hide_transaction:1;
+	unsigned int hdr_changed:1;
 };
 
 #endif

Index: mail-index-transaction.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-transaction.c,v
retrieving revision 1.6
retrieving revision 1.7
diff -u -d -r1.6 -r1.7
--- a/mail-index-transaction.c	22 May 2004 21:30:42 -0000	1.6
+++ b/mail-index-transaction.c	24 May 2004 01:50:16 -0000	1.7
@@ -426,3 +426,16 @@
 	buffer_insert(t->updates, idx * sizeof(update),
 		      &update, sizeof(update));
 }
+
+void mail_index_update_header(struct mail_index_transaction *t,
+			      size_t offset, const void *data, size_t size)
+{
+	i_assert(offset < sizeof(t->hdr_change));
+	i_assert(size <= sizeof(t->hdr_change) - offset);
+
+	t->hdr_changed = TRUE;
+
+	memcpy(t->hdr_change + offset, data, size);
+	for (; size > 0; size--)
+		t->hdr_mask[offset++] = 1;
+}

Index: mail-index-view-private.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-view-private.h,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -d -r1.5 -r1.6
--- a/mail-index-view-private.h	23 May 2004 16:30:24 -0000	1.5
+++ b/mail-index-view-private.h	24 May 2004 01:50:16 -0000	1.6
@@ -9,6 +9,7 @@
 
 	unsigned int indexid;
 	struct mail_index_map *map;
+	struct mail_index_map *new_map;
 
 	struct mail_index_header tmp_hdr_copy;
 	uint32_t messages_count; /* last synced one, map may be different */

Index: mail-index-view-sync.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-view-sync.c,v
retrieving revision 1.10
retrieving revision 1.11
diff -u -d -r1.10 -r1.11
--- a/mail-index-view-sync.c	23 May 2004 16:30:24 -0000	1.10
+++ b/mail-index-view-sync.c	24 May 2004 01:50:16 -0000	1.11
@@ -10,7 +10,6 @@
 struct mail_index_view_sync_ctx {
 	struct mail_index_view *view;
 	enum mail_index_sync_type sync_mask;
-	struct mail_index_map *sync_map;
 	buffer_t *expunges;
 
 	const struct mail_transaction_header *hdr;
@@ -79,6 +78,7 @@
 {
 	const struct mail_index_header *hdr;
 	struct mail_index_view_sync_ctx *ctx;
+	struct mail_index_map *map;
 	enum mail_transaction_type mask;
 	buffer_t *expunges = NULL;
 
@@ -115,14 +115,20 @@
 	ctx->expunges = expunges;
 
 	if ((sync_mask & MAIL_INDEX_SYNC_TYPE_EXPUNGE) != 0) {
-		ctx->sync_map = view->index->map;
-		ctx->sync_map->refcount++;
+		view->new_map = view->index->map;
+		view->new_map->refcount++;
+
+		/* keep the old mapping without expunges until we're
+		   fully synced */
 	} else {
 		/* we need a private copy of the map if we don't want to
 		   sync expunges */
-		if (MAIL_INDEX_MAP_IS_IN_MEMORY(view->map))
+		if (view->map != view->index->map)
 			ctx->sync_map_update = TRUE;
-		ctx->sync_map = mail_index_map_to_memory(view->map);
+
+		map = mail_index_map_to_memory(view->map);
+		mail_index_unmap(view->index, view->map);
+		view->map = map;
 	}
 
 	view->syncing = TRUE;
@@ -153,112 +159,6 @@
 	return 0;
 }
 
-static int sync_expunge(const struct mail_transaction_expunge *e, void *context)
-{
-        struct mail_index_view_sync_ctx *ctx = context;
-	struct mail_index_map *map = ctx->sync_map;
-	uint32_t idx, count, seq1, seq2;
-	int ret;
-
-	ret = mail_index_lookup_uid_range(ctx->view, e->uid1, e->uid2,
-					  &seq1, &seq2);
-	i_assert(ret == 0);
-
-	if (seq1 == 0)
-		return 1;
-
-	for (idx = seq1-1; idx < seq2; idx++) {
-		mail_index_header_update_counts(&map->hdr_copy,
-						map->records[idx].flags, 0);
-	}
-
-	count = seq2 - seq1 + 1;
-	buffer_delete(map->buffer,
-		      (seq1-1) * sizeof(struct mail_index_record),
-		      count * sizeof(struct mail_index_record));
-	map->records = buffer_get_modifyable_data(map->buffer, NULL);
-
-	map->records_count -= count;
-	map->hdr_copy.messages_count -= count;
-	return 1;
-}
-
-static int sync_append(const struct mail_index_record *rec, void *context)
-{
-        struct mail_index_view_sync_ctx *ctx = context;
-	struct mail_index_map *map = ctx->sync_map;
-
-	buffer_append(map->buffer, rec, sizeof(*rec));
-	map->records = buffer_get_modifyable_data(map->buffer, NULL);
-
-	map->records_count++;
-	map->hdr_copy.messages_count++;
-	map->hdr_copy.next_uid = rec->uid+1;
-
-	mail_index_header_update_counts(&map->hdr_copy, 0, rec->flags);
-	mail_index_header_update_lowwaters(&map->hdr_copy, rec);
-	return 1;
-}
-
-static int sync_flag_update(const struct mail_transaction_flag_update *u,
-			    void *context)
-{
-        struct mail_index_view_sync_ctx *ctx = context;
-	struct mail_index_map *map = ctx->sync_map;
-	struct mail_index_record *rec;
-	uint32_t i, idx, seq1, seq2;
-	uint8_t old_flags;
-	int ret;
-
-	ret = mail_index_lookup_uid_range(ctx->view, u->uid1, u->uid2,
-					  &seq1, &seq2);
-	i_assert(ret == 0);
-
-	if (seq1 == 0)
-		return 1;
-
-	for (idx = seq1-1; idx < seq2; idx++) {
-		rec = &map->records[idx];
-
-		old_flags = rec->flags;
-		rec->flags = (rec->flags & ~u->remove_flags) | u->add_flags;
-		for (i = 0; i < INDEX_KEYWORDS_BYTE_COUNT; i++) {
-			rec->keywords[i] = u->add_keywords[i] |
-				(rec->keywords[i] & ~u->remove_keywords[i]);
-		}
-
-		mail_index_header_update_counts(&map->hdr_copy, old_flags,
-						rec->flags);
-		mail_index_header_update_lowwaters(&map->hdr_copy, rec);
-	}
-	return 1;
-}
-
-static int sync_cache_update(const struct mail_transaction_cache_update *u,
-			     void *context)
-{
-        struct mail_index_view_sync_ctx *ctx = context;
-	uint32_t seq;
-	int ret;
-
-	ret = mail_index_lookup_uid_range(ctx->view, u->uid, u->uid,
-					  &seq, &seq);
-	i_assert(ret == 0);
-
-	if (seq != 0)
-		ctx->sync_map->records[seq-1].cache_offset = u->cache_offset;
-	return 1;
-}
-
-static int mail_index_view_sync_map(struct mail_index_view_sync_ctx *ctx)
-{
-	static struct mail_transaction_map_functions map_funcs = {
-		sync_expunge, sync_append, sync_flag_update, sync_cache_update
-	};
-
-	return mail_transaction_map(ctx->hdr, ctx->data, &map_funcs, ctx);
-}
-
 static int mail_index_view_sync_next_trans(struct mail_index_view_sync_ctx *ctx,
 					   uint32_t *seq_r, uoff_t *offset_r)
 {
@@ -285,8 +185,13 @@
 	if (view_is_transaction_synced(view, *seq_r, *offset_r))
 		return 0;
 
-	if (ctx->sync_map_update) {
-		if (mail_index_view_sync_map(ctx) < 0)
+	/* expunges have to be synced afterwards so that caller can still get
+	   information of the messages. otherwise caller most likely wants to
+	   see only updated information. */
+	if (ctx->sync_map_update &&
+	    (ctx->hdr->type & MAIL_TRANSACTION_EXPUNGE) == 0) {
+		if (mail_transaction_map(ctx->hdr, ctx->data,
+					 &mail_index_map_sync_funcs, view) < 0)
 			return -1;
 	}
 
@@ -408,9 +313,12 @@
 		view->inconsistent = TRUE;
 	}
 
-	mail_index_unmap(view->index, view->map);
-	view->map = ctx->sync_map;
-	view->map_protected = FALSE;
+	if (view->new_map != NULL) {
+		mail_index_unmap(view->index, view->map);
+		view->map = view->new_map;
+		view->new_map = NULL;
+		view->map_protected = FALSE;
+	}
 
 	if ((ctx->sync_mask & MAIL_INDEX_SYNC_TYPE_APPEND) != 0)
 		view->messages_count = view->map->records_count;

Index: mail-index.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index.c,v
retrieving revision 1.123
retrieving revision 1.124
diff -u -d -r1.123 -r1.124
--- a/mail-index.c	23 May 2004 16:29:35 -0000	1.123
+++ b/mail-index.c	24 May 2004 01:50:16 -0000	1.124
@@ -275,7 +275,10 @@
 	int ret;
 
 	map = index->map;
-	if (map != NULL && MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
+	if (map == NULL) {
+		map = i_new(struct mail_index_map, 1);
+		map->refcount = 1;
+	} else if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
 		if (map->write_to_disk) {
 			/* we have modified this mapping and it's waiting to
 			   be written to disk once we drop exclusive lock.
@@ -283,7 +286,7 @@
 			return 1;
 		}
 		/* FIXME: we need to re-read header */
-	} else if (map != NULL) {
+	} else if (map->mmap_base != NULL) {
 		/* see if re-mmaping is needed (file has grown) */
 		hdr = map->mmap_base;
                 used_size = hdr->header_size +
@@ -296,9 +299,6 @@
 		if (munmap(map->mmap_base, map->mmap_size) < 0)
 			mail_index_set_syscall_error(index, "munmap()");
 		map->mmap_base = NULL;
-	} else {
-		map = i_new(struct mail_index_map, 1);
-		map->refcount = 1;
 	}
 
 	index->hdr = NULL;

Index: mail-index.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index.h,v
retrieving revision 1.110
retrieving revision 1.111
diff -u -d -r1.110 -r1.111
--- a/mail-index.h	22 May 2004 22:23:29 -0000	1.110
+++ b/mail-index.h	24 May 2004 01:50:16 -0000	1.111
@@ -197,10 +197,8 @@
 			 struct mail_index_sync_rec *sync_rec);
 /* Returns 1 if there's more to sync, 0 if not. */
 int mail_index_sync_have_more(struct mail_index_sync_ctx *ctx);
-/* End synchronization by unlocking the index and closing the view.
-   sync_stamp/sync_size in header is updated to given values. */
-int mail_index_sync_end(struct mail_index_sync_ctx *ctx,
-			uint32_t sync_stamp, uint64_t sync_size);
+/* End synchronization by unlocking the index and closing the view. */
+int mail_index_sync_end(struct mail_index_sync_ctx *ctx);
 
 /* Mark index file corrupted. Invalidates all views. */
 void mail_index_mark_corrupted(struct mail_index *index);
@@ -254,6 +252,9 @@
 void mail_index_update_flags(struct mail_index_transaction *t, uint32_t seq,
 			     enum modify_type modify_type,
 			     enum mail_flags flags, keywords_mask_t keywords);
+/* Update field in header. */
+void mail_index_update_header(struct mail_index_transaction *t,
+			      size_t offset, const void *data, size_t size);
 
 /* Returns the last error code. */
 enum mail_index_error mail_index_get_last_error(struct mail_index *index);

Index: mail-transaction-log-view.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-transaction-log-view.c,v
retrieving revision 1.11
retrieving revision 1.12
diff -u -d -r1.11 -r1.12
--- a/mail-transaction-log-view.c	22 May 2004 00:48:45 -0000	1.11
+++ b/mail-transaction-log-view.c	24 May 2004 01:50:16 -0000	1.12
@@ -14,7 +14,6 @@
 	uoff_t min_file_offset, max_file_offset;
 
 	enum mail_transaction_type type_mask;
-	buffer_t *expunges_buf;
 	struct mail_transaction_header tmp_hdr;
 
         struct mail_transaction_log_file *file;
@@ -34,8 +33,6 @@
 	view = i_new(struct mail_transaction_log_view, 1);
 	view->log = log;
 	view->broken = TRUE;
-	view->expunges_buf =
-		buffer_create_dynamic(default_pool, 512, (size_t)-1);
 
 	view->next = log->views;
 	log->views = view;
@@ -54,7 +51,6 @@
 	}
 
 	mail_transaction_log_view_unset(view);
-	buffer_free(view->expunges_buf);
 	i_free(view);
 }
 
@@ -150,8 +146,6 @@
 		file = file->next;
 	}
 
-	buffer_set_used_size(view->expunges_buf, 0);
-
 	view->prev_file_seq = 0;
 	view->prev_file_offset = 0;
 

Index: mail-transaction-log.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-transaction-log.c,v
retrieving revision 1.29
retrieving revision 1.30
diff -u -d -r1.29 -r1.30
--- a/mail-transaction-log.c	23 May 2004 17:02:50 -0000	1.29
+++ b/mail-transaction-log.c	24 May 2004 01:50:16 -0000	1.30
@@ -960,6 +960,36 @@
 	return 0;
 }
 
+static const buffer_t *
+log_get_hdr_update_buffer(struct mail_index_transaction *t)
+{
+	buffer_t *buf;
+	struct mail_transaction_header_update u;
+	uint16_t offset;
+	int state = 0;
+
+	memset(&u, 0, sizeof(u));
+
+	buf = buffer_create_dynamic(pool_datastack_create(), 256, (size_t)-1);
+	for (offset = 0; offset <= sizeof(t->hdr_change); offset++) {
+		if (offset < sizeof(t->hdr_change) && t->hdr_mask[offset]) {
+			if (state == 0) {
+				u.offset = offset;
+				state++;
+			}
+		} else {
+			if (state > 0) {
+				u.size = offset - u.offset;
+				buffer_append(buf, &u, sizeof(uint16_t)*2);
+				buffer_append(buf, t->hdr_change + u.offset,
+					      u.size);
+				state = 0;
+			}
+		}
+	}
+	return buf;
+}
+
 int mail_transaction_log_append(struct mail_index_transaction *t,
 				uint32_t *log_file_seq_r,
 				uoff_t *log_file_offset_r)
@@ -976,7 +1006,7 @@
 	log = index->log;
 
 	if (t->updates == NULL && t->cache_updates == NULL &&
-	    t->expunges == NULL && t->appends == NULL) {
+	    t->expunges == NULL && t->appends == NULL && !t->hdr_changed) {
 		/* nothing to append */
 		*log_file_seq_r = 0;
 		*log_file_offset_r = 0;
@@ -1036,6 +1066,11 @@
 					MAIL_TRANSACTION_EXPUNGE,
 					view->external);
 	}
+	if (t->hdr_changed && ret == 0) {
+		ret = log_append_buffer(file, log_get_hdr_update_buffer(t),
+					MAIL_TRANSACTION_HEADER_UPDATE,
+					view->external);
+	}
 
 	if (ret == 0) {
 		/* rewrite used_size */

Index: mail-transaction-log.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-transaction-log.h,v
retrieving revision 1.9
retrieving revision 1.10
diff -u -d -r1.9 -r1.10
--- a/mail-transaction-log.h	22 May 2004 00:48:45 -0000	1.9
+++ b/mail-transaction-log.h	24 May 2004 01:50:16 -0000	1.10
@@ -18,6 +18,7 @@
 	MAIL_TRANSACTION_APPEND		= 0x00000002,
 	MAIL_TRANSACTION_FLAG_UPDATE	= 0x00000004,
 	MAIL_TRANSACTION_CACHE_UPDATE	= 0x00000008,
+	MAIL_TRANSACTION_HEADER_UPDATE	= 0x00000010,
 
 	MAIL_TRANSACTION_TYPE_MASK	= 0x0000ffff,
 
@@ -40,11 +41,6 @@
 	uint32_t uid1, uid2;
 };
 
-struct mail_transaction_cache_update {
-	uint32_t uid;
-	uint32_t cache_offset;
-};
-
 struct mail_transaction_flag_update {
 	uint32_t uid1, uid2;
 	uint8_t add_flags;
@@ -53,6 +49,17 @@
 	keywords_mask_t remove_keywords;
 };
 
+struct mail_transaction_cache_update {
+	uint32_t uid;
+	uint32_t cache_offset;
+};
+
+struct mail_transaction_header_update {
+	uint16_t offset;
+	uint16_t size;
+	unsigned char data[1]; /* variable size */
+};
+
 struct mail_transaction_log *
 mail_transaction_log_open_or_create(struct mail_index *index);
 void mail_transaction_log_close(struct mail_transaction_log *log);

Index: mail-transaction-util.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-transaction-util.c,v
retrieving revision 1.7
retrieving revision 1.8
diff -u -d -r1.7 -r1.8
--- a/mail-transaction-util.c	22 May 2004 00:48:45 -0000	1.7
+++ b/mail-transaction-util.c	24 May 2004 01:50:16 -0000	1.8
@@ -22,6 +22,7 @@
 	  sizeof(struct mail_transaction_flag_update) },
 	{ MAIL_TRANSACTION_CACHE_UPDATE, 0,
 	  sizeof(struct mail_transaction_cache_update) },
+	{ MAIL_TRANSACTION_HEADER_UPDATE, 0, 1 }, /* variable size, use 1 */
 	{ 0, 0, 0 }
 };
 
@@ -115,6 +116,23 @@
 		}
 		break;
 	}
+	case MAIL_TRANSACTION_HEADER_UPDATE: {
+		const struct mail_transaction_header_update *rec;
+		unsigned int i;
+
+		if (map->header_update == NULL)
+			break;
+
+		for (i = 0; i < hdr->size; ) {
+			rec = CONST_PTR_OFFSET(data, i);
+			ret = map->header_update(rec, context);
+			if (ret <= 0)
+				break;
+
+			i += sizeof(uint16_t)*2 + rec->size;
+		}
+		break;
+	}
 	default:
 		i_unreached();
 	}

Index: mail-transaction-util.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-transaction-util.h,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -d -r1.3 -r1.4
--- a/mail-transaction-util.h	22 May 2004 00:48:45 -0000	1.3
+++ b/mail-transaction-util.h	24 May 2004 01:50:16 -0000	1.4
@@ -15,6 +15,8 @@
 			   void *context);
 	int (*cache_update)(const struct mail_transaction_cache_update *u,
 			    void *context);
+	int (*header_update)(const struct mail_transaction_header_update *u,
+			     void *context);
 };
 
 const struct mail_transaction_type_map *

