[dovecot-cvs] dovecot/src/lib-index mail-index-sync-private.h, 1.33, 1.34 mail-index-sync-update.c, 1.108, 1.109 mail-index-view-sync.c, 1.61, 1.62

[tss at dovecot.org]

Update of /var/lib/cvs/dovecot/src/lib-index
In directory talvi:/tmp/cvs-serv14696

Modified Files:
	mail-index-sync-private.h mail-index-sync-update.c 
	mail-index-view-sync.c 
Log Message:
Some mmap_disable=no fixes. Some debug additions. Some generic index fixes.



Index: mail-index-sync-private.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-sync-private.h,v
retrieving revision 1.33
retrieving revision 1.34
diff -u -d -r1.33 -r1.34
--- mail-index-sync-private.h	17 Jan 2007 23:33:29 -0000	1.33
+++ mail-index-sync-private.h	23 Jan 2007 16:10:58 -0000	1.34
@@ -56,6 +56,7 @@
 	unsigned int expunge_handlers_used:1;
 	unsigned int cur_ext_ignore:1;
 	unsigned int unreliable_flags:1;
+	unsigned int sync_only_external:1;
 };
 
 extern struct mail_transaction_map_functions mail_index_map_sync_funcs;

Index: mail-index-sync-update.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-sync-update.c,v
retrieving revision 1.108
retrieving revision 1.109
diff -u -d -r1.108 -r1.109
--- mail-index-sync-update.c	23 Jan 2007 13:29:44 -0000	1.108
+++ mail-index-sync-update.c	23 Jan 2007 16:10:58 -0000	1.109
@@ -12,13 +12,63 @@
 #include "mail-transaction-log-private.h"
 #include "mail-transaction-util.h"
 
+static void
+mail_index_sync_update_log_offset(struct mail_index_sync_map_ctx *ctx,
+				  struct mail_index_map *map)
+{
+	uint32_t prev_seq;
+	uoff_t prev_offset;
+
+	mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
+					       &prev_seq, &prev_offset);
+
+	if (!ctx->sync_only_external)
+		map->hdr.log_file_int_offset = prev_offset;
+	else if (map->hdr.log_file_seq != prev_seq && map == ctx->view->map) {
+		/* log sequence changed. update internal offset to
+		   beginning of the new file. */
+		i_assert(map->hdr.log_file_int_offset ==
+			 ctx->view->index->log->head->hdr.prev_file_offset);
+		map->hdr.log_file_int_offset =
+			ctx->view->index->log->head->hdr.hdr_size;
+	}
+	map->hdr.log_file_seq = prev_seq;
+	map->hdr.log_file_ext_offset = prev_offset;
+}
+
+static int
+mail_index_map_msync(struct mail_index *index, struct mail_index_map *map)
+{
+	unsigned int base_size;
+
+	if (MAIL_INDEX_MAP_IS_IN_MEMORY(map))
+		return 0;
+
+	base_size = I_MIN(map->hdr.base_header_size, sizeof(map->hdr));
+	map->mmap_used_size = index->hdr->header_size +
+		map->records_count * map->hdr.record_size;
+
+	memcpy(map->mmap_base, &map->hdr, base_size);
+	memcpy(PTR_OFFSET(map->mmap_base, base_size),
+	       CONST_PTR_OFFSET(map->hdr_base, base_size),
+	       map->hdr.header_size - base_size);
+	if (msync(map->mmap_base, map->mmap_used_size, MS_SYNC) < 0) {
+		mail_index_set_syscall_error(index, "msync()");
+		return -1;
+	}
+	return 0;
+}
+
 void mail_index_sync_replace_map(struct mail_index_sync_map_ctx *ctx,
 				 struct mail_index_map *map)
 {
         struct mail_index_view *view = ctx->view;
+	struct mail_index_map *old_map = view->map;
 
 	i_assert(view->map != map);
 
+	old_map->refcount++;
+
 	/* if map still exists after this, it's only in views. */
 	view->map->write_to_disk = FALSE;
 	/* keywords aren't parsed for the new map yet */
@@ -28,9 +78,8 @@
 	view->map = map;
 
 	if ((ctx->type & (MAIL_INDEX_SYNC_HANDLER_FILE |
-			  MAIL_INDEX_SYNC_HANDLER_HEAD)) != 0) {
-		i_assert(view->index->map != map);
-
+			  MAIL_INDEX_SYNC_HANDLER_HEAD)) != 0 &&
+	    view->index->map != map) {
 		mail_index_unmap(view->index, &view->index->map);
 		view->index->map = map;
 		view->index->hdr = &map->hdr;
@@ -42,6 +91,15 @@
 		}
 	}
 
+	if (!MAIL_INDEX_MAP_IS_IN_MEMORY(old_map)) {
+		/* other processes may still have the same file mapped,
+		   and since we could have already updated the records, make
+		   sure we leave the header in a valid state as well */
+		mail_index_sync_update_log_offset(ctx, old_map);
+		(void)mail_index_map_msync(view->index, old_map);
+	}
+	mail_index_unmap(view->index, &old_map);
+
 	i_assert(view->hdr.messages_count == map->hdr.messages_count);
 }
 
@@ -700,8 +758,7 @@
 	const struct mail_transaction_header *thdr;
 	const void *data;
 	unsigned int count, old_lock_id;
-	uint32_t seq, i, first_append_uid;
-	uoff_t offset;
+	uint32_t i, first_append_uid;
 	int ret;
 	bool had_dirty, skipped, check_ext_offsets;
 
@@ -775,10 +832,9 @@
 				break;
 			}
 			if (map != index->map) {
-				map = index->map;
-				mail_index_unmap(view->index, &view->map);
-				view->map = map;
-				view->map->refcount++;
+				index->map->refcount++;
+				mail_index_sync_replace_map(&sync_map_ctx,
+							    index->map);
 			}
 		}
 
@@ -791,8 +847,10 @@
 		   make sure we don't accidentally try to use it. */
 		map = NULL;
 	}
-	map = view->map;
-        mail_index_sync_map_deinit(&sync_map_ctx);
+
+	if (ret == 0)
+		mail_index_sync_update_log_offset(&sync_map_ctx, view->map);
+	mail_index_sync_map_deinit(&sync_map_ctx);
 
 	index->sync_update = FALSE;
 
@@ -801,22 +859,8 @@
 		return -1;
 	}
 
+	map = view->map;
 	i_assert(map->records_count == map->hdr.messages_count);
-
-	mail_transaction_log_get_head(index->log, &seq, &offset);
-
-	if (!sync_only_external)
-		map->hdr.log_file_int_offset = offset;
-	else if (map->hdr.log_file_seq != seq) {
-		/* log sequence changed. update internal offset to
-		   beginning of the new file. */
-		i_assert(map->hdr.log_file_int_offset ==
-			 index->log->head->hdr.prev_file_offset);
-		map->hdr.log_file_int_offset = index->log->head->hdr.hdr_size;
-	}
-	map->hdr.log_file_seq = seq;
-	map->hdr.log_file_ext_offset = offset;
-
 	i_assert(map->hdr_copy_buf->used <= map->hdr.header_size);
 
 	if (first_append_uid != 0)
@@ -837,22 +881,8 @@
 		}
 	}
 
-	if (!MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
-		unsigned int base_size;
-
-		base_size = I_MIN(map->hdr.base_header_size, sizeof(map->hdr));
-		map->mmap_used_size = index->hdr->header_size +
-			map->records_count * map->hdr.record_size;
-
-		memcpy(map->mmap_base, &map->hdr, base_size);
-		memcpy(PTR_OFFSET(map->mmap_base, base_size),
-		       CONST_PTR_OFFSET(map->hdr_base, base_size),
-		       map->hdr.header_size - base_size);
-		if (msync(map->mmap_base, map->mmap_used_size, MS_SYNC) < 0) {
-			mail_index_set_syscall_error(index, "msync()");
-			ret = -1;
-		}
-	}
+	if (mail_index_map_msync(index, map) < 0)
+		ret = -1;
 	i_assert(view->map == index->map);
 	view->hdr = map->hdr;
 

Index: mail-index-view-sync.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-view-sync.c,v
retrieving revision 1.61
retrieving revision 1.62
diff -u -d -r1.61 -r1.62
--- mail-index-view-sync.c	17 Jan 2007 18:47:46 -0000	1.61
+++ mail-index-view-sync.c	23 Jan 2007 16:10:58 -0000	1.62
@@ -188,6 +188,42 @@
 	 MAIL_TRANSACTION_FLAG_UPDATE | MAIL_TRANSACTION_KEYWORD_UPDATE | \
 	 MAIL_TRANSACTION_KEYWORD_RESET)
 
+#ifdef DEBUG
+static void mail_index_view_check(struct mail_index_view *view)
+{
+	unsigned int i, del = 0, recent = 0, seen = 0;
+
+	i_assert(view->hdr.deleted_messages_count ==
+		 view->map->hdr.deleted_messages_count);
+	i_assert(view->hdr.recent_messages_count ==
+		 view->map->hdr.recent_messages_count);
+	i_assert(view->hdr.seen_messages_count ==
+		 view->map->hdr.seen_messages_count);
+
+	for (i = 0; i < view->map->records_count; i++) {
+		const struct mail_index_record *rec;
+
+		rec = MAIL_INDEX_MAP_IDX(view->map, i);
+
+		if (rec->flags & MAIL_DELETED) {
+			i_assert(rec->uid >= view->hdr.first_deleted_uid_lowwater);
+			del++;
+		}
+		if (rec->flags & MAIL_RECENT) {
+			i_assert(rec->uid >= view->hdr.first_recent_uid_lowwater);
+			recent++;
+		}
+		if (rec->flags & MAIL_SEEN)
+			seen++;
+		else
+			i_assert(rec->uid >= view->hdr.first_unseen_uid_lowwater);
+	}
+	i_assert(del == view->hdr.deleted_messages_count);
+	i_assert(recent == view->hdr.recent_messages_count);
+	i_assert(seen == view->hdr.seen_messages_count);
+}
+#endif
+
 int mail_index_view_sync_begin(struct mail_index_view *view,
                                enum mail_index_sync_type sync_mask,
 			       struct mail_index_view_sync_ctx **ctx_r)
@@ -271,8 +307,12 @@
 			   update flag counters. note that map->hdr may contain
 			   old information if another process updated the
 			   index file since. */
-			hdr = view->map->mmap_base != NULL ?
-				view->map->mmap_base : &view->map->hdr;
+			if (view->map->mmap_base == NULL)
+				hdr = &view->map->hdr;
+			else {
+				hdr = view->map->mmap_base;
+				view->map->hdr = *hdr;
+			}
 			ctx->sync_map_ctx.unreliable_flags =
 				!(hdr->log_file_seq == view->log_file_seq &&
 				  hdr->log_file_int_offset ==
@@ -284,6 +324,11 @@
 			i_assert(view->map->records_count >=
 				 view->hdr.messages_count);
 			view->map->records_count = view->hdr.messages_count;
+
+#ifdef DEBUG
+			if (!ctx->sync_map_ctx.unreliable_flags)
+				mail_index_view_check(view);
+#endif
 		}
 
 		map = mail_index_map_clone(view->map,
@@ -340,7 +385,7 @@
 	uint32_t seq;
 	uoff_t offset;
 	int ret;
-	bool skipped;
+	bool skipped, synced_to_map;
 
 	for (;;) {
 		/* Get the next transaction from log. */
@@ -374,13 +419,6 @@
 		}
 
 		/* skip everything we've already synced */
-		if (offset < view->hdr.log_file_ext_offset &&
-		    seq == view->hdr.log_file_seq &&
-		    (ctx->hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0) {
-			/* view->log_file_offset contains the minimum of
-			   int/ext offsets. */
-			continue;
-		}
 		if (view_sync_pos_find(&view->syncs_done, seq, offset))
 			continue;
 
@@ -393,9 +431,15 @@
 							       offset);
 		}
 
+		/* view->log_file_offset contains the minimum of
+		   int/ext offsets. */
+		synced_to_map = offset < view->hdr.log_file_ext_offset &&
+			seq == view->hdr.log_file_seq &&
+			(ctx->hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0;
+
 		/* Apply transaction to view's mapping if needed (meaning we
 		   didn't just re-map the view to head mapping). */
-		if (ctx->sync_map_update) {
+		if (ctx->sync_map_update && !synced_to_map) {
 			i_assert((ctx->hdr->type &
 				  MAIL_TRANSACTION_EXPUNGE) == 0);
 
@@ -589,6 +633,11 @@
 	}
 	view->hdr = view->map->hdr;
 
+#ifdef DEBUG
+	if (!view->broken_counters)
+		mail_index_view_check(view);
+#endif
+
 	/* set log view to empty range so unneeded memory gets freed */
 	(void)mail_transaction_log_view_set(view->log_view,
 					    view->log_file_seq,