[dovecot-cvs] dovecot/src/lib-index mail-index-private.h, 1.36, 1.37 mail-index-sync.c, 1.39, 1.40 mail-index-view-sync.c, 1.29, 1.30 mail-index.c, 1.166, 1.167 mail-transaction-log-view.c, 1.34, 1.35

[cras at dovecot.org]

Update of /var/lib/cvs/dovecot/src/lib-index
In directory talvi:/tmp/cvs-serv32286

Modified Files:
	mail-index-private.h mail-index-sync.c mail-index-view-sync.c 
	mail-index.c mail-transaction-log-view.c 
Log Message:
Avoid re-reading index file with mmap_disabled=yes. Instead sync it by
reading transaction log file.



Index: mail-index-private.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-private.h,v
retrieving revision 1.36
retrieving revision 1.37
diff -u -d -r1.36 -r1.37
--- mail-index-private.h	13 Nov 2004 23:08:07 -0000	1.36
+++ mail-index-private.h	14 Nov 2004 01:23:21 -0000	1.37
@@ -107,6 +107,11 @@
 	struct dotlock dotlock;
         enum mail_index_lock_method lock_method;
 
+	/* These are typically same as map->hdr->log_file_*, but with
+	   mmap_disable we may have synced more than index */
+	uint32_t sync_log_file_seq;
+	uoff_t sync_log_file_offset;
+
 	unsigned int last_grow_count;
 
 	char *error;

Index: mail-index-sync.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-sync.c,v
retrieving revision 1.39
retrieving revision 1.40
diff -u -d -r1.39 -r1.40
--- mail-index-sync.c	13 Nov 2004 23:08:07 -0000	1.39
+++ mail-index-sync.c	14 Nov 2004 01:23:21 -0000	1.40
@@ -310,10 +310,29 @@
 		return -1;
 	}
 
-	if (mail_index_map(index, FALSE) <= 0) {
-		mail_transaction_log_sync_unlock(index->log);
-		mail_index_unlock(index, lock_id);
-		return -1;
+	if (index->mmap_disable) {
+		if (index->sync_log_file_seq != seq ||
+		    index->sync_log_file_offset != offset) {
+			/* we may have synced our internal view more than what
+			   is synced in index. re-read the whole index if our
+			   sync seq/offset doesn't match what is in index's
+			   header. */
+			if (mail_index_map(index, TRUE) <= 0) {
+				mail_transaction_log_sync_unlock(index->log);
+				mail_index_unlock(index, lock_id);
+				return -1;
+			}
+		} else {
+			/* the whole log file is synced already. */
+			i_assert(index->map->hdr.log_file_seq == seq);
+			i_assert(index->map->hdr.log_file_ext_offset == offset);
+		}
+	} else {
+		if (mail_index_map(index, FALSE) <= 0) {
+			mail_transaction_log_sync_unlock(index->log);
+			mail_index_unlock(index, lock_id);
+			return -1;
+		}
 	}
 
 	if (!mail_index_need_lock(index, sync_recent,
@@ -504,6 +523,7 @@
 
 int mail_index_sync_commit(struct mail_index_sync_ctx *ctx)
 {
+	struct mail_index *index = ctx->index;
 	const struct mail_index_header *hdr;
 	uint32_t seq, seq2;
 	uoff_t offset, offset2;
@@ -515,9 +535,9 @@
 	/* we have had the transaction log locked since the beginning of sync,
 	   so only external changes could have been committed. write them to
 	   the index here as well. */
-	mail_transaction_log_get_head(ctx->index->log, &seq, &offset);
+	mail_transaction_log_get_head(index->log, &seq, &offset);
 
-	hdr = ctx->index->hdr;
+	hdr = index->hdr;
 	if (ret == 0 && (hdr->log_file_seq != seq ||
 			 hdr->log_file_int_offset != offset ||
 			 hdr->log_file_ext_offset != offset)) {
@@ -529,12 +549,12 @@
 			ret = -1;
 	}
 
-	if (ret == 0 && mail_cache_need_compress(ctx->index->cache)) {
-		if (mail_cache_compress(ctx->index->cache, ctx->view) < 0)
+	if (ret == 0 && mail_cache_need_compress(index->cache)) {
+		if (mail_cache_compress(index->cache, ctx->view) < 0)
 			ret = -1;
 		else {
 			/* cache_offsets have changed, sync them */
-			mail_transaction_log_get_head(ctx->index->log,
+			mail_transaction_log_get_head(index->log,
 						      &seq2, &offset2);
 			if (mail_transaction_log_view_set(ctx->view->log_view,
 					seq, offset, seq2, offset2,
@@ -545,6 +565,9 @@
 		}
 	}
 
+	index->sync_log_file_seq = index->map->hdr.log_file_seq;
+	index->sync_log_file_offset = index->map->hdr.log_file_int_offset;
+
 	mail_index_sync_end(ctx);
 	return ret;
 }

Index: mail-index-view-sync.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-view-sync.c,v
retrieving revision 1.29
retrieving revision 1.30
diff -u -d -r1.29 -r1.30
--- mail-index-view-sync.c	13 Nov 2004 23:08:07 -0000	1.29
+++ mail-index-view-sync.c	14 Nov 2004 01:23:21 -0000	1.30
@@ -342,6 +342,9 @@
 
 	i_assert(view->syncing);
 
+	if (ctx->sync_map_update)
+		mail_index_sync_map_deinit(&ctx->sync_map_ctx);
+
 	if (view->log_syncs != NULL && !ctx->skipped_some)
 		buffer_set_used_size(view->log_syncs, 0);
 

Index: mail-index.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index.c,v
retrieving revision 1.166
retrieving revision 1.167
diff -u -d -r1.166 -r1.167
--- mail-index.c	13 Nov 2004 23:08:07 -0000	1.166
+++ mail-index.c	14 Nov 2004 01:23:21 -0000	1.167
@@ -6,6 +6,7 @@
 #include "read-full.h"
 #include "write-full.h"
 #include "mail-index-private.h"
+#include "mail-index-sync-private.h"
 #include "mail-transaction-log.h"
 #include "mail-cache.h"
 
@@ -432,6 +433,25 @@
 	return 1;
 }
 
+static int mail_index_read_header(struct mail_index *index,
+				  struct mail_index_header *hdr, size_t *pos_r)
+{
+	size_t pos;
+	int ret;
+
+	memset(hdr, 0, sizeof(*hdr));
+
+	ret = 1;
+	for (pos = 0; ret > 0 && pos < sizeof(*hdr); ) {
+		ret = pread(index->fd, PTR_OFFSET(hdr, pos),
+			    sizeof(*hdr) - pos, pos);
+		if (ret > 0)
+			pos += ret;
+	}
+	*pos_r = pos;
+	return ret;
+}
+
 static int mail_index_read_map(struct mail_index *index,
 			       struct mail_index_map *map, int *retry_r)
 {
@@ -443,15 +463,7 @@
 	i_assert(map->mmap_base == NULL);
 
 	*retry_r = FALSE;
-	memset(&hdr, 0, sizeof(hdr));
-
-	ret = 1;
-	for (pos = 0; ret > 0 && pos < sizeof(hdr); ) {
-		ret = pread(index->fd, PTR_OFFSET(&hdr, pos),
-			    sizeof(hdr) - pos, pos);
-		if (ret > 0)
-			pos += ret;
-	}
+	ret = mail_index_read_header(index, &hdr, &pos);
 
 	if (pos > (ssize_t)offsetof(struct mail_index_header, major_version) &&
 	    hdr.major_version != MAIL_INDEX_MAJOR_VERSION) {
@@ -523,14 +535,114 @@
 		memcpy(&map->hdr, &hdr, hdr.base_header_size);
 	}
 	map->hdr_base = map->hdr_copy_buf->data;
+
+	index->sync_log_file_seq = hdr.log_file_seq;
+	index->sync_log_file_offset = hdr.log_file_int_offset;
 	return 1;
 }
 
+static int mail_index_sync_from_transactions(struct mail_index *index,
+					     struct mail_index_map *map,
+					     int sync_to_index)
+{
+	struct mail_index_view *view;
+	struct mail_transaction_log_view *log_view;
+	struct mail_index_sync_map_ctx sync_map_ctx;
+	struct mail_index_header hdr;
+	const struct mail_transaction_header *thdr;
+	const void *tdata;
+	uint32_t max_seq;
+	uoff_t max_offset;
+	size_t pos;
+	int ret, skipped;
+
+	if (sync_to_index) {
+		/* read the real log position where we are supposed to be
+		   synced */
+		ret = mail_index_read_header(index, &hdr, &pos);
+		if (ret < 0 && errno != ESTALE) {
+			mail_index_set_syscall_error(index, "pread()");
+			return -1;
+		}
+		if (pos < MAIL_INDEX_HEADER_MIN_SIZE)
+			return 0;
+
+		if (map->hdr.log_file_seq == hdr.log_file_seq &&
+		    map->hdr.log_file_int_offset == hdr.log_file_int_offset) {
+			/* nothing to do */
+			return 1;
+		}
+
+		if (map->hdr.log_file_seq > hdr.log_file_seq ||
+		    (map->hdr.log_file_seq == hdr.log_file_seq &&
+		     map->hdr.log_file_int_offset > hdr.log_file_int_offset)) {
+			/* we went too far, have to re-read the file */
+			return 0;
+		}
+		if (map->hdr.log_file_ext_offset !=
+		    map->hdr.log_file_int_offset ||
+		    hdr.log_file_ext_offset != hdr.log_file_int_offset) {
+			/* too much trouble to get this right. */
+			return 0;
+		}
+		max_seq = hdr.log_file_seq;
+		max_offset = hdr.log_file_int_offset;
+	} else {
+		/* sync everything there is */
+		max_seq = (uint32_t)-1;
+		max_offset = (uoff_t)-1;
+	}
+
+	log_view = mail_transaction_log_view_open(index->log);
+	if (mail_transaction_log_view_set(log_view,
+					  map->hdr.log_file_seq,
+					  map->hdr.log_file_int_offset,
+					  max_seq, max_offset,
+					  MAIL_TRANSACTION_TYPE_MASK) < 0) {
+		mail_transaction_log_view_close(log_view);
+		return 0;
+	}
+
+	index->map = map;
+
+	view = mail_index_view_open(index);
+	mail_index_sync_map_init(&sync_map_ctx, view,
+				 MAIL_INDEX_SYNC_HANDLER_VIEW);
+
+	while ((ret = mail_transaction_log_view_next(log_view, &thdr, &tdata,
+						     &skipped)) > 0) {
+		if (mail_index_sync_record(&sync_map_ctx, thdr, tdata) < 0) {
+			ret = -1;
+			break;
+		}
+	}
+
+	mail_index_sync_map_deinit(&sync_map_ctx);
+	mail_index_view_close(view);
+	mail_transaction_log_view_close(log_view);
+
+	index->map = NULL;
+	return ret < 0 ? -1 : 1;
+}
+
 static int mail_index_read_map_with_retry(struct mail_index *index,
-					  struct mail_index_map *map)
+					  struct mail_index_map *map,
+					  int sync_to_index)
 {
 	int i, ret, retry;
 
+	if (map->hdr.indexid != 0) {
+		/* sync this as a view from transaction log. */
+		ret = mail_index_sync_from_transactions(index, map,
+							sync_to_index);
+		if (ret != 0)
+			return ret;
+
+		/* transaction log lost/broken, fallback to re-reading it */
+		/* FIXME: file cache need to be reset (except not really with
+		   sync_to_index if we were just rewinding..) */
+	}
+
 	for (i = 0; i < MAIL_INDEX_ESTALE_RETRY_COUNT; i++) {
 		ret = mail_index_read_map(index, map, &retry);
 		if (ret != 0 || !retry)
@@ -598,24 +710,27 @@
 	if (index->map != NULL && index->map->refcount > 1) {
 		/* this map is already used by some views and they may have
 		   pointers into it. leave them and create a new mapping. */
+		if (!index->mmap_disable) {
+			map = NULL;
+		} else {
+			/* create a copy of the mapping instead so we don't
+			   have to re-read it */
+			map = mail_index_map_to_memory(index->map,
+						index->map->hdr.record_size);
+		}
 		index->map->refcount--;
 		index->map = NULL;
+	} else {
+		map = index->map;
 	}
 
-	map = index->map;
 	if (map == NULL) {
 		map = i_new(struct mail_index_map, 1);
 		map->refcount = 1;
 		map->hdr_copy_buf =
 			buffer_create_dynamic(default_pool, sizeof(map->hdr));
 	} else if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
-		if (map->write_to_disk) {
-			/* we have modified this mapping and it's waiting to
-			   be written to disk once we drop exclusive lock.
-			   mapping couldn't have changed, so do nothing. */
-			return 1;
-		}
-		/* FIXME: we need to re-read header */
+		i_assert(!map->write_to_disk);
 	} else if (map->mmap_base != NULL) {
 		i_assert(map->buffer == NULL);
 		if (munmap(map->mmap_base, map->mmap_size) < 0)
@@ -628,8 +743,8 @@
 
 	if (!index->mmap_disable)
 		ret = mail_index_mmap(index, map);
-	 else
-		ret = mail_index_read_map_with_retry(index, map);
+	else
+		ret = mail_index_read_map_with_retry(index, map, force);
 	if (ret <= 0) {
 		mail_index_unmap_forced(index, map);
 		return ret;

Index: mail-transaction-log-view.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-transaction-log-view.c,v
retrieving revision 1.34
retrieving revision 1.35
diff -u -d -r1.34 -r1.35
--- mail-transaction-log-view.c	13 Nov 2004 22:27:57 -0000	1.34
+++ mail-transaction-log-view.c	14 Nov 2004 01:23:21 -0000	1.35
@@ -153,6 +153,12 @@
 	for (seq = min_file_seq+1; seq <= max_file_seq; seq++) {
 		file = file->next;
 		if (file == NULL || file->hdr.file_seq != seq)  {
+			if (file == NULL && max_file_seq == (uint32_t)-1) {
+				/* we just wanted to sync everything */
+				max_file_seq = seq-1;
+				break;
+			}
+
 			mail_index_set_error(view->log->index,
 				"Lost transaction log file %s seq %u",
 				view->log->tail->filepath, seq);