[dovecot-cvs] dovecot/src/lib-index mail-cache-compress.c, 1.24, 1.25 mail-cache-fields.c, 1.8, 1.9 mail-cache-lookup.c, 1.25, 1.26 mail-cache-private.h, 1.17, 1.18 mail-cache-sync-update.c, 1.1, 1.2 mail-cache-transaction.c, 1.31, 1.32 mail-cache.c, 1.51, 1.52

[cras at dovecot.org]

Update of /var/lib/cvs/dovecot/src/lib-index
In directory talvi:/tmp/cvs-serv18728/lib-index

Modified Files:
	mail-cache-compress.c mail-cache-fields.c mail-cache-lookup.c 
	mail-cache-private.h mail-cache-sync-update.c 
	mail-cache-transaction.c mail-cache.c 
Log Message:
Cache file works now with mmap_disable=yes. Still needs a few optimizations.



Index: mail-cache-compress.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-cache-compress.c,v
retrieving revision 1.24
retrieving revision 1.25
diff -u -d -r1.24 -r1.25
--- mail-cache-compress.c	6 Nov 2004 17:20:18 -0000	1.24
+++ mail-cache-compress.c	8 Nov 2004 00:40:41 -0000	1.25
@@ -3,6 +3,7 @@
 #include "lib.h"
 #include "buffer.h"
 #include "ostream.h"
+#include "file-cache.h"
 #include "file-set-size.h"
 #include "mail-cache-private.h"
 
@@ -276,6 +277,9 @@
 			mail_cache_file_close(cache);
 			cache->fd = fd;
 
+			if (cache->file_cache != NULL)
+				file_cache_set_fd(cache->file_cache, cache->fd);
+
 			if (mail_cache_map(cache, 0, 0) < 0)
 				ret = -1;
 			else if (mail_cache_header_fields_read(cache) < 0)

Index: mail-cache-fields.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-cache-fields.c,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -d -r1.8 -r1.9
--- mail-cache-fields.c	8 Oct 2004 17:51:48 -0000	1.8
+++ mail-cache-fields.c	8 Nov 2004 00:40:41 -0000	1.9
@@ -3,6 +3,7 @@
 #include "lib.h"
 #include "buffer.h"
 #include "hash.h"
+#include "file-cache.h"
 #include "write-full.h"
 #include "mail-cache-private.h"
 
@@ -111,7 +112,7 @@
 				   sizeof(*field_hdr) + CACHE_HDR_PREFETCH) < 0)
 			return -1;
 
-		field_hdr = CONST_PTR_OFFSET(cache->mmap_base, offset);
+		field_hdr = CONST_PTR_OFFSET(cache->data, offset);
 		next_offset =
 			mail_index_offset_to_uint32(field_hdr->next_offset);
 	}
@@ -138,7 +139,7 @@
 		return 0;
 	}
 
-	field_hdr = CONST_PTR_OFFSET(cache->mmap_base, offset);
+	field_hdr = CONST_PTR_OFFSET(cache->data, offset);
 	if (offset + field_hdr->size > cache->mmap_length) {
 		mail_cache_set_corrupted(cache,
 					 "field header points outside file");
@@ -157,7 +158,7 @@
 		if (mail_cache_map(cache, offset, field_hdr->size) < 0)
 			return -1;
 	}
-	field_hdr = CONST_PTR_OFFSET(cache->mmap_base, offset);
+	field_hdr = CONST_PTR_OFFSET(cache->data, offset);
 
 	cache->file_field_map =
 		i_realloc(cache->file_field_map,
@@ -267,6 +268,7 @@
 	int locked = cache->locked;
 	buffer_t *buffer;
 	uint32_t i, offset;
+	size_t size;
 	int ret = 0;
 
 	if (!locked) {
@@ -286,15 +288,18 @@
 	copy_to_buf(cache, buffer,
 		    offsetof(struct mail_cache_field_private, last_used),
 		    sizeof(uint32_t));
-	ret = pwrite_full(cache->fd, buffer_get_data(buffer, NULL),
+	size = buffer->used;
+
+	ret = pwrite_full(cache->fd, buffer->data,
 			  sizeof(uint32_t) * cache->file_fields_count,
 			  offset + MAIL_CACHE_FIELD_LAST_USED());
 	if (ret == 0) {
 		buffer_set_used_size(buffer, 0);
 		copy_to_buf_byte(cache, buffer,
 				 offsetof(struct mail_cache_field, decision));
+		size += buffer->used;
 
-		ret = pwrite_full(cache->fd, buffer_get_data(buffer, NULL),
+		ret = pwrite_full(cache->fd, buffer->data,
 			sizeof(uint8_t) * cache->file_fields_count, offset +
 			MAIL_CACHE_FIELD_DECISION(cache->file_fields_count));
 
@@ -305,8 +310,11 @@
 	}
 	t_pop();
 
-	if (ret == 0)
+	if (ret == 0) {
 		cache->field_header_write_pending = FALSE;
+		if (cache->file_cache != NULL)
+			file_cache_invalidate(cache->file_cache, offset, size);
+	}
 
 	if (!locked)
 		mail_cache_unlock(cache);

Index: mail-cache-lookup.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-cache-lookup.c,v
retrieving revision 1.25
retrieving revision 1.26
diff -u -d -r1.25 -r1.26
--- mail-cache-lookup.c	6 Nov 2004 17:20:18 -0000	1.25
+++ mail-cache-lookup.c	8 Nov 2004 00:40:41 -0000	1.26
@@ -9,10 +9,10 @@
 
 #define CACHE_PREFETCH 1024
 
-struct mail_cache_record *
+const struct mail_cache_record *
 mail_cache_get_record(struct mail_cache *cache, uint32_t offset)
 {
-	struct mail_cache_record *cache_rec;
+	const struct mail_cache_record *cache_rec;
 
 	if (offset == 0)
 		return NULL;
@@ -401,10 +401,9 @@
 	}
 	lines_count = i;
 
-	/* FIXME: this relies on mmap() too heavily */
 	hdr_data_rec = t_new(struct header_lookup_data_rec, 1);
 	hdr_data_rec->offset = (const char *)&lines[lines_count+1] -
-		(const char *)view->cache->mmap_base;
+		(const char *)view->cache->data;
 	hdr_data_rec->data_size = (uint32_t)data_size;
 
 	for (i = 0; i < lines_count; i++) {
@@ -487,8 +486,7 @@
 
 	/* then start filling dest buffer from the headers */
 	for (i = 0; i < size; i++) {
-		start = CONST_PTR_OFFSET(cache->mmap_base,
-					 data[i].data->offset);
+		start = CONST_PTR_OFFSET(cache->data, data[i].data->offset);
 		end = start + data[i].data->data_size;
 
 		for (p = start; p != end; p++) {

Index: mail-cache-private.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-cache-private.h,v
retrieving revision 1.17
retrieving revision 1.18
diff -u -d -r1.17 -r1.18
--- mail-cache-private.h	6 Nov 2004 17:20:18 -0000	1.17
+++ mail-cache-private.h	8 Nov 2004 00:40:41 -0000	1.18
@@ -33,7 +33,8 @@
 #define MAIL_CACHE_LOCK_IMMEDIATE_TIMEOUT (5*60)
 
 #define CACHE_RECORD(cache, offset) \
-	((struct mail_cache_record *) ((char *) (cache)->mmap_base + offset))
+	((const struct mail_cache_record *) \
+	 ((const char *) (cache)->data + offset))
 
 #define MAIL_CACHE_IS_UNUSABLE(cache) \
 	((cache)->hdr == NULL)
@@ -121,7 +122,9 @@
 	int fd;
 
 	void *mmap_base;
+	const void *data;
 	size_t mmap_length;
+	struct file_cache *file_cache;
 
 	const struct mail_cache_header *hdr;
 	struct mail_cache_header hdr_copy;
@@ -175,7 +178,7 @@
 int mail_cache_header_fields_get_next_offset(struct mail_cache *cache,
 					     uint32_t *offset_r);
 
-struct mail_cache_record *
+const struct mail_cache_record *
 mail_cache_get_record(struct mail_cache *cache, uint32_t offset);
 
 int mail_cache_foreach(struct mail_cache_view *view, uint32_t seq,

Index: mail-cache-sync-update.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-cache-sync-update.c,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- mail-cache-sync-update.c	6 Nov 2004 17:20:18 -0000	1.1
+++ mail-cache-sync-update.c	8 Nov 2004 00:40:41 -0000	1.2
@@ -1,6 +1,7 @@
 /* Copyright (C) 2004 Timo Sirainen */
 
 #include "lib.h"
+#include "file-cache.h"
 #include "mail-cache-private.h"
 #include "mail-index-view-private.h"
 #include "mail-index-sync-private.h"
@@ -96,6 +97,7 @@
 			    void **context)
 {
 	struct mail_index_view *view = sync_ctx->view;
+	struct mail_cache *cache = view->index->cache;
 	struct mail_cache_sync_context *ctx = *context;
 	const uint32_t *old_cache_offset = old_data;
 	const uint32_t *new_cache_offset = new_data;
@@ -108,11 +110,16 @@
 		return 1;
 	}
 
+	if (cache->file_cache != NULL) {
+		file_cache_invalidate(cache->file_cache, *new_cache_offset,
+				      (size_t)-1);
+	}
+
 	if (*old_cache_offset == 0)
 		return 1;
 
 	/* we'll need to link the old and new cache records */
-	ret = mail_cache_handler_init(&ctx, view->index->cache);
+	ret = mail_cache_handler_init(&ctx, cache);
 	*context = ctx;
 	if (ret <= 0)
 		return ret < 0 ? -1 : 1;
@@ -120,13 +127,12 @@
 	if (!get_cache_file_seq(view, &cache_file_seq))
 		return 1;
 
-	if (cache_file_seq != view->index->cache->hdr->file_seq) {
+	if (cache_file_seq != cache->hdr->file_seq) {
 		/* cache has been compressed, don't modify it */
 		return 1;
 	}
 
-	if (mail_cache_link(view->index->cache,
-			    *old_cache_offset, *new_cache_offset) < 0)
+	if (mail_cache_link(cache, *old_cache_offset, *new_cache_offset) < 0)
 		return -1;
 
 	return 1;

Index: mail-cache-transaction.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-cache-transaction.c,v
retrieving revision 1.31
retrieving revision 1.32
diff -u -d -r1.31 -r1.32
--- mail-cache-transaction.c	6 Nov 2004 17:20:18 -0000	1.31
+++ mail-cache-transaction.c	8 Nov 2004 00:40:41 -0000	1.32
@@ -2,6 +2,7 @@
 
 #include "lib.h"
 #include "buffer.h"
+#include "file-cache.h"
 #include "file-set-size.h"
 #include "read-full.h"
 #include "write-full.h"
@@ -170,6 +171,10 @@
 			mail_cache_set_syscall_error(cache, "pwrite_full()");
 			return FALSE;
 		}
+		if (cache->file_cache != NULL) {
+			file_cache_invalidate(cache->file_cache, prev_offset,
+					      sizeof(hole.next_offset));
+		}
 	}
 	hdr->deleted_space -= hole.size;
 	cache->hdr_modified = TRUE;
@@ -285,6 +290,11 @@
 			return;
 		}
 
+		if (cache->file_cache != NULL) {
+			file_cache_invalidate(cache->file_cache, offset,
+					      sizeof(hole));
+		}
+
 		cache->hdr_copy.deleted_space += size;
 		cache->hdr_copy.hole_offset = offset;
 		cache->hdr_modified = TRUE;
@@ -362,13 +372,62 @@
 	return offset;
 }
 
+static uint32_t
+mail_cache_transaction_update_index(struct mail_cache_transaction_ctx *ctx,
+				    const struct mail_cache_record *rec,
+				    const uint32_t *seq, uint32_t *seq_idx,
+				    uint32_t seq_limit, uint32_t write_offset)
+{
+	struct mail_cache *cache = ctx->cache;
+	uint32_t i, old_offset, orig_write_offset;
+
+	/* write the cache_offsets to index file. records' prev_offset
+	   is updated to point to old cache record when index is being
+	   synced. */
+	orig_write_offset = write_offset;
+	for (i = *seq_idx; i < seq_limit; i++) {
+		mail_index_update_ext(ctx->trans, seq[i], cache->ext_id,
+				      &write_offset, &old_offset);
+		if (old_offset != 0) {
+			/* we added records for this message multiple
+			   times in this same uncommitted transaction.
+			   only the new one will be written to
+			   transaction log, we need to do the linking
+			   ourself here. */
+			if (old_offset > write_offset) {
+				if (mail_cache_link_unlocked(cache, old_offset,
+							     write_offset) < 0)
+					return -1;
+			} else {
+				/* if we're combining multiple transactions,
+				   make sure the one with the smallest offset
+				   is written into index. this is required for
+				   non-file-mmaped cache to work properly. */
+				mail_index_update_ext(ctx->trans, seq[i],
+						      cache->ext_id,
+						      &old_offset, NULL);
+				if (mail_cache_link_unlocked(cache,
+							     write_offset,
+							     old_offset) < 0)
+					return -1;
+			}
+		}
+
+		write_offset += rec->size;
+		rec = CONST_PTR_OFFSET(rec, rec->size);
+	}
+
+	*seq_idx = i;
+	return write_offset - orig_write_offset;
+}
+
 static int
 mail_cache_transaction_flush(struct mail_cache_transaction_ctx *ctx)
 {
 	struct mail_cache *cache = ctx->cache;
 	const struct mail_cache_record *rec, *tmp_rec;
 	const uint32_t *seq;
-	uint32_t write_offset, old_offset, rec_pos, cache_file_seq;
+	uint32_t write_offset, rec_pos, cache_file_seq;
 	size_t size, max_size, seq_idx, seq_limit, seq_count;
 	int commit;
 
@@ -432,29 +491,16 @@
 			mail_cache_set_syscall_error(cache, "pwrite_full()");
 			return -1;
 		}
-
-		/* write the cache_offsets to index file. records' prev_offset
-		   is updated to point to old cache record when index is being
-		   synced. */
-		for (; seq_idx < seq_limit; seq_idx++) {
-			mail_index_update_ext(ctx->trans, seq[seq_idx],
-					      cache->ext_id, &write_offset,
-					      &old_offset);
-			if (old_offset != 0) {
-				/* we added records for this message multiple
-				   times in this same uncommitted transaction.
-				   only the new one will be written to
-				   transaction log, we need to do the linking
-				   ourself here. */
-				if (mail_cache_link_unlocked(cache, old_offset,
-							     write_offset) < 0)
-					return -1;
-			}
-
-			write_offset += rec->size;
-			rec_pos += rec->size;
-			rec = CONST_PTR_OFFSET(rec, rec->size);
+		if (cache->file_cache != NULL) {
+			file_cache_invalidate(cache->file_cache,
+					      write_offset, max_size);
 		}
+
+		size = mail_cache_transaction_update_index(ctx, rec, seq,
+							   &seq_idx, seq_limit,
+							   write_offset);
+		rec_pos += size;
+		rec = CONST_PTR_OFFSET(rec, size);
 	}
 
 	/* drop the written data from buffer */
@@ -608,6 +654,11 @@
 		ret = -1;
 	else {
 		/* after it's guaranteed to be in disk, update header offset */
+		if (cache->file_cache != NULL) {
+			file_cache_invalidate(cache->file_cache, offset, size);
+			file_cache_invalidate(cache->file_cache, hdr_offset,
+					      sizeof(offset));
+		}
 		offset = mail_index_uint32_to_offset(offset);
 		if (pwrite_full(cache->fd, &offset, sizeof(offset),
 				hdr_offset) < 0) {
@@ -699,6 +750,10 @@
 		mail_cache_set_syscall_error(cache, "pwrite_full()");
 		return -1;
 	}
+	if (cache->file_cache != NULL) {
+		file_cache_invalidate(cache->file_cache,
+				      new_offset, sizeof(old_offset));
+	}
 	return 0;
 }
 
@@ -728,7 +783,7 @@
 
 int mail_cache_delete(struct mail_cache *cache, uint32_t offset)
 {
-	struct mail_cache_record *cache_rec;
+	const struct mail_cache_record *cache_rec;
 
 	i_assert(cache->locked);
 

Index: mail-cache.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-cache.c,v
retrieving revision 1.51
retrieving revision 1.52
diff -u -d -r1.51 -r1.52
--- mail-cache.c	6 Nov 2004 17:20:18 -0000	1.51
+++ mail-cache.c	8 Nov 2004 00:40:41 -0000	1.52
@@ -3,6 +3,7 @@
 #include "lib.h"
 #include "buffer.h"
 #include "hash.h"
+#include "file-cache.h"
 #include "mmap-util.h"
 #include "write-full.h"
 #include "mail-cache-private.h"
@@ -46,7 +47,11 @@
 			mail_cache_set_syscall_error(cache, "munmap()");
 	}
 
+	if (cache->file_cache != NULL)
+		file_cache_set_fd(cache->file_cache, -1);
+
 	cache->mmap_base = NULL;
+	cache->data = NULL;
 	cache->hdr = NULL;
 	cache->mmap_length = 0;
 
@@ -78,6 +83,9 @@
 		return -1;
 	}
 
+	if (cache->file_cache != NULL)
+		file_cache_set_fd(cache->file_cache, cache->fd);
+
 	if (mail_cache_map(cache, 0, 0) < 0)
 		return -1;
 
@@ -100,16 +108,15 @@
 	return 1;
 }
 
-static int mmap_verify_header(struct mail_cache *cache)
+static int mail_cache_verify_header(struct mail_cache *cache)
 {
-	const struct mail_cache_header *hdr;
+	const struct mail_cache_header *hdr = cache->data;
 
 	/* check that the header is still ok */
 	if (cache->mmap_length < sizeof(struct mail_cache_header)) {
 		mail_cache_set_corrupted(cache, "File too small");
 		return FALSE;
 	}
-	cache->hdr = hdr = cache->mmap_base;
 
 	if (cache->hdr->version != MAIL_CACHE_VERSION) {
 		/* version changed - upgrade silently */
@@ -135,7 +142,8 @@
 		return FALSE;
 	}
 
-	if (hdr->used_file_size > cache->mmap_length) {
+	if (cache->mmap_base != NULL &&
+	    hdr->used_file_size > cache->mmap_length) {
 		mail_cache_set_corrupted(cache, "used_file_size too large");
 		return FALSE;
 	}
@@ -144,9 +152,33 @@
 
 int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size)
 {
+	ssize_t ret;
+
 	if (size == 0)
 		size = sizeof(struct mail_cache_header);
 
+	if (cache->file_cache != NULL) {
+		cache->data = NULL;
+		cache->hdr = NULL;
+
+		ret = file_cache_read(cache->file_cache, offset, size);
+		if (ret < 0) {
+			// FIXME: ESTALE
+			mail_cache_set_syscall_error(cache, "read()");
+			return -1;
+		}
+
+		cache->data = file_cache_get_map(cache->file_cache,
+						 &cache->mmap_length);
+		cache->hdr = cache->data;
+
+		if (offset == 0 && !mail_cache_verify_header(cache)) {
+			cache->need_compress = TRUE;
+			return -1;
+		}
+		return 0;
+	}
+
 	if (offset < cache->mmap_length &&
 	    size <= cache->mmap_length - offset) {
 		/* already mapped */
@@ -171,11 +203,14 @@
 	cache->mmap_base = mmap_ro_file(cache->fd, &cache->mmap_length);
 	if (cache->mmap_base == MAP_FAILED) {
 		cache->mmap_base = NULL;
+		cache->data = NULL;
 		mail_cache_set_syscall_error(cache, "mmap()");
 		return -1;
 	}
+	cache->data = cache->mmap_base;
+	cache->hdr = cache->mmap_base;
 
-	if (!mmap_verify_header(cache)) {
+	if (!mail_cache_verify_header(cache)) {
 		cache->need_compress = TRUE;
 		return -1;
 	}
@@ -188,6 +223,9 @@
 	cache->filepath = i_strconcat(cache->index->filepath,
 				      MAIL_CACHE_FILE_PREFIX, NULL);
 
+	if (cache->index->mmap_disable || cache->index->mmap_no_write)
+		cache->file_cache = file_cache_new(-1);
+
 	cache->fd = open(cache->filepath, O_RDWR);
 	if (cache->fd == -1) {
 		if (errno == ENOENT) {
@@ -199,6 +237,9 @@
 		return -1;
 	}
 
+	if (cache->file_cache != NULL)
+		file_cache_set_fd(cache->file_cache, cache->fd);
+
 	if (mail_cache_map(cache, 0, sizeof(struct mail_cache_header)) < 0)
 		return -1;
 
@@ -217,12 +258,10 @@
 		hash_create(default_pool, cache->field_pool, 0,
 			    strcase_hash, (hash_cmp_callback_t *)strcasecmp);
 
-	if (!index->mmap_disable && !index->mmap_no_write) {
-		if (mail_cache_open_and_verify(cache) < 0) {
-			/* failed for some reason - doesn't really matter,
-			   it's disabled for now. */
-			mail_cache_file_close(cache);
-		}
+	if (mail_cache_open_and_verify(cache) < 0) {
+		/* failed for some reason - doesn't really matter,
+		   it's disabled for now. */
+		mail_cache_file_close(cache);
 	}
 
 	cache->ext_id =
@@ -237,6 +276,11 @@
 
 void mail_cache_free(struct mail_cache *cache)
 {
+	if (cache->file_cache != NULL) {
+		file_cache_free(cache->file_cache);
+		cache->file_cache = NULL;
+	}
+
 	mail_cache_file_close(cache);
 
 	hash_destroy(cache->field_name_hash);
@@ -299,8 +343,12 @@
 		ret = 0;
 	}
 
-	if (ret > 0)
+	if (ret > 0) {
+		/* make sure our header is up to date */
+		if (mail_cache_map(cache, 0, 0) < 0)
+			ret = -1;
 		cache->hdr_copy = *cache->hdr;
+	}
 
 	mail_index_view_close(view);
 	return ret;
@@ -342,7 +390,11 @@
 		if (pwrite_full(cache->fd, &cache->hdr_copy,
 				sizeof(cache->hdr_copy), 0) < 0)
 			mail_cache_set_syscall_error(cache, "pwrite_full()");
-                mail_cache_update_need_compress(cache);
+		if (cache->file_cache != NULL) {
+			file_cache_invalidate(cache->file_cache, 0,
+					      sizeof(cache->hdr_copy));
+		}
+		mail_cache_update_need_compress(cache);
 	}
 
 	if (mail_index_lock_fd(cache->index, cache->fd, F_UNLCK, 0) <= 0) {