[dovecot-cvs] dovecot/src/lib-index mailbox-list-index-private.h, 1.2, 1.3 mailbox-list-index-sync.c, 1.2, 1.3 mailbox-list-index.c, 1.2, 1.3

tss at dovecot.org tss at dovecot.org
Wed Dec 20 08:16:45 UTC 2006


Update of /var/lib/cvs/dovecot/src/lib-index
In directory talvi:/tmp/cvs-serv9945/lib-index

Modified Files:
	mailbox-list-index-private.h mailbox-list-index-sync.c 
	mailbox-list-index.c 
Log Message:
Compress list index when its deleted space amount gets too high. Also fixed
several bugs.



Index: mailbox-list-index-private.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mailbox-list-index-private.h,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- mailbox-list-index-private.h	10 Dec 2006 16:04:46 -0000	1.2
+++ mailbox-list-index-private.h	20 Dec 2006 08:16:41 -0000	1.3
@@ -6,6 +6,9 @@
 #define MAILBOX_LIST_INDEX_MAJOR_VERSION 1
 #define MAILBOX_LIST_INDEX_MINOR_VERSION 0
 
+#define MAILBOX_LIST_COMPRESS_PERCENTAGE 10
+#define MAILBOX_LIST_COMPRESS_MIN_SIZE 1024
+
 struct mailbox_list_index_header {
 	uint8_t major_version;
 	uint8_t minor_version;
@@ -82,4 +85,10 @@
 			       const struct mailbox_list_dir_record **dir_r);
 int mailbox_list_index_map(struct mailbox_list_index *index);
 
+int mailbox_list_index_file_create(struct mailbox_list_index *index,
+				   uint32_t uid_validity);
+void mailbox_list_index_file_close(struct mailbox_list_index *index);
+
+int mailbox_list_index_refresh(struct mailbox_list_index *index);
+
 #endif

Index: mailbox-list-index-sync.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mailbox-list-index-sync.c,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- mailbox-list-index-sync.c	10 Dec 2006 16:04:46 -0000	1.2
+++ mailbox-list-index-sync.c	20 Dec 2006 08:16:41 -0000	1.3
@@ -69,6 +69,9 @@
 	bool *match;
 };
 
+static bool mailbox_list_index_need_compress(struct mailbox_list_index *index);
+static int mailbox_list_index_compress(struct mailbox_list_index_sync_ctx *ctx);
+
 static struct mailbox_list_sync_dir *
 mailbox_list_alloc_sync_dir(struct mailbox_list_index_sync_ctx *ctx,
 			    unsigned int initial_count)
@@ -105,6 +108,11 @@
 		if (recs[i].deleted)
 			continue;
 
+		if (recs[i].uid == 0) {
+			return mailbox_list_index_set_corrupted(ctx->index,
+							"Record with UID=0");
+		}
+
 		sync_rec = array_append_space(&sync_dir->records);
 		sync_rec->name_hash = recs[i].name_hash;
 		sync_rec->uid = recs[i].uid;
@@ -317,10 +325,9 @@
 					   &ctx->sync_root, &seq);
 }
 
-static int sync_init_mail_sync(struct mailbox_list_index_sync_ctx *ctx)
+static int sync_mail_sync_init(struct mailbox_list_index_sync_ctx *ctx)
 {
 	struct mail_index_sync_rec sync_rec;
-	const struct mail_index_header *hdr;
 
 	if (mail_index_sync_begin(ctx->index->mail_index, &ctx->mail_sync_ctx,
 				  &ctx->view, (uint32_t)-1, 0,
@@ -331,6 +338,14 @@
 	   don't need to do anything but write them to the index */
 	while (mail_index_sync_next(ctx->mail_sync_ctx, &sync_rec) > 0)
 		;
+	return 0;
+}
+
+static int sync_mail_sync_init2(struct mailbox_list_index_sync_ctx *ctx)
+{
+	const struct mail_index_header *hdr;
+
+	ctx->hdr = *ctx->index->hdr;
 
 	hdr = mail_index_get_header(ctx->view);
 	if (hdr->uid_validity != 0) {
@@ -348,7 +363,7 @@
 			TRUE);
 	}
 
-	return mailbox_list_index_get_root(ctx);
+	return 0;
 }
 
 int mailbox_list_index_sync_init(struct mailbox_list_index *index,
@@ -372,11 +387,13 @@
 	ctx->index = index;
 	ctx->sync_path = p_strdup(pool, path);
 	ctx->flags = flags;
-	ctx->hdr = *index->hdr;
 
 	/* mail index syncing acts as the only locking for us */
-	if (sync_init_mail_sync(ctx) < 0) {
-		mailbox_list_index_sync_commit(&ctx);
+	if (sync_mail_sync_init(ctx) < 0 ||
+	    mailbox_list_index_refresh(index) < 0 ||
+	    sync_mail_sync_init2(ctx) < 0 ||
+	    mailbox_list_index_get_root(ctx) < 0) {
+		mailbox_list_index_sync_rollback(&ctx);
 		return -1;
 	}
 
@@ -745,6 +762,7 @@
 {
 	struct mailbox_list_index_header *hdr;
 	bool partial;
+	int ret = 0;
 
 	if (ctx->index->mmap_disable) {
 		ctx->output = o_stream_create_file(ctx->index->fd, default_pool,
@@ -763,29 +781,30 @@
 	}
 
 	if (mailbox_list_index_sync_write_dir(ctx, ctx->root, 0, partial) < 0)
-		return -1;
+		ret = -1;
 
 	if (!ctx->index->mmap_disable) {
 		/* update header */
 		hdr = ctx->index->mmap_base;
-		hdr->next_uid = ctx->hdr.next_uid;
-		hdr->used_space = ctx->hdr.used_space;
-		hdr->deleted_space = ctx->hdr.deleted_space;
+		if (ret == 0)
+			memcpy(hdr, &ctx->hdr, sizeof(*hdr));
 
 		if (msync(ctx->index->mmap_base,
 			  hdr->used_space, MS_SYNC) < 0) {
 			mailbox_list_index_set_syscall_error(ctx->index,
 							     "msync()");
-			return -1;
+			ret = -1;
 		}
 	} else {
-		o_stream_seek(ctx->output, 0);
-		o_stream_send(ctx->output, &ctx->hdr, sizeof(ctx->hdr));
+		if (ret == 0) {
+			o_stream_seek(ctx->output, 0);
+			o_stream_send(ctx->output, &ctx->hdr, sizeof(ctx->hdr));
+		}
 
 		o_stream_destroy(&ctx->output);
 		buffer_free(ctx->output_buf);
 	}
-	return 0;
+	return ret;
 }
 
 int mailbox_list_index_sync_commit(struct mailbox_list_index_sync_ctx **_ctx)
@@ -798,6 +817,8 @@
 	if (!ctx->failed) {
 		/* write all the changes to the index */
 		ret = mailbox_list_index_sync_write(ctx);
+		if (ret == 0 && mailbox_list_index_need_compress(ctx->index))
+			ret = mailbox_list_index_compress(ctx);
 	}
 
 	if (ctx->mail_sync_ctx != NULL) {
@@ -829,3 +850,67 @@
 	(*ctx)->failed = TRUE;
 	(void)mailbox_list_index_sync_commit(ctx);
 }
+
+static bool mailbox_list_index_need_compress(struct mailbox_list_index *index)
+{
+	uoff_t max_del_space;
+
+	max_del_space = index->hdr->used_space / 100 *
+		MAILBOX_LIST_COMPRESS_PERCENTAGE;
+	if (index->hdr->deleted_space >= max_del_space &&
+	    index->hdr->used_space >= MAILBOX_LIST_COMPRESS_MIN_SIZE)
+		return TRUE;
+
+	return FALSE;
+}
+
+static int mailbox_list_copy_to_mem_all(struct mailbox_list_index_sync_ctx *ctx,
+					struct mailbox_list_sync_dir *dir)
+{
+	struct mailbox_list_sync_record *recs;
+	unsigned int i, count;
+
+	/* mark the directories as new */
+	dir->offset = 0;
+	dir->new_records_count = 1;
+
+	recs = array_get_modifiable(&dir->records, &count);
+	for (i = 0; i < count; i++) {
+		recs[i].created = TRUE;
+		recs[i].seen = TRUE;
+
+		if (recs[i].dir == NULL) {
+			if (recs[i].dir_offset == 0)
+				continue;
+
+			if (mailbox_list_copy_sync_dir(ctx, recs[i].dir_offset,
+						       &recs[i].dir) < 0)
+				return -1;
+		}
+		recs[i].dir_offset = 0;
+
+		if (mailbox_list_copy_to_mem_all(ctx, recs[i].dir) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+static int mailbox_list_index_compress(struct mailbox_list_index_sync_ctx *ctx)
+{
+	/* first read everything to memory */
+	if (mailbox_list_copy_to_mem_all(ctx, ctx->root) < 0)
+		return -1;
+
+	/* truncate the index file */
+	mailbox_list_index_file_close(ctx->index);
+	if (mailbox_list_index_file_create(ctx->index,
+					   ctx->hdr.uid_validity) < 0)
+		return -1;
+
+	/* reset header */
+	ctx->hdr.used_space = sizeof(ctx->hdr);
+	ctx->hdr.deleted_space = 0;
+
+	/* and write everything back */
+	return mailbox_list_index_sync_write(ctx);
+}

Index: mailbox-list-index.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mailbox-list-index.c,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- mailbox-list-index.c	10 Dec 2006 16:04:46 -0000	1.2
+++ mailbox-list-index.c	20 Dec 2006 08:16:41 -0000	1.3
@@ -71,7 +71,7 @@
 	index->hdr = NULL;
 }
 
-static void mailbox_list_index_file_close(struct mailbox_list_index *index)
+void mailbox_list_index_file_close(struct mailbox_list_index *index)
 {
 	mailbox_list_index_unmap(index);
 
@@ -80,6 +80,7 @@
 	if (index->fd != -1) {
 		if (close(index->fd) < 0)
 			mailbox_list_index_set_syscall_error(index, "close()");
+		index->fd = -1;
 	}
 }
 
@@ -87,7 +88,7 @@
 				     const char *str)
 {
 	(void)unlink(index->filepath);
-	// FIXME: reopen or something
+	mailbox_list_index_file_close(index);
 
 	i_error("Corrupted mailbox list index file %s: %s",
 		index->filepath, str);
@@ -174,7 +175,8 @@
 }
 
 static void
-mailbox_list_index_init_header(struct mailbox_list_index_header *hdr)
+mailbox_list_index_init_header(struct mailbox_list_index_header *hdr,
+			       uint32_t uid_validity)
 {
 	memset(hdr, 0, sizeof(*hdr));
 	hdr->major_version = MAILBOX_LIST_INDEX_MAJOR_VERSION;
@@ -183,7 +185,7 @@
 	hdr->header_size = sizeof(*hdr);
 	hdr->used_space = hdr->header_size;
 
-	hdr->uid_validity = ioloop_time;
+	hdr->uid_validity = uid_validity;
 	hdr->next_uid = 1;
 }
 
@@ -191,7 +193,13 @@
 {
 	struct stat st1, st2;
 
+	if (index->fd == -1)
+		return 1;
+
 	if (stat(index->filepath, &st1) < 0) {
+		if (errno == ENOENT)
+			return 1;
+
 		mailbox_list_index_set_syscall_error(index, "stat()");
 		return -1;
 	}
@@ -204,8 +212,8 @@
 		!CMP_DEV_T(st1.st_dev, st2.st_dev);
 }
 
-static int
-mailbox_list_index_file_create(struct mailbox_list_index *index)
+int mailbox_list_index_file_create(struct mailbox_list_index *index,
+				   uint32_t uid_validity)
 {
 	struct mailbox_list_index_header hdr;
 	struct dotlock *dotlock;
@@ -228,7 +236,7 @@
 		}
 	}
 
-	mailbox_list_index_init_header(&hdr);
+	mailbox_list_index_init_header(&hdr, uid_validity);
 	if (write_full(fd, &hdr, sizeof(hdr)) < 0) {
 		mailbox_list_index_set_syscall_error(index, "write_full()");
 		(void)file_dotlock_delete(&dotlock);
@@ -278,7 +286,7 @@
 		}
 	}
 
-	ret = mailbox_list_index_file_create(index);
+	ret = mailbox_list_index_file_create(index, ioloop_time);
 	if (ret <= 0)
 		mailbox_list_index_file_close(index);
 	return ret;
@@ -483,12 +491,18 @@
 	return mailbox_list_index_lookup_rec(index, dir_offset, p + 1, rec_r);
 }
 
-static int mailbox_list_index_refresh(struct mailbox_list_index *index)
+int mailbox_list_index_refresh(struct mailbox_list_index *index)
 {
 	int ret;
 
-	if ((ret = mailbox_list_index_is_recreated(index)) <= 0)
+	if ((ret = mailbox_list_index_is_recreated(index)) <= 0) {
+		if (ret < 0)
+			return -1;
+
+		if (mailbox_list_index_map(index) < 0)
+			ret = -1;
 		return ret;
+	}
 
 	mailbox_list_index_file_close(index);
 	return mailbox_list_index_open_or_create(index);
@@ -529,7 +543,9 @@
 		(unsigned int)recurse_level;
 	ctx->name_path = str_new(default_pool, 512);
 
-	if (*path != '\0') {
+	if (mailbox_list_index_refresh(index) < 0)
+		ctx->failed = TRUE;
+	if (!ctx->failed && *path != '\0') {
 		ret = mailbox_list_index_lookup_rec(index, offset, path, &rec);
 		if (ret < 0)
 			ctx->failed = TRUE;



More information about the dovecot-cvs mailing list