[dovecot-cvs] dovecot/src/lib-index mailbox-list-index-private.h, 1.2, 1.3 mailbox-list-index-sync.c, 1.2, 1.3 mailbox-list-index.c, 1.2, 1.3
tss at dovecot.org
tss at dovecot.org
Wed Dec 20 08:16:45 UTC 2006
Update of /var/lib/cvs/dovecot/src/lib-index
In directory talvi:/tmp/cvs-serv9945/lib-index
Modified Files:
mailbox-list-index-private.h mailbox-list-index-sync.c
mailbox-list-index.c
Log Message:
Compress list index when its deleted space amount gets too high. Also fixed
several bugs.
Index: mailbox-list-index-private.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mailbox-list-index-private.h,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- mailbox-list-index-private.h 10 Dec 2006 16:04:46 -0000 1.2
+++ mailbox-list-index-private.h 20 Dec 2006 08:16:41 -0000 1.3
@@ -6,6 +6,9 @@
#define MAILBOX_LIST_INDEX_MAJOR_VERSION 1
#define MAILBOX_LIST_INDEX_MINOR_VERSION 0
+#define MAILBOX_LIST_COMPRESS_PERCENTAGE 10
+#define MAILBOX_LIST_COMPRESS_MIN_SIZE 1024
+
struct mailbox_list_index_header {
uint8_t major_version;
uint8_t minor_version;
@@ -82,4 +85,10 @@
const struct mailbox_list_dir_record **dir_r);
int mailbox_list_index_map(struct mailbox_list_index *index);
+int mailbox_list_index_file_create(struct mailbox_list_index *index,
+ uint32_t uid_validity);
+void mailbox_list_index_file_close(struct mailbox_list_index *index);
+
+int mailbox_list_index_refresh(struct mailbox_list_index *index);
+
#endif
Index: mailbox-list-index-sync.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mailbox-list-index-sync.c,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- mailbox-list-index-sync.c 10 Dec 2006 16:04:46 -0000 1.2
+++ mailbox-list-index-sync.c 20 Dec 2006 08:16:41 -0000 1.3
@@ -69,6 +69,9 @@
bool *match;
};
+static bool mailbox_list_index_need_compress(struct mailbox_list_index *index);
+static int mailbox_list_index_compress(struct mailbox_list_index_sync_ctx *ctx);
+
static struct mailbox_list_sync_dir *
mailbox_list_alloc_sync_dir(struct mailbox_list_index_sync_ctx *ctx,
unsigned int initial_count)
@@ -105,6 +108,11 @@
if (recs[i].deleted)
continue;
+ if (recs[i].uid == 0) {
+ return mailbox_list_index_set_corrupted(ctx->index,
+ "Record with UID=0");
+ }
+
sync_rec = array_append_space(&sync_dir->records);
sync_rec->name_hash = recs[i].name_hash;
sync_rec->uid = recs[i].uid;
@@ -317,10 +325,9 @@
&ctx->sync_root, &seq);
}
-static int sync_init_mail_sync(struct mailbox_list_index_sync_ctx *ctx)
+static int sync_mail_sync_init(struct mailbox_list_index_sync_ctx *ctx)
{
struct mail_index_sync_rec sync_rec;
- const struct mail_index_header *hdr;
if (mail_index_sync_begin(ctx->index->mail_index, &ctx->mail_sync_ctx,
&ctx->view, (uint32_t)-1, 0,
@@ -331,6 +338,14 @@
don't need to do anything but write them to the index */
while (mail_index_sync_next(ctx->mail_sync_ctx, &sync_rec) > 0)
;
+ return 0;
+}
+
+static int sync_mail_sync_init2(struct mailbox_list_index_sync_ctx *ctx)
+{
+ const struct mail_index_header *hdr;
+
+ ctx->hdr = *ctx->index->hdr;
hdr = mail_index_get_header(ctx->view);
if (hdr->uid_validity != 0) {
@@ -348,7 +363,7 @@
TRUE);
}
- return mailbox_list_index_get_root(ctx);
+ return 0;
}
int mailbox_list_index_sync_init(struct mailbox_list_index *index,
@@ -372,11 +387,13 @@
ctx->index = index;
ctx->sync_path = p_strdup(pool, path);
ctx->flags = flags;
- ctx->hdr = *index->hdr;
/* mail index syncing acts as the only locking for us */
- if (sync_init_mail_sync(ctx) < 0) {
- mailbox_list_index_sync_commit(&ctx);
+ if (sync_mail_sync_init(ctx) < 0 ||
+ mailbox_list_index_refresh(index) < 0 ||
+ sync_mail_sync_init2(ctx) < 0 ||
+ mailbox_list_index_get_root(ctx) < 0) {
+ mailbox_list_index_sync_rollback(&ctx);
return -1;
}
@@ -745,6 +762,7 @@
{
struct mailbox_list_index_header *hdr;
bool partial;
+ int ret = 0;
if (ctx->index->mmap_disable) {
ctx->output = o_stream_create_file(ctx->index->fd, default_pool,
@@ -763,29 +781,30 @@
}
if (mailbox_list_index_sync_write_dir(ctx, ctx->root, 0, partial) < 0)
- return -1;
+ ret = -1;
if (!ctx->index->mmap_disable) {
/* update header */
hdr = ctx->index->mmap_base;
- hdr->next_uid = ctx->hdr.next_uid;
- hdr->used_space = ctx->hdr.used_space;
- hdr->deleted_space = ctx->hdr.deleted_space;
+ if (ret == 0)
+ memcpy(hdr, &ctx->hdr, sizeof(*hdr));
if (msync(ctx->index->mmap_base,
hdr->used_space, MS_SYNC) < 0) {
mailbox_list_index_set_syscall_error(ctx->index,
"msync()");
- return -1;
+ ret = -1;
}
} else {
- o_stream_seek(ctx->output, 0);
- o_stream_send(ctx->output, &ctx->hdr, sizeof(ctx->hdr));
+ if (ret == 0) {
+ o_stream_seek(ctx->output, 0);
+ o_stream_send(ctx->output, &ctx->hdr, sizeof(ctx->hdr));
+ }
o_stream_destroy(&ctx->output);
buffer_free(ctx->output_buf);
}
- return 0;
+ return ret;
}
int mailbox_list_index_sync_commit(struct mailbox_list_index_sync_ctx **_ctx)
@@ -798,6 +817,8 @@
if (!ctx->failed) {
/* write all the changes to the index */
ret = mailbox_list_index_sync_write(ctx);
+ if (ret == 0 && mailbox_list_index_need_compress(ctx->index))
+ ret = mailbox_list_index_compress(ctx);
}
if (ctx->mail_sync_ctx != NULL) {
@@ -829,3 +850,67 @@
(*ctx)->failed = TRUE;
(void)mailbox_list_index_sync_commit(ctx);
}
+
+static bool mailbox_list_index_need_compress(struct mailbox_list_index *index)
+{
+ uoff_t max_del_space;
+
+ max_del_space = index->hdr->used_space / 100 *
+ MAILBOX_LIST_COMPRESS_PERCENTAGE;
+ if (index->hdr->deleted_space >= max_del_space &&
+ index->hdr->used_space >= MAILBOX_LIST_COMPRESS_MIN_SIZE)
+ return TRUE;
+
+ return FALSE;
+}
+
+static int mailbox_list_copy_to_mem_all(struct mailbox_list_index_sync_ctx *ctx,
+ struct mailbox_list_sync_dir *dir)
+{
+ struct mailbox_list_sync_record *recs;
+ unsigned int i, count;
+
+ /* mark the directories as new */
+ dir->offset = 0;
+ dir->new_records_count = 1;
+
+ recs = array_get_modifiable(&dir->records, &count);
+ for (i = 0; i < count; i++) {
+ recs[i].created = TRUE;
+ recs[i].seen = TRUE;
+
+ if (recs[i].dir == NULL) {
+ if (recs[i].dir_offset == 0)
+ continue;
+
+ if (mailbox_list_copy_sync_dir(ctx, recs[i].dir_offset,
+ &recs[i].dir) < 0)
+ return -1;
+ }
+ recs[i].dir_offset = 0;
+
+ if (mailbox_list_copy_to_mem_all(ctx, recs[i].dir) < 0)
+ return -1;
+ }
+ return 0;
+}
+
+static int mailbox_list_index_compress(struct mailbox_list_index_sync_ctx *ctx)
+{
+ /* first read everything to memory */
+ if (mailbox_list_copy_to_mem_all(ctx, ctx->root) < 0)
+ return -1;
+
+ /* truncate the index file */
+ mailbox_list_index_file_close(ctx->index);
+ if (mailbox_list_index_file_create(ctx->index,
+ ctx->hdr.uid_validity) < 0)
+ return -1;
+
+ /* reset header */
+ ctx->hdr.used_space = sizeof(ctx->hdr);
+ ctx->hdr.deleted_space = 0;
+
+ /* and write everything back */
+ return mailbox_list_index_sync_write(ctx);
+}
Index: mailbox-list-index.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mailbox-list-index.c,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- mailbox-list-index.c 10 Dec 2006 16:04:46 -0000 1.2
+++ mailbox-list-index.c 20 Dec 2006 08:16:41 -0000 1.3
@@ -71,7 +71,7 @@
index->hdr = NULL;
}
-static void mailbox_list_index_file_close(struct mailbox_list_index *index)
+void mailbox_list_index_file_close(struct mailbox_list_index *index)
{
mailbox_list_index_unmap(index);
@@ -80,6 +80,7 @@
if (index->fd != -1) {
if (close(index->fd) < 0)
mailbox_list_index_set_syscall_error(index, "close()");
+ index->fd = -1;
}
}
@@ -87,7 +88,7 @@
const char *str)
{
(void)unlink(index->filepath);
- // FIXME: reopen or something
+ mailbox_list_index_file_close(index);
i_error("Corrupted mailbox list index file %s: %s",
index->filepath, str);
@@ -174,7 +175,8 @@
}
static void
-mailbox_list_index_init_header(struct mailbox_list_index_header *hdr)
+mailbox_list_index_init_header(struct mailbox_list_index_header *hdr,
+ uint32_t uid_validity)
{
memset(hdr, 0, sizeof(*hdr));
hdr->major_version = MAILBOX_LIST_INDEX_MAJOR_VERSION;
@@ -183,7 +185,7 @@
hdr->header_size = sizeof(*hdr);
hdr->used_space = hdr->header_size;
- hdr->uid_validity = ioloop_time;
+ hdr->uid_validity = uid_validity;
hdr->next_uid = 1;
}
@@ -191,7 +193,13 @@
{
struct stat st1, st2;
+ if (index->fd == -1)
+ return 1;
+
if (stat(index->filepath, &st1) < 0) {
+ if (errno == ENOENT)
+ return 1;
+
mailbox_list_index_set_syscall_error(index, "stat()");
return -1;
}
@@ -204,8 +212,8 @@
!CMP_DEV_T(st1.st_dev, st2.st_dev);
}
-static int
-mailbox_list_index_file_create(struct mailbox_list_index *index)
+int mailbox_list_index_file_create(struct mailbox_list_index *index,
+ uint32_t uid_validity)
{
struct mailbox_list_index_header hdr;
struct dotlock *dotlock;
@@ -228,7 +236,7 @@
}
}
- mailbox_list_index_init_header(&hdr);
+ mailbox_list_index_init_header(&hdr, uid_validity);
if (write_full(fd, &hdr, sizeof(hdr)) < 0) {
mailbox_list_index_set_syscall_error(index, "write_full()");
(void)file_dotlock_delete(&dotlock);
@@ -278,7 +286,7 @@
}
}
- ret = mailbox_list_index_file_create(index);
+ ret = mailbox_list_index_file_create(index, ioloop_time);
if (ret <= 0)
mailbox_list_index_file_close(index);
return ret;
@@ -483,12 +491,18 @@
return mailbox_list_index_lookup_rec(index, dir_offset, p + 1, rec_r);
}
-static int mailbox_list_index_refresh(struct mailbox_list_index *index)
+int mailbox_list_index_refresh(struct mailbox_list_index *index)
{
int ret;
- if ((ret = mailbox_list_index_is_recreated(index)) <= 0)
+ if ((ret = mailbox_list_index_is_recreated(index)) <= 0) {
+ if (ret < 0)
+ return -1;
+
+ if (mailbox_list_index_map(index) < 0)
+ ret = -1;
return ret;
+ }
mailbox_list_index_file_close(index);
return mailbox_list_index_open_or_create(index);
@@ -529,7 +543,9 @@
(unsigned int)recurse_level;
ctx->name_path = str_new(default_pool, 512);
- if (*path != '\0') {
+ if (mailbox_list_index_refresh(index) < 0)
+ ctx->failed = TRUE;
+ if (!ctx->failed && *path != '\0') {
ret = mailbox_list_index_lookup_rec(index, offset, path, &rec);
if (ret < 0)
ctx->failed = TRUE;
More information about the dovecot-cvs
mailing list