[dovecot-cvs] dovecot/src/lib-storage/index/mbox
mbox-sync-private.h, 1.23, 1.24 mbox-sync-rewrite.c, 1.23,
1.24 mbox-sync.c, 1.55, 1.56
cras at procontrol.fi
cras at procontrol.fi
Fri Jul 2 21:30:19 EEST 2004
Update of /home/cvs/dovecot/src/lib-storage/index/mbox
In directory talvi:/tmp/cvs-serv24399/src/lib-storage/index/mbox
Modified Files:
mbox-sync-private.h mbox-sync-rewrite.c mbox-sync.c
Log Message:
Don't keep filling empty spaces which will be overwritten soon anyway. Major
speedup for rewriting large mboxes.
Index: mbox-sync-private.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync-private.h,v
retrieving revision 1.23
retrieving revision 1.24
diff -u -d -r1.23 -r1.24
--- mbox-sync-private.h 28 Jun 2004 22:03:24 -0000 1.23
+++ mbox-sync-private.h 2 Jul 2004 18:30:17 -0000 1.24
@@ -21,6 +21,7 @@
#define MBOX_NONRECENT MAIL_RECENT /* kludgy */
#define MBOX_EXPUNGED 0x40
+#define MBOX_DIRTY_SPACE 0x80
#define STATUS_FLAGS_MASK (MAIL_SEEN|MBOX_NONRECENT)
#define XSTATUS_FLAGS_MASK (MAIL_ANSWERED|MAIL_FLAGGED|MAIL_DRAFT|MAIL_DELETED)
@@ -114,7 +115,8 @@
buffer_t *syncs_buf);
void mbox_sync_update_header_from(struct mbox_sync_mail_context *ctx,
const struct mbox_sync_mail *mail);
-int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx, off_t move_diff);
+int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx, off_t move_diff,
+ int leave_space_hole);
int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx, uoff_t extra_space,
uint32_t first_seq, uint32_t last_seq);
Index: mbox-sync-rewrite.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync-rewrite.c,v
retrieving revision 1.23
retrieving revision 1.24
diff -u -d -r1.23 -r1.24
--- mbox-sync-rewrite.c 28 Jun 2004 22:03:24 -0000 1.23
+++ mbox-sync-rewrite.c 2 Jul 2004 18:30:17 -0000 1.24
@@ -72,7 +72,7 @@
}
static void mbox_sync_headers_add_space(struct mbox_sync_mail_context *ctx,
- size_t size)
+ size_t size, size_t *skip_space_pos_r)
{
size_t data_size, pos, start_pos;
const unsigned char *data;
@@ -105,19 +105,30 @@
}
}
- /* pos points to end of headers now, and start_pos to beginning
+ /* pos points to end of header now, and start_pos to beginning
of whitespace. */
- buffer_copy(ctx->header, pos + size,
- ctx->header, pos, (size_t)-1);
- p = buffer_get_space_unsafe(ctx->header, pos, size);
- memset(p, ' ', size);
- ctx->mail.offset = ctx->hdr_offset + start_pos;
- ctx->mail.space = (pos - start_pos) + size;
+ if (/*size > 1024 && */skip_space_pos_r != NULL) {
+ /* we're rewriting messages and moving space towards beginning
+ of file. it's useless to write lots of spaces because we're
+ overwriting it soon anyway. */
+ *skip_space_pos_r = start_pos;
+ } else {
+ if (skip_space_pos_r != NULL)
+ *skip_space_pos_r = (size_t)-1;
+
+ buffer_copy(ctx->header, pos + size,
+ ctx->header, pos, (size_t)-1);
+ p = buffer_get_space_unsafe(ctx->header, pos, size);
+ memset(p, ' ', size);
+ }
if (ctx->header_first_change > pos)
ctx->header_first_change = pos;
ctx->header_last_change = (size_t)-1;
+
+ ctx->mail.offset = ctx->hdr_offset + start_pos;
+ ctx->mail.space = (pos - start_pos) + size;
}
static void mbox_sync_header_remove_space(struct mbox_sync_mail_context *ctx,
@@ -201,19 +212,23 @@
/* FIXME: see if we could remove X-Keywords header completely */
}
-int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx, off_t move_diff)
+int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx, off_t move_diff,
+ int leave_space_hole)
{
- size_t old_hdr_size, new_hdr_size;
+ size_t old_hdr_size, new_hdr_size, space_pos, hdr_size;
i_assert(ctx->sync_ctx->ibox->mbox_lock_type == F_WRLCK);
old_hdr_size = ctx->body_offset - ctx->hdr_offset;
new_hdr_size = str_len(ctx->header);
+ space_pos = (size_t)-1;
if (new_hdr_size <= old_hdr_size) {
/* add space. note that we must call add_space() even if we're
not adding anything so mail.offset gets fixed. */
- mbox_sync_headers_add_space(ctx, old_hdr_size - new_hdr_size);
+ mbox_sync_headers_add_space(ctx, old_hdr_size - new_hdr_size,
+ leave_space_hole ?
+ &space_pos : NULL);
} else if (new_hdr_size > old_hdr_size) {
/* try removing the space where we can */
mbox_sync_headers_remove_space(ctx,
@@ -259,14 +274,29 @@
ctx->header_last_change != 0)
str_truncate(ctx->header, ctx->header_last_change);
+ i_assert(space_pos >= ctx->header_first_change);
+
+ hdr_size = space_pos == (size_t)-1 ? str_len(ctx->header) : space_pos;
if (pwrite_full(ctx->sync_ctx->fd,
str_data(ctx->header) + ctx->header_first_change,
- str_len(ctx->header) - ctx->header_first_change,
- ctx->hdr_offset + move_diff +
- ctx->header_first_change) < 0) {
+ hdr_size - ctx->header_first_change,
+ ctx->hdr_offset + ctx->header_first_change +
+ move_diff) < 0) {
mbox_set_syscall_error(ctx->sync_ctx->ibox, "pwrite_full()");
return -1;
}
+ if (space_pos != (size_t)-1) {
+ ctx->mail.flags |= MBOX_DIRTY_SPACE;
+ if (pwrite_full(ctx->sync_ctx->fd,
+ str_data(ctx->header) + space_pos,
+ str_len(ctx->header) - space_pos,
+ ctx->mail.offset + ctx->mail.space +
+ move_diff) < 0) {
+ mbox_set_syscall_error(ctx->sync_ctx->ibox,
+ "pwrite_full()");
+ return -1;
+ }
+ }
if (ctx->sync_ctx->dest_first_mail) {
ctx->sync_ctx->base_uid_last =
@@ -286,7 +316,7 @@
struct mbox_sync_mail_context mail_ctx;
uint32_t old_prev_msg_uid;
uoff_t hdr_offset, offset, dest_offset;
- size_t old_hdr_size, need_space;
+ size_t old_hdr_size, need_space, space_pos, hdr_size;
if (mbox_sync_seek(sync_ctx, mails[idx].from_offset) < 0)
return -1;
@@ -295,6 +325,8 @@
mail_ctx.sync_ctx = sync_ctx;
mail_ctx.seq = seq;
mail_ctx.header = sync_ctx->header;
+ mail_ctx.uidl = sync_ctx->uidl;
+ str_truncate(mail_ctx.uidl, 0);
hdr_offset = mails[idx].offset;
mail_ctx.mail.offset = mails[idx].offset;
@@ -328,9 +360,10 @@
if (space_diff - need_space < (uoff_t)mail_ctx.mail.space) {
mbox_sync_headers_remove_space(&mail_ctx, mail_ctx.mail.space -
(space_diff - need_space));
+ space_pos = (size_t)-1;
} else {
mbox_sync_headers_add_space(&mail_ctx, space_diff - need_space -
- mail_ctx.mail.space);
+ mail_ctx.mail.space, &space_pos);
}
mails[idx].offset = mail_ctx.mail.offset;
mails[idx].space = mail_ctx.mail.space;
@@ -343,11 +376,24 @@
end_offset - dest_offset) < 0)
return -1;
+ hdr_size = space_pos == (size_t)-1 ?
+ str_len(mail_ctx.header) : space_pos;
if (pwrite_full(sync_ctx->fd, str_data(mail_ctx.header),
- str_len(mail_ctx.header), hdr_offset) < 0) {
+ hdr_size, hdr_offset) < 0) {
mbox_set_syscall_error(sync_ctx->ibox, "pwrite_full()");
return -1;
}
+ if (space_pos != (size_t)-1) {
+ /* skip over the space in header, it's written later. */
+ mails[idx].flags |= MBOX_DIRTY_SPACE;
+ if (pwrite_full(sync_ctx->fd,
+ str_data(mail_ctx.header) + space_pos,
+ str_len(mail_ctx.header) - space_pos,
+ mails[idx].offset + mails[idx].space) < 0) {
+ mbox_set_syscall_error(sync_ctx->ibox, "pwrite_full()");
+ return -1;
+ }
+ }
if (mails[idx].from_offset == 0) {
sync_ctx->base_uid_last =
@@ -435,18 +481,30 @@
break;
}
- if (mbox_fill_space(sync_ctx, offset,
- dest_offset - offset) < 0) {
- ret = -1;
- break;
- }
-
mails[idx].space += space_diff;
+ mails[idx].flags |= MBOX_DIRTY_SPACE;
}
mails[idx+1].from_offset += space_diff;
+ mails[idx+1].offset += space_diff;
+
+ if ((mails[idx+1].flags & MBOX_DIRTY_SPACE) != 0) {
+ mails[idx+1].flags &= ~MBOX_DIRTY_SPACE;
+ if (mbox_fill_space(sync_ctx, mails[idx+1].offset,
+ mails[idx+1].space) < 0) {
+ ret = -1;
+ break;
+ }
+ }
} while (idx > 0);
+ if ((mails[0].flags & MBOX_DIRTY_SPACE) != 0) {
+ mails[0].flags &= ~MBOX_DIRTY_SPACE;
+ if (mbox_fill_space(sync_ctx, mails[0].offset,
+ mails[0].space) < 0)
+ ret = -1;
+ }
+
istream_raw_mbox_flush(sync_ctx->input);
return ret;
}
Index: mbox-sync.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync.c,v
retrieving revision 1.55
retrieving revision 1.56
diff -u -d -r1.55 -r1.56
--- mbox-sync.c 2 Jul 2004 12:14:13 -0000 1.55
+++ mbox-sync.c 2 Jul 2004 18:30:17 -0000 1.56
@@ -629,7 +629,8 @@
return -1;
mbox_sync_update_header(mail_ctx, sync_ctx->syncs);
- if ((ret = mbox_sync_try_rewrite(mail_ctx, move_diff)) < 0)
+ ret = mbox_sync_try_rewrite(mail_ctx, move_diff, FALSE);
+ if (ret < 0)
return -1;
if (ret > 0) {
@@ -648,7 +649,7 @@
return ret;
mbox_sync_update_header(mail_ctx, sync_ctx->syncs);
- if ((ret = mbox_sync_try_rewrite(mail_ctx, 0)) < 0)
+ if ((ret = mbox_sync_try_rewrite(mail_ctx, 0, FALSE)) < 0)
return -1;
} else {
/* nothing to do */
@@ -929,6 +930,7 @@
struct mbox_sync_mail_context *mail_ctx)
{
uoff_t offset, extra_space, trailer_size;
+ int need_rewrite;
if (!istream_raw_mbox_is_eof(sync_ctx->input)) {
i_assert(sync_ctx->need_space_seq == 0);
@@ -959,10 +961,11 @@
-sync_ctx->space_diff) < 0)
return -1;
- if (mbox_sync_try_rewrite(mail_ctx, 0) < 0)
+ need_rewrite = sync_ctx->seq != sync_ctx->need_space_seq;
+ if (mbox_sync_try_rewrite(mail_ctx, 0, need_rewrite) < 0)
return -1;
- if (sync_ctx->seq != sync_ctx->need_space_seq) {
+ if (need_rewrite) {
buffer_set_used_size(sync_ctx->mails,
(sync_ctx->seq -
sync_ctx->need_space_seq) *
More information about the dovecot-cvs
mailing list