[dovecot-cvs] dovecot/src/lib-storage/index/mbox
mbox-sync-private.h, 1.34, 1.35 mbox-sync-rewrite.c, 1.31,
1.32 mbox-sync.c, 1.95, 1.96
cras at dovecot.org
cras at dovecot.org
Tue Oct 5 03:26:42 EEST 2004
Update of /var/lib/cvs/dovecot/src/lib-storage/index/mbox
In directory talvi:/tmp/cvs-serv1546/lib-storage/index/mbox
Modified Files:
mbox-sync-private.h mbox-sync-rewrite.c mbox-sync.c
Log Message:
Bringing some sanity into mbox rewrites. Fixes expunge crashes and reduces
writes.
Index: mbox-sync-private.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync-private.h,v
retrieving revision 1.34
retrieving revision 1.35
diff -u -d -r1.34 -r1.35
--- mbox-sync-private.h 24 Sep 2004 11:47:30 -0000 1.34
+++ mbox-sync-private.h 5 Oct 2004 00:26:40 -0000 1.35
@@ -29,7 +29,6 @@
#define MBOX_NONRECENT MAIL_RECENT /* kludgy */
#define MBOX_EXPUNGED 0x40
-#define MBOX_DIRTY_SPACE 0x80
#define STATUS_FLAGS_MASK (MAIL_SEEN|MBOX_NONRECENT)
#define XSTATUS_FLAGS_MASK (MAIL_ANSWERED|MAIL_FLAGGED|MAIL_DRAFT|MAIL_DELETED)
@@ -129,9 +128,9 @@
buffer_t *syncs_buf);
void mbox_sync_update_header_from(struct mbox_sync_mail_context *ctx,
const struct mbox_sync_mail *mail);
-int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx, off_t move_diff,
- int leave_space_hole);
-int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx, uoff_t extra_space,
+int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx, off_t move_diff);
+int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx,
+ uoff_t end_offset, uoff_t move_diff, uoff_t extra_space,
uint32_t first_seq, uint32_t last_seq);
int mbox_sync_seek(struct mbox_sync_context *sync_ctx, uoff_t from_offset);
Index: mbox-sync-rewrite.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync-rewrite.c,v
retrieving revision 1.31
retrieving revision 1.32
diff -u -d -r1.31 -r1.32
--- mbox-sync-rewrite.c 23 Sep 2004 22:29:13 -0000 1.31
+++ mbox-sync-rewrite.c 5 Oct 2004 00:26:40 -0000 1.32
@@ -65,8 +65,8 @@
return 0;
}
-static void mbox_sync_headers_add_space(struct mbox_sync_mail_context *ctx,
- size_t size, size_t *skip_space_pos_r)
+static void
+mbox_sync_headers_add_space(struct mbox_sync_mail_context *ctx, size_t size)
{
size_t data_size, pos, start_pos;
const unsigned char *data;
@@ -102,23 +102,10 @@
/* pos points to end of header now, and start_pos to beginning
of whitespace. */
- if (size > 1024 && skip_space_pos_r != NULL) {
- /* we're rewriting messages and moving space towards beginning
- of file. it's useless to write lots of spaces because we're
- overwriting it soon anyway. */
- *skip_space_pos_r = start_pos;
-
- /* we'll have to delete the existing space */
- buffer_delete(ctx->header, start_pos, pos - start_pos);
- } else {
- if (skip_space_pos_r != NULL)
- *skip_space_pos_r = (size_t)-1;
-
- buffer_copy(ctx->header, pos + size,
- ctx->header, pos, (size_t)-1);
- p = buffer_get_space_unsafe(ctx->header, pos, size);
- memset(p, ' ', size);
- }
+ buffer_copy(ctx->header, pos + size,
+ ctx->header, pos, (size_t)-1);
+ p = buffer_get_space_unsafe(ctx->header, pos, size);
+ memset(p, ' ', size);
if (ctx->header_first_change > pos)
ctx->header_first_change = pos;
@@ -209,23 +196,19 @@
/* FIXME: see if we could remove X-Keywords header completely */
}
-int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx, off_t move_diff,
- int leave_space_hole)
+int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx, off_t move_diff)
{
- size_t old_hdr_size, new_hdr_size, space_pos, hdr_size;
+ size_t old_hdr_size, new_hdr_size;
i_assert(ctx->sync_ctx->ibox->mbox_lock_type == F_WRLCK);
old_hdr_size = ctx->body_offset - ctx->hdr_offset;
new_hdr_size = str_len(ctx->header);
- space_pos = (size_t)-1;
if (new_hdr_size <= old_hdr_size) {
/* add space. note that we must call add_space() even if we're
not adding anything so mail.offset gets fixed. */
- mbox_sync_headers_add_space(ctx, old_hdr_size - new_hdr_size,
- leave_space_hole ?
- &space_pos : NULL);
+ mbox_sync_headers_add_space(ctx, old_hdr_size - new_hdr_size);
} else if (new_hdr_size > old_hdr_size) {
/* try removing the space where we can */
mbox_sync_headers_remove_space(ctx,
@@ -271,29 +254,14 @@
ctx->header_last_change != 0)
str_truncate(ctx->header, ctx->header_last_change);
- i_assert(space_pos >= ctx->header_first_change);
-
- hdr_size = space_pos == (size_t)-1 ? str_len(ctx->header) : space_pos;
if (pwrite_full(ctx->sync_ctx->fd,
str_data(ctx->header) + ctx->header_first_change,
- hdr_size - ctx->header_first_change,
+ str_len(ctx->header) - ctx->header_first_change,
ctx->hdr_offset + ctx->header_first_change +
move_diff) < 0) {
mbox_set_syscall_error(ctx->sync_ctx->ibox, "pwrite_full()");
return -1;
}
- if (space_pos != (size_t)-1) {
- ctx->mail.flags |= MBOX_DIRTY_SPACE;
- if (pwrite_full(ctx->sync_ctx->fd,
- str_data(ctx->header) + space_pos,
- str_len(ctx->header) - space_pos,
- ctx->mail.offset + ctx->mail.space +
- move_diff) < 0) {
- mbox_set_syscall_error(ctx->sync_ctx->ibox,
- "pwrite_full()");
- return -1;
- }
- }
if (ctx->sync_ctx->dest_first_mail) {
ctx->sync_ctx->base_uid_last =
@@ -307,13 +275,13 @@
static int mbox_sync_read_and_move(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail *mails,
- uint32_t seq, uint32_t idx,
- uoff_t space_diff, uoff_t end_offset)
+ uint32_t seq, uint32_t idx, uint32_t padding,
+ uoff_t move_diff, uoff_t end_offset)
{
struct mbox_sync_mail_context mail_ctx;
uint32_t old_prev_msg_uid;
uoff_t hdr_offset, offset, dest_offset;
- size_t old_hdr_size, need_space, space_pos, hdr_size;
+ size_t need_space;
if (mbox_sync_seek(sync_ctx, mails[idx].from_offset) < 0)
return -1;
@@ -348,50 +316,39 @@
sync_ctx->prev_msg_uid = old_prev_msg_uid;
sync_ctx->dest_first_mail = FALSE;
- old_hdr_size = mail_ctx.body_offset - mail_ctx.hdr_offset;
need_space = str_len(mail_ctx.header) - mail_ctx.mail.space -
- old_hdr_size;
+ (mail_ctx.body_offset - mail_ctx.hdr_offset);
i_assert(need_space == (uoff_t)-mails[idx].space);
- i_assert(space_diff >= need_space);
- if (space_diff - need_space < (uoff_t)mail_ctx.mail.space) {
+ if (padding < (uoff_t)mail_ctx.mail.space) {
mbox_sync_headers_remove_space(&mail_ctx, mail_ctx.mail.space -
- (space_diff - need_space));
- space_pos = (size_t)-1;
+ padding);
} else {
- mbox_sync_headers_add_space(&mail_ctx, space_diff - need_space -
- mail_ctx.mail.space, &space_pos);
+ mbox_sync_headers_add_space(&mail_ctx, padding -
+ mail_ctx.mail.space);
}
- mails[idx].offset = mail_ctx.mail.offset;
- mails[idx].space = mail_ctx.mail.space;
/* move the body of this message and headers of next message forward,
then write the headers */
offset = sync_ctx->input->v_offset;
- dest_offset = offset + space_diff;
- if (mbox_move(sync_ctx, dest_offset, offset,
- end_offset == (uoff_t)-1 ? mail_ctx.mail.body_size :
- end_offset - dest_offset) < 0)
+ dest_offset = offset + move_diff;
+ i_assert(offset <= end_offset);
+ if (mbox_move(sync_ctx, dest_offset, offset, end_offset - offset) < 0)
return -1;
- hdr_size = space_pos == (size_t)-1 ?
- str_len(mail_ctx.header) : space_pos;
+ /* the header may actually be moved backwards if there was expunged
+ space which we wanted to remove */
+ dest_offset -= str_len(mail_ctx.header);
if (pwrite_full(sync_ctx->fd, str_data(mail_ctx.header),
- hdr_size, hdr_offset) < 0) {
+ str_len(mail_ctx.header), dest_offset) < 0) {
mbox_set_syscall_error(sync_ctx->ibox, "pwrite_full()");
return -1;
}
- if (space_pos != (size_t)-1) {
- /* skip over the space in header, it's written later. */
- mails[idx].flags |= MBOX_DIRTY_SPACE;
- if (pwrite_full(sync_ctx->fd,
- str_data(mail_ctx.header) + space_pos,
- str_len(mail_ctx.header) - space_pos,
- mails[idx].offset + mails[idx].space) < 0) {
- mbox_set_syscall_error(sync_ctx->ibox, "pwrite_full()");
- return -1;
- }
- }
+
+ mails[idx].offset = dest_offset +
+ (mail_ctx.mail.offset - mail_ctx.hdr_offset);
+ mails[idx].space = mail_ctx.mail.space;
+ i_assert(mails[idx].space == padding);
if (mails[idx].from_offset == 0) {
sync_ctx->base_uid_last =
@@ -404,12 +361,13 @@
/* extra_space specifies how many bytes from last_seq's space will be left
over after all the rewrites. */
-int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx, uoff_t extra_space,
+int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx,
+ uoff_t end_offset, uoff_t move_diff, uoff_t extra_space,
uint32_t first_seq, uint32_t last_seq)
{
struct mbox_sync_mail *mails;
- uoff_t offset, end_offset, dest_offset, space_diff;
- uint32_t idx, padding_per_mail;
+ uoff_t offset, dest_offset, next_end_offset, next_move_diff;
+ uint32_t idx, first_nonexpunged_idx, padding_per_mail;
size_t size;
int ret = 0;
@@ -418,105 +376,81 @@
mails = buffer_get_modifyable_data(sync_ctx->mails, &size);
i_assert(size / sizeof(*mails) == last_seq - first_seq + 1);
- if (first_seq == last_seq) {
- /* just move this mail forward */
- if (mbox_sync_read_and_move(sync_ctx, mails, first_seq, 0,
- -mails[0].space + extra_space,
- (uoff_t)-1) < 0)
- return -1;
-
- if ((mails[0].flags & MBOX_DIRTY_SPACE) != 0) {
- mails[0].flags &= ~MBOX_DIRTY_SPACE;
- if (mbox_fill_space(sync_ctx, mails[0].offset,
- mails[0].space) < 0)
- return -1;
- }
-
- istream_raw_mbox_flush(sync_ctx->input);
- return 0;
- }
-
/* if there's expunges in mails[], we would get more correct balancing
by counting only them here. however, that might make us overwrite
data which hasn't yet been copied backwards. to avoid too much
complexity, we just leave all the rest of the extra space to first
mail */
- idx = last_seq - first_seq;
- padding_per_mail = extra_space / (idx + 1);
+ idx = last_seq - first_seq + 1;
+ padding_per_mail = extra_space / idx;
/* after expunge the next mail must have been missing space, or we
would have moved it backwards already */
- i_assert(mails[0].space < 0 || (mails[0].flags & MBOX_EXPUNGED) != 0);
+ for (first_nonexpunged_idx = 0;; first_nonexpunged_idx++) {
+ i_assert(first_nonexpunged_idx != idx);
+ if ((mails[first_nonexpunged_idx].flags & MBOX_EXPUNGED) == 0)
+ break;
+ }
+ i_assert(mails[first_nonexpunged_idx].space < 0);
/* start moving backwards. */
- do {
- /* this message's body is always moved space_diff bytes
- forward along with next message's headers, so current
- message gets temporarily space_diff amount of padding.
-
- the moving stops at next message's beginning of padding.
- each message gets left padding_per_mail bytes of space.
- what gets left over is given to first message */
- i_assert(mails[idx].space > 0);
- space_diff = mails[idx].space;
- end_offset = mails[idx].offset + mails[idx].space;
-
- if ((mails[idx].flags & MBOX_EXPUNGED) == 0) {
- space_diff -= padding_per_mail;
- end_offset -= padding_per_mail;
- mails[idx].space = padding_per_mail;
+ while (idx-- > first_nonexpunged_idx) {
+ if (idx == first_nonexpunged_idx) {
+ /* give the rest of the extra space to first mail */
+ i_assert((uoff_t)-mails[idx].space <= move_diff);
+ i_assert(move_diff + mails[idx].space >=
+ padding_per_mail);
+ padding_per_mail = move_diff + mails[idx].space;
}
- idx--;
+ next_end_offset = mails[idx].offset;
if (mails[idx].space <= 0 &&
(mails[idx].flags & MBOX_EXPUNGED) == 0) {
- /* offset points to beginning of headers. read the
- header again, update it and give enough space to
- fill space_diff */
+ /* give space to this mail */
+ next_move_diff = -mails[idx].space + padding_per_mail;
if (mbox_sync_read_and_move(sync_ctx, mails,
first_seq + idx, idx,
- space_diff,
+ padding_per_mail, move_diff,
end_offset) < 0) {
ret = -1;
break;
}
+ move_diff -= next_move_diff;
} else {
- /* X-Keywords: xx [offset]\n
- ...
- X-Keywords: xx [end_offset] \n
+ /* this mail provides more space. just move it forward
+ from the extra space offset and set end_offset to
+ point to beginning of extra space. that way the
+ header will be moved along with previous mail's
+ body.
- move data forward so mails before us gets the extra
- space (ie. we temporarily get more space to us) */
+ if this is expunged mail, we're moving following
+ mail's From-line and maybe headers. */
offset = mails[idx].offset + mails[idx].space;
- dest_offset = offset + space_diff;
+ dest_offset = offset + move_diff;
+ i_assert(offset <= end_offset);
if (mbox_move(sync_ctx, dest_offset, offset,
- end_offset - dest_offset) < 0) {
+ end_offset - offset) < 0) {
ret = -1;
break;
}
- mails[idx].space += space_diff;
- mails[idx].flags |= MBOX_DIRTY_SPACE;
- }
-
- mails[idx+1].from_offset += space_diff;
- mails[idx+1].offset += space_diff;
-
- if ((mails[idx+1].flags & MBOX_DIRTY_SPACE) != 0) {
- mails[idx+1].flags &= ~MBOX_DIRTY_SPACE;
- if (mbox_fill_space(sync_ctx, mails[idx+1].offset,
- mails[idx+1].space) < 0) {
+ move_diff += mails[idx].space;
+ if ((mails[idx].flags & MBOX_EXPUNGED) == 0) {
+ move_diff -= padding_per_mail;
+ mails[idx].space = padding_per_mail;
+ }
+ if (mbox_fill_space(sync_ctx,
+ mails[idx].offset + move_diff,
+ padding_per_mail) < 0) {
ret = -1;
break;
}
+ mails[idx].offset += move_diff;
}
- } while (idx > 0);
- if ((mails[0].flags & MBOX_DIRTY_SPACE) != 0) {
- mails[0].flags &= ~MBOX_DIRTY_SPACE;
- if (mbox_fill_space(sync_ctx, mails[0].offset,
- mails[0].space) < 0)
- ret = -1;
+ end_offset = next_end_offset;
+ mails[idx].from_offset += move_diff;
+ i_assert(move_diff < OFF_T_MAX);
}
istream_raw_mbox_flush(sync_ctx->input);
Index: mbox-sync.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync.c,v
retrieving revision 1.95
retrieving revision 1.96
diff -u -d -r1.95 -r1.96
--- mbox-sync.c 4 Oct 2004 16:31:51 -0000 1.95
+++ mbox-sync.c 5 Oct 2004 00:26:40 -0000 1.96
@@ -62,31 +62,6 @@
return 0;
}
-static int mbox_sync_grow_file(struct mbox_sync_context *sync_ctx,
- struct mbox_sync_mail_context *mail_ctx,
- uoff_t grow_size)
-{
- uoff_t src_offset, file_size;
-
- i_assert(grow_size > 0);
-
- /* put the padding between last message's header and body */
- file_size = i_stream_get_size(sync_ctx->file_input) + grow_size;
- if (file_set_size(sync_ctx->fd, file_size) < 0) {
- mbox_set_syscall_error(sync_ctx->ibox, "file_set_size()");
- return -1;
- }
-
- src_offset = mail_ctx->body_offset;
- mail_ctx->body_offset += grow_size;
- if (mbox_move(sync_ctx, mail_ctx->body_offset, src_offset,
- file_size - mail_ctx->body_offset) < 0)
- return -1;
-
- istream_raw_mbox_flush(sync_ctx->input);
- return 0;
-}
-
static void mbox_sync_buffer_delete_old(buffer_t *syncs_buf, uint32_t uid)
{
struct mail_index_sync_rec *sync;
@@ -555,7 +530,7 @@
return -1;
mbox_sync_update_header(mail_ctx, sync_ctx->syncs);
- ret = mbox_sync_try_rewrite(mail_ctx, move_diff, FALSE);
+ ret = mbox_sync_try_rewrite(mail_ctx, move_diff);
if (ret < 0)
return -1;
@@ -578,7 +553,7 @@
return 0;
}
- if ((ret = mbox_sync_try_rewrite(mail_ctx, 0, FALSE)) < 0)
+ if ((ret = mbox_sync_try_rewrite(mail_ctx, 0)) < 0)
return -1;
} else {
/* nothing to do */
@@ -611,7 +586,7 @@
mbox_sync_handle_missing_space(struct mbox_sync_mail_context *mail_ctx)
{
struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
- uoff_t padding;
+ uoff_t end_offset, move_diff, extra_space, needed_space;
uint32_t last_seq;
buffer_append(sync_ctx->mails, &mail_ctx->mail, sizeof(mail_ctx->mail));
@@ -621,23 +596,36 @@
return 0;
/* we have enough space now */
- padding = MBOX_HEADER_PADDING *
- (sync_ctx->seq - sync_ctx->need_space_seq + 1);
-
- if (mail_ctx->mail.uid == 0 &&
- (uoff_t)sync_ctx->space_diff > padding) {
- /* don't waste too much on padding */
- sync_ctx->expunged_space = sync_ctx->space_diff - padding;
- sync_ctx->space_diff = padding;
+ if (mail_ctx->mail.uid == 0) {
+ /* this message was expunged. fill more or less of the space. */
+ extra_space = MBOX_HEADER_PADDING *
+ (sync_ctx->seq - sync_ctx->need_space_seq + 1);
+ needed_space = mail_ctx->mail.space - sync_ctx->space_diff;
+ if ((uoff_t)sync_ctx->space_diff > needed_space + extra_space) {
+ /* don't waste too much on padding */
+ sync_ctx->expunged_space = mail_ctx->mail.space -
+ (needed_space + extra_space);
+ sync_ctx->space_diff = needed_space + extra_space;
+ } else {
+ extra_space = sync_ctx->space_diff - needed_space;
+ }
last_seq = sync_ctx->seq - 1;
buffer_set_used_size(sync_ctx->mails, sync_ctx->mails->used -
sizeof(mail_ctx->mail));
+ end_offset = mail_ctx->mail.from_offset;
+ move_diff = sync_ctx->space_diff;
} else {
+ /* this message gave enough space from headers. rewriting stops
+ at the end of this message's headers. */
sync_ctx->expunged_space = 0;
last_seq = sync_ctx->seq;
+ end_offset = mail_ctx->body_offset;
+
+ move_diff = 0;
+ extra_space = sync_ctx->space_diff;
}
- if (mbox_sync_rewrite(sync_ctx, sync_ctx->space_diff,
+ if (mbox_sync_rewrite(sync_ctx, end_offset, move_diff, extra_space,
sync_ctx->need_space_seq, last_seq) < 0)
return -1;
@@ -648,6 +636,7 @@
memset(mail_ctx, 0, sizeof(*mail_ctx));
sync_ctx->need_space_seq = 0;
+ sync_ctx->space_diff = 0;
buffer_set_used_size(sync_ctx->mails, 0);
return 0;
}
@@ -923,8 +912,7 @@
static int mbox_sync_handle_eof_updates(struct mbox_sync_context *sync_ctx,
struct mbox_sync_mail_context *mail_ctx)
{
- uoff_t offset, padding, trailer_size;
- int need_rewrite;
+ uoff_t offset, padding, trailer_size, old_file_size;
if (!istream_raw_mbox_is_eof(sync_ctx->input)) {
i_assert(sync_ctx->need_space_seq == 0);
@@ -950,28 +938,22 @@
if (mail_ctx->have_eoh && !mail_ctx->updated)
str_append_c(mail_ctx->header, '\n');
- if (sync_ctx->space_diff < 0 &&
- mbox_sync_grow_file(sync_ctx, mail_ctx,
- -sync_ctx->space_diff) < 0)
- return -1;
+ i_assert(sync_ctx->space_diff < 0);
- need_rewrite = sync_ctx->seq != sync_ctx->need_space_seq;
- if (mbox_sync_try_rewrite(mail_ctx, 0, need_rewrite) < 0)
+ old_file_size = i_stream_get_size(sync_ctx->file_input);
+ if (file_set_size(sync_ctx->fd,
+ old_file_size + -sync_ctx->space_diff) < 0) {
+ mbox_set_syscall_error(sync_ctx->ibox,
+ "file_set_size()");
return -1;
-
- if (need_rewrite) {
- buffer_set_used_size(sync_ctx->mails,
- (sync_ctx->seq -
- sync_ctx->need_space_seq) *
- sizeof(mail_ctx->mail));
- buffer_append(sync_ctx->mails, &mail_ctx->mail,
- sizeof(mail_ctx->mail));
-
- if (mbox_sync_rewrite(sync_ctx, padding,
- sync_ctx->need_space_seq,
- sync_ctx->seq) < 0)
- return -1;
}
+ istream_raw_mbox_flush(sync_ctx->input);
+
+ if (mbox_sync_rewrite(sync_ctx, old_file_size,
+ -sync_ctx->space_diff, padding,
+ sync_ctx->need_space_seq,
+ sync_ctx->seq) < 0)
+ return -1;
update_from_offsets(sync_ctx);
More information about the dovecot-cvs
mailing list