[dovecot-cvs]
dovecot/src/lib-storage/index/mbox istream-raw-mbox.c, 1.5,
1.6 mbox-sync-private.h, 1.6, 1.7 mbox-sync-rewrite.c, 1.5,
1.6 mbox-sync-update.c, 1.5, 1.6 mbox-sync.c, 1.11, 1.12
cras at procontrol.fi
cras at procontrol.fi
Fri Jun 11 06:20:13 EEST 2004
Update of /home/cvs/dovecot/src/lib-storage/index/mbox
In directory talvi:/tmp/cvs-serv10588/lib-storage/index/mbox
Modified Files:
istream-raw-mbox.c mbox-sync-private.h mbox-sync-rewrite.c
mbox-sync-update.c mbox-sync.c
Log Message:
expunging is somewhat working
Index: istream-raw-mbox.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/istream-raw-mbox.c,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -d -r1.5 -r1.6
--- istream-raw-mbox.c 23 May 2004 01:58:32 -0000 1.5
+++ istream-raw-mbox.c 11 Jun 2004 03:20:10 -0000 1.6
@@ -201,8 +201,9 @@
stream->buffer = buf;
stream->pos = new_pos;
- if (i < pos && new_pos == stream->pos) {
- /* beginning from From-line, try again */
+ if (i < pos) {
+ /* beginning from From-line, try again
+ FIXME: loops forever if we don't skip forward */
ret = 0;
}
Index: mbox-sync-private.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync-private.h,v
retrieving revision 1.6
retrieving revision 1.7
diff -u -d -r1.6 -r1.7
--- mbox-sync-private.h 23 May 2004 01:58:32 -0000 1.6
+++ mbox-sync-private.h 11 Jun 2004 03:20:10 -0000 1.7
@@ -40,7 +40,7 @@
struct mbox_sync_mail mail;
uint32_t seq;
- uoff_t hdr_offset, body_offset;
+ uoff_t from_offset, hdr_offset, body_offset;
size_t header_first_change, header_last_change;
string_t *header;
@@ -59,9 +59,10 @@
struct istream *input, *file_input;
int fd;
- buffer_t *header;
+ string_t *header, *from_line;
uint32_t base_uid_validity, base_uid_last;
uint32_t prev_msg_uid, next_uid;
+ off_t expunged_space;
};
int mbox_sync(struct index_mailbox *ibox, int last_commit);
@@ -72,7 +73,7 @@
buffer_t *syncs_buf);
void mbox_sync_update_header_from(struct mbox_sync_mail_context *ctx,
const struct mbox_sync_mail *mail);
-int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx);
+int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx, off_t move_diff);
int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx, buffer_t *mails_buf,
uint32_t first_seq, uint32_t last_seq, off_t extra_space);
Index: mbox-sync-rewrite.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync-rewrite.c,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -d -r1.5 -r1.6
--- mbox-sync-rewrite.c 23 May 2004 01:58:32 -0000 1.5
+++ mbox-sync-rewrite.c 11 Jun 2004 03:20:10 -0000 1.6
@@ -16,6 +16,9 @@
struct ostream *output;
off_t ret;
+ if (size == 0 || source == dest)
+ return 0;
+
istream_raw_mbox_flush(sync_ctx->input);
output = o_stream_create_file(sync_ctx->fd, default_pool, 4096, FALSE);
@@ -24,15 +27,18 @@
if (size == (uoff_t)-1) {
input = sync_ctx->file_input;
- return o_stream_send_istream(output, input) < 0 ? -1 : 0;
+ ret = o_stream_send_istream(output, input) < 0 ? -1 : 0;
} else {
input = i_stream_create_limit(default_pool,
sync_ctx->file_input,
source, size);
ret = o_stream_send_istream(output, input);
i_stream_unref(input);
- return ret == (off_t)size ? 0 : -1;
+ ret = ret == (off_t)size ? 0 : -1;
}
+
+ o_stream_unref(output);
+ return (int)ret;
}
static void mbox_sync_headers_add_space(struct mbox_sync_mail_context *ctx,
@@ -134,7 +140,7 @@
i_assert(size == 0);
}
-int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx)
+int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx, off_t move_diff)
{
size_t old_hdr_size, new_hdr_size;
const unsigned char *data;
@@ -149,13 +155,27 @@
mbox_sync_headers_add_space(ctx, old_hdr_size - new_hdr_size);
} else if (new_hdr_size > old_hdr_size) {
size_t needed = new_hdr_size - old_hdr_size;
- if (ctx->mail.space < 0)
- return 0;
- mbox_sync_headers_remove_space(ctx, needed);
+ if (ctx->mail.space >= 0)
+ mbox_sync_headers_remove_space(ctx, needed);
+ else if (move_diff < 0 && needed <= -move_diff) {
+ /* moving backwards - we can use the extra space from
+ it, just update expunged_space accordingly */
+ i_assert(ctx->sync_ctx->expunged_space >= needed);
+ ctx->sync_ctx->expunged_space -= needed;
+ } else {
+ return 0;
+ }
}
- i_assert(ctx->header_first_change != (size_t)-1);
+ i_assert(ctx->header_first_change != (size_t)-1 || move_diff != 0);
+
+ if (move_diff != 0) {
+ /* we're moving the header, forget about partial write
+ optimizations */
+ ctx->header_first_change = 0;
+ ctx->header_last_change = 0;
+ }
/* FIXME: last_change should rather just tell if we want to truncate
to beginning of extra whitespace */
@@ -164,10 +184,11 @@
str_truncate(ctx->header, ctx->header_last_change);
data = str_data(ctx->header);
- new_hdr_size = str_len(ctx->header);
+ new_hdr_size = str_len(ctx->header);
if (pwrite_full(ctx->sync_ctx->fd, data + ctx->header_first_change,
new_hdr_size - ctx->header_first_change,
- ctx->hdr_offset + ctx->header_first_change) < 0) {
+ ctx->hdr_offset + move_diff +
+ ctx->header_first_change) < 0) {
// FIXME: error handling
return -1;
}
@@ -253,6 +274,7 @@
uint32_t idx, extra_per_mail;
int ret = 0;
+ i_assert(first_seq != last_seq);
i_assert(sync_ctx->ibox->mbox_lock_type == F_WRLCK);
mails = buffer_get_modifyable_data(mails_buf, &size);
@@ -263,13 +285,16 @@
extra_per_mail = (extra_space / (last_seq - first_seq + 1));
- mails[last_seq-1].space -= extra_per_mail;
- i_assert(mails[last_seq-1].space >= 0);
- end_offset = mails[last_seq-1].offset + mails[last_seq-1].space;
+ last_seq--;
+ idx = last_seq - first_seq;
+
+ mails[idx].space -= extra_per_mail;
+ i_assert(mails[idx].space >= 0);
+ end_offset = mails[idx].offset + mails[idx].space;
/* start moving backwards */
- while (--last_seq >= first_seq) {
- idx = last_seq-1;
+ do {
+ idx--;
if (mails[idx].space <= 0) {
/* offset points to beginning of headers. read the
header again, update it and give enough space to
@@ -303,7 +328,7 @@
i_assert(mails[idx].space > 0);
end_offset = mails[idx].offset + mails[idx].space;
}
- }
+ } while (idx > 0);
istream_raw_mbox_flush(sync_ctx->input);
return ret;
Index: mbox-sync-update.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync-update.c,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -d -r1.5 -r1.6
--- mbox-sync-update.c 22 May 2004 00:48:45 -0000 1.5
+++ mbox-sync-update.c 11 Jun 2004 03:20:10 -0000 1.6
@@ -155,11 +155,6 @@
}
}
- if (ctx->header_first_change == (size_t)-1) {
- /* no headers had to be modified */
- return;
- }
-
if (ctx->have_eoh)
str_append_c(ctx->header, '\n');
}
Index: mbox-sync.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync.c,v
retrieving revision 1.11
retrieving revision 1.12
diff -u -d -r1.11 -r1.12
--- mbox-sync.c 3 Jun 2004 15:01:27 -0000 1.11
+++ mbox-sync.c 11 Jun 2004 03:20:10 -0000 1.12
@@ -113,19 +113,20 @@
mail_ctx->seq = seq;
mail_ctx->header = sync_ctx->header;
- from_offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
+ mail_ctx->from_offset =
+ istream_raw_mbox_get_start_offset(sync_ctx->input);
mail_ctx->mail.offset =
istream_raw_mbox_get_header_offset(sync_ctx->input);
mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx, FALSE);
- i_assert(sync_ctx->input->v_offset != from_offset);
+ i_assert(sync_ctx->input->v_offset != mail_ctx->from_offset);
mail_ctx->mail.body_size =
istream_raw_mbox_get_body_size(sync_ctx->input,
mail_ctx->content_length);
/* save the offset permanently with recent flag state */
- from_offset <<= 1;
+ from_offset = (mail_ctx->from_offset - sync_ctx->expunged_space) << 1;
if ((mail_ctx->mail.flags & MBOX_NONRECENT) == 0) {
/* need to add 'O' flag to Status-header */
mail_ctx->need_rewrite = TRUE;
@@ -148,10 +149,66 @@
mail_index_sync_flags_apply(&sync[i], flags, keywords);
}
+static int mbox_read_from_line(struct mbox_sync_mail_context *ctx)
+{
+ struct istream *input = ctx->sync_ctx->file_input;
+ const unsigned char *data;
+ size_t size, from_line_size;
+
+ buffer_set_used_size(ctx->sync_ctx->from_line, 0);
+ from_line_size = ctx->hdr_offset - ctx->from_offset;
+
+ i_stream_seek(input, ctx->from_offset);
+ for (;;) {
+ data = i_stream_get_data(input, &size);
+ if (size >= from_line_size)
+ size = from_line_size;
+
+ buffer_append(ctx->sync_ctx->from_line, data, size);
+ i_stream_skip(input, size);
+ from_line_size -= size;
+
+ if (from_line_size == 0)
+ break;
+
+ if (i_stream_read(input) < 0)
+ return -1;
+ }
+
+ return 0;
+}
+
+static int
+mbox_write_from_line(struct mbox_sync_mail_context *ctx, off_t move_diff)
+{
+ string_t *str = ctx->sync_ctx->from_line;
+
+ if (move_diff == 0)
+ return 0;
+
+ if (ctx->from_offset + move_diff == 0) {
+ /* FIXME: kludge: we're writing the first header,
+ change the \n prefix into space suffix */
+ buffer_copy(str, 0, str, 1, (size_t)-1);
+ str_truncate(str, str_len(str)-2);
+ str_append(str, " \n");
+ }
+
+ if (pwrite_full(ctx->sync_ctx->fd, str_data(str), str_len(str),
+ ctx->from_offset + move_diff) < 0) {
+ // FIXME: error handling
+ return -1;
+ }
+
+ istream_raw_mbox_flush(ctx->sync_ctx->input);
+ return 0;
+}
+
static int mbox_sync_do(struct index_mailbox *ibox,
struct mail_index_sync_ctx *index_sync_ctx,
struct mail_index_view *sync_view)
{
+ /* FIXME: expunging + mbox_sync_rewrite() is broken within same sync */
struct mbox_sync_context sync_ctx;
struct mbox_sync_mail_context mail_ctx;
struct mail_index_sync_rec sync_rec;
@@ -161,8 +218,8 @@
struct istream *input;
uint32_t seq, need_space_seq, idx_seq, messages_count;
uint8_t new_flags;
- off_t space_diff;
- uoff_t offset, extra_space;
+ off_t space_diff, move_diff;
+ uoff_t offset, extra_space, trailer_size;
buffer_t *mails, *syncs;
size_t size;
struct stat st;
@@ -182,6 +239,7 @@
sync_ctx.file_input = ibox->mbox_file_stream;
sync_ctx.input = ibox->mbox_stream;
sync_ctx.fd = ibox->mbox_fd;
+ sync_ctx.from_line = str_new(default_pool, 256);
sync_ctx.header = str_new(default_pool, 4096);
sync_ctx.next_uid = 1;
@@ -214,10 +272,8 @@
sizeof(sync_rec));
if (sync_rec.type ==
- MAIL_INDEX_SYNC_TYPE_EXPUNGE) {
+ MAIL_INDEX_SYNC_TYPE_EXPUNGE)
sync_expunge = TRUE;
- break;
- }
}
ret = mail_index_sync_next(index_sync_ctx, &sync_rec);
if (ret == 0)
@@ -236,16 +292,45 @@
hdr->uid_validity;
}
- if ((mail_ctx.need_rewrite ||
+ if ((mail_ctx.need_rewrite || sync_ctx.expunged_space > 0 ||
buffer_get_used_size(syncs) != 0) && !ibox->readonly) {
if (ibox->mbox_lock_type == F_RDLCK) {
ret = -2;
break;
}
- mbox_sync_update_header(&mail_ctx, syncs);
- if ((ret = mbox_sync_try_rewrite(&mail_ctx)) < 0)
- return -1;
+ if (sync_expunge) {
+ ret = 1;
+ sync_ctx.expunged_space +=
+ mail_ctx.body_offset -
+ mail_ctx.from_offset +
+ mail_ctx.mail.body_size;
+ } else {
+ move_diff = need_space_seq != 0 ? 0 :
+ -sync_ctx.expunged_space;
+
+ /* read the From-line */
+ if (move_diff != 0 &&
+ mbox_read_from_line(&mail_ctx) < 0) {
+ ret = -1;
+ break;
+ }
+
+ mbox_sync_update_header(&mail_ctx, syncs);
+ ret = mbox_sync_try_rewrite(&mail_ctx,
+ move_diff);
+
+ if (ret > 0) {
+ mail_ctx.mail.offset += move_diff;
+ ret = mbox_write_from_line(&mail_ctx,
+ move_diff);
+ if (ret == 0)
+ ret = 1;
+ }
+
+ if (ret < 0)
+ break;
+ }
if (ret == 0 && need_space_seq == 0) {
/* first mail with no space to write it */
@@ -285,7 +370,8 @@
}
if (sync_expunge) {
- /* .. */
+ if (rec != NULL)
+ mail_index_expunge(t, idx_seq);
} else if (rec != NULL) {
/* see if flags changed */
keywords_mask_t old_keywords;
@@ -325,12 +411,26 @@
istream_raw_mbox_next(input, mail_ctx.mail.body_size);
offset = istream_raw_mbox_get_start_offset(input);
+ if (sync_ctx.expunged_space > 0 && !sync_expunge &&
+ need_space_seq == 0) {
+ /* move the body */
+ if (mbox_move(&sync_ctx,
+ mail_ctx.body_offset -
+ sync_ctx.expunged_space,
+ mail_ctx.body_offset,
+ mail_ctx.mail.body_size) < 0) {
+ ret = -1;
+ break;
+ }
+ i_stream_seek(input, offset);
+ }
+
if (need_space_seq != 0) {
buffer_append(mails, &mail_ctx.mail,
sizeof(mail_ctx.mail));
space_diff += mail_ctx.mail.space;
- if (space_diff >= 0) {
+ if (space_diff + sync_ctx.expunged_space >= 0) {
/* we have enough space now */
if (mbox_sync_rewrite(&sync_ctx, mails,
need_space_seq, seq,
@@ -343,20 +443,27 @@
it */
memset(&mail_ctx, 0, sizeof(mail_ctx));
i_stream_seek(input, offset);
+
need_space_seq = 0;
+ buffer_set_used_size(mails, 0);
}
}
}
+ trailer_size = i_stream_get_size(sync_ctx.file_input) - offset;
if (need_space_seq != 0 && ret >= 0) {
i_assert(space_diff < 0);
extra_space = MBOX_HEADER_EXTRA_SPACE *
(seq - need_space_seq + 1);
+ space_diff -= extra_space;
- if (mbox_sync_grow_file(&sync_ctx, &mail_ctx,
- -space_diff + extra_space) < 0)
+ space_diff += sync_ctx.expunged_space;
+ sync_ctx.expunged_space -= -space_diff;
+
+ if (space_diff < 0 &&
+ mbox_sync_grow_file(&sync_ctx, &mail_ctx, -space_diff) < 0)
ret = -1;
- else if (mbox_sync_try_rewrite(&mail_ctx) < 0)
+ else if (mbox_sync_try_rewrite(&mail_ctx, 0) < 0)
ret = -1;
else if (seq != need_space_seq) {
buffer_set_used_size(mails,
@@ -370,6 +477,21 @@
}
}
+ if (sync_ctx.expunged_space > 0) {
+ /* copy trailer, then truncate the file */
+ offset = i_stream_get_size(sync_ctx.file_input) -
+ sync_ctx.expunged_space - trailer_size;
+
+ if (mbox_move(&sync_ctx, offset,
+ offset + sync_ctx.expunged_space,
+ trailer_size) < 0)
+ ret = -1;
+ else if (ftruncate(ibox->mbox_fd, offset + trailer_size) < 0)
+ ret = -1;
+
+ istream_raw_mbox_flush(input);
+ }
+
if (ret >= 0) {
if (rec != NULL)
mail_index_expunge(t, idx_seq);
@@ -460,6 +582,7 @@
ibox->mbox_data_count = size / sizeof(*ibox->mbox_data);
str_free(sync_ctx.header);
+ str_free(sync_ctx.from_line);
buffer_free(mails);
buffer_free(syncs);
return ret < 0 ? ret : 0;
More information about the dovecot-cvs
mailing list