[dovecot-cvs] dovecot/src/lib-storage/index/mbox mbox-lock.c, 1.1, 1.2 mbox-storage.c, 1.74, 1.75 mbox-sync-parse.c, 1.3, 1.4 mbox-sync-private.h, 1.4, 1.5 mbox-sync-rewrite.c, 1.3, 1.4 mbox-sync-update.c, 1.3, 1.4 mbox-sync.c, 1.4, 1.5

[cras at procontrol.fi]

Update of /home/cvs/dovecot/src/lib-storage/index/mbox
In directory talvi:/tmp/cvs-serv28607/lib-storage/index/mbox

Modified Files:
	mbox-lock.c mbox-storage.c mbox-sync-parse.c 
	mbox-sync-private.h mbox-sync-rewrite.c mbox-sync-update.c 
	mbox-sync.c 
Log Message:
mbox growing and locking works now



Index: mbox-lock.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-lock.c,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- mbox-lock.c	6 May 2004 01:22:25 -0000	1.1
+++ mbox-lock.c	9 May 2004 21:06:48 -0000	1.2
@@ -279,6 +279,7 @@
 	}
 
 	*lock_id_r = ++ibox->mbox_lock_id;
+	ibox->mbox_locks++;
 	return 1;
 }
 

Index: mbox-storage.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-storage.c,v
retrieving revision 1.74
retrieving revision 1.75
diff -u -d -r1.74 -r1.75
--- mbox-storage.c	6 May 2004 01:22:25 -0000	1.74
+++ mbox-storage.c	9 May 2004 21:06:48 -0000	1.75
@@ -420,6 +420,7 @@
 
 	ibox->path = i_strdup(path);
 	ibox->mbox_fd = -1;
+	ibox->mbox_lock_type = F_UNLCK;
 
 	ibox->get_recent_count = mbox_get_recent_count;
 	ibox->mail_interface = &mbox_mail;

Index: mbox-sync-parse.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync-parse.c,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -d -r1.3 -r1.4
--- mbox-sync-parse.c	9 May 2004 17:06:06 -0000	1.3
+++ mbox-sync-parse.c	9 May 2004 21:06:48 -0000	1.4
@@ -247,7 +247,7 @@
 	ctx->hdr_offset = ctx->mail.offset;
 
         ctx->header_first_change = (size_t)-1;
-	ctx->header_last_change = (size_t)-1;
+	ctx->header_last_change = 0;
 
 	for (i = 0; i < MBOX_HDR_COUNT; i++)
 		ctx->hdr_pos[i] = (size_t)-1;

Index: mbox-sync-private.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync-private.h,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -d -r1.4 -r1.5
--- mbox-sync-private.h	9 May 2004 17:06:06 -0000	1.4
+++ mbox-sync-private.h	9 May 2004 21:06:48 -0000	1.5
@@ -59,8 +59,6 @@
 	struct istream *input, *file_input;
 	int fd;
 
-	const struct mail_index_header *hdr;
-
 	buffer_t *header;
 	uint32_t base_uid_validity, base_uid_last;
 	uint32_t prev_msg_uid, next_uid;

Index: mbox-sync-rewrite.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync-rewrite.c,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -d -r1.3 -r1.4
--- mbox-sync-rewrite.c	9 May 2004 17:06:06 -0000	1.3
+++ mbox-sync-rewrite.c	9 May 2004 21:06:48 -0000	1.4
@@ -58,7 +58,10 @@
 	memset(p, ' ', size);
 
 	ctx->mail.offset = ctx->hdr_offset + pos;
-	ctx->mail.space += size;
+	if (ctx->mail.space < 0)
+		ctx->mail.space = size;
+	else
+		ctx->mail.space += size;
 
 	if (ctx->header_first_change > pos)
 		ctx->header_first_change = pos;
@@ -136,15 +139,17 @@
 	size_t old_hdr_size, new_hdr_size;
 	const unsigned char *data;
 
+	i_assert(ctx->sync_ctx->ibox->mbox_lock_type == F_WRLCK);
+
 	old_hdr_size = ctx->body_offset - ctx->hdr_offset;
 	new_hdr_size = str_len(ctx->header);
 
 	/* do we have enough space? */
-	if (new_hdr_size < old_hdr_size)
+	if (new_hdr_size < old_hdr_size) {
 		mbox_sync_headers_add_space(ctx, old_hdr_size - new_hdr_size);
-	else if (new_hdr_size > old_hdr_size) {
+	} else if (new_hdr_size > old_hdr_size) {
 		size_t needed = new_hdr_size - old_hdr_size;
-		if (ctx->mail.space < needed)
+		if (ctx->mail.space < 0)
 			return 0;
 
 		mbox_sync_headers_remove_space(ctx, needed);
@@ -152,7 +157,10 @@
 
 	i_assert(ctx->header_first_change != (size_t)-1);
 
-	if (ctx->header_last_change != (size_t)-1)
+	/* FIXME: last_change should rather just tell if we want to truncate
+	   to beginning of extra whitespace */
+	if (ctx->header_last_change != (size_t)-1 &&
+	    ctx->header_last_change != 0)
 		str_truncate(ctx->header, ctx->header_last_change);
 
 	data = str_data(ctx->header);
@@ -204,10 +212,9 @@
 	/* we're moving next message - update it's from_offset */
 	mbox_sync_fix_from_offset(sync_ctx, idx+1, mails[idx+1].space);
 
-	if (mail_ctx.mail.space <= 0) {
-		mail_ctx.mail.space = 0;
+	if (mail_ctx.mail.space <= 0)
 		mbox_sync_headers_add_space(&mail_ctx, extra_per_mail);
-	} else if (mail_ctx.mail.space <= extra_per_mail) {
+	else if (mail_ctx.mail.space <= extra_per_mail) {
 		mbox_sync_headers_add_space(&mail_ctx, extra_per_mail -
 					    mail_ctx.mail.space);
 	} else {
@@ -246,6 +253,8 @@
 	uint32_t idx, extra_per_mail;
 	int ret = 0;
 
+	i_assert(sync_ctx->ibox->mbox_lock_type == F_WRLCK);
+
 	mails = buffer_get_modifyable_data(mails_buf, &size);
 	size /= sizeof(*mails);
 

Index: mbox-sync-update.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync-update.c,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -d -r1.3 -r1.4
--- mbox-sync-update.c	9 May 2004 17:06:06 -0000	1.3
+++ mbox-sync-update.c	9 May 2004 21:06:48 -0000	1.4
@@ -22,6 +22,9 @@
 	size_t size;
 	int i, need, have;
 
+	if (ctx->header_first_change > pos)
+		ctx->header_first_change = pos;
+
 	/* how many bytes do we need? */
 	for (i = 0, need = 0; flags_list[i].chr != 0; i++) {
 		if ((ctx->mail.flags & flags_list[i].flag) != 0)
@@ -40,7 +43,7 @@
 				break;
 		}
 
-		if (flags_list[i].chr == 0)
+		if (flags_list[i].chr != 0)
 			have++;
 		else {
 			/* save this one */
@@ -52,10 +55,16 @@
 	if (need < have)
 		str_delete(ctx->header, pos, have-need);
 	else if (need > have) {
-		buffer_copy(ctx->header, pos + (have-need),
+		buffer_copy(ctx->header, pos + (need-have),
 			    ctx->header, pos, (size_t)-1);
 	}
 
+	for (i = 0; i < MBOX_HDR_COUNT; i++) {
+		if (ctx->hdr_pos[i] > pos &&
+		    ctx->hdr_pos[i] != (size_t)-1)
+			ctx->hdr_pos[i] += need - have;
+	}
+
 	/* @UNSAFE */
 	data = buffer_get_space_unsafe(ctx->header, pos, need);
 	for (i = 0, need = 0; flags_list[i].chr != 0; i++) {
@@ -76,16 +85,17 @@
 	int i, have_keywords;
 
 	old_hdr_size = ctx->body_offset - ctx->hdr_offset;
-	new_hdr_size = str_len(ctx->header) + ctx->have_eoh;
+	new_hdr_size = str_len(ctx->header);
 
-	if (ctx->seq == 1 && ctx->sync_ctx->base_uid_validity == 0) {
+	if (ctx->seq == 1 && ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] == (size_t)-1) {
 		ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] = str_len(ctx->header);
 		str_printfa(ctx->header, "X-IMAPbase: %u %u",
-			    ctx->sync_ctx->hdr->uid_validity,
-			    ctx->sync_ctx->next_uid);
+			    ctx->sync_ctx->base_uid_validity,
+			    ctx->sync_ctx->next_uid-1);
 		//FIXME:keywords_append(ctx, all_keywords);
 		str_append_c(ctx->header, '\n');
 	}
+	i_assert(ctx->sync_ctx->base_uid_validity != 0);
 
 	if (ctx->hdr_pos[MBOX_HDR_X_UID] == (size_t)-1) {
 		if (ctx->mail.uid == 0)
@@ -97,6 +107,7 @@
 
 	if (ctx->hdr_pos[MBOX_HDR_STATUS] == (size_t)-1 &&
 	    (ctx->mail.flags & STATUS_FLAGS_MASK) != 0) {
+		ctx->mail.flags |= MBOX_NONRECENT;
 		ctx->hdr_pos[MBOX_HDR_STATUS] = str_len(ctx->header);
 		str_append(ctx->header, "Status: ");
 		status_flags_append(ctx, mbox_status_flags);
@@ -135,8 +146,7 @@
 		if (ctx->header_first_change == (size_t)-1)
 			ctx->header_first_change = new_hdr_size;
 		ctx->header_last_change = (size_t)-1;
-		ctx->mail.space -= str_len(ctx->header) -
-			(new_hdr_size - ctx->have_eoh);
+		ctx->mail.space -= str_len(ctx->header) - new_hdr_size;
 		if (ctx->mail.space > 0) {
 			/* we should rewrite this header, so offset
 			   must be broken if it's used anymore. */
@@ -146,7 +156,6 @@
 			   offset to point back to beginning of headers */
 			ctx->mail.offset = ctx->hdr_offset;
 		}
-		new_hdr_size = str_len(ctx->header) + ctx->have_eoh;
 	}
 
 	if (ctx->header_first_change == (size_t)-1) {
@@ -208,6 +217,11 @@
 		if (memcmp(old_keywords, ctx->mail.keywords,
 			   INDEX_KEYWORDS_BYTE_COUNT) != 0)
 			mbox_sync_update_xkeywords(ctx);
+	} else {
+		if ((ctx->mail.flags & MBOX_NONRECENT) == 0) {
+			ctx->mail.flags |= MBOX_NONRECENT;
+			mbox_sync_update_status(ctx);
+		}
 	}
 
         mbox_sync_add_missing_headers(ctx);
@@ -217,9 +231,10 @@
 				  const struct mbox_sync_mail *mail)
 {
 	if ((ctx->mail.flags & STATUS_FLAGS_MASK) !=
-	    (mail->flags & STATUS_FLAGS_MASK)) {
+	    (mail->flags & STATUS_FLAGS_MASK) ||
+	    (ctx->mail.flags & MBOX_NONRECENT) == 0) {
 		ctx->mail.flags = (ctx->mail.flags & ~STATUS_FLAGS_MASK) |
-			(mail->flags & STATUS_FLAGS_MASK);
+			(mail->flags & STATUS_FLAGS_MASK) | MBOX_NONRECENT;
 		mbox_sync_update_status(ctx);
 	}
 	if ((ctx->mail.flags & XSTATUS_FLAGS_MASK) !=

Index: mbox-sync.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync.c,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -d -r1.4 -r1.5
--- mbox-sync.c	9 May 2004 17:06:06 -0000	1.4
+++ mbox-sync.c	9 May 2004 21:06:48 -0000	1.5
@@ -54,64 +54,29 @@
 #include "istream-raw-mbox.h"
 #include "mbox-storage.h"
 #include "mbox-file.h"
+#include "mbox-lock.h"
 #include "mbox-sync-private.h"
 
 #include <sys/stat.h>
 
 static int mbox_sync_grow_file(struct mbox_sync_context *sync_ctx,
-			       struct mbox_sync_mail *mail, uoff_t body_offset,
+			       struct mbox_sync_mail_context *mail_ctx,
 			       uoff_t grow_size)
 {
-	char spaces[1024];
-	uoff_t offset, size;
+	uoff_t src_offset, file_size;
 
 	i_assert(grow_size > 0);
 
-	memset(spaces, ' ', sizeof(spaces));
-
-	size = sync_ctx->input->v_offset + grow_size;
-	if (file_set_size(sync_ctx->fd, size) < 0)
+	/* put the extra space between last message's header and body */
+	file_size = i_stream_get_size(sync_ctx->file_input) + grow_size;
+	if (file_set_size(sync_ctx->fd, file_size) < 0)
 		return -1;
 
-	if (mail->space <= 0) {
-		/* no X-Keywords header - place it at the end. */
-		grow_size += 13;
-
-		offset = body_offset-1;
-		if (mbox_move(sync_ctx, body_offset-1 + size,
-			      offset, (uoff_t)-1) < 0)
-			return -1;
-		if (pwrite_full(sync_ctx->fd, "X-Keywords: ", 12, offset) < 0)
-			return -1;
-		if (pwrite_full(sync_ctx->fd, "\n", 1,
-				offset + grow_size-1) < 0)
-			return -1;
-		grow_size -= 13; offset += 12;
-
-		/* FIXME: can this break anything? X-Keywords text might
-		   have been already included in space calculation. now we
-		   have more.. */
-		mail->offset = offset;
-		mail->space += grow_size;
-	} else {
-		offset = mail->offset;
-		if (mbox_move(sync_ctx, offset + grow_size,
-			      offset, (uoff_t)-1) < 0)
-			return -1;
-	}
-
-	while (grow_size >= sizeof(spaces)) {
-		if (pwrite_full(sync_ctx->fd, spaces,
-				sizeof(spaces), offset) < 0)
-			return -1;
-		grow_size -= sizeof(spaces);
-		offset += sizeof(spaces);
-	}
-
-	if (grow_size > 0) {
-		if (pwrite_full(sync_ctx->fd, spaces, grow_size, offset) < 0)
-			return -1;
-	}
+	src_offset = mail_ctx->body_offset;
+	mail_ctx->body_offset += grow_size;
+	if (mbox_move(sync_ctx, mail_ctx->body_offset, src_offset,
+		      file_size - mail_ctx->body_offset) < 0)
+		return -1;
 
 	istream_raw_mbox_flush(sync_ctx->input);
 	return 0;
@@ -136,7 +101,7 @@
 	buffer_set_used_size(syncs_buf, dest * sizeof(*sync));
 }
 
-static int
+static void
 mbox_sync_next_mail(struct mbox_sync_context *sync_ctx,
 		    struct mbox_sync_mail_context *mail_ctx, uint32_t seq)
 {
@@ -152,10 +117,7 @@
 		istream_raw_mbox_get_header_offset(sync_ctx->input);
 
 	mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx);
-	if (sync_ctx->input->v_offset == from_offset) {
-		/* this was the last mail */
-		return 0;
-	}
+	i_assert(sync_ctx->input->v_offset != from_offset);
 
 	mail_ctx->mail.body_size =
 		istream_raw_mbox_get_body_size(sync_ctx->input,
@@ -163,50 +125,47 @@
 
 	/* save the offset permanently with recent flag state */
 	from_offset <<= 1;
-	if ((mail_ctx->mail.flags & MBOX_NONRECENT) == 0)
+	if ((mail_ctx->mail.flags & MBOX_NONRECENT) == 0) {
+		/* need to add 'O' flag to Status-header */
+		mail_ctx->need_rewrite = TRUE;
 		from_offset |= 1;
+	}
 	buffer_append(sync_ctx->ibox->mbox_data_buf, &from_offset,
 		      sizeof(from_offset));
-	return 1;
 }
 
-int mbox_sync(struct index_mailbox *ibox, int last_commit)
+static void mbox_sync_apply_index_syncs(buffer_t *syncs_buf, uint8_t *flags,
+					keywords_mask_t keywords)
+{
+	const struct mail_index_sync_rec *sync;
+	size_t size, i;
+
+	sync = buffer_get_data(syncs_buf, &size);
+	size /= sizeof(*sync);
+
+	for (i = 0; i < size; i++)
+		mail_index_sync_flags_apply(&sync[i], flags, keywords);
+}
+
+static int mbox_sync_do(struct index_mailbox *ibox,
+			struct mail_index_sync_ctx *index_sync_ctx,
+			struct mail_index_view *sync_view)
 {
 	struct mbox_sync_context sync_ctx;
 	struct mbox_sync_mail_context mail_ctx;
-	struct mail_index_sync_ctx *index_sync_ctx;
 	struct mail_index_sync_rec sync_rec;
-	struct mail_index_view *sync_view;
 	struct mail_index_transaction *t;
 	const struct mail_index_header *hdr;
 	const struct mail_index_record *rec;
 	struct istream *input;
 	uint32_t seq, need_space_seq, idx_seq, messages_count;
+	uint8_t new_flags;
 	off_t space_diff;
 	uoff_t offset, extra_space;
 	buffer_t *mails, *syncs;
 	size_t size;
 	struct stat st;
-	int readonly, ret = 0;
-
-	if (last_commit) {
-		seq = ibox->commit_log_file_seq;
-		offset = ibox->commit_log_file_offset;
-	} else {
-		seq = (uint32_t)-1;
-		offset = (uoff_t)-1;
-	}
-
-	ret = mail_index_sync_begin(ibox->index, &index_sync_ctx, &sync_view,
-				    seq, offset);
-	if (ret <= 0)
-		return ret;
-
-	if (mbox_file_open_stream(ibox) < 0)
-		return -1;
-
-	if (mail_index_get_header(sync_view, &hdr) < 0)
-		return -1;
+	int ret = 0;
 
 	t = mail_index_transaction_begin(sync_view, FALSE);
 
@@ -217,15 +176,11 @@
 		buffer_set_used_size(ibox->mbox_data_buf, 0);
 	}
 
-	readonly = FALSE; // FIXME
-	// FIXME: lock the file
-
 	memset(&sync_ctx, 0, sizeof(sync_ctx));
 	sync_ctx.ibox = ibox;
 	sync_ctx.file_input = ibox->mbox_file_stream;
 	sync_ctx.input = ibox->mbox_stream;
 	sync_ctx.fd = ibox->mbox_fd;
-	sync_ctx.hdr = hdr;
 	sync_ctx.header = str_new(default_pool, 4096);
 
 	input = sync_ctx.input;
@@ -238,27 +193,45 @@
 	messages_count = mail_index_view_get_message_count(sync_view);
 
 	space_diff = 0; need_space_seq = 0; idx_seq = 0; rec = NULL;
-	for (seq = 1; !input->eof; seq++) {
-		ret = 1;
+	for (seq = 1;; seq++) {
+		/* set input->eof */
+		(void)istream_raw_mbox_get_header_offset(input);
+		if (input->eof)
+			break;
 
 		/* get all sync records related to this message */
+		ret = 1;
 		mbox_sync_buffer_delete_old(syncs, seq);
-		while (sync_rec.seq2 <= seq && ret > 0) {
-			if (sync_rec.seq2 != 0) {
+		while (seq >= sync_rec.seq1 && ret > 0) {
+			if (sync_rec.seq1 != 0) {
+				i_assert(seq <= sync_rec.seq2);
 				buffer_append(syncs, &sync_rec,
 					      sizeof(sync_rec));
 			}
 			ret = mail_index_sync_next(index_sync_ctx, &sync_rec);
+			if (ret == 0)
+				memset(&sync_rec, 0, sizeof(sync_rec));
 		}
 		if (ret < 0)
 			break;
 
-		ret = mbox_sync_next_mail(&sync_ctx, &mail_ctx, seq);
-		if (ret <= 0)
-			break;
+		mbox_sync_next_mail(&sync_ctx, &mail_ctx, seq);
+
+		if (seq == 1 && sync_ctx.base_uid_validity == 0) {
+			if (mail_index_get_header(sync_view, &hdr) < 0) {
+				ret = -1;
+				break;
+			}
+			sync_ctx.base_uid_validity = hdr->uid_validity;
+		}
 
 		if ((mail_ctx.need_rewrite ||
-		     buffer_get_used_size(syncs) != 0) && !readonly) {
+		     buffer_get_used_size(syncs) != 0) && !ibox->readonly) {
+			if (ibox->mbox_lock_type == F_RDLCK) {
+				ret = -2;
+				break;
+			}
+
 			mbox_sync_update_header(&mail_ctx, syncs);
 			if ((ret = mbox_sync_try_rewrite(&mail_ctx)) < 0)
 				return -1;
@@ -302,29 +275,42 @@
 
 		if (rec != NULL) {
 			/* see if flags changed */
-			if ((rec->flags & MAIL_FLAGS_MASK) !=
-			    (mail_ctx.mail.flags & MAIL_FLAGS_MASK) ||
-			    memcmp(rec->keywords, mail_ctx.mail.keywords,
+			keywords_mask_t old_keywords;
+			uint8_t old_flags;
+
+			old_flags = rec->flags;
+			memcpy(old_keywords, rec->keywords,
+			       INDEX_KEYWORDS_BYTE_COUNT);
+			mbox_sync_apply_index_syncs(syncs, &old_flags,
+						    old_keywords);
+
+			new_flags = (rec->flags & ~MAIL_FLAGS_MASK) |
+				(mail_ctx.mail.flags &
+				 (MAIL_FLAGS_MASK^MAIL_RECENT));
+
+			if (old_flags != new_flags ||
+			    memcmp(old_keywords, mail_ctx.mail.keywords,
 				   INDEX_KEYWORDS_BYTE_COUNT) != 0) {
-				uint8_t new_flags =
-					(rec->flags & ~MAIL_FLAGS_MASK) |
-					(mail_ctx.mail.flags & MAIL_FLAGS_MASK);
 				mail_index_update_flags(t, idx_seq,
 							MODIFY_REPLACE,
 							new_flags,
 							mail_ctx.mail.keywords);
 			}
+
+			/* we used this record */
 			rec = NULL;
 		} else {
 			/* new message */
 			mail_index_append(t, mail_ctx.mail.uid, &idx_seq);
+			new_flags = mail_ctx.mail.flags &
+				(MAIL_FLAGS_MASK^MAIL_RECENT);
 			mail_index_update_flags(t, idx_seq, MODIFY_REPLACE,
-				mail_ctx.mail.flags & MAIL_FLAGS_MASK,
-				mail_ctx.mail.keywords);
+						new_flags,
+						mail_ctx.mail.keywords);
 		}
 
 		istream_raw_mbox_next(input, mail_ctx.mail.body_size);
-		offset = input->v_offset;
+		offset = istream_raw_mbox_get_start_offset(input);
 
 		if (need_space_seq != 0) {
 			buffer_append(mails, &mail_ctx.mail,
@@ -344,7 +330,7 @@
 				   rewrite, so make sure we don't try to access
 				   it */
 				memset(&mail_ctx, 0, sizeof(mail_ctx));
-				i_stream_seek(input, input->v_offset);
+				i_stream_seek(input, offset);
 				need_space_seq = 0;
 			}
 		}
@@ -354,28 +340,40 @@
 		i_assert(space_diff < 0);
 		extra_space = MBOX_HEADER_EXTRA_SPACE *
 			((seq-1) - need_space_seq);
-		if (mbox_sync_grow_file(&sync_ctx, &mail_ctx.mail,
-					mail_ctx.body_offset,
+		if (mbox_sync_grow_file(&sync_ctx, &mail_ctx,
 					-space_diff + extra_space) < 0)
 			ret = -1;
-		else if (mbox_sync_rewrite(&sync_ctx, mails, need_space_seq,
-					   seq-1, extra_space) < 0)
+		else if (mbox_sync_try_rewrite(&mail_ctx) < 0)
 			ret = -1;
+		else if (--seq != need_space_seq) {
+			buffer_set_used_size(mails,
+					     (seq-1) * sizeof(mail_ctx.mail));
+			buffer_append(mails, &mail_ctx.mail,
+				      sizeof(mail_ctx.mail));
+
+			if (mbox_sync_rewrite(&sync_ctx, mails, need_space_seq,
+					      seq, extra_space) < 0)
+				ret = -1;
+		}
 	}
 
+	if (rec != NULL)
+		mail_index_expunge(t, idx_seq);
+	while (idx_seq < messages_count)
+		mail_index_expunge(t, ++idx_seq);
+
 	if (sync_ctx.base_uid_last+1 != sync_ctx.next_uid) {
 		// FIXME: rewrite X-IMAPbase header
 	}
 
-	/* only syncs left should be just appends which weren't synced yet.
-	   we'll just ignore them, as we've overwritten those above. */
-	while ((ret = mail_index_sync_next(index_sync_ctx, &sync_rec)) > 0) {
-		i_assert(sync_rec.type == MAIL_INDEX_SYNC_TYPE_APPEND);
-	}
-
-	if (fstat(ibox->mbox_fd, &st) < 0) {
-		mbox_set_syscall_error(ibox, "fstat()");
-		ret = -1;
+	if (ret >= 0) {
+		/* only syncs left should be just appends which weren't synced
+		   yet. we'll just ignore them, as we've overwritten those
+		   above. */
+		while ((ret = mail_index_sync_next(index_sync_ctx,
+						   &sync_rec)) > 0) {
+			i_assert(sync_rec.type == MAIL_INDEX_SYNC_TYPE_APPEND);
+		}
 	}
 
 	if (ret < 0)
@@ -389,13 +387,22 @@
 		}
 	}
 
+	if (ret == 0) {
+		if (fstat(ibox->mbox_fd, &st) < 0) {
+			mbox_set_syscall_error(ibox, "fstat()");
+			ret = -1;
+		}
+	}
 	if (ret < 0) {
 		st.st_mtime = 0;
 		st.st_size = 0;
 	}
 
-	if (mail_index_sync_end(index_sync_ctx, st.st_mtime, st.st_size) < 0)
-		ret = -1;
+	if (ret != -2) {
+		if (mail_index_sync_end(index_sync_ctx,
+					st.st_mtime, st.st_size) < 0)
+			ret = -1;
+	}
 
 	if (ret == 0) {
 		ibox->commit_log_file_seq = 0;
@@ -410,7 +417,60 @@
 	str_free(sync_ctx.header);
 	buffer_free(mails);
 	buffer_free(syncs);
-	return ret < 0 ? -1 : 0;
+	return ret < 0 ? ret : 0;
+}
+
+int mbox_sync(struct index_mailbox *ibox, int last_commit)
+{
+	struct mail_index_sync_ctx *index_sync_ctx;
+        struct mail_index_view *sync_view;
+	unsigned int lock_id;
+	uint32_t seq;
+	uoff_t offset;
+	int ret, lock_type;
+
+	if (last_commit) {
+		seq = ibox->commit_log_file_seq;
+		offset = ibox->commit_log_file_offset;
+	} else {
+		seq = (uint32_t)-1;
+		offset = (uoff_t)-1;
+	}
+
+	ret = mail_index_sync_begin(ibox->index, &index_sync_ctx, &sync_view,
+				    seq, offset);
+	if (ret <= 0)
+		return ret;
+
+	lock_type = mail_index_sync_have_more(index_sync_ctx) ?
+		F_WRLCK : F_RDLCK;
+	if (mbox_lock(ibox, lock_type, &lock_id) > 0 &&
+	    mbox_file_open_stream(ibox) == 0) {
+		ret = mbox_sync_do(ibox, index_sync_ctx, sync_view);
+		if (ret == -2) {
+			/* read lock -> write lock. do it again. */
+			(void)mbox_unlock(ibox, lock_id);
+			lock_id = 0;
+			if (mbox_lock(ibox, F_WRLCK, &lock_id) <= 0)
+				ret = -1;
+			else if (mbox_file_open_stream(ibox) < 0)
+				ret = -1;
+			else {
+				ret = mbox_sync_do(ibox, index_sync_ctx,
+						   sync_view);
+			}
+		}
+	} else {
+		(void)mail_index_sync_end(index_sync_ctx, 0, 0);
+		ret = -1;
+	}
+
+	if (lock_id != 0) {
+		if (mbox_unlock(ibox, lock_id) < 0)
+			ret = -1;
+	}
+
+	return ret;
 }
 
 int mbox_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags)