[dovecot-cvs] dovecot/src/lib-storage/index/mbox istream-raw-mbox.c, 1.2, 1.3 istream-raw-mbox.h, 1.2, 1.3 mbox-sync-parse.c, 1.2, 1.3 mbox-sync-private.h, 1.3, 1.4 mbox-sync-rewrite.c, 1.2, 1.3 mbox-sync-update.c, 1.2, 1.3 mbox-sync.c, 1.3, 1.4

cras at procontrol.fi cras at procontrol.fi
Sun May 9 20:06:10 EEST 2004


Update of /home/cvs/dovecot/src/lib-storage/index/mbox
In directory talvi:/tmp/cvs-serv27474/lib-storage/index/mbox

Modified Files:
	istream-raw-mbox.c istream-raw-mbox.h mbox-sync-parse.c 
	mbox-sync-private.h mbox-sync-rewrite.c mbox-sync-update.c 
	mbox-sync.c 
Log Message:
mbox rewriting is almost working - the hard part is done.



Index: istream-raw-mbox.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/istream-raw-mbox.c,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- istream-raw-mbox.c	6 May 2004 01:22:25 -0000	1.2
+++ istream-raw-mbox.c	9 May 2004 17:06:06 -0000	1.3
@@ -12,7 +12,7 @@
 	time_t received_time, next_received_time;
 	char *sender, *next_sender;
 
-	uoff_t from_offset, hdr_offset, next_from_offset, body_size;
+	uoff_t from_offset, hdr_offset, body_offset, mail_size;
 	struct istream *input;
 };
 
@@ -103,7 +103,7 @@
 	struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream;
 	const unsigned char *buf;
 	const char *fromp;
-	char *sender;
+	char *sender, eoh_char;
 	time_t received_time;
 	size_t i, pos;
 	ssize_t ret;
@@ -119,10 +119,12 @@
 		buf = i_stream_get_data(rstream->input, &pos);
 	} while (ret > 0 && pos <= 6);
 
-	if (pos == 1 && buf[0] == '\n') {
+	if (pos == 0 || (pos == 1 && buf[0] == '\n')) {
 		/* EOF */
 		stream->pos = 0;
 		stream->istream.eof = TRUE;
+		rstream->mail_size = stream->istream.v_offset -
+			rstream->hdr_offset;
 		return -1;
 	}
 
@@ -140,6 +142,8 @@
 		    mbox_from_parse(buf+6, pos-6,
 				    &received_time, &sender) == 0) {
 			rstream->next_received_time = received_time;
+			rstream->mail_size = stream->istream.v_offset -
+				rstream->hdr_offset;
 
 			i_free(rstream->next_sender);
 			rstream->next_sender = sender;
@@ -148,9 +152,17 @@
 		}
 	} else if (ret == -1) {
 		/* last few bytes, can't contain From-line */
+		if (buf[pos-1] == '\n') {
+			/* last LF doesn't belong to last message */
+			pos--;
+		}
+
 		ret = pos <= stream->pos ? -1 :
 			(ssize_t) (pos - stream->pos);
 
+		rstream->mail_size = stream->istream.v_offset + pos -
+			rstream->hdr_offset;
+
 		stream->buffer = buf;
 		stream->pos = pos;
 		stream->istream.eof = ret == -1;
@@ -159,7 +171,12 @@
 
 	/* See if we have From-line here - note that it works right only
 	   because all characters are different in mbox_from. */
+	eoh_char = rstream->body_offset == (uoff_t)-1 ? '\n' : '\0';
 	for (i = 0, fromp = mbox_from; i < pos; i++) {
+		if (buf[i] == eoh_char && i > 0 && buf[i-1] == '\n') {
+			rstream->body_offset = stream->istream.v_offset + i + 1;
+			eoh_char = '\0';
+		}
 		if (buf[i] == *fromp) {
 			if (*++fromp == '\0') {
 				/* potential From-line - stop here */
@@ -197,7 +214,10 @@
 	rstream = p_new(pool, struct raw_mbox_istream, 1);
 
 	rstream->input = input;
-	rstream->body_size = (uoff_t)-1;
+	rstream->body_offset = (uoff_t)-1;
+	rstream->mail_size = (uoff_t)-1;
+	rstream->received_time = (time_t)-1;
+	rstream->next_received_time = (time_t)-1;
 
 	rstream->istream.iostream.close = _close;
 	rstream->istream.iostream.destroy = _destroy;
@@ -244,7 +264,26 @@
 	return TRUE;
 }
 
-uoff_t istream_raw_mbox_get_size(struct istream *stream, uoff_t body_size)
+uoff_t istream_raw_mbox_get_start_offset(struct istream *stream)
+{
+	struct raw_mbox_istream *rstream =
+		(struct raw_mbox_istream *)stream->real_stream;
+
+	return rstream->from_offset;
+}
+
+uoff_t istream_raw_mbox_get_header_offset(struct istream *stream)
+{
+	struct raw_mbox_istream *rstream =
+		(struct raw_mbox_istream *)stream->real_stream;
+
+	if (rstream->hdr_offset == rstream->from_offset)
+		(void)_read(&rstream->istream);
+
+	return rstream->hdr_offset;
+}
+
+uoff_t istream_raw_mbox_get_body_size(struct istream *stream, uoff_t body_size)
 {
 	struct raw_mbox_istream *rstream =
 		(struct raw_mbox_istream *)stream->real_stream;
@@ -252,26 +291,29 @@
 	const unsigned char *data;
 	size_t size;
 
-	if (rstream->body_size != (uoff_t)-1)
-		return rstream->body_size;
+	if (rstream->mail_size != (uoff_t)-1) {
+		return rstream->mail_size -
+			(rstream->body_offset - rstream->hdr_offset);
+	}
 
 	if (body_size != (uoff_t)-1) {
-		i_stream_seek(rstream->input, rstream->from_offset + body_size);
+		i_assert(rstream->body_offset != (uoff_t)-1);
+		i_stream_seek(rstream->input, rstream->body_offset + body_size);
 		if (istream_raw_mbox_is_valid_from(rstream) > 0) {
-			rstream->body_size = body_size;
+			rstream->mail_size = body_size +
+				(rstream->body_offset - rstream->hdr_offset);
 			return body_size;
 		}
 	}
 
-	old_offset = stream->v_offset;
-
 	/* have to read through the message body */
 	while (i_stream_read_data(stream, &data, &size, 0) > 0)
 		i_stream_skip(stream, size);
 
-	rstream->body_size = stream->v_offset - old_offset;
+	i_assert(rstream->mail_size != (uoff_t)-1);
 	i_stream_seek(stream, old_offset);
-	return rstream->body_size;
+	return rstream->mail_size -
+		(rstream->body_offset - rstream->hdr_offset);
 }
 
 time_t istream_raw_mbox_get_received_time(struct istream *stream)
@@ -279,7 +321,8 @@
 	struct raw_mbox_istream *rstream =
 		(struct raw_mbox_istream *)stream->real_stream;
 
-	(void)_read(&rstream->istream);
+	if (rstream->received_time == (time_t)-1)
+		(void)_read(&rstream->istream);
 	return rstream->received_time;
 }
 
@@ -288,7 +331,8 @@
 	struct raw_mbox_istream *rstream =
 		(struct raw_mbox_istream *)stream->real_stream;
 
-	(void)_read(&rstream->istream);
+	if (rstream->sender == NULL)
+		(void)_read(&rstream->istream);
 	return rstream->sender == NULL ? "" : rstream->sender;
 }
 
@@ -297,8 +341,8 @@
 	struct raw_mbox_istream *rstream =
 		(struct raw_mbox_istream *)stream->real_stream;
 
-	body_size = istream_raw_mbox_get_size(stream, body_size);
-	rstream->body_size = (uoff_t)-1;
+	body_size = istream_raw_mbox_get_body_size(stream, body_size);
+	rstream->mail_size = (uoff_t)-1;
 
 	rstream->received_time = rstream->next_received_time;
 	rstream->next_received_time = (time_t)-1;
@@ -307,8 +351,9 @@
 	rstream->sender = rstream->next_sender;
 	rstream->next_sender = NULL;
 
-	rstream->from_offset = stream->v_offset + body_size;
+	rstream->from_offset = rstream->body_offset + body_size;
 	rstream->hdr_offset = rstream->from_offset;
+	rstream->body_offset = (uoff_t)-1;
 
 	/* don't clear stream->eof if we don't have to */
 	if (stream->v_offset != rstream->from_offset)
@@ -321,7 +366,8 @@
 	struct raw_mbox_istream *rstream =
 		(struct raw_mbox_istream *)stream->real_stream;
 
-	if (offset == rstream->next_from_offset) {
+	if (rstream->mail_size != (uoff_t)-1 &&
+	    rstream->hdr_offset + rstream->mail_size == offset) {
 		istream_raw_mbox_next(stream, (uoff_t)-1);
 		return;
 	}
@@ -330,7 +376,8 @@
 		/* back to beginning of current message */
 		offset = rstream->hdr_offset;
 	} else {
-		rstream->body_size = (uoff_t)-1;
+		rstream->body_offset = (uoff_t)-1;
+		rstream->mail_size = (uoff_t)-1;
 		rstream->received_time = (time_t)-1;
 		rstream->next_received_time = (time_t)-1;
 
@@ -338,9 +385,11 @@
 		rstream->sender = NULL;
 		i_free(rstream->next_sender);
 		rstream->next_sender = NULL;
+
+                rstream->from_offset = offset;
+		rstream->hdr_offset = offset;
 	}
 
-	rstream->from_offset = rstream->hdr_offset = offset;
 	i_stream_seek(stream, offset);
 	i_stream_seek(rstream->input, offset);
 }

Index: istream-raw-mbox.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/istream-raw-mbox.h,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- istream-raw-mbox.h	6 May 2004 01:22:25 -0000	1.2
+++ istream-raw-mbox.h	9 May 2004 17:06:06 -0000	1.3
@@ -5,10 +5,15 @@
    you'll have to call istream_raw_mbox_next() to get to next message. */
 struct istream *i_stream_create_raw_mbox(pool_t pool, struct istream *input);
 
-/* Return number of bytes in this message after current offset.
-   If body_size isn't (uoff_t)-1, we'll use it as potentially valid body size
-   to avoid actually reading through the whole message. */
-uoff_t istream_raw_mbox_get_size(struct istream *stream, uoff_t body_size);
+/* Return offset to beginning of the "\nFrom"-line. */
+uoff_t istream_raw_mbox_get_start_offset(struct istream *stream);
+/* Return offset to beginning of the headers. */
+uoff_t istream_raw_mbox_get_header_offset(struct istream *stream);
+
+/* Return the number of bytes in the body of this message. If body_size isn't
+   (uoff_t)-1, we'll use it as potentially valid body size to avoid actually
+   reading through the whole message. */
+uoff_t istream_raw_mbox_get_body_size(struct istream *stream, uoff_t body_size);
 
 /* Return received time of current message, or (time_t)-1 if the timestamp is
    broken. */

Index: mbox-sync-parse.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync-parse.c,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- mbox-sync-parse.c	6 May 2004 01:22:25 -0000	1.2
+++ mbox-sync-parse.c	9 May 2004 17:06:06 -0000	1.3
@@ -52,7 +52,7 @@
 	size_t i;
 
 	for (i = 0; i < hdr->full_value_len; i++) {
-		ctx->mail->flags |=
+		ctx->mail.flags |=
 			mbox_flag_find(flags_list, hdr->full_value[i]);
 	}
 }
@@ -79,8 +79,9 @@
 	const char *str;
 	char *end;
 	size_t pos;
+	uint32_t uid_validity, uid_last;
 
-	if (ctx->seq != 1 || ctx->base_uid_validity != 0) {
+	if (ctx->seq != 1 || ctx->seen_imapbase) {
 		/* Valid only in first message */
 		return FALSE;
 	}
@@ -88,15 +89,15 @@
 	/* <uid validity> <last uid> */
 	t_push();
 	str = t_strndup(hdr->full_value, hdr->full_value_len);
-	ctx->base_uid_validity = strtoul(str, &end, 10);
-	ctx->base_uid_last = strtoul(end, &end, 10);
+	uid_validity = strtoul(str, &end, 10);
+	uid_last = strtoul(end, &end, 10);
 	pos = end - str;
 	t_pop();
 
 	while (pos < hdr->full_value_len && IS_LWSP_LF(hdr->full_value[pos]))
 		pos++;
 
-	if (ctx->base_uid_validity == 0) {
+	if (uid_validity == 0) {
 		/* broken */
 		return FALSE;
 	}
@@ -106,7 +107,14 @@
 
 	// FIXME: save keywords
 
+	if (ctx->sync_ctx->base_uid_validity == 0) {
+		ctx->sync_ctx->base_uid_validity = uid_validity;
+		ctx->sync_ctx->base_uid_last = uid_last;
+		ctx->sync_ctx->next_uid = uid_last+1;
+	}
+
 	ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] = str_len(ctx->header);
+	ctx->seen_imapbase = TRUE;
 	return TRUE;
 }
 
@@ -115,15 +123,16 @@
 {
 	size_t i, space = 0;
 
-	for (i = hdr->full_value_len; i > 0; i++) {
+	for (i = hdr->full_value_len; i > 0; i--) {
 		if (!IS_LWSP_LF(hdr->full_value[i-1]))
 			break;
 		space++;
 	}
 
-	if (space > ctx->mail->space) {
-		ctx->mail->space_offset = hdr->full_value_offset + i;
-		ctx->mail->space = space;
+	if (space > ctx->mail.space) {
+		ctx->mail.offset = ctx->hdr_offset +
+			hdr->full_value_offset + i;
+		ctx->mail.space = space;
 	}
 
 	// FIXME: parse them
@@ -138,7 +147,7 @@
 	uint32_t value = 0;
 	size_t i, space_pos, extra_space = 0;
 
-	if (ctx->mail->uid != 0) {
+	if (ctx->mail.uid != 0) {
 		/* duplicate */
 		return FALSE;
 	}
@@ -162,15 +171,17 @@
 		/* broken - UIDs must be growing */
 		return FALSE;
 	}
+	ctx->sync_ctx->prev_msg_uid = value;
 
 	ctx->hdr_pos[MBOX_HDR_X_UID] = str_len(ctx->header);
 
-	ctx->mail->uid = value;
-	if (ctx->mail->space == 0) {
+	ctx->mail.uid = value;
+	if (extra_space != 0 && ctx->mail.space == 0) {
 		/* set it only if X-Keywords hasn't been seen. spaces in X-UID
 		   should be removed when writing X-Keywords. */
-		ctx->mail->space_offset = hdr->full_value_offset + space_pos;
-		ctx->mail->space = extra_space;
+		ctx->mail.offset = ctx->hdr_offset +
+			hdr->full_value_offset + space_pos;
+		ctx->mail.space = extra_space;
 	}
 	return TRUE;
 }
@@ -233,8 +244,7 @@
 	size_t line_start_pos;
 	int i;
 
-	ctx->hdr_offset = input->v_offset;
-        ctx->mail->space_offset = input->v_offset;
+	ctx->hdr_offset = ctx->mail.offset;
 
         ctx->header_first_change = (size_t)-1;
 	ctx->header_last_change = (size_t)-1;
@@ -253,6 +263,12 @@
 			break;
 		}
 
+		if (!hdr->continued) {
+			line_start_pos = str_len(ctx->header);
+			str_append(ctx->header, hdr->name);
+			str_append(ctx->header, ": ");
+		}
+
 		func = header_func_find(hdr->name);
 		if (func != NULL) {
 			if (hdr->continues)
@@ -260,23 +276,15 @@
 			else if (!func->func(ctx, hdr)) {
 				/* this header is broken, remove it */
 				ctx->need_rewrite = TRUE;
-				if (hdr->continued) {
-					str_truncate(ctx->header,
-						     line_start_pos);
-				}
+				str_truncate(ctx->header, line_start_pos);
 				if (ctx->header_first_change == (size_t)-1) {
 					ctx->header_first_change =
-						str_len(ctx->header);
+						line_start_pos;
 				}
 				continue;
 			}
 		}
 
-		if (!hdr->continued) {
-			line_start_pos = str_len(ctx->header);
-			str_append(ctx->header, hdr->name);
-			str_append(ctx->header, ": ");
-		}
 		buffer_append(ctx->header, hdr->full_value,
 			      hdr->full_value_len);
 		if (!hdr->no_newline)
@@ -284,10 +292,14 @@
 	}
 	message_parse_header_deinit(hdr_ctx);
 
-	if (ctx->seq == 1 && ctx->base_uid_validity == 0) {
+	if (ctx->seq == 1 && ctx->sync_ctx->base_uid_validity == 0) {
 		/* missing X-IMAPbase */
 		ctx->need_rewrite = TRUE;
 	}
+	if (ctx->mail.uid == 0) {
+		/* missing X-UID */
+		ctx->need_rewrite = TRUE;
+	}
 
 	ctx->body_offset = input->v_offset;
 }

Index: mbox-sync-private.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync-private.h,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -d -r1.3 -r1.4
--- mbox-sync-private.h	6 May 2004 01:22:25 -0000	1.3
+++ mbox-sync-private.h	9 May 2004 17:06:06 -0000	1.4
@@ -30,14 +30,14 @@
 	uint8_t flags;
 	keywords_mask_t keywords;
 
-	uoff_t space_offset; /* if space is negative, points to beginning */
+	uoff_t offset; /* if space <= 0, points to beginning */
 	off_t space;
 	uoff_t body_size;
 };
 
 struct mbox_sync_mail_context {
 	struct mbox_sync_context *sync_ctx;
-	struct mbox_sync_mail *mail;
+	struct mbox_sync_mail mail;
 
 	uint32_t seq;
 	uoff_t hdr_offset, body_offset;
@@ -45,13 +45,13 @@
 	size_t header_first_change, header_last_change;
 	string_t *header;
 
-	uint32_t base_uid_validity, base_uid_last;
 	uoff_t content_length;
 
 	size_t hdr_pos[MBOX_HDR_COUNT];
 
 	unsigned int have_eoh:1;
 	unsigned int need_rewrite:1;
+	unsigned int seen_imapbase:1;
 };
 
 struct mbox_sync_context {
@@ -61,6 +61,8 @@
 
 	const struct mail_index_header *hdr;
 
+	buffer_t *header;
+	uint32_t base_uid_validity, base_uid_last;
 	uint32_t prev_msg_uid, next_uid;
 };
 
@@ -68,7 +70,9 @@
 void mbox_sync_parse_next_mail(struct istream *input,
 			       struct mbox_sync_mail_context *ctx);
 void mbox_sync_update_header(struct mbox_sync_mail_context *ctx,
-			     struct mail_index_sync_rec *update);
+			     buffer_t *syncs_buf);
+void mbox_sync_update_header_from(struct mbox_sync_mail_context *ctx,
+				  const struct mbox_sync_mail *mail);
 int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx);
 int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx, buffer_t *mails_buf,
 		      uint32_t first_seq, uint32_t last_seq, off_t extra_space);

Index: mbox-sync-rewrite.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync-rewrite.c,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- mbox-sync-rewrite.c	6 May 2004 01:22:25 -0000	1.2
+++ mbox-sync-rewrite.c	9 May 2004 17:06:06 -0000	1.3
@@ -5,6 +5,7 @@
 #include "str.h"
 #include "write-full.h"
 #include "message-parser.h"
+#include "mbox-storage.h"
 #include "mbox-sync-private.h"
 #include "istream-raw-mbox.h"
 
@@ -15,12 +16,12 @@
 	struct ostream *output;
 	off_t ret;
 
+	istream_raw_mbox_flush(sync_ctx->input);
+
 	output = o_stream_create_file(sync_ctx->fd, default_pool, 4096, FALSE);
 	i_stream_seek(sync_ctx->file_input, source);
 	o_stream_seek(output, dest);
 
-	istream_raw_mbox_flush(sync_ctx->file_input);
-
 	if (size == (uoff_t)-1) {
 		input = sync_ctx->file_input;
 		return o_stream_send_istream(output, input) < 0 ? -1 : 0;
@@ -56,6 +57,9 @@
 	p = buffer_get_space_unsafe(ctx->header, pos, size);
 	memset(p, ' ', size);
 
+	ctx->mail.offset = ctx->hdr_offset + pos;
+	ctx->mail.space += size;
+
 	if (ctx->header_first_change > pos)
 		ctx->header_first_change = pos;
 	ctx->header_last_change = (size_t)-1;
@@ -68,7 +72,7 @@
 	size_t data_size, end, nonspace;
 
 	/* find the end of the lwsp */
-	nonspace = pos;
+	nonspace = pos-1;
 	data = str_data(ctx->header);
 	data_size = str_len(ctx->header);
 	for (end = pos; end < data_size; end++) {
@@ -88,7 +92,13 @@
 		*size -= end-nonspace;
 	} else {
 		str_delete(ctx->header, nonspace, *size);
+		end -= *size;
 		*size = 0;
+
+		if (ctx->mail.space < end-nonspace) {
+			ctx->mail.space = end-nonspace;
+			ctx->mail.offset = ctx->hdr_offset + nonspace;
+		}
 	}
 }
 
@@ -103,9 +113,16 @@
         enum header_position pos;
 	int i;
 
+	ctx->header_last_change = (size_t)-1;
+
+	ctx->mail.space = 0;
+	ctx->mail.offset = ctx->hdr_offset;
+
 	for (i = 0; i < 3 && size > 0; i++) {
 		pos = space_positions[i];
 		if (ctx->hdr_pos[pos] != (size_t)-1) {
+			if (ctx->header_first_change > ctx->hdr_pos[pos])
+                                ctx->header_first_change = ctx->hdr_pos[pos];
 			mbox_sync_header_remove_space(ctx, ctx->hdr_pos[pos],
 						      &size);
 		}
@@ -123,17 +140,13 @@
 	new_hdr_size = str_len(ctx->header);
 
 	/* do we have enough space? */
-	if (new_hdr_size < old_hdr_size) {
+	if (new_hdr_size < old_hdr_size)
 		mbox_sync_headers_add_space(ctx, old_hdr_size - new_hdr_size);
-		ctx->mail->space += old_hdr_size - new_hdr_size;
-	} else if (new_hdr_size > old_hdr_size) {
+	else if (new_hdr_size > old_hdr_size) {
 		size_t needed = new_hdr_size - old_hdr_size;
-		if (ctx->mail->space < needed) {
-			ctx->mail->space -= needed;
+		if (ctx->mail.space < needed)
 			return 0;
-		}
 
-		ctx->mail->space -= needed;
 		mbox_sync_headers_remove_space(ctx, needed);
 	}
 
@@ -145,7 +158,7 @@
 	data = str_data(ctx->header);
         new_hdr_size = str_len(ctx->header);
 	if (pwrite_full(ctx->sync_ctx->fd, data + ctx->header_first_change,
-			new_hdr_size,
+			new_hdr_size - ctx->header_first_change,
 			ctx->hdr_offset + ctx->header_first_change) < 0) {
 		// FIXME: error handling
 		return -1;
@@ -154,30 +167,134 @@
 	return 1;
 }
 
+static void mbox_sync_fix_from_offset(struct mbox_sync_context *sync_ctx,
+				      uint32_t idx, off_t diff)
+{
+	uoff_t *offset_p;
+
+	offset_p = buffer_get_space_unsafe(sync_ctx->ibox->mbox_data_buf,
+					   idx * sizeof(*offset_p),
+					   sizeof(*offset_p));
+	*offset_p = (*offset_p & 1) | (((*offset_p >> 1) + diff) << 1);
+}
+
+static int mbox_sync_read_and_move(struct mbox_sync_context *sync_ctx,
+				   struct mbox_sync_mail *mails, uint32_t idx,
+				   uint32_t extra_per_mail,
+				   uoff_t *end_offset)
+{
+	struct mbox_sync_mail_context mail_ctx;
+	uoff_t offset;
+
+	i_stream_seek(sync_ctx->file_input, mails[idx].offset);
+
+	memset(&mail_ctx, 0, sizeof(mail_ctx));
+	mail_ctx.sync_ctx = sync_ctx;
+	mail_ctx.seq = idx+1;
+	mail_ctx.header = sync_ctx->header;
+
+	mail_ctx.mail.offset = mails[idx].offset;
+	mail_ctx.mail.body_size = mails[idx].body_size;
+
+	mbox_sync_parse_next_mail(sync_ctx->file_input, &mail_ctx);
+	mbox_sync_update_header_from(&mail_ctx, &mails[idx]);
+
+	i_assert(mail_ctx.mail.space == mails[idx].space);
+
+	/* we're moving next message - update it's from_offset */
+	mbox_sync_fix_from_offset(sync_ctx, idx+1, mails[idx+1].space);
+
+	if (mail_ctx.mail.space <= 0) {
+		mail_ctx.mail.space = 0;
+		mbox_sync_headers_add_space(&mail_ctx, extra_per_mail);
+	} else if (mail_ctx.mail.space <= extra_per_mail) {
+		mbox_sync_headers_add_space(&mail_ctx, extra_per_mail -
+					    mail_ctx.mail.space);
+	} else {
+		mbox_sync_headers_remove_space(&mail_ctx, mail_ctx.mail.space -
+					       extra_per_mail);
+	}
+
+	/* now we have to move it. first move the body of the message,
+	   then write the header and leave the extra space to beginning of
+	   headers. */
+	offset = sync_ctx->file_input->v_offset;
+	if (mbox_move(sync_ctx, offset + mails[idx+1].space, offset,
+		      *end_offset - offset - mails[idx+1].space)) {
+		// FIXME: error handling
+		return -1;
+	}
+
+	*end_offset = offset + mails[idx+1].space - str_len(mail_ctx.header);
+
+	if (pwrite_full(sync_ctx->fd, str_data(mail_ctx.header),
+			str_len(mail_ctx.header), *end_offset) < 0) {
+		// FIXME: error handling
+		return -1;
+	}
+
+	mails[idx].space += mails[idx+1].space - extra_per_mail;
+	return 0;
+}
+
 int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx, buffer_t *mails_buf,
 		      uint32_t first_seq, uint32_t last_seq, off_t extra_space)
 {
 	struct mbox_sync_mail *mails;
 	size_t size;
-	uint32_t first_idx, last_idx, extra_per_mail;
-
-	first_idx = first_seq-1;
-	last_idx = last_seq-1;
+	uoff_t offset, end_offset;
+	uint32_t idx, extra_per_mail;
+	int ret = 0;
 
 	mails = buffer_get_modifyable_data(mails_buf, &size);
 	size /= sizeof(*mails);
 
 	/* FIXME: see if we can be faster by going back a few mails
-	   (update first_seq and last_seq) */
-	/*while (mails[last_idx].space > 0) {
-	}*/
+	   (update first_seq and last_seq). */
+
+	extra_per_mail = (extra_space / (last_seq - first_seq + 1));
+
+	mails[last_seq-1].space -= extra_per_mail;
+	i_assert(mails[last_seq-1].space > 0);
+	end_offset = mails[last_seq-1].offset + mails[last_seq-1].space;
 
-#if 0
 	/* start moving backwards */
-	extra_per_mail = (extra_space / (last_seq - first_seq + 1)) + 1;
-	space_diff = 0;
-	while (last_seq > first_seq) {
-		dest = mails[last_seq].space_offset + mails[last_seq].space
+	while (--last_seq >= first_seq) {
+		idx = last_seq-1;
+		if (mails[idx].space <= 0) {
+			/* offset points to beginning of headers. read the
+			   header again, update it and give enough space to
+			   it */
+			if (mbox_sync_read_and_move(sync_ctx, mails, idx,
+						    extra_per_mail,
+						    &end_offset) < 0) {
+				ret = -1;
+				break;
+			}
+		} else {
+			/* X-Keywords: xx [offset]     \n
+			   ...
+			   X-Keywords: xx    [end_offset] \n
+
+			   move data forward so mails before us gets the extra
+			   space (ie. we temporarily get more space to us) */
+			offset = mails[idx].offset + mails[idx].space;
+			if (mbox_move(sync_ctx, offset + mails[idx+1].space,
+				      offset, end_offset - offset)) {
+				// FIXME: error handling
+				ret = -1;
+				break;
+			}
+
+			mbox_sync_fix_from_offset(sync_ctx, idx+1,
+						  mails[idx+1].space);
+
+			mails[idx].space += mails[idx+1].space - extra_per_mail;
+			i_assert(mails[idx].space > 0);
+			end_offset = mails[idx].offset + mails[idx].space;
+		}
 	}
-#endif
+
+	istream_raw_mbox_flush(sync_ctx->input);
+	return ret;
 }

Index: mbox-sync-update.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync-update.c,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- mbox-sync-update.c	2 May 2004 20:32:16 -0000	1.2
+++ mbox-sync-update.c	9 May 2004 17:06:06 -0000	1.3
@@ -5,15 +5,65 @@
 #include "mbox-sync-private.h"
 
 static void status_flags_append(struct mbox_sync_mail_context *ctx,
-				struct mbox_flag_type *flags_list)
+				const struct mbox_flag_type *flags_list)
 {
 	int i;
 
 	for (i = 0; flags_list[i].chr != 0; i++) {
-		if ((ctx->mail->flags & flags_list[i].flag) != 0)
+		if ((ctx->mail.flags & flags_list[i].flag) != 0)
 			str_append_c(ctx->header, flags_list[i].chr);
 	}
 }
+
+static void status_flags_replace(struct mbox_sync_mail_context *ctx, size_t pos,
+				const struct mbox_flag_type *flags_list)
+{
+	unsigned char *data;
+	size_t size;
+	int i, need, have;
+
+	/* how many bytes do we need? */
+	for (i = 0, need = 0; flags_list[i].chr != 0; i++) {
+		if ((ctx->mail.flags & flags_list[i].flag) != 0)
+			need++;
+	}
+
+	/* how many bytes do we have now? */
+	data = buffer_get_modifyable_data(ctx->header, &size);
+	for (have = 0; pos < size; pos++) {
+		if (data[pos] == '\n')
+			break;
+
+		/* see if this is unknown flag for us */
+		for (i = 0; flags_list[i].chr != 0; i++) {
+			if (flags_list[i].chr == data[pos])
+				break;
+		}
+
+		if (flags_list[i].chr == 0)
+			have++;
+		else {
+			/* save this one */
+			data[pos-have] = data[pos];
+		}
+	}
+	pos -= have;
+
+	if (need < have)
+		str_delete(ctx->header, pos, have-need);
+	else if (need > have) {
+		buffer_copy(ctx->header, pos + (have-need),
+			    ctx->header, pos, (size_t)-1);
+	}
+
+	/* @UNSAFE */
+	data = buffer_get_space_unsafe(ctx->header, pos, need);
+	for (i = 0, need = 0; flags_list[i].chr != 0; i++) {
+		if ((ctx->mail.flags & flags_list[i].flag) != 0)
+			*data++ = flags_list[i].chr;
+	}
+}
+
 static void keywords_append(struct mbox_sync_mail_context *ctx,
 			    keywords_mask_t keywords)
 {
@@ -28,7 +78,7 @@
 	old_hdr_size = ctx->body_offset - ctx->hdr_offset;
 	new_hdr_size = str_len(ctx->header) + ctx->have_eoh;
 
-	if (ctx->seq == 1 && ctx->base_uid_validity == 0) {
+	if (ctx->seq == 1 && ctx->sync_ctx->base_uid_validity == 0) {
 		ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] = str_len(ctx->header);
 		str_printfa(ctx->header, "X-IMAPbase: %u %u",
 			    ctx->sync_ctx->hdr->uid_validity,
@@ -37,14 +87,16 @@
 		str_append_c(ctx->header, '\n');
 	}
 
-	if (ctx->mail->uid == 0) {
+	if (ctx->hdr_pos[MBOX_HDR_X_UID] == (size_t)-1) {
+		if (ctx->mail.uid == 0)
+			ctx->mail.uid = ctx->sync_ctx->next_uid++;
 		ctx->hdr_pos[MBOX_HDR_X_UID] = str_len(ctx->header);
-		str_printfa(ctx->header, "X-UID: %u\n",
-			    ctx->sync_ctx->next_uid++);
+		str_printfa(ctx->header, "X-UID: %u\n", ctx->mail.uid);
 	}
+	i_assert(ctx->mail.uid != 0);
 
 	if (ctx->hdr_pos[MBOX_HDR_STATUS] == (size_t)-1 &&
-	    (ctx->mail->flags & STATUS_FLAGS_MASK) != 0) {
+	    (ctx->mail.flags & STATUS_FLAGS_MASK) != 0) {
 		ctx->hdr_pos[MBOX_HDR_STATUS] = str_len(ctx->header);
 		str_append(ctx->header, "Status: ");
 		status_flags_append(ctx, mbox_status_flags);
@@ -52,7 +104,7 @@
 	}
 
 	if (ctx->hdr_pos[MBOX_HDR_X_STATUS] == (size_t)-1 &&
-	    (ctx->mail->flags & XSTATUS_FLAGS_MASK) != 0) {
+	    (ctx->mail.flags & XSTATUS_FLAGS_MASK) != 0) {
 		ctx->hdr_pos[MBOX_HDR_X_STATUS] = str_len(ctx->header);
 		str_append(ctx->header, "X-Status: ");
 		status_flags_append(ctx, mbox_xstatus_flags);
@@ -61,7 +113,7 @@
 
 	have_keywords = FALSE;
 	for (i = 0; i < INDEX_KEYWORDS_BYTE_COUNT; i++) {
-		if (ctx->mail->keywords[i] != 0) {
+		if (ctx->mail.keywords[i] != 0) {
 			have_keywords = TRUE;
 			break;
 		}
@@ -70,21 +122,30 @@
 	if (ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] == (size_t)-1 && have_keywords) {
 		ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] = str_len(ctx->header);
 		str_append(ctx->header, "X-Keywords: ");
-		keywords_append(ctx, ctx->mail->keywords);
+		keywords_append(ctx, ctx->mail.keywords);
 		str_append_c(ctx->header, '\n');
 	}
 
 	if (ctx->content_length == (uoff_t)-1) {
 		str_printfa(ctx->header, "Content-Length: %"PRIuUOFF_T"\n",
-			    ctx->mail->body_size);
+			    ctx->mail.body_size);
 	}
 
 	if (str_len(ctx->header) != new_hdr_size) {
 		if (ctx->header_first_change == (size_t)-1)
 			ctx->header_first_change = new_hdr_size;
 		ctx->header_last_change = (size_t)-1;
-		ctx->mail->space -= str_len(ctx->header) -
+		ctx->mail.space -= str_len(ctx->header) -
 			(new_hdr_size - ctx->have_eoh);
+		if (ctx->mail.space > 0) {
+			/* we should rewrite this header, so offset
+			   must be broken if it's used anymore. */
+			ctx->mail.offset = (uoff_t)-1;
+		} else {
+			/* we don't have enough space for this header, change
+			   offset to point back to beginning of headers */
+			ctx->mail.offset = ctx->hdr_offset;
+		}
 		new_hdr_size = str_len(ctx->header) + ctx->have_eoh;
 	}
 
@@ -99,39 +160,82 @@
 
 static void mbox_sync_update_status(struct mbox_sync_mail_context *ctx)
 {
+	if (ctx->hdr_pos[MBOX_HDR_STATUS] != (size_t)-1) {
+		status_flags_replace(ctx, ctx->hdr_pos[MBOX_HDR_STATUS],
+				     mbox_status_flags);
+	}
 }
 
 static void mbox_sync_update_xstatus(struct mbox_sync_mail_context *ctx)
 {
+	if (ctx->hdr_pos[MBOX_HDR_X_STATUS] != (size_t)-1) {
+		status_flags_replace(ctx, ctx->hdr_pos[MBOX_HDR_X_STATUS],
+				     mbox_xstatus_flags);
+	}
 }
 
 static void mbox_sync_update_xkeywords(struct mbox_sync_mail_context *ctx)
 {
+	// FIXME
 }
 
 void mbox_sync_update_header(struct mbox_sync_mail_context *ctx,
-			     struct mail_index_sync_rec *update)
+			     buffer_t *syncs_buf)
 {
+	const struct mail_index_sync_rec *sync;
+	size_t size, i;
 	uint8_t old_flags;
 	keywords_mask_t old_keywords;
 
-	if (update != NULL) {
-		old_flags = ctx->mail->flags;
-		memcpy(old_keywords, ctx->mail->keywords, sizeof(old_keywords));
+	sync = buffer_get_data(syncs_buf, &size);
+	size /= sizeof(*sync);
 
-		mail_index_sync_flags_apply(update, &ctx->mail->flags,
-					    ctx->mail->keywords);
+	if (size != 0) {
+		old_flags = ctx->mail.flags;
+		memcpy(old_keywords, ctx->mail.keywords, sizeof(old_keywords));
+
+		for (i = 0; i < size; i++) {
+			mail_index_sync_flags_apply(&sync[i], &ctx->mail.flags,
+						    ctx->mail.keywords);
+		}
 
 		if ((old_flags & STATUS_FLAGS_MASK) !=
-		    (ctx->mail->flags & STATUS_FLAGS_MASK))
+		    (ctx->mail.flags & STATUS_FLAGS_MASK))
 			mbox_sync_update_status(ctx);
 		if ((old_flags & XSTATUS_FLAGS_MASK) !=
-		    (ctx->mail->flags & XSTATUS_FLAGS_MASK))
+		    (ctx->mail.flags & XSTATUS_FLAGS_MASK))
 			mbox_sync_update_xstatus(ctx);
-		if (memcmp(old_keywords, ctx->mail->keywords,
-			   sizeof(old_keywords)) != 0)
+		if (memcmp(old_keywords, ctx->mail.keywords,
+			   INDEX_KEYWORDS_BYTE_COUNT) != 0)
 			mbox_sync_update_xkeywords(ctx);
 	}
 
         mbox_sync_add_missing_headers(ctx);
 }
+
+void mbox_sync_update_header_from(struct mbox_sync_mail_context *ctx,
+				  const struct mbox_sync_mail *mail)
+{
+	if ((ctx->mail.flags & STATUS_FLAGS_MASK) !=
+	    (mail->flags & STATUS_FLAGS_MASK)) {
+		ctx->mail.flags = (ctx->mail.flags & ~STATUS_FLAGS_MASK) |
+			(mail->flags & STATUS_FLAGS_MASK);
+		mbox_sync_update_status(ctx);
+	}
+	if ((ctx->mail.flags & XSTATUS_FLAGS_MASK) !=
+	    (mail->flags & XSTATUS_FLAGS_MASK)) {
+		ctx->mail.flags = (ctx->mail.flags & ~XSTATUS_FLAGS_MASK) |
+			(mail->flags & XSTATUS_FLAGS_MASK);
+		mbox_sync_update_xstatus(ctx);
+	}
+	if (memcmp(ctx->mail.keywords, mail->keywords,
+		   INDEX_KEYWORDS_BYTE_COUNT) != 0) {
+		memcpy(ctx->mail.keywords, mail->keywords,
+		       INDEX_KEYWORDS_BYTE_COUNT);
+		mbox_sync_update_xkeywords(ctx);
+	}
+
+	i_assert(ctx->mail.uid == 0 || ctx->mail.uid == mail->uid);
+	ctx->mail.uid = mail->uid;
+	mbox_sync_add_missing_headers(ctx);
+}

Index: mbox-sync.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync.c,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -d -r1.3 -r1.4
--- mbox-sync.c	6 May 2004 03:05:42 -0000	1.3
+++ mbox-sync.c	9 May 2004 17:06:06 -0000	1.4
@@ -73,7 +73,7 @@
 	if (file_set_size(sync_ctx->fd, size) < 0)
 		return -1;
 
-	if (mail->space_offset == 0) {
+	if (mail->space <= 0) {
 		/* no X-Keywords header - place it at the end. */
 		grow_size += 13;
 
@@ -91,11 +91,11 @@
 		/* FIXME: can this break anything? X-Keywords text might
 		   have been already included in space calculation. now we
 		   have more.. */
-		mail->space_offset = offset;
+		mail->offset = offset;
 		mail->space += grow_size;
 	} else {
-		offset = mail->space_offset;
-		if (mbox_move(sync_ctx, mail->space_offset + grow_size,
+		offset = mail->offset;
+		if (mbox_move(sync_ctx, offset + grow_size,
 			      offset, (uoff_t)-1) < 0)
 			return -1;
 	}
@@ -117,11 +117,63 @@
 	return 0;
 }
 
+static void mbox_sync_buffer_delete_old(buffer_t *syncs_buf, uint32_t seq)
+{
+	struct mail_index_sync_rec *sync;
+	size_t size, src, dest;
+
+	sync = buffer_get_modifyable_data(syncs_buf, &size);
+	size /= sizeof(*sync);
+
+	for (src = dest = 0; src < size; src++) {
+		if (sync[src].seq2 >= seq) {
+			if (src != dest)
+				sync[dest] = sync[src];
+			dest++;
+		}
+	}
+
+	buffer_set_used_size(syncs_buf, dest * sizeof(*sync));
+}
+
+static int
+mbox_sync_next_mail(struct mbox_sync_context *sync_ctx,
+		    struct mbox_sync_mail_context *mail_ctx, uint32_t seq)
+{
+	uoff_t from_offset;
+
+	memset(mail_ctx, 0, sizeof(*mail_ctx));
+	mail_ctx->sync_ctx = sync_ctx;
+	mail_ctx->seq = seq;
+	mail_ctx->header = sync_ctx->header;
+
+	from_offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
+	mail_ctx->mail.offset =
+		istream_raw_mbox_get_header_offset(sync_ctx->input);
+
+	mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx);
+	if (sync_ctx->input->v_offset == from_offset) {
+		/* this was the last mail */
+		return 0;
+	}
+
+	mail_ctx->mail.body_size =
+		istream_raw_mbox_get_body_size(sync_ctx->input,
+					       mail_ctx->content_length);
+
+	/* save the offset permanently with recent flag state */
+	from_offset <<= 1;
+	if ((mail_ctx->mail.flags & MBOX_NONRECENT) == 0)
+		from_offset |= 1;
+	buffer_append(sync_ctx->ibox->mbox_data_buf, &from_offset,
+		      sizeof(from_offset));
+	return 1;
+}
+
 int mbox_sync(struct index_mailbox *ibox, int last_commit)
 {
 	struct mbox_sync_context sync_ctx;
 	struct mbox_sync_mail_context mail_ctx;
-	struct mbox_sync_mail mail;
 	struct mail_index_sync_ctx *index_sync_ctx;
 	struct mail_index_sync_rec sync_rec;
 	struct mail_index_view *sync_view;
@@ -131,9 +183,8 @@
 	struct istream *input;
 	uint32_t seq, need_space_seq, idx_seq, messages_count;
 	off_t space_diff;
-	uoff_t from_offset, offset;
-	buffer_t *mails;
-	string_t *header;
+	uoff_t offset, extra_space;
+	buffer_t *mails, *syncs;
 	size_t size;
 	struct stat st;
 	int readonly, ret = 0;
@@ -166,64 +217,62 @@
 		buffer_set_used_size(ibox->mbox_data_buf, 0);
 	}
 
-	readonly = TRUE; // FIXME
-
+	readonly = FALSE; // FIXME
 	// FIXME: lock the file
 
 	memset(&sync_ctx, 0, sizeof(sync_ctx));
+	sync_ctx.ibox = ibox;
 	sync_ctx.file_input = ibox->mbox_file_stream;
 	sync_ctx.input = ibox->mbox_stream;
 	sync_ctx.fd = ibox->mbox_fd;
 	sync_ctx.hdr = hdr;
+	sync_ctx.header = str_new(default_pool, 4096);
 
 	input = sync_ctx.input;
 	istream_raw_mbox_seek(input, 0);
 
-	header = str_new(default_pool, 4096);
-
 	mails = buffer_create_dynamic(default_pool, 4096, (size_t)-1);
+	syncs = buffer_create_dynamic(default_pool, 256, (size_t)-1);
+
 	memset(&sync_rec, 0, sizeof(sync_rec));
 	messages_count = mail_index_view_get_message_count(sync_view);
 
 	space_diff = 0; need_space_seq = 0; idx_seq = 0; rec = NULL;
 	for (seq = 1; !input->eof; seq++) {
-		if (sync_rec.seq2 < seq) {
-			// FIXME: we may need more than one..
+		ret = 1;
+
+		/* get all sync records related to this message */
+		mbox_sync_buffer_delete_old(syncs, seq);
+		while (sync_rec.seq2 <= seq && ret > 0) {
+			if (sync_rec.seq2 != 0) {
+				buffer_append(syncs, &sync_rec,
+					      sizeof(sync_rec));
+			}
 			ret = mail_index_sync_next(index_sync_ctx, &sync_rec);
-			if (ret < 0)
-				break;
 		}
+		if (ret < 0)
+			break;
 
-		from_offset = input->v_offset;
-
-		memset(&mail, 0, sizeof(mail));
-		memset(&mail_ctx, 0, sizeof(mail_ctx));
-		mail_ctx.sync_ctx = &sync_ctx;
-		mail_ctx.mail = &mail;
-		mail_ctx.seq = seq;
-		mail_ctx.header = header;
-
-		mbox_sync_parse_next_mail(input, &mail_ctx);
-		if (input->v_offset == from_offset) {
-			/* this was the last mail */
+		ret = mbox_sync_next_mail(&sync_ctx, &mail_ctx, seq);
+		if (ret <= 0)
 			break;
-		}
 
-		mail.body_size =
-			istream_raw_mbox_get_size(input,
-						  mail_ctx.content_length);
-		buffer_append(mails, &mail, sizeof(mail));
+		if ((mail_ctx.need_rewrite ||
+		     buffer_get_used_size(syncs) != 0) && !readonly) {
+			mbox_sync_update_header(&mail_ctx, syncs);
+			if ((ret = mbox_sync_try_rewrite(&mail_ctx)) < 0)
+				return -1;
 
-		/* save the offset permanently with recent flag state */
-		from_offset <<= 1;
-		if ((mail.flags & MBOX_NONRECENT) == 0)
-			from_offset |= 1;
-		buffer_append(ibox->mbox_data_buf,
-			      &from_offset, sizeof(from_offset));
+			if (ret == 0 && need_space_seq == 0) {
+				/* first mail with no space to write it */
+				need_space_seq = seq;
+				space_diff = 0;
+			}
+		}
 
 		/* update index */
 		do {
-			if (rec != NULL && rec->uid >= mail.uid)
+			if (rec != NULL && rec->uid >= mail_ctx.mail.uid)
 				break;
 
 			if (idx_seq >= messages_count) {
@@ -239,12 +288,13 @@
 
 		if (ret < 0)
 			break;
-		if (rec != NULL && rec->uid != mail.uid) {
+		if (rec != NULL && rec->uid != mail_ctx.mail.uid) {
 			/* new UID in the middle of the mailbox -
 			   shouldn't happen */
 			mail_storage_set_critical(ibox->box.storage,
 				"mbox sync: UID inserted in the middle "
-				"of mailbox (%u > %u)", rec->uid, mail.uid);
+				"of mailbox (%u > %u)",
+				rec->uid, mail_ctx.mail.uid);
 			mail_index_mark_corrupted(ibox->index);
 			ret = -1;
 			break;
@@ -253,40 +303,34 @@
 		if (rec != NULL) {
 			/* see if flags changed */
 			if ((rec->flags & MAIL_FLAGS_MASK) !=
-			    (mail.flags & MAIL_FLAGS_MASK) ||
-			    memcmp(rec->keywords, mail.keywords,
+			    (mail_ctx.mail.flags & MAIL_FLAGS_MASK) ||
+			    memcmp(rec->keywords, mail_ctx.mail.keywords,
 				   INDEX_KEYWORDS_BYTE_COUNT) != 0) {
 				uint8_t new_flags =
 					(rec->flags & ~MAIL_FLAGS_MASK) |
-					(mail.flags & MAIL_FLAGS_MASK);
+					(mail_ctx.mail.flags & MAIL_FLAGS_MASK);
 				mail_index_update_flags(t, idx_seq,
 							MODIFY_REPLACE,
 							new_flags,
-							mail.keywords);
+							mail_ctx.mail.keywords);
 			}
 			rec = NULL;
 		} else {
 			/* new message */
-			mail_index_append(t, mail.uid, &idx_seq);
+			mail_index_append(t, mail_ctx.mail.uid, &idx_seq);
 			mail_index_update_flags(t, idx_seq, MODIFY_REPLACE,
-						mail.flags & MAIL_FLAGS_MASK,
-						mail.keywords);
+				mail_ctx.mail.flags & MAIL_FLAGS_MASK,
+				mail_ctx.mail.keywords);
 		}
 
-		if (mail_ctx.need_rewrite && !readonly) {
-			mbox_sync_update_header(&mail_ctx, NULL);
-			if ((ret = mbox_sync_try_rewrite(&mail_ctx)) < 0)
-				break;
-		} else {
-			ret = 1;
-		}
+		istream_raw_mbox_next(input, mail_ctx.mail.body_size);
+		offset = input->v_offset;
 
-		if (ret == 0 && need_space_seq == 0) {
-			/* didn't have space to write it */
-			need_space_seq = seq;
-			space_diff = mail.space;
-		} else if (need_space_seq != 0) {
-			space_diff += mail.space;
+		if (need_space_seq != 0) {
+			buffer_append(mails, &mail_ctx.mail,
+				      sizeof(mail_ctx.mail));
+
+			space_diff += mail_ctx.mail.space;
 			if (space_diff >= 0) {
 				/* we have enough space now */
 				if (mbox_sync_rewrite(&sync_ctx, mails,
@@ -295,25 +339,38 @@
 					ret = -1;
 					break;
 				}
+
+				/* mail_ctx may contain wrong data after
+				   rewrite, so make sure we don't try to access
+				   it */
+				memset(&mail_ctx, 0, sizeof(mail_ctx));
+				i_stream_seek(input, input->v_offset);
 				need_space_seq = 0;
 			}
 		}
-
-		istream_raw_mbox_next(input, mail.body_size);
 	}
 
 	if (need_space_seq != 0) {
 		i_assert(space_diff < 0);
-		if (mbox_sync_grow_file(&sync_ctx, &mail, mail_ctx.body_offset,
-					-space_diff) < 0)
+		extra_space = MBOX_HEADER_EXTRA_SPACE *
+			((seq-1) - need_space_seq);
+		if (mbox_sync_grow_file(&sync_ctx, &mail_ctx.mail,
+					mail_ctx.body_offset,
+					-space_diff + extra_space) < 0)
 			ret = -1;
 		else if (mbox_sync_rewrite(&sync_ctx, mails, need_space_seq,
-					   seq-1, space_diff) < 0)
+					   seq-1, extra_space) < 0)
 			ret = -1;
 	}
 
+	if (sync_ctx.base_uid_last+1 != sync_ctx.next_uid) {
+		// FIXME: rewrite X-IMAPbase header
+	}
+
+	/* only syncs left should be just appends which weren't synced yet.
+	   we'll just ignore them, as we've overwritten those above. */
 	while ((ret = mail_index_sync_next(index_sync_ctx, &sync_rec)) > 0) {
-		// FIXME: should be just appends
+		i_assert(sync_rec.type == MAIL_INDEX_SYNC_TYPE_APPEND);
 	}
 
 	if (fstat(ibox->mbox_fd, &st) < 0) {
@@ -350,7 +407,9 @@
 	ibox->mbox_data = buffer_get_data(ibox->mbox_data_buf, &size);
 	ibox->mbox_data_count = size / sizeof(*ibox->mbox_data);
 
-	str_free(header);
+	str_free(sync_ctx.header);
+	buffer_free(mails);
+	buffer_free(syncs);
 	return ret < 0 ? -1 : 0;
 }
 



More information about the dovecot-cvs mailing list