[dovecot-cvs] dovecot/src/lib-storage/index/mbox istream-raw-mbox.c, NONE, 1.1 istream-raw-mbox.h, NONE, 1.1 mbox-from.c, NONE, 1.1 mbox-from.h, NONE, 1.1 mbox-sync-header.c, NONE, 1.1 mbox-sync-parse.c, NONE, 1.1 mbox-sync-private.h, NONE, 1.1 mbox-sync-rewrite.c, NONE, 1.1 mbox-sync-update.c, NONE, 1.1 mbox-sync.c, NONE, 1.1 Makefile.am, 1.1.1.1, 1.2 mbox-expunge.c, 1.33, 1.34 mbox-list.c, 1.22, 1.23 mbox-save.c, 1.45, 1.46

[cras at procontrol.fi]

Update of /home/cvs/dovecot/src/lib-storage/index/mbox
In directory talvi:/tmp/cvs-serv29236/src/lib-storage/index/mbox

Modified Files:
	Makefile.am mbox-expunge.c mbox-list.c mbox-save.c 
Added Files:
	istream-raw-mbox.c istream-raw-mbox.h mbox-from.c mbox-from.h 
	mbox-sync-header.c mbox-sync-parse.c mbox-sync-private.h 
	mbox-sync-rewrite.c mbox-sync-update.c mbox-sync.c 
Log Message:
importing new index code. mbox still broken.



--- NEW FILE: istream-raw-mbox.c ---
/* Copyright (C) 2003 Timo Sirainen */

#include "lib.h"
#include "buffer.h"
#include "istream-internal.h"
#include "istream-raw-mbox.h"
#include "mbox-from.h"

struct raw_mbox_istream {
	struct _istream istream;

	time_t received_time, next_received_time;
	uoff_t from_offset, body_size;
	struct istream *input;
};

static void _close(struct _iostream *stream __attr_unused__)
{
}

static void _destroy(struct _iostream *stream)
{
	struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream;

	i_stream_seek(rstream->input, rstream->istream.istream.v_offset);
	i_stream_unref(rstream->input);
}

static void _set_max_buffer_size(struct _iostream *stream, size_t max_size)
{
	struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream;

	i_stream_set_max_buffer_size(rstream->input, max_size);
}

static void _set_blocking(struct _iostream *stream, int timeout_msecs,
			  void (*timeout_cb)(void *), void *context)
{
	struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream;

	i_stream_set_blocking(rstream->input, timeout_msecs,
			      timeout_cb, context);
}

static ssize_t _read(struct _istream *stream)
{
	static const char *mbox_from = "\nFrom ";
	struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream;
	const unsigned char *buf, *p;
	const char *fromp;
	time_t received_time;
	size_t i, pos;
	ssize_t ret;

	i_stream_seek(rstream->input, stream->istream.v_offset);

	stream->pos -= stream->skip;
	stream->skip = 0;
	stream->buffer = NULL;

	do {
		ret = i_stream_read(rstream->input);
		buf = i_stream_get_data(rstream->input, &pos);
	} while (ret > 0 && pos <= 6);

	if (pos == 1 && buf[0] == '\n') {
		/* EOF */
		stream->pos = 0;
		stream->istream.eof = TRUE;
		return -1;
	}

	if (stream->istream.v_offset == rstream->from_offset) {
		/* read the full From-line */
		int skip = rstream->from_offset != 0;
		size_t line_pos;

		while ((p = memchr(buf+skip, '\n', pos-skip)) == NULL) {
			if (i_stream_read(rstream->input) < 0) {
				/* EOF - shouldn't happen */
				stream->pos = 0;
				stream->istream.eof = TRUE;
				return -1;
			}
			buf = i_stream_get_data(rstream->input, &pos);
		}
		line_pos = (size_t)(p - buf);

		if (rstream->from_offset != 0) {
			buf++;
			pos--;
		}

		/* beginning of mbox */
		if (memcmp(buf, "From ", 5) != 0)
			received_time = (time_t)-1;
		else
			received_time = mbox_from_parse_date(buf+5, pos-5);

		if (received_time == (time_t)-1) {
			/* broken From - should happen only at beginning of
			   file if this isn't a mbox.. */
			stream->pos = 0;
			stream->istream.eof = TRUE;
			return -1;
		}

		if (rstream->from_offset == 0)
			rstream->received_time = received_time;
		else
			rstream->next_received_time = received_time;

		/* we'll skip over From-line and try again */
		stream->istream.v_offset += line_pos+1;
		return _read(stream);
	}

	if (pos >= 31) {
		if (memcmp(buf, "\nFrom ", 6) == 0) {
			received_time = mbox_from_parse_date(buf+6, pos-6);
			if (received_time != (time_t)-1) {
				rstream->next_received_time = received_time;
				i_assert(stream->pos == 0);
				return -1;
			}
		}
	} else if (ret == -1) {
		/* last few bytes, can't contain From-line */
		ret = pos <= stream->pos ? -1 :
			(ssize_t) (pos - stream->pos);

		stream->buffer = buf;
		stream->pos = pos;
		stream->istream.eof = ret == -1;
		return ret;
	}

	/* See if we have From-line here - note that it works right only
	   because all characters are different in mbox_from. */
	for (i = 0, fromp = mbox_from; i < pos; i++) {
		if (buf[i] == *fromp) {
			if (*++fromp == '\0') {
				/* potential From-line - stop here */
				i++;
				break;
			}
		} else {
			fromp = mbox_from;
			if (buf[i] == *fromp)
				fromp++;
		}
	}
	pos = i - (fromp - mbox_from);

	ret = pos <= stream->pos ? -1 :
		(ssize_t) (pos - stream->pos);
	stream->buffer = buf;
	stream->pos = pos;
	return ret;
}

static void _seek(struct _istream *stream, uoff_t v_offset)
{
	stream->istream.v_offset = v_offset;
	stream->skip = stream->pos = 0;
	stream->buffer = NULL;
}

struct istream *i_stream_create_raw_mbox(pool_t pool, struct istream *input)
{
	struct raw_mbox_istream *rstream;

	i_stream_ref(input);

	rstream = p_new(pool, struct raw_mbox_istream, 1);

	rstream->input = input;
	rstream->body_size = (uoff_t)-1;

	rstream->istream.iostream.close = _close;
	rstream->istream.iostream.destroy = _destroy;
	rstream->istream.iostream.set_max_buffer_size = _set_max_buffer_size;
	rstream->istream.iostream.set_blocking = _set_blocking;

	rstream->istream.read = _read;
	rstream->istream.seek = _seek;

	return _i_stream_create(&rstream->istream, pool, -1,
				input->real_stream->abs_start_offset);
}

static int istream_raw_mbox_is_valid_from(struct raw_mbox_istream *rstream)
{
	const unsigned char *data;
	size_t size;
	time_t received_time;

	/* minimal: "From x Thu Nov 29 22:33:52 2001" = 31 chars */
	if (i_stream_read_data(rstream->input, &data, &size, 30) == -1)
		return -1;

	if (size == 1 && data[0] == '\n') {
		/* EOF */
		return TRUE;
	}

	if (size < 31 || memcmp(data, "\nFrom ", 6) != 0)
		return FALSE;

	while (memchr(data+1, '\n', size-1) == NULL) {
		if (i_stream_read_data(rstream->input, &data, &size, size) < 0)
			break;
	}

	received_time = mbox_from_parse_date(data+6, size-6);
	if (received_time == (time_t)-1)
		return FALSE;

	rstream->next_received_time = received_time;
	return TRUE;
}

uoff_t istream_raw_mbox_get_size(struct istream *stream, uoff_t body_size)
{
	struct raw_mbox_istream *rstream =
		(struct raw_mbox_istream *)stream->real_stream;
	uoff_t old_offset;
	const unsigned char *data;
	size_t size;

	if (rstream->body_size != (uoff_t)-1)
		return rstream->body_size;

	if (body_size != (uoff_t)-1) {
		i_stream_seek(rstream->input, rstream->from_offset + body_size);
		if (istream_raw_mbox_is_valid_from(rstream) > 0) {
			rstream->body_size = body_size;
			return body_size;
		}
	}

	old_offset = stream->v_offset;

	/* have to read through the message body */
	while (i_stream_read_data(stream, &data, &size, 0) > 0)
		i_stream_skip(stream, size);

	rstream->body_size = stream->v_offset - old_offset;
	i_stream_seek(stream, old_offset);
	return rstream->body_size;
}

void istream_raw_mbox_next(struct istream *stream, uoff_t body_size)
{
	struct raw_mbox_istream *rstream =
		(struct raw_mbox_istream *)stream->real_stream;

	body_size = istream_raw_mbox_get_size(stream, body_size);
	rstream->body_size = (uoff_t)-1;

	rstream->received_time = rstream->next_received_time;
	rstream->next_received_time = (time_t)-1;

	rstream->from_offset = stream->v_offset + body_size;
	i_stream_seek(rstream->input, rstream->from_offset);
}

void istream_raw_mbox_flush(struct istream *stream)
{
	struct raw_mbox_istream *rstream =
		(struct raw_mbox_istream *)stream->real_stream;

	/* kludgy */
	rstream->input->real_stream->skip = 0;
	rstream->input->real_stream->pos = 0;

	rstream->istream.skip = 0;
	rstream->istream.pos = 0;
}

--- NEW FILE: istream-raw-mbox.h ---
#ifndef __ISTREAM_RAW_MBOX_H
#define __ISTREAM_RAW_MBOX_H

/* Create a mbox stream for parsing mbox. Reading stops before From-line,
   you'll have to call istream_raw_mbox_next() to get to next message. */
struct istream *i_stream_create_raw_mbox(pool_t pool, struct istream *input);

/* Return number of bytes in this message after current offset.
   If body_size isn't (uoff_t)-1, we'll use it as potentially valid body size
   to avoid actually reading through the whole message. */
uoff_t istream_raw_mbox_get_size(struct istream *stream, uoff_t body_size);

/* Jump to next message. If body_size isn't (uoff_t)-1, we'll use it as
   potentially valid body size. */
void istream_raw_mbox_next(struct istream *stream, uoff_t body_size);

/* Flush all buffering. Call if you modify the mbox. */
void istream_raw_mbox_flush(struct istream *stream);

#endif

--- NEW FILE: mbox-from.c ---
/* Copyright (C) 2002 Timo Sirainen */

#include "lib.h"
#include "str.h"
#include "utc-mktime.h"
#include "mbox-from.h"

#include <time.h>
#include <ctype.h>

static const char *weekdays[] = {
	"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"
};

static const char *months[] = {
	"Jan", "Feb", "Mar", "Apr", "May", "Jun",
	"Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
};

time_t mbox_from_parse_date(const unsigned char *msg, size_t size)
{
	const unsigned char *msg_end;
	struct tm tm;
	int i, timezone;
	time_t t;

	/* <sender> <date> <moreinfo> */
	msg_end = msg + size;

	/* skip sender */
	while (msg < msg_end && *msg != ' ') {
		if (*msg == '\r' || *msg == '\n')
			return (time_t)-1;
		msg++;
	}
	while (msg < msg_end && *msg == ' ') msg++;

	/* next 24 chars should be in the date in asctime() format, eg.
	   "Thu Nov 29 22:33:52 2001 +0300"

	   Some also include named timezone, which we ignore:

	   "Thu Nov 29 22:33:52 EEST 2001"
	*/
	if (msg+24 > msg_end)
		return (time_t)-1;

	memset(&tm, 0, sizeof(tm));

	/* skip weekday */
	msg += 4;

	/* month */
	for (i = 0; i < 12; i++) {
		if (memcasecmp(months[i], msg, 3) == 0) {
			tm.tm_mon = i;
			break;
		}
	}

	if (i == 12 && memcmp(msg, "???", 3) == 0) {
		/* just a hack to parse one special mbox I have :) */
		i = 0;
	}

	if (i == 12 || msg[3] != ' ')
		return (time_t)-1;
	msg += 4;

	/* day */
	if (msg[0] == ' ') {
		if (!i_isdigit(msg[1]) || msg[2] != ' ')
			return (time_t)-1;
		tm.tm_mday = msg[1]-'0';
	} else {
		if (!i_isdigit(msg[0]) || !i_isdigit(msg[1]) || msg[2] != ' ')
			return (time_t)-1;
		tm.tm_mday = (msg[0]-'0') * 10 + (msg[1]-'0');
	}
	if (tm.tm_mday == 0)
		tm.tm_mday = 1;
	msg += 3;

	/* hour */
	if (!i_isdigit(msg[0]) || !i_isdigit(msg[1]) || msg[2] != ':')
		return (time_t)-1;
	tm.tm_hour = (msg[0]-'0') * 10 + (msg[1]-'0');
	msg += 3;

	/* minute */
	if (!i_isdigit(msg[0]) || !i_isdigit(msg[1]) || msg[2] != ':')
		return (time_t)-1;
	tm.tm_min = (msg[0]-'0') * 10 + (msg[1]-'0');
	msg += 3;

	/* second */
	if (!i_isdigit(msg[0]) || !i_isdigit(msg[1]) || msg[2] != ' ')
		return (time_t)-1;
	tm.tm_sec = (msg[0]-'0') * 10 + (msg[1]-'0');
	msg += 3;

	/* optional named timezone */
	if (!i_isdigit(msg[0]) || !i_isdigit(msg[1]) ||
	    !i_isdigit(msg[2]) || !i_isdigit(msg[3])) {
		/* skip to next space */
		while (msg < msg_end && *msg != ' ') {
			if (*msg == '\r' || *msg == '\n')
				return (time_t)-1;
			msg++;
		}
		if (msg+5 > msg_end)
			return (time_t)-1;
		msg++;
	}

	/* year */
	if (!i_isdigit(msg[0]) || !i_isdigit(msg[1]) ||
	    !i_isdigit(msg[2]) || !i_isdigit(msg[3]))
		return (time_t)-1;

	tm.tm_year = (msg[0]-'0') * 1000 + (msg[1]-'0') * 100 +
		(msg[2]-'0') * 10 + (msg[3]-'0') - 1900;
	msg += 4;

	tm.tm_isdst = -1;
	if (msg[0] == ' ' && (msg[1] == '-' || msg[1] == '+') &&
	    i_isdigit(msg[2]) && i_isdigit(msg[3]) &&
	    i_isdigit(msg[4]) && i_isdigit(msg[5])) {
		timezone = (msg[2]-'0') * 1000 + (msg[3]-'0') * 100 +
			(msg[4]-'0') * 10 +(msg[5]-'0');
		if (msg[1] == '-') timezone = -timezone;

		t = utc_mktime(&tm);
		if (t == (time_t)-1)
			return (time_t)-1;

		t -= timezone * 60;
		return t;
	} else {
		/* assume local timezone */
		return mktime(&tm);
	}
}

const char *mbox_from_create(const char *sender, time_t time)
{
	string_t *str;
	struct tm *tm;
	int year;

	str = t_str_new(256);
	str_append(str, "From ");
	str_append(str, sender);
	str_append(str, "  ");

	/* we could use simply asctime(), but i18n etc. may break it.
	   Example: "Thu Nov 29 22:33:52 2001" */
	tm = localtime(&time);

	/* week day */
	str_append(str, weekdays[tm->tm_wday]);
	str_append_c(str, ' ');

	/* month */
	str_append(str, months[tm->tm_mon]);
	str_append_c(str, ' ');

	/* day */
	str_append_c(str, (tm->tm_mday / 10) + '0');
	str_append_c(str, (tm->tm_mday % 10) + '0');
	str_append_c(str, ' ');

	/* hour */
	str_append_c(str, (tm->tm_hour / 10) + '0');
	str_append_c(str, (tm->tm_hour % 10) + '0');
	str_append_c(str, ':');

	/* minute */
	str_append_c(str, (tm->tm_min / 10) + '0');
	str_append_c(str, (tm->tm_min % 10) + '0');
	str_append_c(str, ':');

	/* second */
	str_append_c(str, (tm->tm_sec / 10) + '0');
	str_append_c(str, (tm->tm_sec % 10) + '0');
	str_append_c(str, ' ');

	/* year */
	year = tm->tm_year + 1900;
	str_append_c(str, (year / 1000) + '0');
	str_append_c(str, ((year / 100) % 10) + '0');
	str_append_c(str, ((year / 10) % 10) + '0');
	str_append_c(str, (year % 10) + '0');

	str_append_c(str, '\n');
	return str_c(str);
}

--- NEW FILE: mbox-from.h ---
#ifndef __MBOX_FROM_H
#define __MBOX_FROM_H

time_t mbox_from_parse_date(const unsigned char *msg, size_t size);
const char *mbox_from_create(const char *sender, time_t time);

#endif

--- NEW FILE: mbox-sync-header.c ---
/* Copyright (C) 2004 Timo Sirainen */

#include "lib.h"
#include "buffer.h"
#include "istream.h"
#include "str.h"
#include "write-full.h"
#include "message-parser.h"
#include "mail-index.h"
#include "mbox-sync-private.h"

#include <unistd.h>

struct mbox_flag_type {
	char chr;
	enum mail_flags flag;
};

#define MBOX_NONRECENT MAIL_RECENT /* kludgy */

#define STATUS_FLAGS_MASK (MAIL_SEEN|MBOX_NONRECENT)
static struct mbox_flag_type status_flags[] = {
	{ 'R', MAIL_SEEN },
	{ 'O', MBOX_NONRECENT },
	{ 0, 0 }
};

#define XSTATUS_FLAGS_MASK (MAIL_ANSWERED|MAIL_FLAGGED|MAIL_DRAFT|MAIL_DELETED)
static struct mbox_flag_type xstatus_flags[] = {
	{ 'A', MAIL_ANSWERED },
	{ 'F', MAIL_FLAGGED },
	{ 'T', MAIL_DRAFT },
	{ 'D', MAIL_DELETED },
	{ 0, 0 }
};

struct header_func {
	const char *header;
	int (*func)(struct mbox_sync_mail_context *ctx,
		    struct message_header_line *hdr);
};

static enum mail_flags mbox_flag_find(struct mbox_flag_type *flags, char chr)
{
	int i;

	for (i = 0; flags[i].chr != 0; i++) {
		if (flags[i].chr == chr)
			return flags[i].flag;
	}

	return 0;
}

static void status_flags_append(struct mbox_sync_mail_context *ctx,
				struct mbox_flag_type *flags_list)
{
	int i;

	for (i = 0; flags_list[i].chr != 0; i++) {
		if ((ctx->mail_flags & flags_list[i].flag) != 0) {
			str_append_c(ctx->header, flags_list[i].chr);
			ctx->mail_flags &= ~flags_list[i].flag;
		}
	}
}

static int parse_status_flags(struct mbox_sync_mail_context *ctx,
			      struct message_header_line *hdr,
			      struct mbox_flag_type *flags_list)
{
	size_t i, start, end;
	int j;
        enum mail_flags flags, flags_mask;

	flags = 0;
	for (i = 0; i < hdr->full_value_len; i++)
		flags |= mbox_flag_find(flags_list, hdr->full_value[i]);

	flags_mask = 0;
	for (j = 0; flags_list[j].chr != 0; j++)
		flags_mask |= flags_list[j].flag;

	/* see if anything changed */
	if (flags == (ctx->mail_flags & flags_mask)) {
		ctx->mail_flags &= ~flags_mask;
		return FALSE;
	}

	start = str_len(ctx->header);
	str_append(ctx->header, hdr->name);
	str_append(ctx->header, ": ");
	end = str_len(ctx->header);

	status_flags_append(ctx, flags_list);

	for (i = 0; i < hdr->full_value_len; i++) {
		switch (hdr->full_value[i]) {
		case ' ':
		case '\t':
		case '\r':
		case '\n':
			break;
		default:
			if (mbox_flag_find(flags_list, hdr->full_value[i]) != 0)
				break;

			/* unknown, keep it */
			str_append_c(ctx->header, hdr->full_value[i]);
			break;
		}
	}

	if (str_len(ctx->header) != end)
		str_append_c(ctx->header, '\n');
	else
		str_truncate(ctx->header, start);
	return TRUE;
}

static int parse_status(struct mbox_sync_mail_context *ctx,
			struct message_header_line *hdr)
{
	return parse_status_flags(ctx, hdr, status_flags);
}

static int parse_x_status(struct mbox_sync_mail_context *ctx,
			  struct message_header_line *hdr)
{
	return parse_status_flags(ctx, hdr, xstatus_flags);
}

static int parse_x_imap_base(struct mbox_sync_mail_context *ctx,
			     struct message_header_line *hdr)
{
	ctx->ximapbase_pos = buffer_get_used_size(ctx->header);
	// FIXME: check it
	//ctx->extra_space += 1;
	return FALSE;
}

static int parse_x_keywords(struct mbox_sync_mail_context *ctx,
			    struct message_header_line *hdr)
{
	ctx->xkeywords_pos = buffer_get_used_size(ctx->header);
	// FIXME: update it
        //ctx->extra_space += 1;
	return FALSE;
}

static int parse_content_length(struct mbox_sync_mail_context *ctx,
				struct message_header_line *hdr)
{
	uoff_t value = 0;
	size_t i;

	if (ctx->content_length != (uoff_t)-1) {
		/* duplicate */
		return TRUE;
	}

	for (i = 0; i < hdr->full_value_len; i++) {
		if (hdr->full_value[i] < '0' || hdr->full_value[i] > '9')
			break;
		value = value*10 + (hdr->full_value[i] - '0');
	}

	for (; i < hdr->full_value_len; i++) {
		if (hdr->full_value[i] != ' ' && hdr->full_value[i] != '\t') {
			/* broken value */
			return TRUE;
		}
	}

	ctx->content_length = value;
	return FALSE;
}

static int parse_x_uid(struct mbox_sync_mail_context *ctx,
		       struct message_header_line *hdr)
{
	uint32_t value = 0;
	size_t i, extra_space = 0;

	if (ctx->uid != 0) {
		/* duplicate */
		return TRUE;
	}

	for (i = 0; i < hdr->full_value_len; i++) {
		if (hdr->full_value[i] < '0' || hdr->full_value[i] > '9')
			break;
		value = value*10 + (hdr->full_value[i] - '0');
	}

	for (; i < hdr->full_value_len; i++) {
		if (hdr->full_value[i] != ' ' && hdr->full_value[i] != '\t') {
			/* broken value */
			return TRUE;
		}
		extra_space++;
	}

	if (value <= ctx->parent->prev_msg_uid) {
		/* broken - UIDs must be growing */
		return TRUE;
	}

	ctx->uid = value;
	ctx->extra_space += extra_space;
	ctx->xuid_pos = buffer_get_used_size(ctx->header);
	return FALSE;
}

static struct header_func header_funcs[] = {
	{ "Content-Length", parse_content_length },
	{ "Status", parse_status },
	{ "X-IMAPbase", parse_x_imap_base },
	{ "X-Keywords", parse_x_keywords },
	{ "X-Status", parse_x_status },
	{ "X-UID", parse_x_uid },
	{ NULL, NULL }
};

static struct header_func *header_func_find(const char *header)
{
	int i;

	for (i = 0; header_funcs[i].header != NULL; i++) {
		if (strcasecmp(header_funcs[i].header, header) == 0)
			return &header_funcs[i];
	}
	return NULL;
}

void mbox_sync_mail_parse_headers(struct mbox_sync_mail_context *ctx)
{
	struct message_header_parser_ctx *hdr_ctx;
	struct message_header_line *hdr;
        struct header_func *func;

        ctx->header_first_change = (size_t)-1;
	ctx->header_last_change = (size_t)-1;

	ctx->xuid_pos = (size_t)-1;
	ctx->xkeywords_pos = (size_t)-1;

	ctx->content_length = (uoff_t)-1;
	str_truncate(ctx->header, 0);

	hdr_ctx = message_parse_header_init(ctx->parent->input, NULL);
	while ((hdr = message_parse_header_next(hdr_ctx)) != NULL) {
		if (hdr->eoh) {
			ctx->have_eoh = 1;
			break;
		}

		func = header_func_find(hdr->name);
		if (func != NULL) {
			if (hdr->continues) {
				hdr->use_full_value = TRUE;
				continue;
			}
			if (func->func(ctx, hdr)) {
				/* we modified this header */
				if (ctx->header_first_change == (size_t)-1) {
					ctx->header_first_change =
					      buffer_get_used_size(ctx->header);
				}
				ctx->header_last_change = (size_t)-1;
			} else {
				func = NULL;
			}
		}

		if (func == NULL) {
			if (ctx->header_last_change == (size_t)-1) {
				/* we may be able to stop rewriting here */
				ctx->header_last_change =
					buffer_get_used_size(ctx->header);
			}
			if (!hdr->continued) {
				str_append(ctx->header, hdr->name);
				str_append(ctx->header, ": ");
			}
			buffer_append(ctx->header, hdr->full_value,
				      hdr->full_value_len);
			if (!hdr->no_newline)
				str_append_c(ctx->header, '\n');
		}
	}
	message_parse_header_deinit(hdr_ctx);
}

void mbox_sync_mail_add_missing_headers(struct mbox_sync_mail_context *ctx)
{
	size_t old_hdr_size, new_hdr_size, size;
	const char *str;
	void *p;
	int changed;

	old_hdr_size = ctx->body_offset - ctx->hdr_offset;
	new_hdr_size = str_len(ctx->header) + ctx->have_eoh;

	changed = FALSE;
	if (ctx->uid == 0) {
		ctx->xuid_pos = buffer_get_used_size(ctx->header);
		str_printfa(ctx->header, "X-UID: %u\n",
			    ctx->parent->next_uid++);
	}

	if ((ctx->mail_flags & STATUS_FLAGS_MASK) != 0) {
		str_append(ctx->header, "Status: ");
		status_flags_append(ctx, status_flags);
		str_append_c(ctx->header, '\n');
	}

	if ((ctx->mail_flags & XSTATUS_FLAGS_MASK) != 0) {
		str_append(ctx->header, "X-Status: ");
		status_flags_append(ctx, xstatus_flags);
		str_append_c(ctx->header, '\n');
	}

	if (ctx->seq == 1 && ctx->base_uidvalidity == 0) {
		ctx->ximapbase_pos = buffer_get_used_size(ctx->header);
		str_printfa(ctx->header, "X-IMAPbase: %u %u",
			    ctx->parent->hdr->uid_validity,
			    ctx->parent->next_uid);

		/* if we can get away by adding only a little space, do it.
		   otherwise give a lot of extra */
		size = str_len(ctx->header) + ctx->have_eoh + 1 -
			ctx->extra_space/2;
		if (size <= old_hdr_size)
			size = old_hdr_size - size;
		else
			size = 256;

		p = buffer_append_space_unsafe(ctx->header, size);
		memset(p, ' ', size);
		str_append_c(ctx->header, '\n');
	}

	/* write Content-Length if we have space */
	if (ctx->content_length == (uoff_t)-1) {
		str = t_strdup_printf("Content-Length: %"PRIuUOFF_T"\n",
				      ctx->body_size);
		size = buffer_get_used_size(ctx->header) + ctx->have_eoh -
			ctx->extra_space;
		if (size > old_hdr_size || size + strlen(str) <= old_hdr_size)
			str_append(ctx->header, str);
	}

	/* Create X-Keywords header if it's not there and we have space */
	if (ctx->xkeywords_pos == (size_t)-1) {
		size = buffer_get_used_size(ctx->header) + ctx->have_eoh -
			ctx->extra_space;
		if (size > old_hdr_size ||
		    size + sizeof("X-Keywords: ") <= old_hdr_size) {
			ctx->xkeywords_pos = buffer_get_used_size(ctx->header);
			str_append(ctx->header, "X-Keywords: \n");
		}
	}

	if (buffer_get_used_size(ctx->header) != new_hdr_size) {
		if (ctx->header_first_change == (size_t)-1)
			ctx->header_first_change = new_hdr_size;
		ctx->header_last_change = (size_t)-1;
		new_hdr_size = buffer_get_used_size(ctx->header) +
			ctx->have_eoh;
	}

	if (ctx->header_first_change == (size_t)-1) {
		/* no headers had to be modified */
		return;
	}

	if (ctx->have_eoh)
		str_append_c(ctx->header, '\n');
}

static void mbox_sync_headers_add_space(struct mbox_sync_mail_context *ctx,
					size_t size)
{
	size_t data_size, pos;
	const unsigned char *data;
	void *p;

	/* Append at the end of X-Keywords header,
	   or X-UID if it doesn't exist */
	pos = ctx->xkeywords_pos != (size_t)-1 ?
		ctx->xkeywords_pos : ctx->xuid_pos;

	data = buffer_get_data(ctx->header, &data_size);
	while (pos < data_size && data[pos] != '\n')
		pos++;

	buffer_copy(ctx->header, pos + size,
		    ctx->header, pos, (size_t)-1);
	p = buffer_get_space_unsafe(ctx->header, pos, size);
	memset(p, ' ', size);
	ctx->extra_space += size;

	if (ctx->header_first_change > pos)
		ctx->header_first_change = pos;
	ctx->header_last_change = (size_t)-1;
}

static void mbox_sync_header_remove_space(struct mbox_sync_mail_context *ctx,
					  size_t pos, size_t *size)
{
	const unsigned char *data;
	size_t data_size, end, nonspace;

	/* find the end of the lwsp */
	nonspace = pos;
	data = str_data(ctx->header);
	data_size = str_len(ctx->header);
	for (end = pos; end < data_size; end++) {
		if (data[end] == '\n') {
			if (end+1 == data_size || !IS_LWSP(data[end+1]))
				break;
		} else {
			if (!IS_LWSP(data[end]))
				nonspace = end;
		}
	}

	/* and remove what we can */
	nonspace++;
	if (end-nonspace < *size) {
		str_delete(ctx->header, nonspace, end-nonspace);
		*size -= end-nonspace;
	} else {
		str_delete(ctx->header, nonspace, *size);
		*size = 0;
	}
}

static void mbox_sync_headers_remove_space(struct mbox_sync_mail_context *ctx,
					   size_t size)
{
	if (ctx->xkeywords_pos != (size_t)-1)
		mbox_sync_header_remove_space(ctx, ctx->xkeywords_pos, &size);
	if (ctx->xuid_pos != (size_t)-1 && size > 0)
		mbox_sync_header_remove_space(ctx, ctx->xuid_pos, &size);
	if (ctx->ximapbase_pos != (size_t)-1 && size > 0)
		mbox_sync_header_remove_space(ctx, ctx->ximapbase_pos, &size);
	i_assert(size == 0);
}

int mbox_sync_try_rewrite_headers(struct mbox_sync_mail_context *ctx,
				  uoff_t *missing_space_r)
{
	size_t old_hdr_size, new_hdr_size;
	const unsigned char *data;

	old_hdr_size = ctx->body_offset - ctx->hdr_offset;
	new_hdr_size = str_len(ctx->header);

	/* do we have enough space? */
	if (new_hdr_size < old_hdr_size)
		mbox_sync_headers_add_space(ctx, old_hdr_size - new_hdr_size);
	else if (new_hdr_size > old_hdr_size) {
		if (ctx->extra_space < new_hdr_size - old_hdr_size) {
			*missing_space_r = new_hdr_size - old_hdr_size -
				ctx->extra_space;
			return 0;
		}

		ctx->extra_space -= new_hdr_size - old_hdr_size;
		mbox_sync_headers_remove_space(ctx, new_hdr_size -
					       old_hdr_size);
	}

	i_assert(ctx->header_first_change != (size_t)-1);

	if (ctx->header_last_change != (size_t)-1)
		str_truncate(ctx->header, ctx->header_last_change);

	data = str_data(ctx->header);
        new_hdr_size = str_len(ctx->header);
	if (pwrite_full(ctx->parent->fd, data + ctx->header_first_change,
			new_hdr_size,
			ctx->hdr_offset + ctx->header_first_change) < 0) {
		// FIXME: error handling
		return -1;
	}
	*missing_space_r = 0;
	return 0;
}

--- NEW FILE: mbox-sync-parse.c ---
/* Copyright (C) 2004 Timo Sirainen */

#include "lib.h"
#include "buffer.h"
#include "istream.h"
#include "str.h"
#include "write-full.h"
#include "message-parser.h"
#include "mail-index.h"
#include "mbox-sync-private.h"

#include <stdlib.h>

#define IS_LWSP_LF(c) (IS_LWSP(c) || (c) == '\n')

struct mbox_flag_type mbox_status_flags[] = {
	{ 'R', MAIL_SEEN },
	{ 'O', MBOX_NONRECENT },
	{ 0, 0 }
};

struct mbox_flag_type mbox_xstatus_flags[] = {
	{ 'A', MAIL_ANSWERED },
	{ 'F', MAIL_FLAGGED },
	{ 'T', MAIL_DRAFT },
	{ 'D', MAIL_DELETED },
	{ 0, 0 }
};

struct header_func {
	const char *header;
	int (*func)(struct mbox_sync_mail_context *ctx,
		    struct message_header_line *hdr);
};

static enum mail_flags mbox_flag_find(struct mbox_flag_type *flags, char chr)
{
	int i;

	for (i = 0; flags[i].chr != 0; i++) {
		if (flags[i].chr == chr)
			return flags[i].flag;
	}

	return 0;
}

static void parse_status_flags(struct mbox_sync_mail_context *ctx,
			       struct message_header_line *hdr,
			       struct mbox_flag_type *flags_list)
{
	size_t i;

	for (i = 0; i < hdr->full_value_len; i++) {
		ctx->mail->flags |=
			mbox_flag_find(flags_list, hdr->full_value[i]);
	}
}

static int parse_status(struct mbox_sync_mail_context *ctx,
			struct message_header_line *hdr)
{
	parse_status_flags(ctx, hdr, mbox_status_flags);
	ctx->hdr_pos[MBOX_HDR_STATUS] = str_len(ctx->header);
	return TRUE;
}

static int parse_x_status(struct mbox_sync_mail_context *ctx,
			  struct message_header_line *hdr)
{
	parse_status_flags(ctx, hdr, mbox_xstatus_flags);
	ctx->hdr_pos[MBOX_HDR_X_STATUS] = str_len(ctx->header);
	return TRUE;
}

static int parse_x_imap_base(struct mbox_sync_mail_context *ctx,
			     struct message_header_line *hdr)
{
	const char *str;
	char *end;
	size_t pos;

	if (ctx->seq != 1 || ctx->base_uid_validity != 0) {
		/* Valid only in first message */
		return FALSE;
	}

	t_push();

	/* <uid validity> <last uid> */
	str = t_strndup(hdr->full_value, hdr->full_value_len);
	ctx->base_uid_validity = strtoul(str, &end, 10);
	ctx->base_uid_last = strtoul(end, &end, 10);
	pos = end - str;

	while (pos < hdr->full_value_len && IS_LWSP_LF(hdr->full_value[pos]))
		pos++;

	if (ctx->base_uid_validity == 0) {
		/* broken */
		t_pop();
		return FALSE;
	}

	if (pos == hdr->full_value_len) {
		t_pop();
		return TRUE;
	}

	// FIXME: save keywords

	ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] = str_len(ctx->header);
	return TRUE;
}

static int parse_x_keywords(struct mbox_sync_mail_context *ctx,
			    struct message_header_line *hdr)
{
	size_t i, space = 0;

	for (i = hdr->full_value_len; i > 0; i++) {
		if (!IS_LWSP_LF(hdr->full_value[i-1]))
			break;
		space++;
	}

	if (space > ctx->mail->space) {
		ctx->mail->space_offset = hdr->full_value_offset + i;
		ctx->mail->space = space;
	}

	// FIXME: parse them

	ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] = str_len(ctx->header);
	return TRUE;
}

static int parse_x_uid(struct mbox_sync_mail_context *ctx,
		       struct message_header_line *hdr)
{
	uint32_t value = 0;
	size_t i, space_pos, extra_space = 0;

	if (ctx->mail->uid != 0) {
		/* duplicate */
		return FALSE;
	}

	for (i = 0; i < hdr->full_value_len; i++) {
		if (hdr->full_value[i] < '0' || hdr->full_value[i] > '9')
			break;
		value = value*10 + (hdr->full_value[i] - '0');
	}

	space_pos = i;
	for (; i < hdr->full_value_len; i++) {
		if (!IS_LWSP_LF(hdr->full_value[i])) {
			/* broken value */
			return FALSE;
		}
		extra_space++;
	}

	if (value <= ctx->sync_ctx->prev_msg_uid) {
		/* broken - UIDs must be growing */
		return FALSE;
	}

	ctx->hdr_pos[MBOX_HDR_X_UID] = str_len(ctx->header);

	ctx->mail->uid = value;
	if (ctx->mail->space == 0) {
		/* set it only if X-Keywords hasn't been seen. spaces in X-UID
		   should be removed when writing X-Keywords. */
		ctx->mail->space_offset = hdr->full_value_offset + space_pos;
		ctx->mail->space = extra_space;
	}
	return TRUE;
}

static int parse_content_length(struct mbox_sync_mail_context *ctx,
				struct message_header_line *hdr)
{
	uoff_t value = 0;
	size_t i;

	if (ctx->content_length != (uoff_t)-1) {
		/* duplicate */
		return FALSE;
	}

	for (i = 0; i < hdr->full_value_len; i++) {
		if (hdr->full_value[i] < '0' || hdr->full_value[i] > '9')
			break;
		value = value*10 + (hdr->full_value[i] - '0');
	}

	for (; i < hdr->full_value_len; i++) {
		if (!IS_LWSP_LF(hdr->full_value[i])) {
			/* broken value */
			return FALSE;
		}
	}

	ctx->content_length = value;
	return TRUE;
}

static struct header_func header_funcs[] = {
	{ "Content-Length", parse_content_length },
	{ "Status", parse_status },
	{ "X-IMAPbase", parse_x_imap_base },
	{ "X-Keywords", parse_x_keywords },
	{ "X-Status", parse_x_status },
	{ "X-UID", parse_x_uid },
	{ NULL, NULL }
};

static struct header_func *header_func_find(const char *header)
{
	int i;

	for (i = 0; header_funcs[i].header != NULL; i++) {
		if (strcasecmp(header_funcs[i].header, header) == 0)
			return &header_funcs[i];
	}
	return NULL;
}

void mbox_sync_parse_next_mail(struct istream *input,
			       struct mbox_sync_mail_context *ctx)
{
	struct message_header_parser_ctx *hdr_ctx;
	struct message_header_line *hdr;
	struct header_func *func;
	size_t line_start_pos;
	int i;

	ctx->hdr_offset = input->v_offset;
        ctx->mail->space_offset = input->v_offset;

        ctx->header_first_change = (size_t)-1;
	ctx->header_last_change = (size_t)-1;

	for (i = 0; i < MBOX_HDR_COUNT; i++)
		ctx->hdr_pos[i] = (size_t)-1;

	ctx->content_length = (uoff_t)-1;
	str_truncate(ctx->header, 0);

        line_start_pos = 0;
	hdr_ctx = message_parse_header_init(input, NULL);
	while ((hdr = message_parse_header_next(hdr_ctx)) != NULL) {
		if (hdr->eoh) {
			ctx->have_eoh = TRUE;
			break;
		}

		func = header_func_find(hdr->name);
		if (func != NULL) {
			if (hdr->continues)
				hdr->use_full_value = TRUE;
			else if (!func->func(ctx, hdr)) {
				/* this header is broken, remove it */
				ctx->need_rewrite = TRUE;
				if (hdr->continued) {
					str_truncate(ctx->header,
						     line_start_pos);
				}
				if (ctx->header_first_change == (size_t)-1) {
					ctx->header_first_change =
						str_len(ctx->header);
				}
				continue;
			}
		}

		if (!hdr->continued) {
			line_start_pos = str_len(ctx->header);
			str_append(ctx->header, hdr->name);
			str_append(ctx->header, ": ");
		}
		buffer_append(ctx->header, hdr->full_value,
			      hdr->full_value_len);
		if (!hdr->no_newline)
			str_append_c(ctx->header, '\n');
	}
	message_parse_header_deinit(hdr_ctx);

	if (ctx->seq == 1 && ctx->base_uid_validity == 0) {
		/* missing X-IMAPbase */
		ctx->need_rewrite = TRUE;
	}

	ctx->body_offset = input->v_offset;
}

--- NEW FILE: mbox-sync-private.h ---
#ifndef __MBOX_SYNC_PRIVATE_H
#define __MBOX_SYNC_PRIVATE_H

#include "mail-index.h"

struct mbox_flag_type {
	char chr;
	enum mail_flags flag;
};

enum header_position {
	MBOX_HDR_STATUS,
	MBOX_HDR_X_IMAPBASE,
	MBOX_HDR_X_KEYWORDS,
	MBOX_HDR_X_STATUS,
	MBOX_HDR_X_UID,

        MBOX_HDR_COUNT
};

#define MBOX_NONRECENT MAIL_RECENT /* kludgy */

#define STATUS_FLAGS_MASK (MAIL_SEEN|MBOX_NONRECENT)
#define XSTATUS_FLAGS_MASK (MAIL_ANSWERED|MAIL_FLAGGED|MAIL_DRAFT|MAIL_DELETED)
extern struct mbox_flag_type mbox_status_flags[];
extern struct mbox_flag_type mbox_xstatus_flags[];

struct mbox_mail {
	uint32_t uid;
	uint8_t flags;
	custom_flags_mask_t custom_flags;

	uoff_t space_offset; /* if space is negative, points to beginning */
	off_t space;
	uoff_t body_size;
};

struct mbox_sync_mail_context {
	struct mbox_sync_context *sync_ctx;
	struct mbox_mail *mail;

	uint32_t seq;
	uoff_t hdr_offset, body_offset;

	size_t header_first_change, header_last_change;
	string_t *header;

	uint32_t base_uid_validity, base_uid_last;
	uoff_t content_length;

	size_t hdr_pos[MBOX_HDR_COUNT];

	unsigned int have_eoh:1;
	unsigned int need_rewrite:1;
};

struct mbox_sync_context {
	struct istream *file_input;
	struct istream *input;
	int fd;

	const struct mail_index_header *hdr;

	uint32_t prev_msg_uid, next_uid;
};

void mbox_sync_parse_next_mail(struct istream *input,
			       struct mbox_sync_mail_context *ctx);
void mbox_sync_update_header(struct mbox_sync_mail_context *ctx,
			     struct mail_index_sync_rec *update);
int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx);
int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx, buffer_t *mails_buf,
		      uint32_t first_seq, uint32_t last_seq, off_t extra_space);

int mbox_move(struct mbox_sync_context *sync_ctx,
	      uoff_t dest, uoff_t source, uoff_t size);

#endif

--- NEW FILE: mbox-sync-rewrite.c ---
#include "lib.h"
#include "buffer.h"
#include "istream.h"
#include "ostream.h"
#include "str.h"
#include "write-full.h"
#include "message-parser.h"
#include "mbox-sync-private.h"
#include "istream-raw-mbox.h"

int mbox_move(struct mbox_sync_context *sync_ctx,
	      uoff_t dest, uoff_t source, uoff_t size)
{
	struct istream *input;
	struct ostream *output;
	off_t ret;

	output = o_stream_create_file(sync_ctx->fd, default_pool, 4096, FALSE);
	i_stream_seek(sync_ctx->file_input, source);
	o_stream_seek(output, dest);

	istream_raw_mbox_flush(sync_ctx->input);

	if (size == (uoff_t)-1) {
		input = sync_ctx->file_input;
		return o_stream_send_istream(output, input) < 0 ? -1 : 0;
	} else {
		input = i_stream_create_limit(default_pool,
					      sync_ctx->file_input,
					      source, size);
		ret = o_stream_send_istream(output, input);
		i_stream_unref(input);
		return ret == (off_t)size ? 0 : -1;
	}
}

static void mbox_sync_headers_add_space(struct mbox_sync_mail_context *ctx,
					size_t size)
{
	size_t data_size, pos;
	const unsigned char *data;
	void *p;

	/* Append at the end of X-Keywords header,
	   or X-UID if it doesn't exist */
	pos = ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] != (size_t)-1 ?
		ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] :
		ctx->hdr_pos[MBOX_HDR_X_UID];

	data = buffer_get_data(ctx->header, &data_size);
	while (pos < data_size && data[pos] != '\n')
		pos++;

	buffer_copy(ctx->header, pos + size,
		    ctx->header, pos, (size_t)-1);
	p = buffer_get_space_unsafe(ctx->header, pos, size);
	memset(p, ' ', size);

	if (ctx->header_first_change > pos)
		ctx->header_first_change = pos;
	ctx->header_last_change = (size_t)-1;
}

static void mbox_sync_header_remove_space(struct mbox_sync_mail_context *ctx,
					  size_t pos, size_t *size)
{
	const unsigned char *data;
	size_t data_size, end, nonspace;

	/* find the end of the lwsp */
	nonspace = pos;
	data = str_data(ctx->header);
	data_size = str_len(ctx->header);
	for (end = pos; end < data_size; end++) {
		if (data[end] == '\n') {
			if (end+1 == data_size || !IS_LWSP(data[end+1]))
				break;
		} else {
			if (!IS_LWSP(data[end]))
				nonspace = end;
		}
	}

	/* and remove what we can */
	nonspace++;
	if (end-nonspace < *size) {
		str_delete(ctx->header, nonspace, end-nonspace);
		*size -= end-nonspace;
	} else {
		str_delete(ctx->header, nonspace, *size);
		*size = 0;
	}
}

static void mbox_sync_headers_remove_space(struct mbox_sync_mail_context *ctx,
					   size_t size)
{
	static enum header_position space_positions[] = {
                MBOX_HDR_X_KEYWORDS,
                MBOX_HDR_X_UID,
                MBOX_HDR_X_IMAPBASE
	};
        enum header_position pos;
	int i;

	for (i = 0; i < 3 && size > 0; i++) {
		pos = space_positions[i];
		if (ctx->hdr_pos[pos] != (size_t)-1) {
			mbox_sync_header_remove_space(ctx, ctx->hdr_pos[pos],
						      &size);
		}
	}

	i_assert(size == 0);
}

int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx)
{
	size_t old_hdr_size, new_hdr_size;
	const unsigned char *data;

	old_hdr_size = ctx->body_offset - ctx->hdr_offset;
	new_hdr_size = str_len(ctx->header);

	/* do we have enough space? */
	if (new_hdr_size < old_hdr_size) {
		mbox_sync_headers_add_space(ctx, old_hdr_size - new_hdr_size);
		ctx->mail->space += old_hdr_size - new_hdr_size;
	} else if (new_hdr_size > old_hdr_size) {
		size_t needed = new_hdr_size - old_hdr_size;
		if (ctx->mail->space < needed) {
			ctx->mail->space -= needed;
			return 0;
		}

		ctx->mail->space -= needed;
		mbox_sync_headers_remove_space(ctx, needed);
	}

	i_assert(ctx->header_first_change != (size_t)-1);

	if (ctx->header_last_change != (size_t)-1)
		str_truncate(ctx->header, ctx->header_last_change);

	data = str_data(ctx->header);
        new_hdr_size = str_len(ctx->header);
	if (pwrite_full(ctx->sync_ctx->fd, data + ctx->header_first_change,
			new_hdr_size,
			ctx->hdr_offset + ctx->header_first_change) < 0) {
		// FIXME: error handling
		return -1;
	}
	istream_raw_mbox_flush(ctx->sync_ctx->input);
	return 1;
}

int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx, buffer_t *mails_buf,
		      uint32_t first_seq, uint32_t last_seq, off_t extra_space)
{
	struct mbox_mail *mails;
	size_t size;
	uint32_t first_idx, last_idx, extra_per_mail;

	first_idx = first_seq-1;
	last_idx = last_seq-1;

	mails = buffer_get_modifyable_data(mails_buf, &size);
	size /= sizeof(*mails);

	/* FIXME: see if we can be faster by going back a few mails
	   (update first_seq and last_seq) */
	/*while (mails[last_idx].space > 0) {
	}*/

#if 0
	/* start moving backwards */
	extra_per_mail = (extra_space / (last_seq - first_seq + 1)) + 1;
	space_diff = 0;
	while (last_seq > first_seq) {
		dest = mails[last_seq].space_offset + mails[last_seq].space
	}
#endif
}

--- NEW FILE: mbox-sync-update.c ---
#include "lib.h"
#include "buffer.h"
#include "str.h"
#include "message-parser.h"
#include "mbox-sync-private.h"

static void status_flags_append(struct mbox_sync_mail_context *ctx,
				struct mbox_flag_type *flags_list)
{
	int i;

	for (i = 0; flags_list[i].chr != 0; i++) {
		if ((ctx->mail->flags & flags_list[i].flag) != 0)
			str_append_c(ctx->header, flags_list[i].chr);
	}
}
static void keywords_append(struct mbox_sync_mail_context *ctx,
			    custom_flags_mask_t custom_flags)
{
	// FIXME
}

static void mbox_sync_add_missing_headers(struct mbox_sync_mail_context *ctx)
{
	size_t old_hdr_size, new_hdr_size;
	int i, have_keywords;

	old_hdr_size = ctx->body_offset - ctx->hdr_offset;
	new_hdr_size = str_len(ctx->header) + ctx->have_eoh;

	if (ctx->seq == 1 && ctx->base_uid_validity == 0) {
		ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] = str_len(ctx->header);
		str_printfa(ctx->header, "X-IMAPbase: %u %u",
			    ctx->sync_ctx->hdr->uid_validity,
			    ctx->sync_ctx->next_uid);
		//FIXME:keywords_append(ctx, all_custom_flags);
		str_append_c(ctx->header, '\n');
	}

	if (ctx->mail->uid == 0) {
		ctx->hdr_pos[MBOX_HDR_X_UID] = str_len(ctx->header);
		str_printfa(ctx->header, "X-UID: %u\n",
			    ctx->sync_ctx->next_uid++);
	}

	if (ctx->hdr_pos[MBOX_HDR_STATUS] == (size_t)-1 &&
	    (ctx->mail->flags & STATUS_FLAGS_MASK) != 0) {
		ctx->hdr_pos[MBOX_HDR_STATUS] = str_len(ctx->header);
		str_append(ctx->header, "Status: ");
		status_flags_append(ctx, mbox_status_flags);
		str_append_c(ctx->header, '\n');
	}

	if (ctx->hdr_pos[MBOX_HDR_X_STATUS] == (size_t)-1 &&
	    (ctx->mail->flags & XSTATUS_FLAGS_MASK) != 0) {
		ctx->hdr_pos[MBOX_HDR_X_STATUS] = str_len(ctx->header);
		str_append(ctx->header, "X-Status: ");
		status_flags_append(ctx, mbox_xstatus_flags);
		str_append_c(ctx->header, '\n');
	}

	have_keywords = FALSE;
	for (i = 0; i < INDEX_CUSTOM_FLAGS_BYTE_COUNT; i++) {
		if (ctx->mail->custom_flags[i] != 0) {
			have_keywords = TRUE;
			break;
		}
	}

	if (ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] == (size_t)-1 && have_keywords) {
		ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] = str_len(ctx->header);
		str_append(ctx->header, "X-Keywords: ");
		keywords_append(ctx, ctx->mail->custom_flags);
		str_append_c(ctx->header, '\n');
	}

	if (ctx->content_length == (uoff_t)-1) {
		str_printfa(ctx->header, "Content-Length: %"PRIuUOFF_T"\n",
			    ctx->mail->body_size);
	}

	if (str_len(ctx->header) != new_hdr_size) {
		if (ctx->header_first_change == (size_t)-1)
			ctx->header_first_change = new_hdr_size;
		ctx->header_last_change = (size_t)-1;
		ctx->mail->space -= str_len(ctx->header) -
			(new_hdr_size - ctx->have_eoh);
		new_hdr_size = str_len(ctx->header) + ctx->have_eoh;
	}

	if (ctx->header_first_change == (size_t)-1) {
		/* no headers had to be modified */
		return;
	}

	if (ctx->have_eoh)
		str_append_c(ctx->header, '\n');
}

static void mbox_sync_update_status(struct mbox_sync_mail_context *ctx)
{
}

static void mbox_sync_update_xstatus(struct mbox_sync_mail_context *ctx)
{
}

static void mbox_sync_update_xkeywords(struct mbox_sync_mail_context *ctx)
{
}

void mbox_sync_update_header(struct mbox_sync_mail_context *ctx,
			     struct mail_index_sync_rec *update)
{
	uint8_t old_flags;
	custom_flags_mask_t old_custom_flags;

	if (update != NULL) {
		old_flags = ctx->mail->flags;
		memcpy(old_custom_flags, ctx->mail->custom_flags,
		       sizeof(old_custom_flags));

		mail_index_sync_flags_apply(update, &ctx->mail->flags,
					    ctx->mail->custom_flags);

		if ((old_flags & STATUS_FLAGS_MASK) !=
		    (ctx->mail->flags & STATUS_FLAGS_MASK))
			mbox_sync_update_status(ctx);
		if ((old_flags & XSTATUS_FLAGS_MASK) !=
		    (ctx->mail->flags & XSTATUS_FLAGS_MASK))
			mbox_sync_update_xstatus(ctx);
		if (memcmp(old_custom_flags, ctx->mail->custom_flags,
			   sizeof(old_custom_flags)) != 0)
			mbox_sync_update_xkeywords(ctx);
	}

        mbox_sync_add_missing_headers(ctx);
}

--- NEW FILE: mbox-sync.c ---
/* Copyright (C) 2004 Timo Sirainen */

/*
   Modifying mbox can be slow, so we try to do it all at once minimizing the
   required disk I/O. We may need to:

   - Update message flags in Status, X-Status and X-Keywords headers
   - Write missing X-UID and X-IMAPbase headers
   - Write missing or broken Content-Length header if there's space
   - Expunge specified messages

   Here's how we do it:

   - Start reading the mails mail headers from the beginning
   - X-Keywords and X-UID headers may contain extra spaces at the end of them,
     remember how much extra each message has and offset to beginning of the
     spaces
   - If message flags are dirty and there's enough space to write them, do it
   - If we didn't have enough space, remember how much was missing and keep
     the total amount of them
   - When we encounter expunged message, check if the amount of empty space in
     previous messages plus size of expunged message is enough to cover the
     missing space. If yes,
       - execute the rewrite plan
       - forget all the messages before the expunged message. only remember
         how much data we still have to move to cover the expunged message
   - If we encounter end of file, grow the file and execute the rewrite plan

   Rewrite plan goes:

   - Start from the first message that needs more space
   - If there's expunged messages before us, we have to write over them.
       - Move all messages after it backwards to fill it
       - Each moved message's X-Keywords header should have n bytes extra
         space, unless there's not enough space to do it.
   - If there's no expunged messages, we can move data either forward or
     backward to get it. Calculate which requires less moving. Forward
     counting may encounter more messages which require extra space, count
     that too.
       - If we decide to move forwards and we had to go through dirty
         messages, do the moving from last to first dirty message
   - If we encounter end of file, grow the file enough to get the required
     amount of space plus enough space to fill X-Keywords headers full of
     spaces.
*/

#include "lib.h"
#include "buffer.h"
#include "istream.h"
#include "file-set-size.h"
#include "str.h"
#include "write-full.h"
#include "istream-raw-mbox.h"
#include "mbox-sync-private.h"

static int mbox_sync_grow_file(struct mbox_sync_context *sync_ctx,
			       struct mbox_mail *mail, uoff_t body_offset,
			       uoff_t grow_size)
{
	char spaces[1024];
	uoff_t offset, size;

	i_assert(grow_size > 0);

	memset(spaces, ' ', sizeof(spaces));

	size = sync_ctx->input->v_offset + grow_size;
	if (file_set_size(sync_ctx->fd, size) < 0)
		return -1;

	if (mail->space_offset == 0) {
		/* no X-Keywords header - place it at the end. */
		grow_size += 13;

		offset = body_offset-1;
		if (mbox_move(sync_ctx, body_offset-1 + size,
			      offset, (uoff_t)-1) < 0)
			return -1;
		if (pwrite_full(sync_ctx->fd, "X-Keywords: ", 12, offset) < 0)
			return -1;
		if (pwrite_full(sync_ctx->fd, "\n", 1,
				offset + grow_size-1) < 0)
			return -1;
		grow_size -= 13; offset += 12;

		/* FIXME: can this break anything? X-Keywords text might
		   have been already included in space calculation. now we
		   have more.. */
		mail->space_offset = offset;
		mail->space += grow_size;
	} else {
		offset = mail->space_offset;
		if (mbox_move(sync_ctx, mail->space_offset + grow_size,
			      offset, (uoff_t)-1) < 0)
			return -1;
	}

	while (grow_size >= sizeof(spaces)) {
		if (pwrite_full(sync_ctx->fd, spaces,
				sizeof(spaces), offset) < 0)
			return -1;
		grow_size -= sizeof(spaces);
		offset += sizeof(spaces);
	}

	if (grow_size > 0) {
		if (pwrite_full(sync_ctx->fd, spaces, grow_size, offset) < 0)
			return -1;
	}

	istream_raw_mbox_flush(sync_ctx->input);
	return 0;
}

int mbox_sync(struct istream *input)
{
	struct mbox_sync_context sync_ctx;
	struct mbox_sync_mail_context mail_ctx;
	struct mbox_mail mail;
	uint32_t seq, need_space_seq;
	off_t space_diff;
	buffer_t *mails;
	int ret = 0;

	mails = buffer_create_dynamic(default_pool, 4096, (size_t)-1);

	memset(&sync_ctx, 0, sizeof(sync_ctx));
	sync_ctx.file_input = input;
	sync_ctx.input = i_stream_create_raw_mbox(default_pool, input);
	sync_ctx.fd = i_stream_get_fd(input);
	//sync_ctx.hdr = ;

	input = sync_ctx.input;

	space_diff = 0; need_space_seq = 0; seq = 1;
	for (seq = 1; !input->eof; seq++) {
		memset(&mail, 0, sizeof(mail));
		memset(&mail_ctx, 0, sizeof(mail_ctx));
		mail_ctx.sync_ctx = &sync_ctx;
		mail_ctx.mail = &mail;
		mail_ctx.seq = seq;

		mbox_sync_parse_next_mail(input, &mail_ctx);
		mail.body_size =
			istream_raw_mbox_get_size(input,
						  mail_ctx.content_length);
		buffer_append(mails, &mail, sizeof(mail));

		if (mail_ctx.need_rewrite) {
			mbox_sync_update_header(&mail_ctx, NULL);
			if ((ret = mbox_sync_try_rewrite(&mail_ctx)) < 0)
				break;
		} else {
			ret = 1;
		}

		if (ret == 0 && need_space_seq == 0) {
			/* didn't have space to write it */
			need_space_seq = seq;
			space_diff = mail.space;
		} else if (need_space_seq != 0) {
			space_diff += mail.space;
			if (space_diff >= 0) {
				/* we have enough space now */
				if (mbox_sync_rewrite(&sync_ctx, mails,
						      need_space_seq, seq,
						      space_diff) < 0) {
					ret = -1;
					break;
				}
				need_space_seq = 0;
			}
		}

		istream_raw_mbox_next(input, mail.body_size);
	}

	if (need_space_seq != 0) {
		i_assert(space_diff < 0);
		if (mbox_sync_grow_file(&sync_ctx, &mail, mail_ctx.body_offset,
					-space_diff) < 0)
			ret = -1;
		else if (mbox_sync_rewrite(&sync_ctx, mails, need_space_seq,
					   seq-1, space_diff) < 0)
			ret = -1;
	}

	i_stream_unref(input);
	return ret < 0 ? -1 : 0;
}

#if 0
int mbox_sync(void)
{
	struct mail_index_view *sync_view;
	struct mail_index_sync_ctx *sync_ctx;
	struct mail_index_sync_rec sync_rec;
	struct mbox_sync_context ctx;
	struct mbox_sync_mail_context mail_ctx;
	struct mbox_mail mail;
	string_t *header;
	uint32_t seq;
	unsigned int need_space_seq;
	uoff_t missing_space;
	buffer_t *mails;
	int ret;

	memset(&ctx, 0, sizeof(ctx));
	/*ctx.index = storage->index;
	ctx.input = storage->input;*/
	ctx.fd = i_stream_get_fd(ctx.input);

	header = str_new(default_pool, 4096);

	if (mail_index_sync_begin(ctx.index, &sync_ctx, &sync_view, 0, 0) < 0)
		return -1;

	ctx.hdr = mail_index_get_header(sync_view);
	ctx.next_uid = ctx.hdr->next_uid;

	seq = 1;
	while ((ret = mail_index_sync_next(sync_ctx, &sync_rec)) > 0) {
		while (seq < sync_rec.seq1) {
			seq++;
		}
		switch (sync_rec.type) {
		case MAIL_INDEX_SYNC_TYPE_EXPUNGE:
			break;
		case MAIL_INDEX_SYNC_TYPE_FLAGS:
			break;
		}
	}

	while (!ctx.input->eof) {
		memset(&mail_ctx, 0, sizeof(mail_ctx));
		mail_ctx.parent = &ctx;
		mail_ctx.header = header;
		mail_ctx.seq = seq;

		mail_ctx.hdr_offset = ctx.input->v_offset;
		mbox_sync_mail_parse_headers(&mail_ctx);
		mail_ctx.body_offset = ctx.input->v_offset;
		mail_ctx.body_size =
			istream_raw_mbox_get_size(ctx.input,
						  mail_ctx.content_length);

                mbox_sync_mail_add_missing_headers(&mail_ctx);

		ret = mbox_sync_try_rewrite_headers(&mail_ctx, &missing_space);
		if (ret < 0)
			break;
		if (missing_space != 0) {
			ctx.space_diff -= missing_space;
		} else {
			ctx.space_diff += mail_ctx.extra_space;
		}

		if (ctx.first_spacy_msg_offset == 0)
                        ctx.first_spacy_msg_offset = mail_ctx.hdr_offset;

		ctx.prev_msg_uid = mail_ctx.uid;
		istream_raw_mbox_next(ctx.input, mail_ctx.content_length);
	}
	str_free(header);
	return 0;
}
#endif

Index: Makefile.am
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/Makefile.am,v
retrieving revision 1.1.1.1
retrieving revision 1.2
diff -u -d -r1.1.1.1 -r1.2
--- Makefile.am	9 Aug 2002 09:16:01 -0000	1.1.1.1
+++ Makefile.am	27 Apr 2004 20:25:54 -0000	1.2
@@ -10,10 +10,19 @@
 	-I$(top_srcdir)/src/lib-storage/index
 
 libstorage_mbox_a_SOURCES = \
+	istream-raw-mbox.c \
 	mbox-expunge.c \
+	mbox-from.c \
 	mbox-list.c \
 	mbox-save.c \
+	mbox-sync-parse.c \
+	mbox-sync-rewrite.c \
+	mbox-sync-update.c \
+	mbox-sync.c
 	mbox-storage.c
 
 noinst_HEADERS = \
-	mbox-storage.h
+	istream-raw-mbox.h \
+	mbox-from.h \
+	mbox-storage.h \
+	mbox-sync-private.h

Index: mbox-expunge.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-expunge.c,v
retrieving revision 1.33
retrieving revision 1.34
diff -u -d -r1.33 -r1.34
--- mbox-expunge.c	9 Nov 2003 18:26:26 -0000	1.33
+++ mbox-expunge.c	27 Apr 2004 20:25:54 -0000	1.34
@@ -1,5 +1,6 @@
 /* Copyright (C) 2002-2003 Timo Sirainen */
 
+#if 0
 #include "lib.h"
 #include "istream.h"
 #include "ostream.h"
@@ -210,3 +211,4 @@
 
 	return index_storage_expunge(mail, ctx->ctx, seq_r, notify);
 }
+#endif

Index: mbox-list.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-list.c,v
retrieving revision 1.22
retrieving revision 1.23
diff -u -d -r1.22 -r1.23
--- mbox-list.c	27 Jul 2003 03:12:13 -0000	1.22
+++ mbox-list.c	27 Apr 2004 20:25:54 -0000	1.23
@@ -1,5 +1,6 @@
 /* Copyright (C) 2002-2003 Timo Sirainen */
 
+#if 0
 #include "lib.h"
 #include "unlink-directory.h"
 #include "imap-match.h"
@@ -453,3 +454,4 @@
 {
 	return NULL;
 }
+#endif

Index: mbox-save.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-save.c,v
retrieving revision 1.45
retrieving revision 1.46
diff -u -d -r1.45 -r1.46
--- mbox-save.c	29 Sep 2003 14:15:05 -0000	1.45
+++ mbox-save.c	27 Apr 2004 20:25:54 -0000	1.46
@@ -1,5 +1,6 @@
 /* Copyright (C) 2002 Timo Sirainen */
 
+#if 0
 #include "lib.h"
 #include "hostpid.h"
 #include "ostream.h"
@@ -371,3 +372,4 @@
 	i_free(ctx);
 	return !failed;
 }
+#endif