[dovecot-cvs] dovecot/src/lib-storage/index/mbox mbox-mail.c, 1.1, 1.2 mbox-sync.c, 1.2, 1.3

[cras at procontrol.fi]

Update of /home/cvs/dovecot/src/lib-storage/index/mbox
In directory talvi:/tmp/cvs-serv21027/lib-storage/index/mbox

Modified Files:
	mbox-mail.c mbox-sync.c 
Log Message:
Some mbox fixes. Kind of works now.



Index: mbox-mail.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-mail.c,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- mbox-mail.c	6 May 2004 01:22:25 -0000	1.1
+++ mbox-mail.c	6 May 2004 03:05:42 -0000	1.2
@@ -15,13 +15,13 @@
 {
 	i_assert(mail->mail.seq <= mail->ibox->mbox_data_count);
 
-	// FIXME: lock the file
+	// FIXME: lock the file. sync if needed.
 
 	if (mbox_file_open_stream(mail->ibox) < 0)
 		return -1;
 
-	i_stream_seek(mail->ibox->mbox_stream,
-		      mail->ibox->mbox_data[mail->mail.seq-1] >> 1);
+	istream_raw_mbox_seek(mail->ibox->mbox_stream,
+			      mail->ibox->mbox_data[mail->mail.seq-1] >> 1);
 	return 0;
 }
 
@@ -83,12 +83,19 @@
 {
 	struct index_mail *mail = (struct index_mail *)_mail;
 	struct index_mail_data *data = &mail->data;
+	struct istream *raw_stream;
 
 	if (data->stream == NULL) {
 		if (mbox_mail_seek(mail) < 0)
 			return NULL;
 
-		data->stream = mail->ibox->mbox_stream;
+		// FIXME: need to hide the headers
+		raw_stream = mail->ibox->mbox_stream;
+		(void)i_stream_read(raw_stream); /* fix v_offset */
+		data->stream = i_stream_create_limit(default_pool,
+						     raw_stream,
+						     raw_stream->v_offset,
+						     (uoff_t)-1);
 	}
 
 	return index_mail_init_stream(mail, hdr_size, body_size);

Index: mbox-sync.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync.c,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- mbox-sync.c	6 May 2004 01:22:25 -0000	1.2
+++ mbox-sync.c	6 May 2004 03:05:42 -0000	1.3
@@ -123,14 +123,18 @@
 	struct mbox_sync_mail_context mail_ctx;
 	struct mbox_sync_mail mail;
 	struct mail_index_sync_ctx *index_sync_ctx;
+	struct mail_index_sync_rec sync_rec;
 	struct mail_index_view *sync_view;
+	struct mail_index_transaction *t;
 	const struct mail_index_header *hdr;
+	const struct mail_index_record *rec;
 	struct istream *input;
-	uint32_t seq, need_space_seq;
+	uint32_t seq, need_space_seq, idx_seq, messages_count;
 	off_t space_diff;
 	uoff_t from_offset, offset;
 	buffer_t *mails;
 	string_t *header;
+	size_t size;
 	struct stat st;
 	int readonly, ret = 0;
 
@@ -138,8 +142,8 @@
 		seq = ibox->commit_log_file_seq;
 		offset = ibox->commit_log_file_offset;
 	} else {
-		seq = 0;
-		offset = 0;
+		seq = (uint32_t)-1;
+		offset = (uoff_t)-1;
 	}
 
 	ret = mail_index_sync_begin(ibox->index, &index_sync_ctx, &sync_view,
@@ -153,6 +157,8 @@
 	if (mail_index_get_header(sync_view, &hdr) < 0)
 		return -1;
 
+	t = mail_index_transaction_begin(sync_view, FALSE);
+
 	if (ibox->mbox_data_buf == NULL) {
 		ibox->mbox_data_buf =
 			buffer_create_dynamic(default_pool, 512, (size_t)-1);
@@ -164,8 +170,6 @@
 
 	// FIXME: lock the file
 
-	mails = buffer_create_dynamic(default_pool, 4096, (size_t)-1);
-
 	memset(&sync_ctx, 0, sizeof(sync_ctx));
 	sync_ctx.file_input = ibox->mbox_file_stream;
 	sync_ctx.input = ibox->mbox_stream;
@@ -173,10 +177,23 @@
 	sync_ctx.hdr = hdr;
 
 	input = sync_ctx.input;
+	istream_raw_mbox_seek(input, 0);
+
 	header = str_new(default_pool, 4096);
 
-	space_diff = 0; need_space_seq = 0; seq = 1;
+	mails = buffer_create_dynamic(default_pool, 4096, (size_t)-1);
+	memset(&sync_rec, 0, sizeof(sync_rec));
+	messages_count = mail_index_view_get_message_count(sync_view);
+
+	space_diff = 0; need_space_seq = 0; idx_seq = 0; rec = NULL;
 	for (seq = 1; !input->eof; seq++) {
+		if (sync_rec.seq2 < seq) {
+			// FIXME: we may need more than one..
+			ret = mail_index_sync_next(index_sync_ctx, &sync_rec);
+			if (ret < 0)
+				break;
+		}
+
 		from_offset = input->v_offset;
 
 		memset(&mail, 0, sizeof(mail));
@@ -199,11 +216,63 @@
 
 		/* save the offset permanently with recent flag state */
 		from_offset <<= 1;
-		if ((mail.flags & MBOX_NONRECENT) != 0)
+		if ((mail.flags & MBOX_NONRECENT) == 0)
 			from_offset |= 1;
 		buffer_append(ibox->mbox_data_buf,
 			      &from_offset, sizeof(from_offset));
 
+		/* update index */
+		do {
+			if (rec != NULL && rec->uid >= mail.uid)
+				break;
+
+			if (idx_seq >= messages_count) {
+				rec = NULL;
+				break;
+			}
+
+			if (rec != NULL)
+				mail_index_expunge(t, idx_seq);
+
+			ret = mail_index_lookup(sync_view, ++idx_seq, &rec);
+		} while (ret == 0);
+
+		if (ret < 0)
+			break;
+		if (rec != NULL && rec->uid != mail.uid) {
+			/* new UID in the middle of the mailbox -
+			   shouldn't happen */
+			mail_storage_set_critical(ibox->box.storage,
+				"mbox sync: UID inserted in the middle "
+				"of mailbox (%u > %u)", rec->uid, mail.uid);
+			mail_index_mark_corrupted(ibox->index);
+			ret = -1;
+			break;
+		}
+
+		if (rec != NULL) {
+			/* see if flags changed */
+			if ((rec->flags & MAIL_FLAGS_MASK) !=
+			    (mail.flags & MAIL_FLAGS_MASK) ||
+			    memcmp(rec->keywords, mail.keywords,
+				   INDEX_KEYWORDS_BYTE_COUNT) != 0) {
+				uint8_t new_flags =
+					(rec->flags & ~MAIL_FLAGS_MASK) |
+					(mail.flags & MAIL_FLAGS_MASK);
+				mail_index_update_flags(t, idx_seq,
+							MODIFY_REPLACE,
+							new_flags,
+							mail.keywords);
+			}
+			rec = NULL;
+		} else {
+			/* new message */
+			mail_index_append(t, mail.uid, &idx_seq);
+			mail_index_update_flags(t, idx_seq, MODIFY_REPLACE,
+						mail.flags & MAIL_FLAGS_MASK,
+						mail.keywords);
+		}
+
 		if (mail_ctx.need_rewrite && !readonly) {
 			mbox_sync_update_header(&mail_ctx, NULL);
 			if ((ret = mbox_sync_try_rewrite(&mail_ctx)) < 0)
@@ -243,11 +312,26 @@
 			ret = -1;
 	}
 
+	while ((ret = mail_index_sync_next(index_sync_ctx, &sync_rec)) > 0) {
+		// FIXME: should be just appends
+	}
+
 	if (fstat(ibox->mbox_fd, &st) < 0) {
 		mbox_set_syscall_error(ibox, "fstat()");
 		ret = -1;
 	}
 
+	if (ret < 0)
+		mail_index_transaction_rollback(t);
+	else {
+		if (mail_index_transaction_commit(t, &seq, &offset) < 0)
+			ret = -1;
+		else {
+			ibox->commit_log_file_seq = seq;
+			ibox->commit_log_file_offset = offset;
+		}
+	}
+
 	if (ret < 0) {
 		st.st_mtime = 0;
 		st.st_size = 0;
@@ -256,6 +340,16 @@
 	if (mail_index_sync_end(index_sync_ctx, st.st_mtime, st.st_size) < 0)
 		ret = -1;
 
+	if (ret == 0) {
+		ibox->commit_log_file_seq = 0;
+		ibox->commit_log_file_offset = 0;
+	} else {
+		mail_storage_set_index_error(ibox);
+	}
+
+	ibox->mbox_data = buffer_get_data(ibox->mbox_data_buf, &size);
+	ibox->mbox_data_count = size / sizeof(*ibox->mbox_data);
+
 	str_free(header);
 	return ret < 0 ? -1 : 0;
 }