[dovecot-cvs] dovecot/src/lib-index/mbox mbox-append.c,1.31,1.32 mbox-index.c,1.41,1.42 mbox-index.h,1.18,1.19 mbox-open.c,1.17,1.18 mbox-rebuild.c,1.17,1.18 mbox-rewrite.c,1.31,1.32 mbox-sync-full.c,1.1,1.2 mbox-sync.c,1.15,1.16

[cras at procontrol.fi]

Update of /home/cvs/dovecot/src/lib-index/mbox
In directory danu:/tmp/cvs-serv17951/lib-index/mbox

Modified Files:
	mbox-append.c mbox-index.c mbox-index.h mbox-open.c 
	mbox-rebuild.c mbox-rewrite.c mbox-sync-full.c mbox-sync.c 
Log Message:
Moved several fields from .imap.index file to .imap.index.data file. Fixed
code so that most of the fields do not need to be set when building index,
allowing the index building to be fast (just readdir()s with maildir). This
still needs some configuration and ability to update the fields whenever it
can grab exclusive lock.

Also fixed SEARCH LARGER, SMALLER and KEYWORD.



Index: mbox-append.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mbox/mbox-append.c,v
retrieving revision 1.31
retrieving revision 1.32
diff -u -d -r1.31 -r1.32
--- mbox-append.c	25 Oct 2002 02:31:19 -0000	1.31
+++ mbox-append.c	27 Oct 2002 06:37:18 -0000	1.32
@@ -8,21 +8,6 @@
 #include "mbox-index.h"
 #include "mail-index-util.h"
 
-static MailIndexRecord *mail_index_record_append_begin(MailIndex *index,
-						       time_t internal_date)
-{
-	MailIndexRecord trec, *rec;
-
-	memset(&trec, 0, sizeof(MailIndexRecord));
-	trec.internal_date = internal_date;
-
-	rec = &trec;
-	if (!index->append_begin(index, &rec))
-		return NULL;
-
-	return rec;
-}
-
 static int mbox_index_append_next(MailIndex *index, IBuffer *inbuf)
 {
 	MailIndexRecord *rec;
@@ -72,14 +57,17 @@
 	eoh_offset = inbuf->v_offset;
 
 	/* add message to index */
-	rec = mail_index_record_append_begin(index, internal_date);
+	rec = index->append_begin(index);
 	if (rec == NULL)
 		return FALSE;
 
 	update = index->update_begin(index, rec);
 
+	index->update_field_raw(update, DATA_HDR_INTERNAL_DATE,
+				&internal_date, sizeof(internal_date));
+
 	/* location = offset to beginning of headers in message */
-	index->update_field_raw(update, FIELD_TYPE_LOCATION,
+	index->update_field_raw(update, DATA_FIELD_LOCATION,
 				&abs_start_offset, sizeof(uoff_t));
 
 	/* parse the header and cache wanted fields. get the message flags
@@ -102,7 +90,7 @@
 
 	/* save MD5 */
 	md5_final(&ctx.md5, md5_digest);
-	index->update_field_raw(update, FIELD_TYPE_MD5,
+	index->update_field_raw(update, DATA_FIELD_MD5,
 				md5_digest, sizeof(md5_digest));
 
 	if (!index->update_end(update))

Index: mbox-index.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mbox/mbox-index.c,v
retrieving revision 1.41
retrieving revision 1.42
diff -u -d -r1.41 -r1.42
--- mbox-index.c	26 Oct 2002 19:38:37 -0000	1.41
+++ mbox-index.c	27 Oct 2002 06:37:18 -0000	1.42
@@ -616,28 +616,58 @@
 		(size >= 5 && strncmp((const char *) data, "From ", 5) == 0);
 }
 
-int mbox_mail_get_start_offset(MailIndex *index, MailIndexRecord *rec,
-			       uoff_t *offset)
+int mbox_mail_get_location(MailIndex *index, MailIndexRecord *rec,
+			   uoff_t *offset, uoff_t *hdr_size, uoff_t *body_size)
 {
+	MailIndexDataRecordHeader *data_hdr;
 	const uoff_t *location;
 	size_t size;
 
-	location = index->lookup_field_raw(index, rec,
-					   FIELD_TYPE_LOCATION, &size);
-	if (location == NULL) {
-		index_data_set_corrupted(index->data, "Missing location field "
-					 "for record %u", rec->uid);
-		*offset = 0;
-		return FALSE;
-	} else if (size < sizeof(uoff_t) || *location > OFF_T_MAX) {
-		index_data_set_corrupted(index->data, "Invalid location field "
-					 "for record %u", rec->uid);
-		*offset = 0;
-		return FALSE;
-	} else {
+	if (offset != NULL) {
+		location = index->lookup_field_raw(index, rec,
+						   DATA_FIELD_LOCATION, &size);
+		if (location == NULL) {
+			index_data_set_corrupted(index->data,
+				"Missing location field for record %u",
+				rec->uid);
+			return FALSE;
+		} else if (size != sizeof(uoff_t) || *location > OFF_T_MAX) {
+			index_data_set_corrupted(index->data,
+				"Invalid location field for record %u",
+				rec->uid);
+			return FALSE;
+		}
+
 		*offset = *location;
-		return TRUE;
 	}
+
+	if (hdr_size != NULL || body_size != NULL) {
+		data_hdr = mail_index_data_lookup_header(index->data, rec);
+		if (data_hdr == NULL) {
+			index_set_corrupted(index,
+				"Missing data header for record %u", rec->uid);
+			return FALSE;
+		}
+
+		if ((rec->data_fields & DATA_HDR_HEADER_SIZE) == 0) {
+			index_set_corrupted(index,
+				"Missing header size for record %u", rec->uid);
+			return FALSE;
+		}
+
+		if ((rec->data_fields & DATA_HDR_BODY_SIZE) == 0) {
+			index_set_corrupted(index,
+				"Missing body size for record %u", rec->uid);
+			return FALSE;
+		}
+
+		if (hdr_size != NULL)
+			*hdr_size = data_hdr->header_size;
+		if (body_size != NULL)
+			*body_size = data_hdr->body_size;
+	}
+
+	return TRUE;
 }
 
 MailIndex *mbox_index_alloc(const char *dir, const char *mbox_path)
@@ -702,6 +732,7 @@
 	mail_index_lookup_field_raw,
 	mail_index_cache_fields_later,
 	mbox_open_mail,
+	mail_get_internal_date,
 	mail_index_expunge,
 	mbox_index_update_flags,
 	mail_index_append_begin,

Index: mbox-index.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mbox/mbox-index.h,v
retrieving revision 1.18
retrieving revision 1.19
diff -u -d -r1.18 -r1.19
--- mbox-index.h	26 Oct 2002 19:38:37 -0000	1.18
+++ mbox-index.h	27 Oct 2002 06:37:18 -0000	1.19
@@ -43,14 +43,15 @@
 void mbox_skip_header(IBuffer *inbuf);
 void mbox_skip_message(IBuffer *inbuf);
 int mbox_verify_end_of_body(IBuffer *inbuf, uoff_t end_offset);
-int mbox_mail_get_start_offset(MailIndex *index, MailIndexRecord *rec,
-			       uoff_t *offset);
+int mbox_mail_get_location(MailIndex *index, MailIndexRecord *rec,
+			   uoff_t *offset, uoff_t *hdr_size, uoff_t *body_size);
 
 MailIndex *mbox_index_alloc(const char *dir, const char *mbox_path);
 int mbox_index_rebuild(MailIndex *index);
 int mbox_index_sync(MailIndex *index);
 int mbox_sync_full(MailIndex *index);
-IBuffer *mbox_open_mail(MailIndex *index, MailIndexRecord *rec, int *deleted);
+IBuffer *mbox_open_mail(MailIndex *index, MailIndexRecord *rec,
+			time_t *internal_date, int *deleted);
 
 int mbox_index_append(MailIndex *index, IBuffer *inbuf);
 

Index: mbox-open.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mbox/mbox-open.c,v
retrieving revision 1.17
retrieving revision 1.18
diff -u -d -r1.17 -r1.18
--- mbox-open.c	26 Oct 2002 19:38:37 -0000	1.17
+++ mbox-open.c	27 Oct 2002 06:37:18 -0000	1.18
@@ -3,16 +3,18 @@
 #include "lib.h"
 #include "ibuffer.h"
 #include "mbox-index.h"
+#include "mail-index-data.h"
 #include "mail-index-util.h"
 
 #include <stdlib.h>
 #include <unistd.h>
 #include <fcntl.h>
 
-IBuffer *mbox_open_mail(MailIndex *index, MailIndexRecord *rec, int *deleted)
+IBuffer *mbox_open_mail(MailIndex *index, MailIndexRecord *rec,
+			time_t *internal_date, int *deleted)
 {
 	IBuffer *inbuf;
-	uoff_t offset;
+	uoff_t offset, hdr_size, body_size;
 
 	i_assert(index->lock_type != MAIL_LOCK_UNLOCK);
 
@@ -22,15 +24,18 @@
 	if (index->inconsistent)
 		return NULL;
 
-	if (!mbox_mail_get_start_offset(index, rec, &offset))
+	if (!mbox_mail_get_location(index, rec, &offset, &hdr_size, &body_size))
 		return NULL;
 
 	inbuf = mbox_get_inbuf(index, offset, MAIL_LOCK_SHARED);
 	if (inbuf == NULL)
 		return NULL;
 
+	if (internal_date != NULL)
+		*internal_date = mail_get_internal_date(index, rec);
+
 	i_assert(index->mbox_sync_counter == index->mbox_lock_counter);
 
-	i_buffer_set_read_limit(inbuf, rec->header_size + rec->body_size);
+	i_buffer_set_read_limit(inbuf, hdr_size + body_size);
 	return inbuf;
 }

Index: mbox-rebuild.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mbox/mbox-rebuild.c,v
retrieving revision 1.17
retrieving revision 1.18
diff -u -d -r1.17 -r1.18
--- mbox-rebuild.c	26 Oct 2002 19:38:37 -0000	1.17
+++ mbox-rebuild.c	27 Oct 2002 06:37:18 -0000	1.18
@@ -28,8 +28,8 @@
 	index->mmap_used_length = index->header->used_file_size;
 
 	/* require these fields */
-	index->header->cache_fields |= FIELD_TYPE_LOCATION |
-		FIELD_TYPE_MESSAGEPART | FIELD_TYPE_MD5;
+	index->header->cache_fields |= DATA_FIELD_LOCATION |
+		DATA_FIELD_MESSAGEPART | DATA_FIELD_MD5;
 
 	/* update indexid, which also means that our state has completely
 	   changed */

Index: mbox-rewrite.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mbox/mbox-rewrite.c,v
retrieving revision 1.31
retrieving revision 1.32
diff -u -d -r1.31 -r1.32
--- mbox-rewrite.c	26 Oct 2002 19:38:37 -0000	1.31
+++ mbox-rewrite.c	27 Oct 2002 06:37:18 -0000	1.32
@@ -295,8 +295,8 @@
 
 static int mbox_write_header(MailIndex *index,
 			     MailIndexRecord *rec, unsigned int seq,
-			     IBuffer *inbuf, OBuffer *outbuf,
-			     uoff_t end_offset)
+			     IBuffer *inbuf, OBuffer *outbuf, uoff_t end_offset,
+			     uoff_t hdr_size, uoff_t body_size)
 {
 	/* We need to update fields that define message flags. Standard fields
 	   are stored in Status and X-Status. For custom flags we use
@@ -310,7 +310,7 @@
 	   Last used UID is also not updated, and set to 0 initially.
 	*/
 	MboxRewriteContext ctx;
-	MessageSize hdr_size;
+	MessageSize hdr_parsed_size;
 
 	if (inbuf->v_offset >= end_offset) {
 		/* fsck should have noticed it.. */
@@ -325,16 +325,16 @@
 	memset(&ctx, 0, sizeof(ctx));
 	ctx.outbuf = outbuf;
 	ctx.seq = seq;
-	ctx.content_length = rec->body_size;
+	ctx.content_length = body_size;
 	ctx.msg_flags = rec->msg_flags;
 	ctx.uid_validity = index->header->uid_validity-1;
 	ctx.custom_flags = mail_custom_flags_list_get(index->custom_flags);
 
-	i_buffer_set_read_limit(inbuf, inbuf->v_offset + rec->header_size);
-	message_parse_header(NULL, inbuf, &hdr_size, header_func, &ctx);
+	i_buffer_set_read_limit(inbuf, inbuf->v_offset + hdr_size);
+	message_parse_header(NULL, inbuf, &hdr_parsed_size, header_func, &ctx);
 	i_buffer_set_read_limit(inbuf, 0);
 
-	i_assert(hdr_size.physical_size == rec->header_size);
+	i_assert(hdr_parsed_size.physical_size == hdr_size);
 
 	/* append the flag fields */
 	if (seq == 1 && !ctx.ximapbase_found) {
@@ -402,7 +402,7 @@
 	MailIndexRecord *rec;
 	IBuffer *inbuf;
 	OBuffer *outbuf;
-	uoff_t offset, dirty_offset;
+	uoff_t offset, hdr_size, body_size, dirty_offset;
 	const char *path;
 	unsigned int seq;
 	int tmp_fd, failed, dirty_found, rewrite;
@@ -456,6 +456,8 @@
 	}
 	dirty_offset = 0;
 
+	//offset = hdr_size = body_size = 0; /* just to keep compiler happy */
+
 	t_push();
 	outbuf = o_buffer_create_file(tmp_fd, data_stack_pool, 8192,
 				      IO_PRIORITY_DEFAULT, FALSE);
@@ -465,21 +467,19 @@
 	while (rec != NULL) {
 		if (dirty_found || (rec->index_flags & INDEX_MAIL_FLAG_DIRTY)) {
 			/* get offset to beginning of mail headers */
-			if (!mbox_mail_get_start_offset(index, rec, &offset)) {
+			if (!mbox_mail_get_location(index, rec, &offset,
+						    &hdr_size, &body_size)) {
 				/* fsck should have fixed it */
 				failed = TRUE;
 				break;
 			}
 
-			if (offset + rec->header_size +
-			    rec->body_size > inbuf->v_size) {
+			if (offset + hdr_size + body_size > inbuf->v_size) {
 				index_set_corrupted(index,
 						    "Invalid message size");
 				failed = TRUE;
 				break;
 			}
-		} else {
-			offset = 0;
 		}
 
 		if (!dirty_found &&
@@ -499,15 +499,15 @@
 			}
 
 			/* write header, updating flag fields */
-			offset += rec->header_size;
+			offset += hdr_size;
 			if (!mbox_write_header(index, rec, seq, inbuf, outbuf,
-					       offset)) {
+					       offset, hdr_size, body_size)) {
 				failed = TRUE;
 				break;
 			}
 
 			/* write body */
-			offset += rec->body_size;
+			offset += body_size;
 			if (!mbox_write(index, inbuf, outbuf, offset)) {
 				failed = TRUE;
 				break;

Index: mbox-sync-full.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mbox/mbox-sync-full.c,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- mbox-sync-full.c	26 Oct 2002 19:38:37 -0000	1.1
+++ mbox-sync-full.c	27 Oct 2002 06:37:18 -0000	1.2
@@ -36,7 +36,7 @@
 	size_t size;
 
 	/* MD5 sums must match */
-	old_digest = index->lookup_field_raw(index, rec, FIELD_TYPE_MD5, &size);
+	old_digest = index->lookup_field_raw(index, rec, DATA_FIELD_MD5, &size);
 	return old_digest != NULL && size >= 16 &&
                 memcmp(old_digest, current_digest, 16) == 0;
 }
@@ -49,14 +49,16 @@
 	void *part_data_copy;
 	size_t size;
 
-	/* update index record */
-	rec->header_size = hdr_size->physical_size;
+	/* update FIELD_HDR_HEADER_SIZE */
+	index->update_field_raw(update, DATA_HDR_HEADER_SIZE,
+				&hdr_size->physical_size,
+				sizeof(hdr_size->physical_size));
 
-	if ((rec->cached_fields & FIELD_TYPE_MESSAGEPART) == 0)
+	if ((rec->data_fields & DATA_FIELD_MESSAGEPART) == 0)
 		return TRUE;
 
-	/* update FIELD_TYPE_MESSAGEPART */
-	part_data = index->lookup_field_raw(index, rec, FIELD_TYPE_MESSAGEPART,
+	/* update DATA_FIELD_MESSAGEPART */
+	part_data = index->lookup_field_raw(index, rec, DATA_FIELD_MESSAGEPART,
 					    &size);
 	if (part_data == NULL) {
 		/* well, this wasn't expected but don't bother failing */
@@ -77,7 +79,7 @@
 
 	t_pop();
 
-	index->update_field_raw(update, FIELD_TYPE_MESSAGEPART,
+	index->update_field_raw(update, DATA_FIELD_MESSAGEPART,
 				part_data_copy, size);
 	return TRUE;
 }
@@ -87,19 +89,21 @@
 			     MailIndexRecord **next_rec, int *dirty)
 {
         MailIndexUpdate *update;
-	MessageSize hdr_size;
+	MessageSize hdr_parsed_size;
 	MboxHeaderContext ctx;
-	uoff_t header_offset, body_offset, offset;
+	uoff_t header_offset, body_offset, offset, hdr_size, body_size;
 	unsigned char current_digest[16];
 
 	*next_rec = NULL;
 
 	/* skip the From-line */
 	skip_line(inbuf);
-
 	header_offset = inbuf->v_offset;
 
-	if (rec->body_size == 0) {
+	if (!mbox_mail_get_location(index, rec, NULL, &hdr_size, &body_size))
+		return FALSE;
+
+	if (body_size == 0) {
 		/* possibly broken message, find the next From-line and make
 		   sure header parser won't pass it. */
 		mbox_skip_header(inbuf);
@@ -110,7 +114,8 @@
 	/* get the MD5 sum of fixed headers and the current message flags
 	   in Status and X-Status fields */
         mbox_header_init_context(&ctx, index, inbuf);
-	message_parse_header(NULL, inbuf, &hdr_size, mbox_header_func, &ctx);
+	message_parse_header(NULL, inbuf, &hdr_parsed_size,
+			     mbox_header_func, &ctx);
 	md5_final(&ctx.md5, current_digest);
 
 	mbox_header_free_context(&ctx);
@@ -119,8 +124,7 @@
 	body_offset = inbuf->v_offset;
 	do {
 		if (verify_header_md5sum(index, rec, current_digest) &&
-		    mbox_verify_end_of_body(inbuf,
-					    body_offset + rec->body_size)) {
+		    mbox_verify_end_of_body(inbuf, body_offset + body_size)) {
 			/* valid message */
 			update = index->update_begin(index, rec);
 
@@ -138,19 +142,20 @@
 			}
 
 			/* update location */
-			if (!mbox_mail_get_start_offset(index, rec, &offset))
+			if (!mbox_mail_get_location(index, rec, &offset,
+						    NULL, NULL))
 				return FALSE;
 			if (offset != header_offset) {
 				index->update_field_raw(update,
-							FIELD_TYPE_LOCATION,
+							DATA_FIELD_LOCATION,
 							&header_offset,
 							sizeof(uoff_t));
 			}
 
 			/* update size */
-			if (rec->header_size != hdr_size.physical_size ) {
-				if (!mail_update_header_size(index, rec,
-							     update, &hdr_size))
+			if (hdr_size != hdr_parsed_size.physical_size ) {
+				if (!mail_update_header_size(index, rec, update,
+							     &hdr_parsed_size))
 					return FALSE;
 			}
 

Index: mbox-sync.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mbox/mbox-sync.c,v
retrieving revision 1.15
retrieving revision 1.16
diff -u -d -r1.15 -r1.16
--- mbox-sync.c	26 Oct 2002 19:38:37 -0000	1.15
+++ mbox-sync.c	27 Oct 2002 06:37:18 -0000	1.16
@@ -13,7 +13,7 @@
 static uoff_t get_indexed_mbox_size(MailIndex *index)
 {
 	MailIndexRecord *rec;
-	uoff_t offset;
+	uoff_t offset, hdr_size, body_size;
 
 	if (index->lock_type == MAIL_LOCK_UNLOCK) {
 		if (!mail_index_set_lock(index, MAIL_LOCK_SHARED))
@@ -28,8 +28,9 @@
 	if (rec != NULL) {
 		/* get the offset + size of last message, which tells the
 		   last known mbox file size */
-		if (mbox_mail_get_start_offset(index, rec, &offset))
-			offset += rec->header_size + rec->body_size;
+		if (mbox_mail_get_location(index, rec, &offset,
+					   &hdr_size, &body_size))
+			offset += hdr_size + body_size;
 	}
 
 	if (index->lock_type == MAIL_LOCK_SHARED)