[dovecot-cvs] dovecot/src/lib-index mail-index-compress.c,1.18,1.19 mail-index-data.c,1.26,1.27 mail-index-data.h,1.10,1.11 mail-index-update-cache.c,1.8,1.9 mail-index-update.c,1.25,1.26 mail-index-util.c,1.13,1.14 mail-index-util.h,1.5,1.6 mail-index.c,1.63,1.64 Message-Id: <20021027063720.9E65223863@danu.procontrol.fi>

cras at procontrol.fi cras at procontrol.fi
Sun Oct 27 08:37:20 EET 2002


Update of /home/cvs/dovecot/src/lib-index
In directory danu:/tmp/cvs-serv17951/lib-index

Modified Files:
	mail-index-compress.c mail-index-data.c mail-index-data.h 
	mail-index-update-cache.c mail-index-update.c 
	mail-index-util.c mail-index-util.h mail-index.c mail-index.h 
Log Message:
Moved several fields from .imap.index file to .imap.index.data file. Fixed
code so that most of the fields do not need to be set when building index,
allowing the index building to be fast (just readdir()s with maildir). This
still needs some configuration and ability to update the fields whenever it
can grab exclusive lock.

Also fixed SEARCH LARGER, SMALLER and KEYWORD.



Index: mail-index-compress.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-compress.c,v
retrieving revision 1.18
retrieving revision 1.19
diff -u -d -r1.18 -r1.19
--- mail-index-compress.c	24 Oct 2002 17:24:00 -0000	1.18
+++ mail-index-compress.c	27 Oct 2002 06:37:18 -0000	1.19
@@ -120,6 +120,7 @@
 static int mail_index_copy_data(MailIndex *index, int fd, const char *path)
 {
 	MailIndexDataHeader data_hdr;
+	MailIndexDataRecordHeader *rec_hdr;
 	MailIndexRecord *rec;
 	unsigned char *mmap_data;
 	size_t mmap_data_size;
@@ -149,14 +150,24 @@
 	offset = sizeof(data_hdr);
 	rec = index->lookup(index, 1);
 	while (rec != NULL) {
-		if (rec->data_position + rec->data_size > mmap_data_size) {
-			index_set_corrupted(index, "data_position+data_size "
-					    "points outside file");
+		if (rec->data_position +
+		    sizeof(MailIndexDataRecordHeader) > mmap_data_size) {
+			index_set_corrupted(index,
+				"data_position points outside file");
+			return FALSE;
+		}
+
+		rec_hdr = (MailIndexDataRecordHeader *) (mmap_data +
+							 rec_hdr->data_size);
+
+		if (rec->data_position + rec_hdr->data_size > mmap_data_size) {
+			index_set_corrupted(index,
+				"data_size points outside file");
 			return FALSE;
 		}
 
 		if (write_full(fd, mmap_data + rec->data_position,
-			       rec->data_size) < 0) {
+			       rec_hdr->data_size) < 0) {
 			if (errno == ENOSPC)
 				index->nodiskspace = TRUE;
 
@@ -166,7 +177,7 @@
 		}
 
 		rec->data_position = offset;
-		offset += rec->data_size;
+		offset += rec_hdr->data_size;
 
 		rec = index->next(index, rec);
 	}
@@ -222,7 +233,7 @@
 		/* now, rename the temp file to new data file. but before that
 		   reset indexid to make sure that other processes know the
 		   data file is closed. */
-		(void)mail_index_data_mark_deleted(index->data);
+		(void)mail_index_data_mark_file_deleted(index->data);
 
 		mail_index_data_free(index->data);
 

Index: mail-index-data.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-data.c,v
retrieving revision 1.26
retrieving revision 1.27
diff -u -d -r1.26 -r1.27
--- mail-index-data.c	23 Oct 2002 19:49:23 -0000	1.26
+++ mail-index-data.c	27 Oct 2002 06:37:18 -0000	1.27
@@ -368,7 +368,7 @@
 	return mmap_update(data, 0, 0);
 }
 
-int mail_index_data_mark_deleted(MailIndexData *data)
+int mail_index_data_mark_file_deleted(MailIndexData *data)
 {
 	if (data->anon_mmap)
 		return TRUE;
@@ -459,13 +459,19 @@
 	return offset;
 }
 
-int mail_index_data_add_deleted_space(MailIndexData *data, size_t data_size)
+int mail_index_data_delete(MailIndexData *data, MailIndexRecord *index_rec)
 {
+        MailIndexDataRecordHeader *rec_hdr;
 	uoff_t max_del_space;
 
 	i_assert(data->index->lock_type == MAIL_LOCK_EXCLUSIVE);
 
-	data->header->deleted_space += data_size;
+	rec_hdr = mail_index_data_lookup_header(data, index_rec);
+	if (rec_hdr == NULL)
+		return FALSE;
+
+	/* just mark it deleted. */
+	data->header->deleted_space += rec_hdr->data_size;
 
 	/* see if we've reached the max. deleted space in file */
 	if (data->header->used_file_size >= COMPRESS_MIN_SIZE &&
@@ -502,42 +508,75 @@
 	return TRUE;
 }
 
+MailIndexDataRecordHeader *
+mail_index_data_lookup_header(MailIndexData *data, MailIndexRecord *index_rec)
+{
+	uoff_t pos;
+
+	pos = index_rec->data_position;
+	if (pos == 0) {
+		/* data not yet written to record */
+		return NULL;
+	}
+
+	if (!mmap_update(data, pos, sizeof(MailIndexDataRecordHeader)))
+		return NULL;
+
+	if (pos + sizeof(MailIndexDataRecordHeader) > data->mmap_used_length) {
+		index_data_set_corrupted(data,
+			"Data position of record %u points outside file "
+			"(%"PRIuUOFF_T" + %"PRIuSIZE_T" > %"PRIuSIZE_T")",
+			index_rec->uid, pos, sizeof(MailIndexDataRecordHeader),
+			data->mmap_used_length);
+		return NULL;
+	}
+
+	if ((pos % MEM_ALIGN_SIZE) != 0) {
+		index_data_set_corrupted(data,
+			"Data position (%"PRIuUOFF_T") is not memory aligned "
+			"for record %u", pos, index_rec->uid);
+		return NULL;
+	}
+
+	return (MailIndexDataRecordHeader *) ((char *) data->mmap_base + pos);
+}
+
 MailIndexDataRecord *
 mail_index_data_lookup(MailIndexData *data, MailIndexRecord *index_rec,
-		       MailField field)
+		       MailDataField field)
 {
+        MailIndexDataRecordHeader *rec_hdr;
 	MailIndexDataRecord *rec;
 	uoff_t pos, max_pos;
 
+	index_reset_error(data->index);
+
 	if (index_rec->data_position == 0) {
-		/* data not yet written to record - FIXME: is this an error? */
+		/* data not yet written to record */
 		return NULL;
 	}
 
-	if (!mmap_update(data, index_rec->data_position, index_rec->data_size))
+	rec_hdr = mail_index_data_lookup_header(data, index_rec);
+	if (rec_hdr == NULL)
+		return NULL;
+
+	if (!mmap_update(data, index_rec->data_position, rec_hdr->data_size))
 		return NULL;
 
 	pos = index_rec->data_position;
-	max_pos = pos + index_rec->data_size;
+	max_pos = index_rec->data_position + rec_hdr->data_size;
 
 	if (pos > data->mmap_used_length ||
-	    (data->mmap_used_length - pos < index_rec->data_size)) {
+	    (data->mmap_used_length - pos < rec_hdr->data_size)) {
 		index_data_set_corrupted(data,
 			"Given data size larger than file size "
 			"(%"PRIuUOFF_T" + %u > %"PRIuSIZE_T") for record %u",
-			index_rec->data_position, index_rec->data_size,
+			index_rec->data_position, rec_hdr->data_size,
 			data->mmap_used_length, index_rec->uid);
 		return NULL;
 	}
 
-	if ((pos % MEM_ALIGN_SIZE) != 0) {
-		index_data_set_corrupted(data,
-			"Data position (%"PRIuUOFF_T") is not memory aligned "
-			"for record %u", index_rec->data_position,
-			index_rec->uid);
-		return NULL;
-	}
-
+	pos += sizeof(MailIndexDataRecordHeader);
 	do {
 		rec = (MailIndexDataRecord *) ((char *) data->mmap_base + pos);
 
@@ -579,14 +618,20 @@
 mail_index_data_next(MailIndexData *data, MailIndexRecord *index_rec,
 		     MailIndexDataRecord *rec)
 {
+        MailIndexDataRecordHeader *rec_hdr;
 	uoff_t pos, end_pos, max_pos;
 
+	index_reset_error(data->index);
+
 	if (rec == NULL)
 		return NULL;
 
+	rec_hdr = (MailIndexDataRecordHeader *) ((char *) data->mmap_base +
+						 index_rec->data_position);
+
 	/* get position to next record */
 	pos = DATA_FILE_POSITION(data, rec) + DATA_RECORD_SIZE(rec);
-	max_pos = index_rec->data_position + index_rec->data_size;
+	max_pos = index_rec->data_position + rec_hdr->data_size;
 
 	/* make sure it's within range */
 	if (pos >= max_pos)

Index: mail-index-data.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-data.h,v
retrieving revision 1.10
retrieving revision 1.11
diff -u -d -r1.10 -r1.11
--- mail-index-data.h	23 Oct 2002 19:49:23 -0000	1.10
+++ mail-index-data.h	27 Oct 2002 06:37:18 -0000	1.11
@@ -12,7 +12,7 @@
 
 /* Set indexid to 0 to notify other processes using this file that they should
    re-open it. */
-int mail_index_data_mark_deleted(MailIndexData *data);
+int mail_index_data_mark_file_deleted(MailIndexData *data);
 
 /* Mark the file as being modified */
 void mail_index_data_mark_modified(MailIndexData *data);
@@ -22,17 +22,22 @@
 uoff_t mail_index_data_append(MailIndexData *data, const void *buffer,
 			      size_t size);
 
-/* Increase header->deleted_space field */
-int mail_index_data_add_deleted_space(MailIndexData *data, size_t data_size);
+/* Mark the given record deleted. */
+int mail_index_data_delete(MailIndexData *data, MailIndexRecord *index_rec);
 
 /* Synchronize the data into disk */
 int mail_index_data_sync_file(MailIndexData *data, int *fsync_fd);
 
+/* Looks up a record header from data file. Returns NULL if not found or
+   if error occured. */
+MailIndexDataRecordHeader *
+mail_index_data_lookup_header(MailIndexData *data, MailIndexRecord *index_rec);
+
 /* Looks up a field from data file. If field is 0, returns the first field
    found. Returns NULL if not found or if error occured. */
 MailIndexDataRecord *
 mail_index_data_lookup(MailIndexData *data, MailIndexRecord *index_rec,
-		       MailField field);
+		       MailDataField field);
 
 /* Returns the next record in data file, or NULL if there's no more. */
 MailIndexDataRecord *

Index: mail-index-update-cache.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-update-cache.c,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -d -r1.8 -r1.9
--- mail-index-update-cache.c	20 Oct 2002 00:17:21 -0000	1.8
+++ mail-index-update-cache.c	27 Oct 2002 06:37:18 -0000	1.9
@@ -7,19 +7,22 @@
 #include <unistd.h>
 
 static int cache_record(MailIndex *index, MailIndexRecord *rec,
-			MailField cache_fields)
+			MailDataField cache_fields)
 {
 	MailIndexUpdate *update;
 	IBuffer *inbuf;
+	time_t internal_date;
 	int failed, deleted;
 
-	inbuf = index->open_mail(index, rec, &deleted);
+	inbuf = index->open_mail(index, rec, &internal_date, &deleted);
 	if (inbuf == NULL)
 		return deleted;
 
-	cache_fields &= ~rec->cached_fields;
+	cache_fields &= ~rec->data_fields;
 
 	update = index->update_begin(index, rec);
+	index->update_field_raw(update, DATA_HDR_INTERNAL_DATE,
+				&internal_date, sizeof(internal_date));
 	mail_index_update_headers(update, inbuf, cache_fields, NULL, NULL);
 	failed = !index->update_end(update);
 
@@ -30,7 +33,7 @@
 int mail_index_update_cache(MailIndex *index)
 {
 	MailIndexRecord *rec;
-	MailField cache_fields;
+	MailDataField cache_fields;
 
 	if (!index->set_lock(index, MAIL_LOCK_EXCLUSIVE))
 		return FALSE;
@@ -43,7 +46,7 @@
 
 	rec = index->lookup(index, 1);
 	while (rec != NULL) {
-		if ((rec->cached_fields & cache_fields) != cache_fields) {
+		if ((rec->data_fields & cache_fields) != cache_fields) {
 			if (!cache_record(index, rec, cache_fields))
 				return FALSE;
 		}

Index: mail-index-update.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-update.c,v
retrieving revision 1.25
retrieving revision 1.26
diff -u -d -r1.25 -r1.26
--- mail-index-update.c	23 Oct 2002 19:49:23 -0000	1.25
+++ mail-index-update.c	27 Oct 2002 06:37:18 -0000	1.26
@@ -19,17 +19,19 @@
 
 	MailIndex *index;
 	MailIndexRecord *rec;
+	MailIndexDataRecordHeader data_hdr;
 
 	unsigned int updated_fields;
-	void *fields[FIELD_TYPE_MAX_BITS];
-	size_t field_sizes[FIELD_TYPE_MAX_BITS];
-	size_t field_extra_sizes[FIELD_TYPE_MAX_BITS];
+	void *fields[DATA_FIELD_MAX_BITS];
+	size_t field_sizes[DATA_FIELD_MAX_BITS];
+	size_t field_extra_sizes[DATA_FIELD_MAX_BITS];
 };
 
 MailIndexUpdate *mail_index_update_begin(MailIndex *index, MailIndexRecord *rec)
 {
 	Pool pool;
 	MailIndexUpdate *update;
+	MailIndexDataRecordHeader *data_hdr;
 
 	i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
 
@@ -39,14 +41,18 @@
 	update->pool = pool;
 	update->index = index;
 	update->rec = rec;
+
+	data_hdr = mail_index_data_lookup_header(index->data, rec);
+	if (data_hdr != NULL)
+		memcpy(&update->data_hdr, data_hdr, sizeof(*data_hdr));
 	return update;
 }
 
-static int mail_field_get_index(MailField field)
+static int mail_field_get_index(MailDataField field)
 {
 	unsigned int i, mask;
 
-	for (i = 0, mask = 1; i < FIELD_TYPE_MAX_BITS; i++, mask <<= 1) {
+	for (i = 0, mask = 1; i < DATA_FIELD_MAX_BITS; i++, mask <<= 1) {
 		if (field == mask)
 			return i;
 	}
@@ -54,132 +60,100 @@
 	return -1;
 }
 
-static int have_new_fields(MailIndexUpdate *update)
+static void get_changed_field_sizes(MailIndexUpdate *update,
+				    size_t *min_size, size_t *max_size)
 {
-	MailField field;
-
-	if (update->rec->cached_fields == 0) {
-		/* new record */
-		return TRUE;
-	}
+	int i;
 
-	for (field = 1; field != FIELD_TYPE_LAST; field <<= 1) {
-		if ((update->updated_fields & field) &&
-		    (update->rec->cached_fields & field) == 0)
-			return TRUE;
+	for (i = 0; i < DATA_FIELD_MAX_BITS; i++) {
+		if (update->fields[i] != NULL) {
+			*min_size += SIZEOF_MAIL_INDEX_DATA +
+				MEM_ALIGN(update->field_sizes[i]);
+			*max_size += SIZEOF_MAIL_INDEX_DATA +
+				MEM_ALIGN(update->field_sizes[i] +
+					  update->field_extra_sizes[i]);
+		}
 	}
-
-	return FALSE;
 }
 
-static int have_too_large_fields(MailIndexUpdate *update)
+static void get_data_block_sizes(MailIndexUpdate *update,
+				 size_t *min_size, size_t *max_size)
 {
 	MailIndexDataRecord *rec;
-	unsigned int size_left;
-	int index;
 
-	size_left = update->rec->data_size;
+	/* first get size of new fields */
+	*min_size = *max_size = sizeof(MailIndexDataRecordHeader);
+	get_changed_field_sizes(update, min_size, max_size);
 
-	/* start from the first data field - it's required to exist */
+	/* then the size of unchanged fields */
 	rec = mail_index_data_lookup(update->index->data, update->rec, 0);
 	while (rec != NULL) {
-		if (rec->full_field_size > size_left) {
-			/* corrupted */
-			update->index->header->flags |= MAIL_INDEX_FLAG_REBUILD;
-			return TRUE;
+		if ((rec->field & update->updated_fields) == 0) {
+			*min_size += SIZEOF_MAIL_INDEX_DATA +
+				rec->full_field_size;
+			*max_size += SIZEOF_MAIL_INDEX_DATA +
+				rec->full_field_size;
 		}
-		size_left -= rec->full_field_size;
-
-		if (rec->field & update->updated_fields) {
-			/* field was changed */
-			index = mail_field_get_index(rec->field);
-			i_assert(index >= 0);
 
-			if (update->field_sizes[index] +
-			    update->field_extra_sizes[index] >
-			    rec->full_field_size)
-				return TRUE;
-		}
 		rec = mail_index_data_next(update->index->data,
 					   update->rec, rec);
 	}
-
-	return FALSE;
 }
 
 /* Append all the data at the end of the data file and update 
    the index's data position */
-static int update_by_append(MailIndexUpdate *update)
+static int update_by_append(MailIndexUpdate *update, size_t data_size)
 {
+        MailIndexDataRecordHeader *dest_hdr;
         MailIndexDataRecord *rec, *destrec;
-	MailField field;
+	MailDataField field;
 	uoff_t fpos;
 	void *mem;
 	const void *src;
-	size_t max_size, pos, src_size;
+	size_t pos, src_size;
 	int i;
 
-	/* allocate the old size + also the new size of all changed or added
-	   fields. this is more than required, but it's much easier than
-	   calculating the exact size.
+	i_assert(data_size <= UINT_MAX);
 
-	   If this calculation overflows (no matter what value), it doesn't
-	   really matter as it's later checked anyway. */
-	max_size = update->rec->data_size;
-	for (i = 0; i < FIELD_TYPE_MAX_BITS; i++) {
-		max_size += SIZEOF_MAIL_INDEX_DATA +
-			update->field_sizes[i] +
-			update->field_extra_sizes[i] + MEM_ALIGN_SIZE-1;
-	}
+	mem = p_malloc(update->pool, data_size);
 
-	if (max_size > INT_MAX) {
-		/* rec->data_size most likely corrupted */
-		index_set_corrupted(update->index,
-				    "data_size points outside file");
-		return FALSE;
-	}
+	/* set header */
+	dest_hdr = (MailIndexDataRecordHeader *) mem;
+	pos = sizeof(MailIndexDataRecordHeader);
 
-	/* allocate two extra records to avoid overflows in case of bad
-	   rec->full_field_size which itself fits into max_size, but
-	   either the record part would make it point ouside allocate memory,
-	   or the next field's record would do that */
-	mem = p_malloc(update->pool, max_size + sizeof(MailIndexDataRecord)*2);
-	pos = 0;
+	memcpy(dest_hdr, &update->data_hdr, sizeof(*dest_hdr));
+	dest_hdr->data_size = data_size;
 
+	/* set fields */
 	rec = mail_index_data_lookup(update->index->data, update->rec, 0);
-	for (i = 0, field = 1; field != FIELD_TYPE_LAST; i++, field <<= 1) {
+	for (i = 0, field = 1; field != DATA_FIELD_LAST; i++, field <<= 1) {
 		destrec = (MailIndexDataRecord *) ((char *) mem + pos);
 
 		if (update->fields[i] != NULL) {
 			/* value was modified - use it */
-			destrec->full_field_size = update->field_sizes[i] +
-				update->field_extra_sizes[i];
+			destrec->full_field_size =
+				MEM_ALIGN(update->field_sizes[i] +
+					  update->field_extra_sizes[i]);
 			src = update->fields[i];
 			src_size = update->field_sizes[i];
 		} else if (rec != NULL && rec->field == field) {
 			/* use the old value */
 			destrec->full_field_size = rec->full_field_size;
 			src = rec->data;
-			src_size = rec->full_field_size;
+			src_size = destrec->full_field_size;
 		} else {
 			/* the field doesn't exist, jump to next */
 			continue;
 		}
+		i_assert((destrec->full_field_size % MEM_ALIGN_SIZE) == 0);
 
-		if (src_size > max_size || max_size - src_size < pos) {
-			/* corrupted data file - old value had a field
-			   larger than expected */
-			index_set_corrupted(update->index,
-				"full_field_size points outside data_size "
-				"(field %u?)", update->index->filepath,
-				rec == NULL ? 0 : rec->field);
-			return FALSE;
+		/* make sure we don't overflow our buffer */
+		if (src_size > data_size || data_size - src_size < pos) {
+			i_panic("data file for index %s unexpectedly modified",
+				update->index->filepath);
 		}
 		memcpy(destrec->data, src, src_size);
 
-		/* memory alignment fix */
-		destrec->full_field_size = MEM_ALIGN(destrec->full_field_size);
-
 		destrec->field = field;
 		pos += DATA_RECORD_SIZE(destrec);
 
@@ -189,7 +163,7 @@
 		}
 	}
 
-	i_assert(pos <= max_size);
+	i_assert(pos == data_size);
 
 	/* append the data at the end of the data file */
 	fpos = mail_index_data_append(update->index->data, mem, pos);
@@ -197,13 +171,11 @@
 		return FALSE;
 
 	/* the old data is discarded */
-	(void)mail_index_data_add_deleted_space(update->index->data,
-						update->rec->data_size);
+	(void)mail_index_data_delete(update->index->data, update->rec);
 
 	/* update index file position - it's mmap()ed so it'll be written
 	   into disk when index is unlocked. */
 	update->rec->data_position = fpos;
-	update->rec->data_size = pos;
 	return TRUE;
 }
 
@@ -214,7 +186,9 @@
 	MailIndexDataRecord *rec;
 	int index;
 
-	/* start from the first data field - it's required to exist */
+	// FIXME: 1) this doesn't work, 2) we need to handle optimally the
+	// writing of extra_space
+
 	rec = mail_index_data_lookup(update->index->data, update->rec, 0);
 	while (rec != NULL) {
 		if (rec->field & update->updated_fields) {
@@ -237,22 +211,27 @@
 
 int mail_index_update_end(MailIndexUpdate *update)
 {
+	MailIndexDataRecordHeader *data_hdr;
+	size_t min_size, max_size;
 	int failed = FALSE;
 
 	i_assert(update->index->lock_type == MAIL_LOCK_EXCLUSIVE);
 
 	if (update->updated_fields != 0) {
-		/* if any of the fields were newly added, or have grown larger
-		   than their old max. size, we need to move the record to end
-		   of file. */
-		if (have_new_fields(update) || have_too_large_fields(update))
-			failed = !update_by_append(update);
-		else
+		/* if fields don't fit to allocated data block, we have
+		   to move it to end of file */
+		get_data_block_sizes(update, &min_size, &max_size);
+		data_hdr = mail_index_data_lookup_header(update->index->data,
+							 update->rec);
+
+		if (data_hdr != NULL && min_size <= data_hdr->data_size)
 			update_by_replace(update);
+		else
+			failed = !update_by_append(update, max_size);
 
 		if (!failed) {
 			/* update cached fields mask */
-			update->rec->cached_fields |= update->updated_fields;
+			update->rec->data_fields |= update->updated_fields;
 		}
 	}
 
@@ -260,7 +239,7 @@
 	return !failed;
 }
 
-static void update_field_full(MailIndexUpdate *update, MailField field,
+static void update_field_full(MailIndexUpdate *update, MailDataField field,
 			      const void *value, size_t size,
 			      size_t extra_space)
 {
@@ -276,16 +255,46 @@
 	memcpy(update->fields[index], value, size);
 }
 
-void mail_index_update_field(MailIndexUpdate *update, MailField field,
+static void update_header_field(MailIndexUpdate *update, MailDataField field,
+				const void *value, size_t size)
+{
+	switch (field) {
+	case DATA_HDR_INTERNAL_DATE:
+		i_assert(size == sizeof(time_t));
+		update->data_hdr.internal_date = *((time_t *) value);
+		break;
+	case DATA_HDR_VIRTUAL_SIZE:
+		i_assert(size == sizeof(uoff_t));
+		update->data_hdr.virtual_size = *((uoff_t *) value);
+		break;
+	case DATA_HDR_HEADER_SIZE:
+		i_assert(size == sizeof(uoff_t));
+		update->data_hdr.header_size = *((uoff_t *) value);
+		break;
+	case DATA_HDR_BODY_SIZE:
+		i_assert(size == sizeof(uoff_t));
+		update->data_hdr.body_size = *((uoff_t *) value);
+		break;
+	default:
+		i_assert(0);
+	}
+
+	update->updated_fields |= field;
+}
+
+void mail_index_update_field(MailIndexUpdate *update, MailDataField field,
 			     const char *value, size_t extra_space)
 {
 	update_field_full(update, field, value, strlen(value) + 1, extra_space);
 }
 
-void mail_index_update_field_raw(MailIndexUpdate *update, MailField field,
+void mail_index_update_field_raw(MailIndexUpdate *update, MailDataField field,
 				 const void *value, size_t size)
 {
-	update_field_full(update, field, value, size, 0);
+	if (field >= DATA_FIELD_LAST)
+		update_header_field(update, field, value, size);
+	else
+		update_field_full(update, field, value, size, 0);
 }
 
 typedef struct {
@@ -308,7 +317,7 @@
 		return;
 
 	/* see if we can do anything with this field */
-	if (ctx->update->index->header->cache_fields & FIELD_TYPE_ENVELOPE) {
+	if (ctx->update->index->header->cache_fields & DATA_FIELD_ENVELOPE) {
 		if (ctx->envelope_pool == NULL) {
 			ctx->envelope_pool =
 				pool_create("index envelope", 2048, FALSE);
@@ -326,7 +335,7 @@
 }
 
 void mail_index_update_headers(MailIndexUpdate *update, IBuffer *inbuf,
-                               MailField cache_fields,
+                               MailDataField cache_fields,
 			       MessageHeaderFunc header_func, void *context)
 {
 	HeaderUpdateContext ctx;
@@ -335,7 +344,7 @@
 	Pool pool;
 	const char *value;
 	size_t size;
-	uoff_t start_offset;
+	uoff_t start_offset, uoff_size;
 
 	ctx.update = update;
 	ctx.envelope_pool = NULL;
@@ -354,7 +363,7 @@
 
 		value = update->index->lookup_field_raw(update->index,
 							update->rec,
-							FIELD_TYPE_MESSAGEPART,
+							DATA_FIELD_MESSAGEPART,
 							&size);
 		if (value == NULL)
 			part = NULL;
@@ -381,40 +390,49 @@
 		}
 
 		/* update our sizes */
-		update->rec->header_size = part->header_size.physical_size;
-		update->rec->body_size = part->body_size.physical_size;
+		update->index->update_field_raw(update, DATA_HDR_HEADER_SIZE,
+			&part->header_size.physical_size,
+			sizeof(part->header_size.physical_size));
+		update->index->update_field_raw(update, DATA_HDR_BODY_SIZE,
+			&part->body_size.physical_size,
+			sizeof(part->body_size.physical_size));
+
+		uoff_size = part->header_size.virtual_size +
+			part->body_size.virtual_size;
+		update->index->update_field_raw(update, DATA_HDR_VIRTUAL_SIZE,
+						&uoff_size, sizeof(uoff_size));
 
 		/* don't save both BODY + BODYSTRUCTURE since BODY can be
 		   generated from BODYSTRUCTURE. FIXME: However that takes
 		   CPU, maybe this should be configurable (I/O vs. CPU)? */
-		if ((cache_fields & FIELD_TYPE_BODY) &&
-		    ((update->rec->cached_fields | cache_fields) &
-		     FIELD_TYPE_BODYSTRUCTURE) == 0) {
+		if ((cache_fields & DATA_FIELD_BODY) &&
+		    ((update->rec->data_fields | cache_fields) &
+		     DATA_FIELD_BODYSTRUCTURE) == 0) {
 			t_push();
 			i_buffer_seek(inbuf, start_offset);
 			value = imap_part_get_bodystructure(pool, &part,
 							    inbuf, FALSE);
-			update->index->update_field(update, FIELD_TYPE_BODY,
+			update->index->update_field(update, DATA_FIELD_BODY,
 						    value, 0);
 			t_pop();
 		}
 
-		if (cache_fields & FIELD_TYPE_BODYSTRUCTURE) {
+		if (cache_fields & DATA_FIELD_BODYSTRUCTURE) {
 			t_push();
 			i_buffer_seek(inbuf, start_offset);
 			value = imap_part_get_bodystructure(pool, &part,
 							    inbuf, TRUE);
 			update->index->update_field(update,
-						    FIELD_TYPE_BODYSTRUCTURE,
+						    DATA_FIELD_BODYSTRUCTURE,
 						    value, 0);
 			t_pop();
 		}
 
-		if (cache_fields & FIELD_TYPE_MESSAGEPART) {
+		if (cache_fields & DATA_FIELD_MESSAGEPART) {
 			t_push();
 			value = message_part_serialize(part, &size);
 			update->index->update_field_raw(update,
-							FIELD_TYPE_MESSAGEPART,
+							DATA_FIELD_MESSAGEPART,
 							value, size);
 			t_pop();
 		}
@@ -424,14 +442,19 @@
 		message_parse_header(NULL, inbuf, &hdr_size,
 				     update_header_func, &ctx);
 
-		update->rec->header_size = hdr_size.physical_size;
-		update->rec->body_size = inbuf->v_size - inbuf->v_offset;
+		update->index->update_field_raw(update, DATA_HDR_HEADER_SIZE,
+			&hdr_size.physical_size,
+			sizeof(hdr_size.physical_size));
+
+		uoff_size = inbuf->v_size - inbuf->v_offset;
+		update->index->update_field_raw(update, DATA_HDR_BODY_SIZE,
+						&uoff_size, sizeof(uoff_size));
 	}
 
 	if (ctx.envelope != NULL) {
 		t_push();
 		value = imap_envelope_get_part_data(ctx.envelope);
-		update->index->update_field(update, FIELD_TYPE_ENVELOPE,
+		update->index->update_field(update, DATA_FIELD_ENVELOPE,
 					    value, 0);
 		t_pop();
 

Index: mail-index-util.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-util.c,v
retrieving revision 1.13
retrieving revision 1.14
diff -u -d -r1.13 -r1.14
--- mail-index-util.c	20 Oct 2002 01:36:46 -0000	1.13
+++ mail-index-util.c	27 Oct 2002 06:37:18 -0000	1.14
@@ -105,61 +105,3 @@
 
 	return fd;
 }
-
-
-int mail_index_get_virtual_size(MailIndex *index, MailIndexRecord *rec,
-				int fastscan, uoff_t *virtual_size)
-{
-	MessageSize hdr_size, body_size;
-	IBuffer *inbuf;
-	const void *part_data;
-	size_t size;
-	int deleted;
-
-	if ((rec->index_flags & INDEX_MAIL_FLAG_BINARY_HEADER) &&
-	    (rec->index_flags & INDEX_MAIL_FLAG_BINARY_BODY)) {
-		/* virtual size == physical size */
-		*virtual_size += rec->header_size + rec->body_size;
-		return TRUE;
-	}
-
-	part_data = index->lookup_field_raw(index, rec,
-					    FIELD_TYPE_MESSAGEPART, &size);
-	if (part_data == NULL)
-		index->cache_fields_later(index, FIELD_TYPE_MESSAGEPART);
-	else {
-		/* get sizes from preparsed message structure */
-		if (!message_part_deserialize_size(part_data, size,
-						   &hdr_size, &body_size)) {
-			/* corrupted, ignore */
-			index_set_corrupted(index,
-				"Corrupted cached MessagePart data");
-		} else {
-			*virtual_size = hdr_size.virtual_size +
-				body_size.virtual_size;
-			return TRUE;
-		}
-	}
-
-	/* only way left is to actually parse the message */
-	*virtual_size = 0;
-
-	if (fastscan) {
-		/* and we don't want that */
-		return FALSE;
-	}
-
-	inbuf = index->open_mail(index, rec, &deleted);
-	if (inbuf == NULL) {
-		/* If we were deleted, just return TRUE with 0 size. */
-		return deleted;
-	}
-
-	/* we don't care about the difference in header/body,
-	   so parse the whole message as a "body" */
-	message_get_body_size(inbuf, &body_size, (uoff_t)-1);
-	*virtual_size = body_size.virtual_size;
-
-	i_buffer_unref(inbuf);
-	return TRUE;
-}

Index: mail-index-util.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-util.h,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -d -r1.5 -r1.6
--- mail-index-util.h	20 Oct 2002 00:17:21 -0000	1.5
+++ mail-index-util.h	27 Oct 2002 06:37:18 -0000	1.6
@@ -22,11 +22,4 @@
    and sets *path to the full path of the created file.  */
 int mail_index_create_temp_file(MailIndex *index, const char **path);
 
-/* Calculates virtual size for specified message. If the fastscan is FALSE
-   and the size can't be figured out from headers, the message is opened and
-   fully scanned to calculate the size. Returns TRUE if size was successfully
-   got. If mail was just deleted, returns TRUE and sets virtual_size to 0. */
-int mail_index_get_virtual_size(MailIndex *index, MailIndexRecord *rec,
-				int fastscan, uoff_t *virtual_size);
-
 #endif

Index: mail-index.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index.c,v
retrieving revision 1.63
retrieving revision 1.64
diff -u -d -r1.63 -r1.64
--- mail-index.c	23 Oct 2002 20:41:35 -0000	1.63
+++ mail-index.c	27 Oct 2002 06:37:18 -0000	1.64
@@ -580,11 +580,11 @@
 }
 
 const char *mail_index_lookup_field(MailIndex *index, MailIndexRecord *rec,
-				    MailField field)
+				    MailDataField field)
 {
 	MailIndexDataRecord *datarec;
 
-	datarec = (rec->cached_fields & field) == 0 ? NULL :
+	datarec = (rec->data_fields & field) == 0 ? NULL :
 		mail_index_data_lookup(index->data, rec, field);
 	if (datarec == NULL)
 		return NULL;
@@ -598,22 +598,55 @@
 }
 
 const void *mail_index_lookup_field_raw(MailIndex *index, MailIndexRecord *rec,
-					MailField field, size_t *size)
+					MailDataField field, size_t *size)
 {
+	MailIndexDataRecordHeader *datahdr;
 	MailIndexDataRecord *datarec;
 
-	datarec = (rec->cached_fields & field) == 0 ? NULL :
-		mail_index_data_lookup(index->data, rec, field);
-	if (datarec == NULL) {
+	if ((rec->data_fields & field) == 0) {
 		*size = 0;
 		return NULL;
 	}
 
-	*size = datarec->full_field_size;
-	return datarec->data;
+	if (field < DATA_FIELD_LAST) {
+		/* read data field */
+		datarec = mail_index_data_lookup(index->data, rec, field);
+		if (datarec == NULL) {
+			*size = 0;
+			return NULL;
+		}
+
+		*size = datarec->full_field_size;
+		return datarec->data;
+	}
+
+	/* read header field */
+	datahdr = mail_index_data_lookup_header(index->data, rec);
+	if (datahdr == NULL) {
+		*size = 0;
+		return NULL;
+	}
+
+	switch (field) {
+	case DATA_HDR_INTERNAL_DATE:
+		*size = sizeof(datahdr->internal_date);
+		return &datahdr->internal_date;
+	case DATA_HDR_VIRTUAL_SIZE:
+		*size = sizeof(datahdr->virtual_size);
+		return &datahdr->virtual_size;
+	case DATA_HDR_HEADER_SIZE:
+		*size = sizeof(datahdr->header_size);
+		return &datahdr->header_size;
+	case DATA_HDR_BODY_SIZE:
+		*size = sizeof(datahdr->body_size);
+		return &datahdr->body_size;
+	default:
+		*size = 0;
+		return NULL;
+	}
 }
 
-void mail_index_cache_fields_later(MailIndex *index, MailField field)
+void mail_index_cache_fields_later(MailIndex *index, MailDataField field)
 {
 	i_assert(index->lock_type != MAIL_LOCK_UNLOCK);
 
@@ -635,6 +668,21 @@
 	}
 }
 
+time_t mail_get_internal_date(MailIndex *index, MailIndexRecord *rec)
+{
+	const time_t *date;
+	size_t size;
+
+	date = index->lookup_field_raw(index, rec,
+				       DATA_HDR_INTERNAL_DATE, &size);
+	if (date == NULL)
+		return (time_t)-1;
+	else {
+		i_assert(size == sizeof(*date));
+		return *date;
+	}
+}
+
 void mail_index_mark_flag_changes(MailIndex *index, MailIndexRecord *rec,
 				  MailFlags old_flags, MailFlags new_flags)
 {
@@ -776,7 +824,7 @@
 	hdr->messages_count--;
 	mail_index_mark_flag_changes(index, rec, rec->msg_flags, 0);
 
-	(void)mail_index_data_add_deleted_space(index->data, rec->data_size);
+	(void)mail_index_data_delete(index->data, rec);
 
 	records = MAIL_INDEX_RECORD_COUNT(index);
 	if (hdr->first_hole_index + hdr->first_hole_records == records) {
@@ -866,31 +914,29 @@
 	return TRUE;
 }
 
-int mail_index_append_begin(MailIndex *index, MailIndexRecord **rec)
+MailIndexRecord *mail_index_append_begin(MailIndex *index)
 {
-	MailIndexRecord *destrec;
+	MailIndexRecord *rec;
 
 	i_assert(index->lock_type == MAIL_LOCK_EXCLUSIVE);
-	i_assert((*rec)->uid == 0);
-	i_assert((*rec)->msg_flags == 0);
 
 	if (index->mmap_used_length == index->mmap_full_length) {
 		if (!mail_index_grow(index))
-			return FALSE;
+			return NULL;
 	}
 
 	i_assert(index->header->used_file_size == index->mmap_used_length);
 	i_assert(index->mmap_used_length + sizeof(MailIndexRecord) <=
 		 index->mmap_full_length);
 
-	destrec = (MailIndexRecord *) ((char *) index->mmap_base +
-				       index->mmap_used_length);
-	memcpy(destrec, *rec, sizeof(MailIndexRecord));
-	*rec = destrec;
+	rec = (MailIndexRecord *) ((char *) index->mmap_base +
+				   index->mmap_used_length);
+	memset(rec, 0, sizeof(MailIndexRecord));
 
 	index->header->used_file_size += sizeof(MailIndexRecord);
 	index->mmap_used_length += sizeof(MailIndexRecord);
-	return TRUE;
+
+	return rec;
 }
 
 int mail_index_append_end(MailIndex *index, MailIndexRecord *rec)

Index: mail-index.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index.h,v
retrieving revision 1.46
retrieving revision 1.47
diff -u -d -r1.46 -r1.47
--- mail-index.h	26 Oct 2002 19:38:37 -0000	1.46
+++ mail-index.h	27 Oct 2002 06:37:18 -0000	1.47
@@ -27,16 +27,26 @@
 };
 
 typedef enum {
-	FIELD_TYPE_LOCATION		= 0x0001,
-	FIELD_TYPE_ENVELOPE		= 0x0002,
-	FIELD_TYPE_BODY			= 0x0004,
-	FIELD_TYPE_BODYSTRUCTURE	= 0x0008,
-	FIELD_TYPE_MD5			= 0x0010,
-	FIELD_TYPE_MESSAGEPART		= 0x0020,
+	DATA_FIELD_LOCATION		= 0x00000001,
+	DATA_FIELD_ENVELOPE		= 0x00000002,
+	DATA_FIELD_BODY			= 0x00000004,
+	DATA_FIELD_BODYSTRUCTURE	= 0x00000008,
+	DATA_FIELD_MD5			= 0x00000010,
+	DATA_FIELD_MESSAGEPART		= 0x00000020,
 
-	FIELD_TYPE_LAST			= 0x0040,
-	FIELD_TYPE_MAX_BITS		= 6
-} MailField;
+	DATA_FIELD_LAST			= 0x00000040,
+	DATA_FIELD_MAX_BITS		= 6,
+
+	/* separate from above, but in same bitmask */
+	DATA_HDR_INTERNAL_DATE		= 0x80000000,
+	DATA_HDR_VIRTUAL_SIZE		= 0x40000000,
+	DATA_HDR_HEADER_SIZE		= 0x20000000,
+	DATA_HDR_BODY_SIZE		= 0x10000000
+} MailDataField;
+
+#define IS_BODYSTRUCTURE_FIELD(field) \
+	(((field) & (DATA_FIELD_BODY | DATA_FIELD_BODYSTRUCTURE | \
+		     DATA_FIELD_MESSAGEPART)) != 0)
 
 typedef enum {
 	/* If binary flags are set, it's not checked whether mail is
@@ -51,14 +61,6 @@
 	INDEX_MAIL_FLAG_DIRTY		= 0x0004
 } MailIndexMailFlags;
 
-#define IS_HEADER_FIELD(field) \
-	(((field) & (FIELD_TYPE_FROM | FIELD_TYPE_TO | FIELD_TYPE_CC | \
-		     FIELD_TYPE_BCC | FIELD_TYPE_SUBJECT)) != 0)
-
-#define IS_BODYSTRUCTURE_FIELD(field) \
-	(((field) & (FIELD_TYPE_BODY|FIELD_TYPE_BODYSTRUCTURE| \
-		     FIELD_TYPE_MESSAGEPART)) != 0)
-
 typedef enum {
 	MAIL_LOCK_UNLOCK = 0,
 	MAIL_LOCK_SHARED,
@@ -76,6 +78,7 @@
 
 typedef struct _MailIndexRecord MailIndexRecord;
 typedef struct _MailIndexDataRecord MailIndexDataRecord;
+typedef struct _MailIndexDataRecordHeader MailIndexDataRecordHeader;
 
 typedef struct _MailIndexUpdate MailIndexUpdate;
 
@@ -122,23 +125,27 @@
 };
 
 struct _MailIndexRecord {
-	/* remember to keep uoff_t's 8 byte aligned so we don't waste space */
 	unsigned int uid;
 	unsigned int msg_flags; /* MailFlags */
-	time_t internal_date;
 
 	unsigned int index_flags; /* MailIndexMailFlags */
-	unsigned int cached_fields; /* MailField */
+	unsigned int data_fields; /* MailDataField */
 
-	unsigned int data_size;
 	uoff_t data_position;
+};
 
-	uoff_t header_size; /* 0 if not known yet */
-	uoff_t body_size; /* if header_size == 0, the size of full message */
+struct _MailIndexDataRecordHeader {
+	unsigned int data_size; /* including this header */
+
+	time_t internal_date;
+	uoff_t virtual_size;
+
+	uoff_t header_size;
+	uoff_t body_size;
 };
 
 struct _MailIndexDataRecord {
-	unsigned int field; /* MailField */
+	unsigned int field; /* MailDataField */
 	unsigned int full_field_size;
 	char data[MEM_ALIGN_SIZE]; /* variable size */
 };
@@ -221,22 +228,26 @@
 	/* Find field from specified record, or NULL if it's not in index.
 	   Makes sure that the field ends with \0. */
 	const char *(*lookup_field)(MailIndex *index, MailIndexRecord *rec,
-				    MailField field);
+				    MailDataField field);
 
 	/* Find field from specified record, or NULL if it's not in index. */
 	const void *(*lookup_field_raw)(MailIndex *index, MailIndexRecord *rec,
-					MailField field, size_t *size);
+					MailDataField field, size_t *size);
 
 	/* Mark the fields to be cached later. If any of them is already
 	   set in hdr->cache_fields, mark the caching to happen next time
 	   index is opened. */
-	void (*cache_fields_later)(MailIndex *index, MailField field);
+	void (*cache_fields_later)(MailIndex *index, MailDataField field);
 
-	/* Open mail file and return it as mmap()ed IBuffer. If we fails,
+	/* Open mail file and return it as mmap()ed IBuffer. If we fail,
 	   we return NULL and set deleted = TRUE if failure was because the
-	   mail was just deleted (ie. not an error). */
+	   mail was just deleted (ie. not an error). internal_date is set
+	   if it's non-NULL. */
 	IBuffer *(*open_mail)(MailIndex *index, MailIndexRecord *rec,
-			      int *deleted);
+			      time_t *internal_date, int *deleted);
+
+	/* Returns internal date of message, or (time_t)-1 if error occured. */
+	time_t (*get_internal_date)(MailIndex *index, MailIndexRecord *rec);
 
 	/* Expunge a mail from index. Tree and modifylog is also updated. The
 	   index must be exclusively locked before calling this function.
@@ -259,10 +270,9 @@
 			    int external_change);
 
 	/* Append a new record to index. The index must be exclusively
-	   locked before calling this function. The record pointer is
-	   updated to the mmap()ed record. rec->uid is updated in
+	   locked before calling this function. rec->uid is updated in
 	   append_end(). */
-	int (*append_begin)(MailIndex *index, MailIndexRecord **rec);
+	MailIndexRecord *(*append_begin)(MailIndex *index);
 	int (*append_end)(MailIndex *index, MailIndexRecord *rec);
 
 	/* Updating fields happens by calling update_begin(), one or more
@@ -282,11 +292,11 @@
 					 MailIndexRecord *rec);
 	int (*update_end)(MailIndexUpdate *update);
 
-	void (*update_field)(MailIndexUpdate *update, MailField field,
+	void (*update_field)(MailIndexUpdate *update, MailDataField field,
 			     const char *value, size_t extra_space);
 	/* Just remember that full_field_size will be MEM_ALIGNed, so
 	   it may differer from the given size parameter. */
-	void (*update_field_raw)(MailIndexUpdate *update, MailField field,
+	void (*update_field_raw)(MailIndexUpdate *update, MailDataField field,
 				 const void *value, size_t size);
 
 	/* Returns last error message */
@@ -311,7 +321,7 @@
 
 	char *dir; /* directory where to place the index files */
 	char *filepath; /* index file path */
-	MailField default_cache_fields, never_cache_fields;
+	MailDataField default_cache_fields, never_cache_fields;
 	unsigned int indexid;
 	unsigned int sync_id;
 
@@ -377,24 +387,25 @@
 					     unsigned int last_uid,
 					     unsigned int *seq_r);
 const char *mail_index_lookup_field(MailIndex *index, MailIndexRecord *rec,
-				    MailField field);
+				    MailDataField field);
 const void *mail_index_lookup_field_raw(MailIndex *index, MailIndexRecord *rec,
-					MailField field, size_t *size);
-void mail_index_cache_fields_later(MailIndex *index, MailField field);
+					MailDataField field, size_t *size);
+void mail_index_cache_fields_later(MailIndex *index, MailDataField field);
 int mail_index_expunge(MailIndex *index, MailIndexRecord *rec,
 		       unsigned int seq, int external_change);
 int mail_index_update_flags(MailIndex *index, MailIndexRecord *rec,
 			    unsigned int seq, MailFlags flags,
 			    int external_change);
-int mail_index_append_begin(MailIndex *index, MailIndexRecord **rec);
+MailIndexRecord *mail_index_append_begin(MailIndex *index);
 int mail_index_append_end(MailIndex *index, MailIndexRecord *rec);
 MailIndexUpdate *mail_index_update_begin(MailIndex *index,
 					 MailIndexRecord *rec);
 int mail_index_update_end(MailIndexUpdate *update);
-void mail_index_update_field(MailIndexUpdate *update, MailField field,
+void mail_index_update_field(MailIndexUpdate *update, MailDataField field,
 			     const char *value, size_t extra_space);
-void mail_index_update_field_raw(MailIndexUpdate *update, MailField field,
+void mail_index_update_field_raw(MailIndexUpdate *update, MailDataField field,
 				 const void *value, size_t size);
+time_t mail_get_internal_date(MailIndex *index, MailIndexRecord *rec);
 const char *mail_index_get_last_error(MailIndex *index);
 int mail_index_is_diskspace_error(MailIndex *index);
 int mail_index_is_inconsistency_error(MailIndex *index);
@@ -408,7 +419,7 @@
 void mail_index_mark_flag_changes(MailIndex *index, MailIndexRecord *rec,
 				  MailFlags old_flags, MailFlags new_flags);
 void mail_index_update_headers(MailIndexUpdate *update, IBuffer *inbuf,
-                               MailField cache_fields,
+                               MailDataField cache_fields,
 			       MessageHeaderFunc header_func, void *context);
 int mail_index_update_cache(MailIndex *index);
 int mail_index_compress(MailIndex *index);




More information about the dovecot-cvs mailing list