[dovecot-cvs] dovecot/src/lib-index mail-index-lock.c, 1.20, 1.21 mail-index-private.h, 1.19, 1.20 mail-index-sync-update.c, 1.23, 1.24 mail-index-transaction.c, 1.9, 1.10 mail-index-view.c, 1.11, 1.12 mail-index.c, 1.130, 1.131 mail-index.h, 1.114, 1.115 mail-transaction-log.c, 1.36, 1.37 mail-transaction-util.c, 1.10, 1.11

Mon Jun 14 07:27:46 EEST 2004

Update of /home/cvs/dovecot/src/lib-index
In directory talvi:/tmp/cvs-serv21591/lib-index

Modified Files:
	mail-index-lock.c mail-index-private.h 
	mail-index-sync-update.c mail-index-transaction.c 
	mail-index-view.c mail-index.c mail-index.h 
	mail-transaction-log.c mail-transaction-util.c 
Log Message:
Fixes for extra_records



Index: mail-index-lock.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-lock.c,v
retrieving revision 1.20
retrieving revision 1.21
diff -u -d -r1.20 -r1.21

--- mail-index-lock.c	13 Jun 2004 23:38:47 -0000	1.20
+++ mail-index-lock.c	14 Jun 2004 04:27:44 -0000	1.21
@@ -255,7 +255,7 @@
 	/* if header size is smaller than what we have, we'll have to recreate
 	   the index to grow it. so don't even try regular locking. */
 	if (index->map->hdr == &index->map->hdr_copy &&
-	    index->map->hdr->header_size < sizeof(*index->hdr)) {
+	    index->map->hdr->base_header_size < sizeof(*index->hdr)) {
 		/* wait two seconds for exclusive lock */
 		ret = mail_index_lock(index, F_WRLCK, 2, TRUE, lock_id_r);
 		if (ret > 0)

Index: mail-index-private.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-private.h,v
retrieving revision 1.19
retrieving revision 1.20
diff -u -d -r1.19 -r1.20
--- mail-index-private.h	14 Jun 2004 02:07:33 -0000	1.19
+++ mail-index-private.h	14 Jun 2004 04:27:44 -0000	1.20
@@ -6,10 +6,9 @@
 
 struct mail_transaction_header;
 
-/* Maximum number of extra record data items we allowed. Currently maximum
-   would be 32767 because of how transaction log is implemented. Raising this
-   limit only means it takes a few bytes more memory, but 32 should be enough
-   for a long time, right? :) */
+/* Maximum number of extra record data items we allowed. Raising this limit
+   only means it takes a few bytes more memory, but 32 should be enough for a
+   long time, right? :) */
 #define MAIL_INDEX_MAX_EXTRA_RECORDS 32
 
 /* Index file is grown exponentially when we're adding less than this many
@@ -47,6 +46,12 @@
 	unsigned int write_to_disk:1;
 };
 
+struct mail_index_extra_record_info {
+	const char *name;
+	uint16_t offset;
+	uint16_t size;
+};
+
 struct mail_index {
 	char *dir, *prefix;
 
@@ -56,9 +61,11 @@
 	mode_t mode;
 	gid_t gid;
 
-	uint16_t extra_record_offsets[MAIL_INDEX_MAX_EXTRA_RECORDS];
-	uint16_t extra_record_sizes[MAIL_INDEX_MAX_EXTRA_RECORDS];
+	pool_t extra_records_pool;
+	buffer_t *extra_records_buf;
+	const struct mail_index_extra_record_info *extra_records;
 	unsigned int extra_records_count;
+
 	unsigned int record_size;
 
 	char *filepath;

Index: mail-index-sync-update.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-sync-update.c,v
retrieving revision 1.23
retrieving revision 1.24
diff -u -d -r1.23 -r1.24
--- mail-index-sync-update.c	14 Jun 2004 02:07:33 -0000	1.23
+++ mail-index-sync-update.c	14 Jun 2004 04:27:44 -0000	1.24
@@ -225,13 +225,15 @@
 	uint16_t offset, size;
 	int ret;
 
+	/* FIXME: do data_id mapping conversion */
+
 	ret = mail_index_lookup_uid_range(view, u->uid, u->uid,
 					  &seq, &seq);
 	i_assert(ret == 0);
 
 	if (seq != 0) {
-		offset = view->index->extra_record_offsets[hdr->idx];
-		size = view->index->extra_record_sizes[hdr->idx];
+		offset = view->index->extra_records[hdr->idx].offset;
+		size = view->index->extra_records[hdr->idx].size;
 
 		rec = MAIL_INDEX_MAP_IDX(view->index, view->map, seq-1);
 		memcpy(PTR_OFFSET(rec, offset), u->data, size);

Index: mail-index-transaction.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-transaction.c,v
retrieving revision 1.9
retrieving revision 1.10
diff -u -d -r1.9 -r1.10
--- mail-index-transaction.c	14 Jun 2004 02:07:33 -0000	1.9
+++ mail-index-transaction.c	14 Jun 2004 04:27:44 -0000	1.10
@@ -10,6 +10,8 @@
 #include "mail-transaction-log.h"
 #include "mail-index-transaction-private.h"
 
+#include <stddef.h>
+
 static void mail_index_transaction_add_last(struct mail_index_transaction *t);
 
 struct mail_index_transaction *
@@ -63,10 +65,12 @@
 	data = buffer_get_modifyable_data(buf, &size);
 	for (i = 0; i < size; i += record_size) {
 		seq = (uint32_t *)&data[i];
+		i_assert(seq[0] <= view->map->records_count);
 
 		seq[0] = MAIL_INDEX_MAP_IDX(view->index, view->map,
 					    seq[0]-1)->uid;
 		if (range) {
+			i_assert(seq[1] <= view->map->records_count);
 			seq[1] = MAIL_INDEX_MAP_IDX(view->index, view->map,
 						    seq[1]-1)->uid;
 		}
@@ -83,9 +87,13 @@
 		return -1;
 
 	for (i = 0; i < index->extra_records_count; i++) {
+		if (t->extra_rec_updates[i] == NULL)
+			continue;
+
 		mail_index_buffer_convert_to_uids(t->view,
 						  t->extra_rec_updates[i],
-						  index->extra_record_sizes[i],
+                                                  sizeof(uint32_t) +
+						  index->extra_records[i].size,
 						  FALSE);
 	}
 
@@ -449,27 +457,67 @@
 	}
 
 	idx *= full_record_size;
-	buffer_copy(*buffer, idx + full_record_size, *buffer, idx, (size_t)-1);
+	if (idx != size) {
+		buffer_copy(*buffer, idx + full_record_size,
+			    *buffer, idx, (size_t)-1);
+	}
 	seq_p = buffer_get_space_unsafe(*buffer, idx, full_record_size);
 
 	*seq_p = seq;
 	memcpy(seq_p+1, record, record_size);
 }
 
+static void mail_index_update_record(struct mail_index_transaction *t,
+				     uint32_t seq, size_t offset,
+				     const void *record, size_t record_size)
+{
+	struct mail_index *index = t->view->index;
+	struct mail_index_record *rec;
+	size_t pos;
+
+	i_assert(seq > 0 && seq <= t->last_new_seq);
+
+	pos = (seq - t->first_new_seq) * index->record_size;
+	rec = buffer_get_space_unsafe(t->appends, pos,
+				      index->record_size);
+
+	memcpy(PTR_OFFSET(rec, offset), record, record_size);
+}
+
 void mail_index_update_cache(struct mail_index_transaction *t,
 			     uint32_t seq, uint32_t offset)
 {
-	mail_index_update_seq_buffer(&t->cache_updates, seq,
-				     &offset, sizeof(offset));
+	if (t->first_new_seq != 0 && seq >= t->first_new_seq) {
+		/* just appended message, modify it directly */
+		size_t rec_offset;
+
+		rec_offset = offsetof(struct mail_index_record, cache_offset);
+		mail_index_update_record(t, seq, rec_offset,
+					 &offset, sizeof(offset));
+	} else {
+		mail_index_update_seq_buffer(&t->cache_updates, seq,
+					     &offset, sizeof(offset));
+	}
 }
 
 void mail_index_update_extra_rec(struct mail_index_transaction *t,
-				 uint32_t seq, uint32_t idx, const void *data)
+				 uint32_t seq, uint32_t data_id,
+				 const void *data)
 {
-	i_assert(idx < t->view->index->extra_records_count);
+	struct mail_index *index = t->view->index;
 
-	mail_index_update_seq_buffer(&t->extra_rec_updates[idx], seq, data,
-				     t->view->index->extra_record_sizes[idx]);
+	i_assert(data_id < index->extra_records_count);
+
+	if (t->first_new_seq != 0 && seq >= t->first_new_seq) {
+		/* just appended message, modify it directly */
+		/* FIXME: do data_id mapping conversion */
+		mail_index_update_record(t, seq,
+			index->extra_records[data_id].offset, data,
+			index->extra_records[data_id].size);
+	} else {
+		mail_index_update_seq_buffer(&t->extra_rec_updates[data_id],
+			seq, data, index->extra_records[data_id].size);
+	}
 }
 
 void mail_index_update_header(struct mail_index_transaction *t,

Index: mail-index-view.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index-view.c,v
retrieving revision 1.11
retrieving revision 1.12
diff -u -d -r1.11 -r1.12
--- mail-index-view.c	13 Jun 2004 23:38:47 -0000	1.11
+++ mail-index-view.c	14 Jun 2004 04:27:44 -0000	1.12
@@ -160,11 +160,12 @@
 	return 0;
 }
 
-int mail_index_lookup(struct mail_index_view *view, uint32_t seq,
-		      const struct mail_index_record **rec_r)
+static int mail_index_lookup_int(struct mail_index_view *view, uint32_t seq,
+				 struct mail_index_map **map_r,
+				 const struct mail_index_record **rec_r)
 {
 	struct mail_index_map *map;
-	const struct mail_index_record *rec;
+	const struct mail_index_record *rec, *n_rec;
 	uint32_t uid;
 
 	i_assert(seq > 0);
@@ -175,6 +176,7 @@
 
 	rec = MAIL_INDEX_MAP_IDX(view->index, view->map, seq-1);
 	if (view->map == view->index->map) {
+		*map_r = view->map;
 		*rec_r = rec;
 		return 0;
 	}
@@ -183,23 +185,37 @@
 		return -1;
 
 	/* look for it in the head mapping */
+	*map_r = map = view->index->map;
+
 	uid = rec->uid;
 	if (seq > view->index->hdr->messages_count)
 		seq = view->index->hdr->messages_count;
 
-	map = view->index->map;
-	while (seq > 0) {
+	if (seq == 0) {
+		*rec_r = NULL;
+		return 0;
+	}
+
+	do {
 		// FIXME: we could be skipping more by uid diff
 		seq--;
-		if (MAIL_INDEX_MAP_IDX(view->index, map, seq-1)->uid <= uid)
+		n_rec = MAIL_INDEX_MAP_IDX(view->index, map, seq);
+		if (n_rec->uid <= uid)
 			break;
-	}
+	} while (seq > 0);
 
-	*rec_r = MAIL_INDEX_MAP_IDX(view->index, map, seq)->uid == uid ?
-		MAIL_INDEX_MAP_IDX(view->index, map, seq) : rec;
+	*rec_r = n_rec->uid == uid ? n_rec : rec;
 	return 0;
 }
 
+int mail_index_lookup(struct mail_index_view *view, uint32_t seq,
+		      const struct mail_index_record **rec_r)
+{
+	struct mail_index_map *map;
+
+        return mail_index_lookup_int(view, seq, &map, rec_r);
+}
+
 int mail_index_lookup_uid(struct mail_index_view *view, uint32_t seq,
 			  uint32_t *uid_r)
 {
@@ -213,6 +229,28 @@
 	return 0;
 }
 
+int mail_index_lookup_extra(struct mail_index_view *view, uint32_t seq,
+			    uint32_t data_id, const void **data_r)
+{
+	const struct mail_index_record *rec;
+	struct mail_index_map *map;
+	uint32_t offset;
+
+	if (mail_index_lookup_int(view, seq, &map, &rec) < 0)
+		return -1;
+
+	if (rec == NULL) {
+		*data_r = NULL;
+		return 0;
+	}
+
+	/* FIXME: do data_id mapping conversion */
+
+	offset = view->index->extra_records[data_id].offset;
+	*data_r = CONST_PTR_OFFSET(rec, offset);
+	return 0;
+}
+
 static uint32_t mail_index_bsearch_uid(struct mail_index_view *view,
 				       uint32_t uid, uint32_t *left_idx_p,
 				       int nearest_side)

Index: mail-index.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index.c,v
retrieving revision 1.130
retrieving revision 1.131
diff -u -d -r1.130 -r1.131
--- mail-index.c	14 Jun 2004 02:07:33 -0000	1.130
+++ mail-index.c	14 Jun 2004 04:27:44 -0000	1.131
@@ -24,6 +24,12 @@
 	index->dir = i_strdup(dir);
 	index->prefix = i_strdup(prefix);
 	index->fd = -1;
+
+	index->extra_records_pool =
+		pool_alloconly_create("extra_record_pool", 256);
+	index->extra_records_buf =
+		buffer_create_dynamic(index->extra_records_pool,
+				      64, (size_t)-1);
 	index->record_size = sizeof(struct mail_index_record);
 
 	index->mode = 0600;
@@ -34,6 +40,7 @@
 void mail_index_free(struct mail_index *index)
 {
 	mail_index_close(index);
+	pool_unref(index->extra_records_pool);
 
 	i_free(index->error);
 	i_free(index->dir);
@@ -42,13 +49,23 @@
 }
 
 uint32_t mail_index_register_record_extra(struct mail_index *index,
-					  uint16_t size, uint32_t *offset_r)
+					  const char *name, uint16_t size)
 {
-	uint16_t offset;
+	struct mail_index_extra_record_info info;
+	size_t buf_size;
+	unsigned int i;
+
+	/* see if it's there already */
+	for (i = 0; i < index->extra_records_count; i++) {
+		if (strcmp(index->extra_records[i].name, name) == 0) {
+			i_assert(index->extra_records[i].size == size);
+			return i;
+		}
+	}
 
+	i_assert(size % 4 == 0);
 	i_assert(!index->opened);
 	i_assert(index->record_size + size <= 65535);
-	i_assert(size % 4 == 0);
 
 	if (index->extra_records_count >= MAIL_INDEX_MAX_EXTRA_RECORDS) {
 		i_panic("Maximum extra record count reached, "
@@ -57,13 +74,18 @@
 			MAIL_INDEX_MAX_EXTRA_RECORDS);
 	}
 
-	offset = index->record_size;
-	index->record_size += size;
-	*offset_r = offset;
+	memset(&info, 0, sizeof(info));
+	info.name = p_strdup(index->extra_records_pool, name);
+	info.size = size;
+	info.offset = index->record_size;
 
-	index->extra_record_offsets[index->extra_records_count] = offset;
-	index->extra_record_sizes[index->extra_records_count] = size;
-	return index->extra_records_count++;
+	buffer_append(index->extra_records_buf, &info, sizeof(info));
+	index->extra_records =
+		buffer_get_data(index->extra_records_buf, &buf_size);
+	index->extra_records_count = buf_size / sizeof(info);
+
+	index->record_size += size;
+	return index->extra_records_count-1;
 }
 
 static int mail_index_check_header(struct mail_index *index,
@@ -210,10 +232,10 @@
 	}
 
 	map->hdr = hdr;
-	if (map->hdr->header_size < sizeof(*map->hdr)) {
+	if (map->hdr->base_header_size < sizeof(*map->hdr)) {
 		/* header smaller than ours, make a copy so our newer headers
 		   won't have garbage in them */
-		memcpy(&map->hdr_copy, map->hdr, map->hdr->header_size);
+		memcpy(&map->hdr_copy, map->hdr, map->hdr->base_header_size);
 		map->hdr = &map->hdr_copy;
 	}
 
@@ -403,7 +425,7 @@
 
 	hdr = map->mmap_base;
 	memcpy(&mem_map->hdr_copy, map->mmap_base,
-	       I_MIN(hdr->header_size, sizeof(mem_map->hdr_copy)));
+	       I_MIN(hdr->base_header_size, sizeof(mem_map->hdr_copy)));
 	mem_map->hdr = &mem_map->hdr_copy;
 	return mem_map;
 }
@@ -574,6 +596,7 @@
 
 	hdr->major_version = MAIL_INDEX_MAJOR_VERSION;
 	hdr->minor_version = MAIL_INDEX_MINOR_VERSION;
+	hdr->base_header_size = sizeof(*hdr);
 	hdr->header_size = sizeof(*hdr);
 	hdr->record_size = index->record_size;
 	hdr->keywords_mask_size = sizeof(keywords_mask_t);

Index: mail-index.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-index.h,v
retrieving revision 1.114
retrieving revision 1.115
diff -u -d -r1.114 -r1.115
--- mail-index.h	14 Jun 2004 02:07:33 -0000	1.114
+++ mail-index.h	14 Jun 2004 04:27:44 -0000	1.115
@@ -6,7 +6,7 @@
 #define MAIL_INDEX_MAJOR_VERSION 4
 #define MAIL_INDEX_MINOR_VERSION 0
 
-#define MAIL_INDEX_HEADER_MIN_SIZE 80
+#define MAIL_INDEX_HEADER_MIN_SIZE 88
 
 /* Number of keywords in mail_index_record. */
 #define INDEX_KEYWORDS_COUNT (3*8)
@@ -63,7 +63,8 @@
 	uint8_t major_version;
 	uint8_t minor_version;
 
-	uint16_t header_size;
+	uint16_t base_header_size;
+	uint32_t header_size; /* base + extended header size */
 	uint16_t record_size;
 	uint16_t keywords_mask_size;
 
@@ -96,6 +97,7 @@
 	uint32_t sync_stamp;
 
 	uint32_t cache_file_seq;
+	uint32_t extra_records_hdr_offset;
 };
 
 struct mail_index_record {
@@ -136,13 +138,11 @@
 struct mail_index *mail_index_alloc(const char *dir, const char *prefix);
 void mail_index_free(struct mail_index *index);
 
-/* register extra data to be used in mail_index_record. calls to this function
-   must remain in same order as long as the index exists, or it breaks.
-
-   returns the index number, and sets *offset_r to relative offset in
-   mail_index_record for the data. */
+/* register extra data to be used in mail_index_record. name is a unique
+   identifier for the data. if same name is tried to be registered multiple
+   times, the rest are ignored. returns identifier for the name. */
 uint32_t mail_index_register_record_extra(struct mail_index *index,
-					  uint16_t size, uint32_t *offset_r);
+					  const char *name, uint16_t size);
 
 int mail_index_open(struct mail_index *index, enum mail_index_open_flags flags);
 void mail_index_close(struct mail_index *index);
@@ -245,6 +245,10 @@
    mail_index_lookup()->uid */
 int mail_index_lookup_uid(struct mail_index_view *view, uint32_t seq,
 			  uint32_t *uid_r);
+/* Returns the wanted extra data for given message. If it doesn't exist,
+   *data_r is set to NULL. */
+int mail_index_lookup_extra(struct mail_index_view *view, uint32_t seq,
+			    uint32_t data_id, const void **data_r);
 /* Convert UID range to sequence range. If no UIDs are found, sequences are
    set to 0. Note that any of the returned sequences may have been expunged
    already. */
@@ -269,8 +273,10 @@
 /* Update field in header. */
 void mail_index_update_header(struct mail_index_transaction *t,
 			      size_t offset, const void *data, size_t size);
+/* Update extra record field. */
 void mail_index_update_extra_rec(struct mail_index_transaction *t,
-				 uint32_t seq, uint32_t idx, const void *data);
+				 uint32_t seq, uint32_t data_id,
+				 const void *data);
 
 /* Returns the last error code. */
 enum mail_index_error mail_index_get_last_error(struct mail_index *index);

Index: mail-transaction-log.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-transaction-log.c,v
retrieving revision 1.36
retrieving revision 1.37
diff -u -d -r1.36 -r1.37
--- mail-transaction-log.c	14 Jun 2004 02:07:33 -0000	1.36
+++ mail-transaction-log.c	14 Jun 2004 04:27:44 -0000	1.37
@@ -941,8 +941,8 @@
 			     enum mail_transaction_type type, int external)
 {
 	struct mail_transaction_header hdr;
-	const void *data;
-	size_t size;
+	const void *data, *hdr_data;
+	size_t size, hdr_size;
 
 	i_assert((type & MAIL_TRANSACTION_TYPE_MASK) != 0);
 
@@ -950,28 +950,29 @@
 	if (size == 0)
 		return 0;
 
+	if (hdr_buf != NULL)
+		hdr_data = buffer_get_data(hdr_buf, &hdr_size);
+	else {
+		hdr_data = NULL;
+		hdr_size = 0;
+	}
+
 	hdr.type = type;
 	if (type == MAIL_TRANSACTION_EXPUNGE)
 		hdr.type |= MAIL_TRANSACTION_EXPUNGE_PROT;
 	if (external)
 		hdr.type |= MAIL_TRANSACTION_EXTERNAL;
-	hdr.size = size;
+	hdr.size = size + hdr_size;
 
 	if (pwrite_full(file->fd, &hdr, sizeof(hdr), file->hdr.used_size) < 0)
 		return -1;
 	file->hdr.used_size += sizeof(hdr);
 
-	if (hdr_buf != NULL) {
-		const void *hdr_data;
-		size_t hdr_size;
-
-		hdr_data = buffer_get_data(buf, &hdr_size);
-		if (hdr_size > 0) {
-			if (pwrite_full(file->fd, hdr_data, hdr_size,
-					file->hdr.used_size) < 0)
-				return -1;
-			file->hdr.used_size += hdr_size;
-		}
+	if (hdr_size > 0) {
+		if (pwrite_full(file->fd, hdr_data, hdr_size,
+				file->hdr.used_size) < 0)
+			return -1;
+		file->hdr.used_size += hdr_size;
 	}
 
 	if (pwrite_full(file->fd, data, size, file->hdr.used_size) < 0)
@@ -1095,10 +1096,12 @@
 
 	hdr_buf = buffer_create_data(pool_datastack_create(),
 				     &extra_rec_hdr, sizeof(extra_rec_hdr));
+	buffer_set_used_size(hdr_buf, sizeof(extra_rec_hdr));
 	for (i = 0; i < view->index->extra_records_count; i++) {
 		if (t->extra_rec_updates[i] == NULL || ret != 0)
 			continue;
 
+		/* FIXME: do data_id mapping conversion */
 		extra_rec_hdr.idx = i;
 		ret = log_append_buffer(file, t->extra_rec_updates[i], hdr_buf,
 					MAIL_TRANSACTION_EXTRA_REC_UPDATE,

Index: mail-transaction-util.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-transaction-util.c,v
retrieving revision 1.10
retrieving revision 1.11
diff -u -d -r1.10 -r1.11
--- mail-transaction-util.c	14 Jun 2004 02:07:33 -0000	1.10
+++ mail-transaction-util.c	14 Jun 2004 04:27:44 -0000	1.11
@@ -140,18 +140,20 @@
 	}
 	case MAIL_TRANSACTION_EXTRA_REC_UPDATE: {
 		const struct mail_transaction_extra_rec_header *ehdr;
-		const struct mail_transaction_extra_rec_update *rec;
-		unsigned int i, record_size;
+		const struct mail_transaction_extra_rec_update *rec, *end;
+		unsigned int record_size;
 
 		if (map->extra_rec_update == NULL)
 			break;
 
 		ehdr = data;
 		i_assert(ehdr->idx < index->extra_records_count);
-		record_size = index->extra_record_sizes[ehdr->idx];
+		record_size = sizeof(uint32_t) +
+			index->extra_records[ehdr->idx].size;
 
-		rec = CONST_PTR_OFFSET(data, sizeof(*hdr));
-		for (i = 0; i < hdr->size; ) {
+		rec = CONST_PTR_OFFSET(data, sizeof(*ehdr));
+		end = CONST_PTR_OFFSET(data, hdr->size);
+		while (rec != end) {
 			ret = map->extra_rec_update(ehdr, rec, context);
 			if (ret <= 0)
 				break;

