[dovecot-cvs] dovecot/src/lib-index mail-cache.c,1.4,1.5

[cras at procontrol.fi]

Update of /home/cvs/dovecot/src/lib-index
In directory danu:/tmp/cvs-serv24519/lib-index

Modified Files:
	mail-cache.c 
Log Message:
Changed how cached headers are stored. Previously when we added new headers,
all the old headers were duplicated as well which wasted space. Also fixed
several bugs in caching.



Index: mail-cache.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-index/mail-cache.c,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -d -r1.4 -r1.5
--- mail-cache.c	11 Aug 2003 14:28:27 -0000	1.4
+++ mail-cache.c	18 Aug 2003 03:24:37 -0000	1.5
@@ -493,25 +493,24 @@
 
 static const struct mail_cache_record *
 mail_cache_compress_record(struct mail_cache *cache,
-			   struct mail_index_record *rec,
-			   enum mail_cache_field remove_fields,
+			   struct mail_index_record *rec, int header_idx,
 			   uint32_t *size_r)
 {
-	enum mail_cache_field cached_fields, field;
+	enum mail_cache_field orig_cached_fields, cached_fields, field;
 	struct mail_cache_record cache_rec;
 	buffer_t *buffer;
 	const void *data;
-	size_t size;
+	size_t size, pos;
 	uint32_t nb_size;
 	int i;
 
 	memset(&cache_rec, 0, sizeof(cache_rec));
 	buffer = buffer_create_dynamic(data_stack_pool, 4096, (size_t)-1);
 
-	cached_fields = mail_cache_get_fields(cache, rec) & ~remove_fields;
+        orig_cached_fields = mail_cache_get_fields(cache, rec);
+	cached_fields = orig_cached_fields & ~MAIL_CACHE_HEADERS_MASK;
 	buffer_append(buffer, &cache_rec, sizeof(cache_rec));
-	for (i = 0; i < 32; i++) {
-		field = nbo32_bitmasks[i];
+	for (i = 0, field = 1; i < 31; i++, field <<= 1) {
 		if ((cached_fields & field) == 0)
 			continue;
 
@@ -529,6 +528,32 @@
 			buffer_append(buffer, null4, 4 - (size & 3));
 	}
 
+	/* now merge all the headers if we have them all */
+	if ((orig_cached_fields & mail_cache_header_fields[header_idx]) != 0) {
+		nb_size = 0;
+		pos = buffer_get_used_size(buffer);
+		buffer_append(buffer, &nb_size, sizeof(nb_size));
+
+		for (i = 0; i <= header_idx; i++) {
+			field = mail_cache_header_fields[i];
+			if (mail_cache_lookup_field(cache, rec, field,
+						    &data, &size) && size > 1) {
+				size--; /* terminating \0 */
+				buffer_append(buffer, data, size);
+				nb_size += size;
+			}
+		}
+		buffer_append(buffer, "", 1);
+		nb_size++;
+		if ((nb_size & 3) != 0)
+			buffer_append(buffer, null4, 4 - (nb_size & 3));
+
+		nb_size = uint32_to_nbo(nb_size);
+		buffer_write(buffer, pos, &nb_size, sizeof(nb_size));
+
+		cached_fields |= MAIL_CACHE_HEADERS1;
+	}
+
 	cache_rec.fields = cached_fields;
 	cache_rec.size = uint32_to_nbo(buffer_get_used_size(buffer));
 	buffer_write(buffer, 0, &cache_rec, sizeof(cache_rec));
@@ -543,11 +568,11 @@
 	struct mail_cache_header *hdr;
 	const struct mail_cache_record *cache_rec;
 	struct mail_index_record *rec;
-        enum mail_cache_field used_fields, remove_fields;
+        enum mail_cache_field used_fields;
 	unsigned char *mmap_base;
 	const char *str;
 	uint32_t new_file_size, offset, size, nb_size;
-	int i;
+	int i, header_idx;
 
 	/* pick some reasonably good file size */
 	new_file_size = cache->used_file_size -
@@ -568,27 +593,28 @@
 	hdr = (struct mail_cache_header *) mmap_base;
 	offset = sizeof(*hdr);
 
-	/* get the newest message header list */
-	remove_fields = 0;
+	/* merge all the header pieces into one. if some message doesn't have
+	   all the required pieces, we'll just have to drop them all. */
 	for (i = MAIL_CACHE_HEADERS_COUNT-1; i >= 0; i--) {
 		str = mail_cache_get_header_fields_str(cache, i);
-		if (str != NULL) {
-			hdr->header_offsets[0] = uint32_to_nbo(offset | 2);
-
-			size = strlen(str) + 1;
-			nb_size = uint32_to_nbo(size);
-
-			memcpy(mmap_base + offset, &nb_size, sizeof(nb_size));
-			offset += sizeof(nb_size);
-			memcpy(mmap_base + offset, str, size);
-			offset += (size + 3) & ~3;
+		if (str != NULL)
 			break;
-		}
 	}
 
-	/* remove other headers */
-	for (i--; i >= 0; i--)
-		remove_fields |= mail_cache_header_fields[i];
+	if (str == NULL)
+		header_idx = -1;
+	else {
+		hdr->header_offsets[0] = uint32_to_nbo(offset | 2);
+		header_idx = i;
+
+		size = strlen(str) + 1;
+		nb_size = uint32_to_nbo(size);
+
+		memcpy(mmap_base + offset, &nb_size, sizeof(nb_size));
+		offset += sizeof(nb_size);
+		memcpy(mmap_base + offset, str, size);
+		offset += (size + 3) & ~3;
+	}
 
 	used_fields = 0;
 	rec = cache->index->lookup(cache->index, 1);
@@ -596,8 +622,7 @@
 		cache_rec = mail_cache_lookup(cache, rec);
 		if (cache_rec == NULL)
 			rec->cache_offset = 0;
-		else if ((nbo_to_uint32(cache_rec->next_offset) & 2) == 0 &&
-			 remove_fields == 0) {
+		else if ((nbo_to_uint32(cache_rec->next_offset) & 2) == 0) {
 			/* just one unmodified block, copy it */
 			size = nbo_to_uint32(cache_rec->size);
 			i_assert(offset + size <= new_file_size);
@@ -611,7 +636,7 @@
 			/* multiple blocks, sort them into buffer */
 			t_push();
 			cache_rec = mail_cache_compress_record(cache, rec,
-							       remove_fields,
+							       header_idx,
 							       &size);
 			i_assert(offset + size <= new_file_size);
 			memcpy(mmap_base + offset, cache_rec, size);
@@ -1013,6 +1038,12 @@
 	if (!commit_all_changes(ctx))
 		ret = FALSE;
 
+	if (ctx->next_unused_header_lowwater == MAIL_CACHE_HEADERS_COUNT) {
+		/* they're all used - compress the cache to get more */
+		ctx->cache->index->set_flags |=
+			MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE;
+	}
+
 	mail_cache_transaction_flush(ctx);
 	return ret;
 }
@@ -1189,7 +1220,7 @@
 			cache->split_offsets[i] =
 				cache->header->header_offsets[i];
 
-			str = mail_cache_get_header_fields_str(cache, idx);
+			str = mail_cache_get_header_fields_str(cache, i);
 			cache->split_headers[i] = split_header(cache, str);
 		}
 		t_pop();
@@ -1237,6 +1268,11 @@
 
 		cache->header->header_offsets[idx] = uint32_to_nbo(offset);
 
+		/* the above value may actually be in split_offsets[] if
+		   last transaction was rolled back. make sure the next
+		   get_header_fields() notices it's changed */
+                cache->split_offsets[idx] = 0;
+
 		/* mark used-bit to be updated later. not really needed for
 		   read-safety, but if transaction get rolled back we can't let
 		   this point to invalid location. */
@@ -1279,10 +1315,14 @@
 	cache_rec = CACHE_RECORD(cache, offset);
 
 	size = nbo_to_uint32(cache_rec->size);
-	if (!mmap_update(cache, offset, sizeof(*cache_rec) + size))
+	if (size < sizeof(*cache_rec)) {
+		mail_cache_set_corrupted(cache, "invalid record size");
+		return NULL;
+	}
+	if (!mmap_update(cache, offset, size))
 		return NULL;
 
-	if (offset + sizeof(*cache_rec) + size > cache->mmap_length) {
+	if (offset + size > cache->mmap_length) {
 		mail_cache_set_corrupted(cache, "record points outside file");
 		return NULL;
 	}
@@ -1391,10 +1431,11 @@
 
 static int get_field_num(enum mail_cache_field field)
 {
+	unsigned int mask;
 	int i;
 
-	for (i = 0; i < 32; i++) {
-		if ((field & nbo32_bitmasks[i]) != 0)
+	for (i = 0, mask = 1; i < 31; i++, mask <<= 1) {
+		if ((field & mask) != 0)
 			return i;
 	}
 
@@ -1412,8 +1453,7 @@
 
 	buf = buffer_get_data(ctx->cache_data, NULL);
 
-	for (i = 0; i < 32; i++) {
-		mask = nbo32_bitmasks[i];
+	for (i = 0, mask = 1; i < 31; i++, mask <<= 1) {
 		if ((field & mask) != 0)
 			return offset;
 
@@ -1558,17 +1598,25 @@
 	unsigned char *buf;
 	unsigned int mask;
 	uint32_t rec_size, data_size;
-	size_t offset = 0;
+	size_t offset, next_offset;
 	int i;
 
 	rec_size = nbo_to_uint32(cache_rec->size);
-	buf = (unsigned char *) cache_rec + sizeof(*cache_rec);
+	buf = (unsigned char *) cache_rec;
+	offset = sizeof(*cache_rec);
 
-	for (i = 0; i < 32; i++) {
-		mask = nbo32_bitmasks[i];
+	for (i = 0, mask = 1; i < 31; i++, mask <<= 1) {
 		if ((cache_rec->fields & mask) == 0)
 			continue;
 
+		/* all records are at least 32bit. we have to check this
+		   before getting data_size. */
+		if (offset + sizeof(uint32_t) > rec_size) {
+			mail_cache_set_corrupted(cache,
+				"Record continues outside it's allocated size");
+			return FALSE;
+		}
+
 		if ((mask & MAIL_CACHE_FIXED_MASK) != 0)
 			data_size = mail_cache_field_sizes[i];
 		else {
@@ -1577,6 +1625,13 @@
 			offset += sizeof(data_size);
 		}
 
+		next_offset = offset + ((data_size + 3) & ~3);
+		if (next_offset > rec_size) {
+			mail_cache_set_corrupted(cache,
+				"Record continues outside it's allocated size");
+			return FALSE;
+		}
+
 		if (field == mask) {
 			if (data_size == 0) {
 				mail_cache_set_corrupted(cache,
@@ -1587,14 +1642,7 @@
 			*size_r = data_size;
 			return TRUE;
 		}
-
-		offset += (data_size + 3) & ~3;
-
-		if (offset >= rec_size) {
-			mail_cache_set_corrupted(cache,
-				"Record continues outside it's allocated size");
-			return FALSE;
-		}
+		offset = next_offset;
 	}
 
 	i_unreached();