[dovecot-cvs] dovecot/src/lib-index mail-index-private.h, 1.47, 1.48 mail-index-sync-keywords.c, 1.2, 1.3 mail-index-sync-private.h, 1.21, 1.22 mail-index-sync-update.c, 1.79, 1.80 mail-index-transaction-private.h, 1.21, 1.22 mail-index-transaction.c, 1.51, 1.52 mail-index-view-sync.c, 1.35, 1.36 mail-index.h, 1.142, 1.143 mail-transaction-log.c, 1.84, 1.85 mail-transaction-log.h, 1.23, 1.24 mail-transaction-util.c, 1.22, 1.23

cras at dovecot.org cras at dovecot.org
Sat Feb 5 14:01:51 EET 2005 

Update of /var/lib/cvs/dovecot/src/lib-index
In directory talvi:/tmp/cvs-serv21606/lib-index

Modified Files:
	mail-index-private.h mail-index-sync-keywords.c 
	mail-index-sync-private.h mail-index-sync-update.c 
	mail-index-transaction-private.h mail-index-transaction.c 
	mail-index-view-sync.c mail-index.h mail-transaction-log.c 
	mail-transaction-log.h mail-transaction-util.c 
Log Message:
Internal changes in how keywords are handled. struct mail_keywords isn't
automatically freed anymore, added *_keywords_free() for that.



Index: mail-index-private.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-private.h,v
retrieving revision 1.47
retrieving revision 1.48
diff -u -d -r1.47 -r1.48
--- mail-index-private.h	16 Jan 2005 19:18:24 -0000	1.47
+++ mail-index-private.h	5 Feb 2005 12:01:49 -0000	1.48
@@ -60,14 +60,11 @@
 };
 
 struct mail_keywords {
-	/* linked list of keyword transactions which use this structure.
-	   the list may contain pointers to different transactions. this
-	   structure should be freed when there are no more transactions. */
-	struct mail_keyword_transaction *kt;
-
-	unsigned int start, end;
+	struct mail_index *index;
 	unsigned int count;
-	uint8_t bitmask[4]; /* variable size */
+
+        /* variable sized list of keyword indexes */
+	uint32_t idx[1];
 };
 
 struct mail_index_keyword_header {

Index: mail-index-sync-keywords.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-sync-keywords.c,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -d -r1.2 -r1.3
--- mail-index-sync-keywords.c	10 Jan 2005 17:37:22 -0000	1.2
+++ mail-index-sync-keywords.c	5 Feb 2005 12:01:49 -0000	1.3
@@ -6,56 +6,11 @@
 #include "mail-index-sync-private.h"
 #include "mail-transaction-log.h"
 
-static const char *const *
-keywords_get_from_header(const struct mail_transaction_keyword_update *rec,
-			 size_t size, const uint32_t **uid_r)
-{
-	buffer_t *buf;
-	const unsigned char *p, *end;
-	const char *name;
-	uint32_t i, diff;
-
-	if (size / sizeof(rec->name_size) < rec->keywords_count) {
-		/* keyword_count is badly broken */
-		return NULL;
-	}
-
-	buf = buffer_create_static_hard(pool_datastack_create(),
-					(rec->keywords_count + 1) *
-					sizeof(const char *));
-	p = (const unsigned char *)(rec->name_size + rec->keywords_count);
-	end = CONST_PTR_OFFSET(rec, size);
-
-	if (p >= end)
-		return NULL;
-
-	for (i = 0; i < rec->keywords_count; i++) {
-		if (p + rec->name_size[i] >= end)
-			return NULL;
-
-		name = t_strndup(p, rec->name_size[i]);
-		buffer_append(buf, &name, sizeof(name));
-		p += rec->name_size[i];
-	}
-
-	diff = (p - (const unsigned char *)rec) % 4;
-	if (diff != 0)
-		p += 4 - diff;
-
-	*uid_r = (const uint32_t *)p;
-
-	name = NULL;
-	buffer_append(buf, &name, sizeof(name));
-	return buf->data;
-}
-
-static int keywords_get_missing(struct mail_index_sync_map_ctx *ctx,
-				const char *const *keywords,
-                                const char *const **missing_r)
+static int
+keyword_lookup(struct mail_index_sync_map_ctx *ctx,
+	       const char *keyword_name, unsigned int *idx_r)
 {
-	struct mail_index_map *map = ctx->view->index->map;
-	const char *name;
-	buffer_t *missing_buf;
+	struct mail_index_map *map = ctx->view->map;
 	unsigned int i;
 
 	if (!ctx->keywords_read) {
@@ -64,23 +19,14 @@
 		ctx->keywords_read = TRUE;
 	}
 
-        missing_buf = buffer_create_dynamic(pool_datastack_create(), 64);
-	for (; *keywords != NULL; keywords++) {
-		for (i = 0; i < map->keywords_count; i++) {
-			if (strcmp(map->keywords[i], *keywords) == 0)
-				break;
+	for (i = 0; i < map->keywords_count; i++) {
+		if (strcmp(map->keywords[i], keyword_name) == 0) {
+			*idx_r = i;
+			return 1;
 		}
-		if (i == map->keywords_count)
-			buffer_append(missing_buf, keywords, sizeof(*keywords));
 	}
 
-	if (missing_buf->used == 0)
-		*missing_r = NULL;
-	else {
-		name = NULL;
-		buffer_append(missing_buf, &name, sizeof(name));
-		*missing_r = missing_buf->data;
-	}
+	*idx_r = (unsigned int)-1;
 	return 0;
 }
 
@@ -155,43 +101,35 @@
 }
 
 static int
-keywords_update_header(struct mail_index_sync_map_ctx *ctx,
-		       const char *const *keywords)
+keywords_header_add(struct mail_index_sync_map_ctx *ctx,
+		    const char *keyword_name, unsigned int *keyword_idx_r)
 {
 	struct mail_index_map *map = ctx->view->map;
         const struct mail_index_ext *ext = NULL;
 	struct mail_index_keyword_header *kw_hdr;
 	struct mail_index_keyword_header_rec kw_rec;
 	uint32_t ext_id;
-	const char *const *missing = keywords;
 	buffer_t *buf = NULL;
-	size_t rec_offset, name_offset, name_offset_root;
+	size_t keyword_len, rec_offset, name_offset, name_offset_root;
 	unsigned int keywords_count;
-	int ret = 0;
+	int ret;
 
 	ext_id = mail_index_map_lookup_ext(map, "keywords");
 	if (ext_id != (uint32_t)-1) {
-		/* make sure all keywords exist in the header */
+		/* update existing header */
 		ext = map->extensions->data;
 		ext += ext_id;
 
-		ret = keywords_get_missing(ctx, keywords, &missing);
-		if (ret == 0 && missing != NULL) {
-			/* update existing header */
-			buf = keywords_get_header_buf(map, ext,
-						      strarray_length(missing),
-						      &keywords_count,
-						      &rec_offset,
-						      &name_offset_root,
-						      &name_offset);
-		}
+		buf = keywords_get_header_buf(map, ext, 1, &keywords_count,
+					      &rec_offset, &name_offset_root,
+					      &name_offset);
 	}
 
 	if (buf == NULL) {
 		/* create new / replace broken header */
 		buf = buffer_create_dynamic(pool_datastack_create(), 512);
 		kw_hdr = buffer_append_space_unsafe(buf, sizeof(*kw_hdr));
-		kw_hdr->keywords_count = strarray_length(missing);
+		kw_hdr->keywords_count = 1;
 
                 keywords_count = kw_hdr->keywords_count;
 		rec_offset = buf->used;
@@ -200,23 +138,17 @@
 		name_offset = 0;
 	}
 
-	if (missing == NULL)
-		return 1;
-
-	/* missing some keywords - add them */
+	/* add the keyword */
 	memset(&kw_rec, 0, sizeof(kw_rec));
 	kw_rec.name_offset = name_offset;
 
-	for (; *missing != NULL; missing++) {
-		size_t len = strlen(*missing) + 1;
-
-		buffer_write(buf, rec_offset, &kw_rec, sizeof(kw_rec));
-		buffer_write(buf, name_offset_root, *missing, len);
+	keyword_len = strlen(keyword_name);
+	buffer_write(buf, rec_offset, &kw_rec, sizeof(kw_rec));
+	buffer_write(buf, name_offset_root, keyword_name, keyword_len);
 
-		rec_offset += sizeof(kw_rec);
-		kw_rec.name_offset += len;
-		name_offset_root += len;
-	}
+	rec_offset += sizeof(kw_rec);
+	kw_rec.name_offset += keyword_len;
+	name_offset_root += keyword_len;
 
 	if ((buf->used % 4) != 0)
 		buffer_append_zero(buf, 4 - (buf->used % 4));
@@ -249,43 +181,24 @@
 		    buf, 0, buf->used);
 	map->hdr_base = map->hdr_copy_buf->data;
 
+	*keyword_idx_r = keywords_count - 1;
         ctx->keywords_read = FALSE;
 	return 1;
 }
 
-static const unsigned char *
-keywords_make_mask(struct mail_index_map *map, const struct mail_index_ext *ext,
-		   const char *const *keywords)
-{
-	const char *const *n;
-	unsigned char *mask;
-	unsigned int i;
-
-	mask = t_malloc0(ext->record_size);
-
-	for (i = 0; i < map->keywords_count; i++) {
-		for (n = keywords; *n != NULL; n++) {
-			if (strcmp(map->keywords[i], *n) == 0) {
-				mask[i / CHAR_BIT] |= 1 << (i % CHAR_BIT);
-				break;
-			}
-		}
-	}
-
-	return mask;
-}
-
 static int
 keywords_update_records(struct mail_index_view *view,
 			const struct mail_index_ext *ext,
-			const unsigned char *mask,
+			unsigned int keyword_idx,
 			enum modify_type type,
 			uint32_t uid1, uint32_t uid2)
 {
 	struct mail_index_record *rec;
-	unsigned char *data;
+	unsigned char *data, data_mask;
+	unsigned int data_offset;
 	uint32_t seq1, seq2;
-	unsigned int i;
+
+	i_assert(keyword_idx != (unsigned int)-1);
 
 	if (mail_index_lookup_uid_range(view, uid1, uid2, &seq1, &seq2) < 0)
 		return -1;
@@ -293,25 +206,30 @@
 	if (seq1 == 0)
 		return 1;
 
-	for (; seq1 <= seq2; seq1++) {
-		rec = MAIL_INDEX_MAP_IDX(view->map, seq1-1);
-		data = PTR_OFFSET(rec, ext->record_offset);
+	data_offset = keyword_idx / CHAR_BIT;
+	data_mask = 1 << (keyword_idx % CHAR_BIT);
 
-		switch (type) {
-		case MODIFY_ADD:
-			for (i = 0; i < ext->record_size; i++)
-				data[i] |= mask[i];
-			break;
-		case MODIFY_REMOVE:
-		for (i = 0; i < ext->record_size; i++)
-			data[i] &= ~mask[i];
+	i_assert(data_offset < ext->record_size);
+	data_offset += ext->record_offset;
+
+	switch (type) {
+	case MODIFY_ADD:
+		for (seq1--; seq1 < seq2; seq1++) {
+			rec = MAIL_INDEX_MAP_IDX(view->map, seq1);
+			data = PTR_OFFSET(rec, data_offset);
+			*data |= data_mask;
+		}
 		break;
-		case MODIFY_REPLACE:
-			memcpy(data, mask, ext->record_size);
-			break;
-		default:
-			i_unreached();
+	case MODIFY_REMOVE:
+		data_mask = ~data_mask;
+		for (seq1--; seq1 < seq2; seq1++) {
+			rec = MAIL_INDEX_MAP_IDX(view->map, seq1);
+			data = PTR_OFFSET(rec, data_offset);
+			*data &= data_mask;
 		}
+		break;
+	default:
+		i_unreached();
 	}
 	return 1;
 }
@@ -320,28 +238,37 @@
 			     const struct mail_transaction_header *hdr,
 			     const struct mail_transaction_keyword_update *rec)
 {
-	const uint32_t *uid, *end;
-	const char *const *keywords;
+	const char *keyword_name;
 	const struct mail_index_ext *ext;
-	const unsigned char *mask;
-	uint32_t ext_id;
+	const uint32_t *uid, *end;
+	uint32_t seqset_offset, ext_id;
+	unsigned int keyword_idx;
 	int ret;
 
-	keywords = keywords_get_from_header(rec, hdr->size, &uid);
-	if (keywords == NULL) {
+	seqset_offset = sizeof(*rec) + rec->name_size;
+	if ((seqset_offset % 4) != 0)
+		seqset_offset += 4 - (seqset_offset % 4);
+
+	if (seqset_offset > hdr->size) {
 		mail_transaction_log_view_set_corrupted(ctx->view->log_view,
 			"Keyword header ended unexpectedly");
 		return -1;
 	}
+
+	uid = CONST_PTR_OFFSET(rec, seqset_offset);
 	end = CONST_PTR_OFFSET(rec, hdr->size);
 
-	if (*keywords == NULL && rec->modify_type != MODIFY_REPLACE) {
-		/* adding/removing empty keywords list - do nothing */
-		return 1;
+	if (uid == end) {
+		mail_transaction_log_view_set_corrupted(ctx->view->log_view,
+			"Keyword sequence list empty");
+		return -1;
 	}
 
-	if (rec->modify_type != MODIFY_REMOVE) {
-		ret = keywords_update_header(ctx, keywords);
+	keyword_name = t_strndup(rec + 1, rec->name_size);
+	if (keyword_lookup(ctx, keyword_name, &keyword_idx) < 0)
+		return -1;
+	if (keyword_idx == (unsigned int)-1) {
+		ret = keywords_header_add(ctx, keyword_name, &keyword_idx);
 		if (ret <= 0)
 			return ret;
 	}
@@ -358,8 +285,7 @@
 
 	if (ext->record_size == 0) {
 		/* nothing to do */
-		i_assert(*keywords == NULL);
-		i_assert(rec->modify_type == MODIFY_REPLACE);
+		i_assert(rec->modify_type == MODIFY_REMOVE);
 		return 1;
 	}
 
@@ -370,8 +296,6 @@
 		ctx->keywords_read = TRUE;
 	}
 
-	mask = keywords_make_mask(ctx->view->map, ext, keywords);
-
 	while (uid+2 <= end) {
 		if (uid[0] > uid[1] || uid[0] == 0) {
 			mail_transaction_log_view_set_corrupted(
@@ -380,7 +304,7 @@
 			return -1;
 		}
 
-		ret = keywords_update_records(ctx->view, ext, mask,
+		ret = keywords_update_records(ctx->view, ext, keyword_idx,
 					      rec->modify_type,
 					      uid[0], uid[1]);
 		if (ret <= 0)
@@ -391,3 +315,41 @@
 
 	return 1;
 }
+
+
+int
+mail_index_sync_keywords_reset(struct mail_index_sync_map_ctx *ctx,
+			       const struct mail_transaction_header *hdr,
+			       const struct mail_transaction_keyword_reset *r)
+{
+	struct mail_index_map *map = ctx->view->map;
+	struct mail_index_record *rec;
+	const struct mail_index_ext *ext;
+	const struct mail_transaction_keyword_reset *end;
+	uint32_t ext_id, seq1, seq2;
+
+	ext_id = mail_index_map_lookup_ext(map, "keywords");
+	if (ext_id == (uint32_t)-1) {
+		/* nothing to do */
+		return 1;
+	}
+
+	ext = map->extensions->data;
+	ext += ext_id;
+
+	end = CONST_PTR_OFFSET(r, hdr->size);
+	for (; r != end; r++) {
+		if (mail_index_lookup_uid_range(ctx->view, r->uid1, r->uid2,
+						&seq1, &seq2) < 0)
+			return -1;
+		if (seq1 == 0)
+			continue;
+
+		for (seq1--; seq1 < seq2; seq1++) {
+			rec = MAIL_INDEX_MAP_IDX(map, seq1);
+			memset(PTR_OFFSET(rec, ext->record_offset),
+			       0, ext->record_size);
+		}
+	}
+	return 1;
+}

Index: mail-index-sync-private.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-sync-private.h,v
retrieving revision 1.21
retrieving revision 1.22
diff -u -d -r1.21 -r1.22
--- mail-index-sync-private.h	31 Jan 2005 19:43:20 -0000	1.21
+++ mail-index-sync-private.h	5 Feb 2005 12:01:49 -0000	1.22
@@ -94,5 +94,9 @@
 int mail_index_sync_keywords(struct mail_index_sync_map_ctx *ctx,
 			     const struct mail_transaction_header *hdr,
 			     const struct mail_transaction_keyword_update *rec);
+int
+mail_index_sync_keywords_reset(struct mail_index_sync_map_ctx *ctx,
+			       const struct mail_transaction_header *hdr,
+			       const struct mail_transaction_keyword_reset *r);
 
 #endif

Index: mail-index-sync-update.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-sync-update.c,v
retrieving revision 1.79
retrieving revision 1.80
diff -u -d -r1.79 -r1.80
--- mail-index-sync-update.c	31 Jan 2005 19:43:20 -0000	1.79
+++ mail-index-sync-update.c	5 Feb 2005 12:01:49 -0000	1.80
@@ -499,6 +499,12 @@
 		ret = mail_index_sync_keywords(ctx, hdr, rec);
 		break;
 	}
+	case MAIL_TRANSACTION_KEYWORD_RESET: {
+		const struct mail_transaction_keyword_reset *rec = data;
+
+		ret = mail_index_sync_keywords_reset(ctx, hdr, rec);
+		break;
+	}
 	default:
 		i_unreached();
 	}

Index: mail-index-transaction-private.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-transaction-private.h,v
retrieving revision 1.21
retrieving revision 1.22
diff -u -d -r1.21 -r1.22
--- mail-index-transaction-private.h	22 Jan 2005 16:45:24 -0000	1.21
+++ mail-index-transaction-private.h	5 Feb 2005 12:01:49 -0000	1.22
@@ -3,18 +3,6 @@
 
 #include "mail-transaction-log.h"
 
-struct mail_keyword_transaction {
-	/* mail_keywords points to first mail_index_keyword_transaction.
-	   this points to next keyword transaction using the same keywords */
-	struct mail_keyword_transaction *next;
-
-	enum modify_type modify_type;
-	struct mail_index_transaction *transaction;
-
-	struct mail_keywords *keywords;
-	buffer_t *messages;
-};
-
 struct mail_index_transaction {
 	int refcount;
 	struct mail_index_view *view;
@@ -30,10 +18,12 @@
 	unsigned char hdr_change[sizeof(struct mail_index_header)];
 	unsigned char hdr_mask[sizeof(struct mail_index_header)];
 
-	buffer_t *ext_rec_updates; /* buffer[] */
+	buffer_t *ext_rec_updates; /* buffer_t*[] */
 	buffer_t *ext_resizes; /* struct mail_transaction_ext_intro[] */
 	buffer_t *ext_resets; /* uint32_t[] */
-	buffer_t *keyword_updates; /* struct mail_keyword_transaction[] */
+
+	buffer_t *keyword_updates; /* buffer_t*[] */
+	buffer_t *keyword_resets; /* buffer_t*[] */
 
         struct mail_cache_transaction_ctx *cache_trans_ctx;
 

Index: mail-index-transaction.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-transaction.c,v
retrieving revision 1.51
retrieving revision 1.52
diff -u -d -r1.51 -r1.52
--- mail-index-transaction.c	22 Jan 2005 23:15:25 -0000	1.51
+++ mail-index-transaction.c	5 Feb 2005 12:01:49 -0000	1.52
@@ -40,28 +40,6 @@
 	return t;
 }
 
-static void mail_keyword_transaction_free(struct mail_keyword_transaction *kt)
-{
-	struct mail_keyword_transaction **p;
-
-	for (p = &kt->keywords->kt; *p != NULL; p = &(*p)->next) {
-		if (*p == kt) {
-			*p = kt->next;
-			break;
-		}
-	}
-
-	if (*p == NULL) {
-		/* no transactions left, free mail_keywords */
-		i_assert(kt->keywords->kt == NULL);
-		i_free(kt->keywords);
-	}
-
-	if (kt->messages != NULL)
-		buffer_free(kt->messages);
-	i_free(kt);
-}
-
 static void mail_index_transaction_free(struct mail_index_transaction *t)
 {
 	buffer_t **recs;
@@ -79,15 +57,19 @@
 	}
 
 	if (t->keyword_updates != NULL) {
-		struct mail_keyword_transaction **kt;
+		buffer_t **buf;
 
-		kt = buffer_get_modifyable_data(t->keyword_updates, &size);
-		size /= sizeof(*kt);
+		buf = buffer_get_modifyable_data(t->keyword_updates, &size);
+		size /= sizeof(*buf);
 
-		for (i = 0; i < size; i++)
-			mail_keyword_transaction_free(kt[i]);
+		for (i = 0; i < size; i++) {
+			if (buf[i] != NULL)
+				buffer_free(buf[i]);
+		}
 		buffer_free(t->keyword_updates);
 	}
+	if (t->keyword_resets != NULL)
+		buffer_free(t->keyword_resets);
 
 	if (t->appends != NULL)
 		buffer_free(t->appends);
@@ -176,16 +158,16 @@
 	}
 
 	if (t->keyword_updates != NULL) {
-		struct mail_keyword_transaction **kt;
+		buffer_t **buf;
 
-		kt = buffer_get_modifyable_data(t->keyword_updates, &size);
-		size /= sizeof(*kt);
+		buf = buffer_get_modifyable_data(t->keyword_updates, &size);
+		size /= sizeof(*buf);
 
 		for (i = 0; i < size; i++) {
-			if (kt[i]->messages == NULL)
+			if (buf[i] == NULL)
 				continue;
 
-			mail_index_buffer_convert_to_uids(t, kt[i]->messages,
+			mail_index_buffer_convert_to_uids(t, buf[i],
 				sizeof(uint32_t) * 2, TRUE);
 		}
 	}
@@ -299,12 +281,23 @@
 	uint32_t seq1, seq2;
 };
 
-static void mail_index_update_seq_range_buffer(buffer_t *buffer, uint32_t seq)
+static void
+mail_index_seq_range_buffer_add(buffer_t **buffer_p, size_t initial_size,
+				uint32_t seq)
 {
-        struct seq_range *data, value;
+	buffer_t *buffer;
+	struct seq_range *data, value;
 	unsigned int idx, left_idx, right_idx;
 	size_t size;
 
+	if (*buffer_p == NULL) {
+		*buffer_p = buffer_create_dynamic(default_pool, initial_size);
+		buffer_append(*buffer_p, &seq, sizeof(seq));
+		buffer_append(*buffer_p, &seq, sizeof(seq));
+		return;
+	}
+	buffer = *buffer_p;
+
 	value.seq1 = value.seq2 = seq;
 
 	data = buffer_get_modifyable_data(buffer, &size);
@@ -339,7 +332,7 @@
 
 		if (data[idx].seq1 <= seq) {
 			if (data[idx].seq2 >= seq) {
-				/* it's already expunged */
+				/* it's already in the range */
 				return;
 			}
 			left_idx = idx+1;
@@ -378,6 +371,75 @@
 	}
 }
 
+static void mail_index_seq_range_buffer_remove(buffer_t *buffer, uint32_t seq)
+{
+	struct seq_range *data, value;
+	unsigned int idx, left_idx, right_idx;
+	size_t size;
+
+	if (buffer == NULL)
+		return;
+
+	data = buffer_get_modifyable_data(buffer, &size);
+	size /= sizeof(*data);
+	i_assert(size > 0);
+
+	/* quick checks */
+	if (seq > data[size-1].seq2 || seq < data[0].seq1) {
+		/* outside the range */
+		return;
+	}
+	if (data[size-1].seq2 == seq) {
+		/* shrink last range */
+		data[size-1].seq2--;
+		return;
+	}
+	if (data[0].seq1 == seq) {
+		/* shrink up first range */
+		data[0].seq1++;
+		return;
+	}
+
+	/* somewhere in the middle, array is sorted so find it with
+	   binary search */
+	idx = 0; left_idx = 0; right_idx = size;
+	while (left_idx < right_idx) {
+		idx = (left_idx + right_idx) / 2;
+
+		if (data[idx].seq1 > seq)
+			right_idx = idx;
+		else if (data[idx].seq2 < seq)
+			left_idx = idx+1;
+		else {
+			/* found it */
+			if (data[idx].seq1 == seq) {
+				if (data[idx].seq1 == data[idx].seq2) {
+					/* a single sequence range.
+					   remove it entirely */
+					buffer_delete(buffer,
+						      idx * sizeof(*data),
+						      sizeof(*data));
+				} else {
+					/* shrink the range */
+					data[idx].seq1++;
+				}
+			} else if (data[idx].seq2 == seq) {
+				/* shrink the range */
+				data[idx].seq2--;
+			} else {
+				/* split the sequence range */
+				value.seq1 = seq + 1;
+				value.seq2 = data[idx].seq2;
+				data[idx].seq2 = seq - 1;
+
+				buffer_insert(buffer, idx * sizeof(*data),
+					      &value, sizeof(value));
+			}
+			break;
+		}
+	}
+}
+
 void mail_index_expunge(struct mail_index_transaction *t, uint32_t seq)
 {
 	i_assert(seq > 0 && seq <= mail_index_view_get_messages_count(t->view));
@@ -385,14 +447,7 @@
 	t->log_updates = TRUE;
 
 	/* expunges is a sorted array of {seq1, seq2, ..}, .. */
-	if (t->expunges == NULL) {
-		t->expunges = buffer_create_dynamic(default_pool, 1024);
-		buffer_append(t->expunges, &seq, sizeof(seq));
-		buffer_append(t->expunges, &seq, sizeof(seq));
-		return;
-	}
-
-	mail_index_update_seq_range_buffer(t->expunges, seq);
+	mail_index_seq_range_buffer_add(&t->expunges, 512, seq);
 }
 
 static void
@@ -651,9 +706,9 @@
 	return FALSE;
 }
 
-static int mail_index_update_seq_buffer(buffer_t **buffer, uint32_t seq,
-					const void *record, size_t record_size,
-					void *old_record)
+static int mail_index_seq_buffer_add(buffer_t **buffer, uint32_t seq,
+				     const void *record, size_t record_size,
+				     void *old_record)
 {
 	void *p;
 	size_t pos;
@@ -804,86 +859,48 @@
 				      sizeof(buffer_t *));
 
 	/* @UNSAFE */
-	if (!mail_index_update_seq_buffer(buf, seq, data, record_size,
-					  old_data_r)) {
+	if (!mail_index_seq_buffer_add(buf, seq, data, record_size,
+				       old_data_r)) {
 		if (old_data_r != NULL)
 			memset(old_data_r, 0, record_size);
 	}
 }
 
-static struct mail_keyword_transaction *
-mail_keyword_transaction_new(struct mail_index_transaction *t,
-			     struct mail_keywords *keywords)
-{
-	struct mail_keyword_transaction *kt;
-
-	if (t->keyword_updates == NULL)
-                t->keyword_updates = buffer_create_dynamic(default_pool, 512);
-
-	kt = i_new(struct mail_keyword_transaction, 1);
-	kt->transaction = t;
-	kt->keywords = keywords;
-
-	kt->next = keywords->kt;
-	keywords->kt = kt;
-
-	buffer_append(t->keyword_updates, &kt, sizeof(kt));
-	return kt;
-}
-
-static struct mail_keywords *
-mail_index_keywords_build(struct mail_index *index,
-			  const char *const keywords[], unsigned int count)
+struct mail_keywords *
+mail_index_keywords_create(struct mail_index_transaction *t,
+			   const char *const keywords[])
 {
-	struct mail_keywords k;
+	/* @UNSAFE */
+	struct mail_index *index = t->view->index;
+	struct mail_keywords *k;
 	const char **missing_keywords, *keyword;
-	buffer_t *keyword_buf;
-	unsigned int i, j, bitmask_offset, missing_count = 0;
+	unsigned int count, i, j, k_pos = 0, missing_count = 0;
 	size_t size;
-	uint8_t *b;
 
-	if (count == 0)
+	if (keywords == NULL)
 		return i_new(struct mail_keywords, 1);
+	count = strarray_length(keywords);
 
-	/* @UNSAFE */
-	t_push();
+	k = i_malloc(sizeof(struct mail_keywords) +
+		     (sizeof(k->idx) * (count-1)));
+	k->index = t->view->index;
+	k->count = count;
 
+	t_push();
 	missing_keywords = t_new(const char *, count + 1);
-	memset(&k, 0, sizeof(k));
 
-	/* keywords are sorted in index. look up the existing ones and add
-	   new ones. build a bitmap pointing to them. keywords are never
-	   removed from index's keyword list. */
-	bitmask_offset = sizeof(k) - sizeof(k.bitmask);
-	keyword_buf = buffer_create_dynamic(default_pool, bitmask_offset +
-					    (count + 7) / 8 + 8);
+	/* look up the keywords from index. they're never removed from there
+	   so we can permanently store indexes to them. */
 	for (i = 0; i < count; i++) {
 		for (j = 0; index->keywords[j] != NULL; j++) {
 			if (strcasecmp(keywords[i], index->keywords[j]) == 0)
 				break;
 		}
 
-		if (index->keywords[j] != NULL) {
-			if (keyword_buf->used == 0) {
-				/* first one */
-				k.start = j;
-			} else if (j < k.start) {
-				buffer_copy(keyword_buf,
-					    bitmask_offset + k.start - j,
-					    keyword_buf, bitmask_offset,
-					    (size_t)-1);
-				k.start = j;
-			}
-			b = buffer_get_space_unsafe(keyword_buf,
-						    bitmask_offset +
-						    (j - k.start) / 8, 1);
-			*b |= 1 << ((j - k.start) % 8);
-			k.end = j;
-			k.count++;
-		} else {
-			/* arrays are sorted, can't match anymore */
+		if (index->keywords[j] != NULL)
+			k->idx[k_pos++] = j;
+		else
 			missing_keywords[missing_count++] = keywords[i];
-		}
 	}
 
 	if (missing_count > 0) {
@@ -898,67 +915,90 @@
 			buffer_append(index->keywords_buf,
 				      &keyword, sizeof(keyword));
 
-			b = buffer_get_space_unsafe(keyword_buf,
-						    bitmask_offset +
-						    (j - k.start) / 8, 1);
-			*b |= 1 << ((j - k.start) % 8);
-			k.end = j;
-			k.count++;
+			k->idx[k_pos++] = j;
 		}
 
 		buffer_append_zero(index->keywords_buf, sizeof(const char *));
 		index->keywords = index->keywords_buf->data;
 	}
-	buffer_write(keyword_buf, 0, &k, bitmask_offset);
+	i_assert(k_pos == count);
 
 	t_pop();
-	return buffer_free_without_data(keyword_buf);
+	return k;
 }
 
-struct mail_keywords *
-mail_index_keywords_create(struct mail_index_transaction *t,
-			   const char *const keywords[])
+void mail_index_keywords_free(struct mail_keywords *keywords)
 {
-	struct mail_keywords *k;
-	const char *const null_keywords[] = { NULL };
-
-	if (keywords == NULL)
-		keywords = null_keywords;
-
-	k = mail_index_keywords_build(t->view->index, keywords,
-				      strarray_length(keywords));
-	(void)mail_keyword_transaction_new(t, k);
-	return k;
+	i_free(keywords);
 }
 
 void mail_index_update_keywords(struct mail_index_transaction *t, uint32_t seq,
 				enum modify_type modify_type,
 				struct mail_keywords *keywords)
 {
-	struct mail_keyword_transaction *kt;
+	buffer_t **buf;
+	unsigned int i;
+	size_t pos;
 
 	i_assert(seq > 0 &&
 		 (seq <= mail_index_view_get_messages_count(t->view) ||
 		  seq <= t->last_new_seq));
 	i_assert(keywords->count > 0 || modify_type == MODIFY_REPLACE);
-	i_assert(keywords->kt->transaction->view->index == t->view->index);
+	i_assert(keywords->index == t->view->index);
 
-	for (kt = keywords->kt; kt != NULL; kt = kt->next) {
-		if (kt->transaction == t &&
-		    (kt->modify_type == modify_type || kt->messages == NULL))
-			break;
+	/* keyword_updates is an array of
+	   { buffer_t *add_seq; buffer_t *remove_seq; }
+	   which is why there's the multiplication by 2.
+
+	   If t->keyword_resets is set for the sequence, there's no need to
+	   update remove_seq as it will remove all keywords. */
+
+	if (t->keyword_updates == NULL) {
+		uint32_t max_idx = keywords->idx[keywords->count-1];
+
+		t->keyword_updates =
+			buffer_create_dynamic(default_pool,
+					      max_idx * 2 * sizeof(buffer_t *));
 	}
 
-	if (kt == NULL)
-		kt = mail_keyword_transaction_new(t, keywords);
+	switch (modify_type) {
+	case MODIFY_ADD:
+		for (i = 0; i < keywords->count; i++) {
+			pos = keywords->idx[i] * 2 * sizeof(buffer_t *);
+			buf = buffer_get_space_unsafe(t->keyword_updates, pos,
+						      sizeof(buffer_t *));
+			mail_index_seq_range_buffer_add(buf, 64, seq);
 
-	if (kt->messages == NULL) {
-		kt->messages = buffer_create_dynamic(default_pool, 32);
-		kt->modify_type = modify_type;
-		buffer_append(kt->messages, &seq, sizeof(seq));
-		buffer_append(kt->messages, &seq, sizeof(seq));
-	} else {
-		mail_index_update_seq_range_buffer(kt->messages, seq);
+			buf = buffer_get_space_unsafe(t->keyword_updates,
+						      pos + sizeof(buffer_t *),
+						      sizeof(buffer_t *));
+			mail_index_seq_range_buffer_remove(*buf, seq);
+		}
+		break;
+	case MODIFY_REMOVE:
+		for (i = 0; i < keywords->count; i++) {
+			pos = keywords->idx[i] * 2 * sizeof(buffer_t *);
+			buf = buffer_get_space_unsafe(t->keyword_updates, pos,
+						      sizeof(buffer_t *));
+			mail_index_seq_range_buffer_remove(*buf, seq);
+
+			buf = buffer_get_space_unsafe(t->keyword_updates,
+						      pos + sizeof(buffer_t *),
+						      sizeof(buffer_t *));
+			mail_index_seq_range_buffer_add(buf, 64, seq);
+		}
+		break;
+	case MODIFY_REPLACE:
+		for (i = 0; i < keywords->count; i++) {
+			pos = keywords->idx[i] * 2 * sizeof(buffer_t *);
+			buf = buffer_get_space_unsafe(t->keyword_updates, pos,
+						      sizeof(buffer_t *));
+			mail_index_seq_range_buffer_add(buf, 64, seq);
+		}
+
+		mail_index_seq_range_buffer_add(&t->keyword_resets, 64, seq);
+		break;
 	}
+
 	t->log_updates = TRUE;
 }

Index: mail-index-view-sync.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index-view-sync.c,v
retrieving revision 1.35
retrieving revision 1.36
diff -u -d -r1.35 -r1.36
--- mail-index-view-sync.c	26 Dec 2004 12:40:10 -0000	1.35
+++ mail-index-view-sync.c	5 Feb 2005 12:01:49 -0000	1.36
@@ -285,18 +285,10 @@
 	}
 	case MAIL_TRANSACTION_KEYWORD_UPDATE: {
 		const struct mail_transaction_keyword_update *update = data;
-		const unsigned char *p;
 		const uint32_t *uids;
-		uint32_t i;
 
 		if (ctx->data_offset == 0) {
-			p = (const unsigned char *)
-				(update->name_size + update->keywords_count);
-
-			for (i = 0; i < update->keywords_count; i++)
-				p += update->name_size[i];
-
-			ctx->data_offset = p - (const unsigned char *)update;
+			ctx->data_offset = sizeof(*update) + update->name_size;
 			if ((ctx->data_offset % 4) != 0)
 				ctx->data_offset += 4 - (ctx->data_offset % 4);
 		}

Index: mail-index.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-index.h,v
retrieving revision 1.142
retrieving revision 1.143
diff -u -d -r1.142 -r1.143
--- mail-index.h	22 Jan 2005 16:45:24 -0000	1.142
+++ mail-index.h	5 Feb 2005 12:01:49 -0000	1.143
@@ -293,6 +293,8 @@
 struct mail_keywords *
 mail_index_keywords_create(struct mail_index_transaction *t,
 			   const char *const keywords[]);
+/* Free the keywords. */
+void mail_index_keywords_free(struct mail_keywords *keywords);
 /* Update keywords for given message. */
 void mail_index_update_keywords(struct mail_index_transaction *t, uint32_t seq,
 				enum modify_type modify_type,

Index: mail-transaction-log.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-transaction-log.c,v
retrieving revision 1.84
retrieving revision 1.85
diff -u -d -r1.84 -r1.85
--- mail-transaction-log.c	22 Jan 2005 23:25:51 -0000	1.84
+++ mail-transaction-log.c	5 Feb 2005 12:01:49 -0000	1.85
@@ -1198,58 +1198,30 @@
 				      struct mail_index_transaction *t)
 {
 	struct mail_index *index = t->view->index;
-	struct mail_keyword_transaction **kt;
 	struct mail_transaction_keyword_update kt_hdr;
-	buffer_t *buf;
-	size_t i, size, size_offset, name_offset;
-	unsigned int idx, last_idx, first_keyword;
+	buffer_t *hdr_buf, **buf;
+	size_t i, size;
 
-	buf = buffer_create_dynamic(pool_datastack_create(), 128);
+	hdr_buf = buffer_create_dynamic(pool_datastack_create(), 64);
 
-	kt = buffer_get_modifyable_data(t->keyword_updates, &size);
-	size /= sizeof(*kt);
+	buf = buffer_get_modifyable_data(t->keyword_updates, &size);
+	size /= sizeof(*buf);
 	for (i = 0; i < size; i++) {
-		buffer_set_used_size(buf, 0);
-
-		memset(&kt_hdr, 0, sizeof(kt_hdr));
-		kt_hdr.keywords_count = kt[i]->keywords->count;
-		kt_hdr.modify_type = kt[i]->modify_type;
-		buffer_append(buf, &kt_hdr,
-			      sizeof(kt_hdr) - sizeof(kt_hdr.name_size));
-
-		size_offset = buf->used;
-		name_offset = buf->used +
-			kt[i]->keywords->count * sizeof(uint16_t);
-
-		idx = 0;
-		first_keyword = kt[i]->keywords->start;
-		last_idx = kt[i]->keywords->end - first_keyword;
-
-		for (; idx <= last_idx; idx++) {
-			uint16_t name_size;
-			const char *keyword;
-
-			if ((kt[i]->keywords->bitmask[idx / 8] &
-			     (1 << (idx % 8))) == 0)
-				continue;
-
-			i_assert(first_keyword + idx <
-				 index->keywords_buf->used / sizeof(keyword));
-			keyword = index->keywords[first_keyword + idx];
-
-			name_size = strlen(keyword);
-			buffer_write(buf, size_offset,
-				     &name_size, sizeof(name_size));
-			size_offset += sizeof(name_size);
+		if (buf[i] == NULL)
+			continue;
 
-			buffer_write(buf, name_offset, keyword, name_size);
-			name_offset += name_size;
-		}
+		buffer_set_used_size(hdr_buf, 0);
 
-		if ((buf->used % 4) != 0)
-			buffer_append_zero(buf, 4 - (buf->used % 4));
+		memset(&kt_hdr, 0, sizeof(kt_hdr));
+		kt_hdr.modify_type = (i & 1) == 0 ? MODIFY_ADD : MODIFY_REMOVE;
+		kt_hdr.name_size = strlen(index->keywords[i / 2]);
+		buffer_append(hdr_buf, &kt_hdr, sizeof(kt_hdr));
+		buffer_append(hdr_buf, index->keywords[i / 2],
+			      kt_hdr.name_size);
+		if ((hdr_buf->used % 4) != 0)
+			buffer_append_zero(hdr_buf, 4 - (hdr_buf->used % 4));
 
-		if (log_append_buffer(file, kt[i]->messages, buf, 
+		if (log_append_buffer(file, buf[i], hdr_buf,
 				      MAIL_TRANSACTION_KEYWORD_UPDATE,
 				      t->external) < 0)
 			return -1;
@@ -1342,6 +1314,12 @@
 	if (t->ext_rec_updates != NULL && ret == 0)
 		ret = log_append_ext_rec_updates(file, t);
 
+	/* keyword resets before updates */
+	if (t->keyword_resets != NULL && ret == 0) {
+		ret = log_append_buffer(file, t->keyword_resets, NULL,
+					MAIL_TRANSACTION_KEYWORD_RESET,
+					t->external);
+	}
 	if (t->keyword_updates != NULL && ret == 0)
 		ret = log_append_keyword_updates(file, t);
 

Index: mail-transaction-log.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-transaction-log.h,v
retrieving revision 1.23
retrieving revision 1.24
diff -u -d -r1.23 -r1.24
--- mail-transaction-log.h	22 Jan 2005 16:39:20 -0000	1.23
+++ mail-transaction-log.h	5 Feb 2005 12:01:49 -0000	1.24
@@ -22,6 +22,7 @@
 	MAIL_TRANSACTION_EXT_HDR_UPDATE		= 0x00000100,
 	MAIL_TRANSACTION_EXT_REC_UPDATE		= 0x00000200,
 	MAIL_TRANSACTION_KEYWORD_UPDATE		= 0x00000400,
+	MAIL_TRANSACTION_KEYWORD_RESET		= 0x00000800,
 
 	MAIL_TRANSACTION_TYPE_MASK		= 0x0000ffff,
 
@@ -52,15 +53,18 @@
 };
 
 struct mail_transaction_keyword_update {
-	uint32_t keywords_count;
-	uint8_t modify_type; /* enum modify_type */
+	uint8_t modify_type; /* enum modify_type : MODIFY_ADD / MODIFY_REMOVE */
 	uint8_t padding;
-	uint16_t name_size[1]; /* [keywords_count] */
-	/* unsigned char name[keywords_count][name_size];
+	uint16_t name_size;
+	/* unsigned char name[];
 	   array of { uint32_t uid1, uid2; }
 	*/
 };
 
+struct mail_transaction_keyword_reset {
+	uint32_t uid1, uid2;
+};
+
 struct mail_transaction_header_update {
 	uint16_t offset;
 	uint16_t size;

Index: mail-transaction-util.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-index/mail-transaction-util.c,v
retrieving revision 1.22
retrieving revision 1.23
diff -u -d -r1.22 -r1.23
--- mail-transaction-util.c	26 Dec 2004 09:12:40 -0000	1.22
+++ mail-transaction-util.c	5 Feb 2005 12:01:49 -0000	1.23
@@ -20,6 +20,7 @@
 	{ MAIL_TRANSACTION_EXT_HDR_UPDATE, 0, 1 },
 	{ MAIL_TRANSACTION_EXT_REC_UPDATE, 0, 1 },
 	{ MAIL_TRANSACTION_KEYWORD_UPDATE, MAIL_INDEX_SYNC_TYPE_KEYWORDS, 1 },
+	{ MAIL_TRANSACTION_KEYWORD_RESET, 0, 1 },
 	{ 0, 0, 0 }
 };

More information about the dovecot-cvs mailing list