[dovecot-cvs] dovecot/src/lib-storage/index/mbox mbox-save.c, 1.76, 1.77 mbox-sync-parse.c, 1.36, 1.37 mbox-sync-private.h, 1.46, 1.47 mbox-sync-rewrite.c, 1.43, 1.44 mbox-sync-update.c, 1.29, 1.30 mbox-sync.c, 1.143, 1.144

cras at dovecot.org cras at dovecot.org
Sun Apr 3 00:09:10 EEST 2005 

Update of /var/lib/cvs/dovecot/src/lib-storage/index/mbox
In directory talvi:/tmp/cvs-serv14722/lib-storage/index/mbox

Modified Files:
	mbox-save.c mbox-sync-parse.c mbox-sync-private.h 
	mbox-sync-rewrite.c mbox-sync-update.c mbox-sync.c 
Log Message:
Keywords are now stored in X-Keywords headers in mbox. Did several related
API changes to get better performance.



Index: mbox-save.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/mbox/mbox-save.c,v
retrieving revision 1.76
retrieving revision 1.77
diff -u -d -r1.76 -r1.77
--- mbox-save.c	29 Mar 2005 19:27:37 -0000	1.76
+++ mbox-save.c	2 Apr 2005 21:09:07 -0000	1.77
@@ -232,13 +232,21 @@
 {
 	unsigned char space[MBOX_HEADER_PADDING+1 +
 			    sizeof("Content-Length: \n")-1 + MAX_INT_STRLEN];
-	unsigned int i;
+	const array_t *keyword_names_list;
+	ARRAY_SET_TYPE(keyword_names_list, const char *);
+	const char *const *keyword_names;
+	unsigned int i, keyword_names_count;
+
+	keyword_names_list = mail_index_get_keywords(ctx->ibox->index);
+	keyword_names = array_get(keyword_names_list, &keyword_names_count);
 
 	str_append(ctx->headers, "X-Keywords:");
-	/*FIXME:for (i = 0; i < count; i++) {
+	for (i = 0; i < keywords->count; i++) {
+		i_assert(keywords->idx[i] < keyword_names_count);
+
 		str_append_c(ctx->headers, ' ');
-		str_append(ctx->headers, keywords[i]);
-	}*/
+		str_append(ctx->headers, keyword_names[keywords->idx[i]]);
+	}
 
 	memset(space, ' ', sizeof(space));
 	str_append_n(ctx->headers, space, sizeof(space));

Index: mbox-sync-parse.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync-parse.c,v
retrieving revision 1.36
retrieving revision 1.37
diff -u -d -r1.36 -r1.37
--- mbox-sync-parse.c	29 Mar 2005 19:27:37 -0000	1.36
+++ mbox-sync-parse.c	2 Apr 2005 21:09:07 -0000	1.37
@@ -107,6 +107,46 @@
 	return TRUE;
 }
 
+static void
+parse_imap_keywords_list(struct mbox_sync_mail_context *ctx,
+                         struct message_header_line *hdr, size_t pos)
+{
+	const char *keyword;
+	size_t keyword_start;
+	unsigned int idx, count;
+
+	count = 0;
+	while (pos < hdr->full_value_len) {
+		if (IS_LWSP_LF(hdr->full_value[pos])) {
+                        pos++;
+			continue;
+		}
+
+		/* read the keyword */
+		keyword_start = pos;
+		for (; pos < hdr->full_value_len; pos++) {
+			if (IS_LWSP_LF(hdr->full_value[pos]))
+				break;
+		}
+
+		/* add it to index's keyword list if it's not there already */
+		t_push();
+		keyword = t_strndup(hdr->full_value + keyword_start,
+				    pos - keyword_start);
+		(void)mail_index_keyword_lookup(ctx->sync_ctx->ibox->index,
+						keyword, TRUE, &idx);
+		t_pop();
+
+		count++;
+	}
+
+	if (count != array_count(ctx->sync_ctx->ibox->keyword_names)) {
+		/* need to update this list */
+		ctx->update_imapbase_keywords = TRUE;
+		ctx->need_rewrite = TRUE;
+	}
+}
+
 static int parse_x_imap_base(struct mbox_sync_mail_context *ctx,
 			     struct message_header_line *hdr)
 {
@@ -128,17 +168,16 @@
 	pos = end - str;
 	t_pop();
 
-	while (pos < hdr->full_value_len && IS_LWSP_LF(hdr->full_value[pos]))
-		pos++;
-
 	if (uid_validity == 0) {
 		/* broken */
 		return FALSE;
 	}
 
 	if (ctx->sync_ctx->base_uid_validity == 0) {
+		/* first time parsing this. save the values. */
 		ctx->sync_ctx->base_uid_validity = uid_validity;
 		ctx->sync_ctx->base_uid_last = uid_last;
+
 		if (ctx->sync_ctx->next_uid-1 <= uid_last)
 			ctx->sync_ctx->next_uid = uid_last+1;
 		else {
@@ -156,12 +195,8 @@
 	ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] = str_len(ctx->header);
 	ctx->seen_imapbase = TRUE;
 
-	if (pos == hdr->full_value_len)
-		return TRUE;
-
-	// FIXME: save keywords
-
-        parse_trailing_whitespace(ctx, hdr);
+	parse_imap_keywords_list(ctx, hdr, pos);
+	parse_trailing_whitespace(ctx, hdr);
 	return TRUE;
 }
 
@@ -180,10 +215,73 @@
 static int parse_x_keywords(struct mbox_sync_mail_context *ctx,
 			    struct message_header_line *hdr)
 {
-	// FIXME: parse them
+	array_t ARRAY_DEFINE(keyword_list, unsigned int);
+	string_t *keyword;
+	size_t keyword_start;
+	unsigned int idx;
+	size_t pos;
+
+	if (array_is_created(&ctx->mail.keywords))
+		return FALSE; /* duplicate header, delete */
+
+	/* read keyword indexes to temporary array first */
+	t_push();
+	keyword = t_str_new(128);
+	ARRAY_CREATE(&keyword_list, pool_datastack_create(), unsigned int, 16);
+
+	for (pos = 0; pos < hdr->full_value_len; ) {
+		if (IS_LWSP_LF(hdr->full_value[pos])) {
+                        pos++;
+			continue;
+		}
+
+		/* read the keyword string */
+		keyword_start = pos;
+		for (; pos < hdr->full_value_len; pos++) {
+			if (IS_LWSP_LF(hdr->full_value[pos]))
+				break;
+		}
+
+		str_truncate(keyword, 0);
+		str_append_n(keyword, hdr->full_value + keyword_start,
+			     pos - keyword_start);
+		if (!mail_index_keyword_lookup(ctx->sync_ctx->ibox->index,
+					       str_c(keyword), FALSE, &idx)) {
+			if (ctx->sync_ctx->ibox->mbox_sync_dirty &&
+			    !ctx->sync_ctx->dest_first_mail &&
+			    !ctx->sync_ctx->seen_first_mail) {
+				/* we'll have to read the X-IMAP header to
+				   make sure we have the latest list of
+				   keywords */
+				i_assert(!ctx->sync_ctx->sync_restart);
+				ctx->sync_ctx->sync_restart = TRUE;
+				t_pop();
+				return FALSE;
+			}
+
+			/* index is fully up-to-date and the keyword wasn't
+			   found. that means the sent mail originally
+			   contained X-Keywords header. Delete it. */
+			t_pop();
+			return FALSE;
+		}
+
+		array_append(&keyword_list, &idx, 1);
+	}
+
+	/* once we know how many keywords there are, we can allocate the array
+	   from mail_keyword_pool without wasting memory. */
+	if (array_count(&keyword_list) > 0) {
+		ARRAY_CREATE(&ctx->mail.keywords,
+			     ctx->sync_ctx->mail_keyword_pool,
+			     unsigned int, array_count(&keyword_list));
+		array_append_array(&ctx->mail.keywords, &keyword_list);
+	}
 
 	ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] = str_len(ctx->header);
 	parse_trailing_whitespace(ctx, hdr);
+
+	t_pop();
 	return TRUE;
 }
 

Index: mbox-sync-private.h
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync-private.h,v
retrieving revision 1.46
retrieving revision 1.47
diff -u -d -r1.46 -r1.47
--- mbox-sync-private.h	29 Mar 2005 19:27:37 -0000	1.46
+++ mbox-sync-private.h	2 Apr 2005 21:09:07 -0000	1.47
@@ -42,7 +42,7 @@
 	uint32_t uid;
 	uint32_t idx_seq;
 	uint8_t flags;
-	uint32_t keywords_idx; /* +1 */
+	array_t ARRAY_DEFINE(keywords, unsigned int);
 
 	uoff_t from_offset;
 	uoff_t body_size;
@@ -85,6 +85,7 @@
 	unsigned int recent:1;
 	unsigned int dirty:1;
 	unsigned int uid_broken:1;
+	unsigned int update_imapbase_keywords:1;
 };
 
 struct mbox_sync_context {
@@ -109,6 +110,8 @@
 	array_t ARRAY_DEFINE(syncs, struct mail_index_sync_rec);
 	struct mail_index_sync_rec sync_rec;
 
+	pool_t mail_keyword_pool;
+
 	uint32_t prev_msg_uid, next_uid;
 	uint32_t seq, idx_seq, need_space_seq;
 	off_t expunged_space, space_diff;
@@ -129,8 +132,7 @@
 int mbox_sync_parse_match_mail(struct index_mailbox *ibox,
 			       struct mail_index_view *view, uint32_t seq);
 
-void mbox_sync_update_header(struct mbox_sync_mail_context *ctx,
-			     array_t *syncs_arr);
+void mbox_sync_update_header(struct mbox_sync_mail_context *ctx);
 void mbox_sync_update_header_from(struct mbox_sync_mail_context *ctx,
 				  const struct mbox_sync_mail *mail);
 int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx, off_t move_diff);
@@ -138,11 +140,15 @@
 		      uoff_t end_offset, off_t move_diff, uoff_t extra_space,
 		      uint32_t first_seq, uint32_t last_seq);
 
-void mbox_sync_apply_index_syncs(array_t *syncs_arr, uint8_t *flags);
+void mbox_sync_apply_index_syncs(struct mbox_sync_context *sync_ctx,
+				 struct mbox_sync_mail *mail,
+				 int *keywords_changed_r);
 int mbox_sync_seek(struct mbox_sync_context *sync_ctx, uoff_t from_offset);
 int mbox_move(struct mbox_sync_context *sync_ctx,
 	      uoff_t dest, uoff_t source, uoff_t size);
 void mbox_sync_move_buffer(struct mbox_sync_mail_context *ctx,
 			   size_t pos, size_t need, size_t have);
+void mbox_sync_headers_add_space(struct mbox_sync_mail_context *ctx,
+				 size_t size);
 
 #endif

Index: mbox-sync-rewrite.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync-rewrite.c,v
retrieving revision 1.43
retrieving revision 1.44
diff -u -d -r1.43 -r1.44
--- mbox-sync-rewrite.c	29 Mar 2005 10:30:20 -0000	1.43
+++ mbox-sync-rewrite.c	2 Apr 2005 21:09:07 -0000	1.44
@@ -73,8 +73,8 @@
 	return 0;
 }
 
-static void
-mbox_sync_headers_add_space(struct mbox_sync_mail_context *ctx, size_t size)
+void mbox_sync_headers_add_space(struct mbox_sync_mail_context *ctx,
+				 size_t size)
 {
 	size_t data_size, pos, start_pos;
 	const unsigned char *data;
@@ -322,9 +322,13 @@
 	}
 
 	mbox_sync_parse_next_mail(sync_ctx->input, &mail_ctx);
-	if (mails[idx].space != 0)
+	if (mails[idx].space != 0) {
+		if (mails[idx].space < 0) {
+			/* remove all possible spacing before updating */
+			mbox_sync_headers_remove_space(&mail_ctx, (size_t)-1);
+		}
 		mbox_sync_update_header_from(&mail_ctx, &mails[idx]);
-	else {
+	} else {
 		/* updating might just try to add headers and mess up our
 		   calculations completely. so only add the EOH here. */
 		if (mail_ctx.have_eoh)

Index: mbox-sync-update.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync-update.c,v
retrieving revision 1.29
retrieving revision 1.30
diff -u -d -r1.29 -r1.30
--- mbox-sync-update.c	29 Mar 2005 19:27:37 -0000	1.29
+++ mbox-sync-update.c	2 Apr 2005 21:09:07 -0000	1.30
@@ -103,6 +103,68 @@
 	}
 }
 
+static void keywords_append(struct mbox_sync_context *sync_ctx, string_t *dest,
+			    const array_t *keyword_indexes_arr)
+{
+	ARRAY_SET_TYPE(keyword_indexes_arr, unsigned int);
+	const char *const *keyword_names;
+	const unsigned int *keyword_indexes;
+	unsigned int i, idx_count, keywords_count;
+	size_t last_break;
+
+	keyword_names = array_get(sync_ctx->ibox->keyword_names,
+				  &keywords_count);
+	keyword_indexes = array_get(keyword_indexes_arr, &idx_count);
+
+	for (i = 0, last_break = 0; i < idx_count; i++) {
+		i_assert(keyword_indexes[i] < keywords_count);
+
+		/* try avoid overly long lines but cutting them
+		   every 70 chars or so */
+		if (str_len(dest) - last_break < 70) {
+			if (i > 0)
+				str_append_c(dest, ' ');
+		} else {
+			str_append(dest, "\n\t");
+			last_break = str_len(dest);
+		}
+		str_append(dest, keyword_names[keyword_indexes[i]]);
+	}
+}
+
+static void
+keywords_append_all(struct mbox_sync_mail_context *ctx, string_t *dest)
+{
+	const char *const *names;
+	const unsigned char *p;
+	unsigned int i, count;
+	size_t last_break;
+
+	p = str_data(dest);
+	if (str_len(dest) < 70)
+		last_break = 0;
+	else {
+		/* set last_break to beginning of line */
+		for (last_break = str_len(dest); last_break > 0; last_break--) {
+			if (p[last_break-1] == '\n')
+				break;
+		}
+	}
+
+	names = array_get(ctx->sync_ctx->ibox->keyword_names, &count);
+	for (i = 0; i < count; i++) {
+		/* try avoid overly long lines but cutting them
+		   every 70 chars or so */
+		if (str_len(dest) - last_break < 70)
+			str_append_c(dest, ' ');
+		else {
+			str_append(dest, "\n\t");
+			last_break = str_len(dest);
+		}
+		str_append(dest, names[i]);
+	}
+}
+
 static void mbox_sync_add_missing_headers(struct mbox_sync_mail_context *ctx)
 {
 	size_t old_hdr_size, new_hdr_size;
@@ -130,7 +192,7 @@
 		str_printfa(ctx->header, "%u %010u",
 			    ctx->sync_ctx->base_uid_validity,
 			    ctx->sync_ctx->next_uid-1);
-		//FIXME:keywords_append(ctx, all_keywords);
+		keywords_append_all(ctx, ctx->header);
 		str_append_c(ctx->header, '\n');
 	}
 
@@ -157,10 +219,12 @@
 	}
 
 	if (ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] == (size_t)-1 &&
-	    ctx->mail.keywords_idx != 0) {
+	    array_is_created(&ctx->mail.keywords) &&
+	    array_count(&ctx->mail.keywords) > 0) {
 		str_append(ctx->header, "X-Keywords: ");
 		ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] = str_len(ctx->header);
-		//keywords_append(ctx, ctx->mail.keywords);
+		keywords_append(ctx->sync_ctx, ctx->header,
+				&ctx->mail.keywords);
 		str_append_c(ctx->header, '\n');
 	}
 
@@ -196,14 +260,11 @@
 	}
 }
 
-static void mbox_sync_update_xkeywords(struct mbox_sync_mail_context *ctx)
-{
-}
-
 static void mbox_sync_update_line(struct mbox_sync_mail_context *ctx,
 				  size_t pos, string_t *new_line)
 {
 	const char *hdr, *p;
+	uoff_t file_pos;
 
 	if (ctx->header_first_change > pos)
 		ctx->header_first_change = pos;
@@ -213,32 +274,61 @@
 
 	if (p == NULL) {
 		/* shouldn't really happen, but allow anyway.. */
-		ctx->header_last_change = (size_t)-1;
-		str_truncate(ctx->header, pos);
-		str_append_str(ctx->header, new_line);
-	} else {
-		mbox_sync_move_buffer(ctx, pos, str_len(new_line), p - hdr + 1);
-		buffer_copy(ctx->header, pos, new_line, 0, (size_t)-1);
+		p = hdr + strlen(hdr);
+	}
+
+	file_pos = pos + ctx->hdr_offset;
+	if (ctx->mail.space > 0 && ctx->mail.offset >= file_pos &&
+	    ctx->mail.offset < file_pos + (p - hdr)) {
+		/* extra space points to this line. remove it. */
+		ctx->mail.offset = ctx->hdr_offset;
+		ctx->mail.space = 0;
 	}
+
+	mbox_sync_move_buffer(ctx, pos, str_len(new_line), p - hdr + 1);
+	buffer_copy(ctx->header, pos, new_line, 0, (size_t)-1);
+}
+
+static void mbox_sync_update_xkeywords(struct mbox_sync_mail_context *ctx)
+{
+	string_t *str;
+
+	if (ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] == (size_t)-1)
+		return;
+
+	t_push();
+	str = t_str_new(256);
+	keywords_append(ctx->sync_ctx, str, &ctx->mail.keywords);
+	str_append_c(str, '\n');
+	mbox_sync_update_line(ctx, ctx->hdr_pos[MBOX_HDR_X_KEYWORDS], str);
+	t_pop();
 }
 
 static void mbox_sync_update_x_imap_base(struct mbox_sync_mail_context *ctx)
 {
+	struct mbox_sync_context *sync_ctx = ctx->sync_ctx;
 	string_t *str;
 
-	if (!ctx->sync_ctx->dest_first_mail ||
-	    ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] == (size_t)-1 ||
-	    ctx->sync_ctx->update_base_uid_last == 0 ||
-	    ctx->sync_ctx->update_base_uid_last < ctx->sync_ctx->base_uid_last)
+	if (!sync_ctx->dest_first_mail ||
+	    ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] == (size_t)-1)
+		return;
+
+	if (sync_ctx->update_base_uid_last <= sync_ctx->base_uid_last)
+                sync_ctx->update_base_uid_last = 0;
+
+	/* see if anything changed */
+	if (!(ctx->update_imapbase_keywords ||
+	      sync_ctx->update_base_uid_last != 0))
 		return;
 
 	/* update uid-last field in X-IMAPbase */
 	t_push();
 
 	str = t_str_new(200);
-	str_printfa(str, "%u %010u", ctx->sync_ctx->base_uid_validity,
-		    ctx->sync_ctx->update_base_uid_last);
-	//FIXME:keywords_append(ctx, all_keywords);
+	str_printfa(str, "%u %010u", sync_ctx->base_uid_validity,
+		    sync_ctx->update_base_uid_last != 0 ?
+		    sync_ctx->update_base_uid_last : sync_ctx->base_uid_last);
+	keywords_append_all(ctx, str);
 	str_append_c(str, '\n');
 
         mbox_sync_update_line(ctx, ctx->hdr_pos[MBOX_HDR_X_IMAPBASE], str);
@@ -260,30 +350,24 @@
 	t_pop();
 }
 
-void mbox_sync_update_header(struct mbox_sync_mail_context *ctx,
-			     array_t *syncs_arr)
+void mbox_sync_update_header(struct mbox_sync_mail_context *ctx)
 {
-	ARRAY_SET_TYPE(syncs_arr, struct mail_index_sync_rec);
-	const struct mail_index_sync_rec *syncs;
 	uint8_t old_flags;
-	uint32_t old_keywords_idx;
-	unsigned int i, count;
+	int keywords_changed;
 
 	i_assert(ctx->mail.uid != 0 || ctx->pseudo);
 
-	syncs = array_get(syncs_arr, &count);
 	old_flags = ctx->mail.flags;
 
-	if (count != 0) {
-		old_keywords_idx = ctx->mail.keywords_idx;
-                mbox_sync_apply_index_syncs(syncs_arr, &ctx->mail.flags);
+	if (array_count(&ctx->sync_ctx->syncs) > 0) {
+		mbox_sync_apply_index_syncs(ctx->sync_ctx, &ctx->mail,
+					    &keywords_changed);
 
 		if ((old_flags & XSTATUS_FLAGS_MASK) !=
 		    (ctx->mail.flags & XSTATUS_FLAGS_MASK))
 			mbox_sync_update_xstatus(ctx);
-		/*FIXME:if (memcmp(old_keywords, ctx->mail.keywords,
-			   INDEX_KEYWORDS_BYTE_COUNT) != 0)
-			mbox_sync_update_xkeywords(ctx);*/
+		if (keywords_changed)
+			mbox_sync_update_xkeywords(ctx);
 	}
 
 	if (!ctx->sync_ctx->ibox->keep_recent)
@@ -317,12 +401,30 @@
 			(mail->flags & XSTATUS_FLAGS_MASK);
 		mbox_sync_update_xstatus(ctx);
 	}
-	/*FIXME:if (memcmp(ctx->mail.keywords, mail->keywords,
-		   INDEX_KEYWORDS_BYTE_COUNT) != 0) {
-		memcpy(ctx->mail.keywords, mail->keywords,
-		       INDEX_KEYWORDS_BYTE_COUNT);
+	if (!array_is_created(&mail->keywords) ||
+	    array_count(&mail->keywords) == 0) {
+		/* no keywords for this mail */
+		if (array_is_created(&ctx->mail.keywords)) {
+			array_clear(&ctx->mail.keywords);
+			mbox_sync_update_xkeywords(ctx);
+		}
+	} else if (!array_is_created(&ctx->mail.keywords)) {
+		/* adding first keywords */
+		ARRAY_CREATE(&ctx->mail.keywords,
+			     ctx->sync_ctx->mail_keyword_pool,
+			     unsigned int,
+			     array_count(&mail->keywords));
+		array_append_array(&ctx->mail.keywords,
+				   &mail->keywords);
 		mbox_sync_update_xkeywords(ctx);
-	}*/
+	} else if (!buffer_cmp(ctx->mail.keywords.buffer,
+			       mail->keywords.buffer)) {
+		/* keywords changed. */
+		array_clear(&ctx->mail.keywords);
+		array_append_array(&ctx->mail.keywords,
+				   &mail->keywords);
+		mbox_sync_update_xkeywords(ctx);
+	}
 
 	i_assert(ctx->mail.uid == 0 || ctx->mail.uid == mail->uid);
 	ctx->mail.uid = mail->uid;

Index: mbox-sync.c
===================================================================
RCS file: /var/lib/cvs/dovecot/src/lib-storage/index/mbox/mbox-sync.c,v
retrieving revision 1.143
retrieving revision 1.144
diff -u -d -r1.143 -r1.144
--- mbox-sync.c	29 Mar 2005 19:27:37 -0000	1.143
+++ mbox-sync.c	2 Apr 2005 21:09:07 -0000	1.144
@@ -205,16 +205,43 @@
 	return 0;
 }
 
-void mbox_sync_apply_index_syncs(array_t *syncs_arr, uint8_t *flags)
+void mbox_sync_apply_index_syncs(struct mbox_sync_context *sync_ctx,
+				 struct mbox_sync_mail *mail,
+				 int *keywords_changed_r)
 {
-	ARRAY_SET_TYPE(syncs_arr, struct mail_index_sync_rec);
 	const struct mail_index_sync_rec *syncs;
 	unsigned int i, count;
 
-	syncs = array_get(syncs_arr, &count);
+	*keywords_changed_r = FALSE;
+
+	syncs = array_get(&sync_ctx->syncs, &count);
 	for (i = 0; i < count; i++) {
-		if (syncs[i].type == MAIL_INDEX_SYNC_TYPE_FLAGS)
-			mail_index_sync_flags_apply(&syncs[i], flags);
+		switch (syncs[i].type) {
+		case MAIL_INDEX_SYNC_TYPE_FLAGS:
+			mail_index_sync_flags_apply(&syncs[i], &mail->flags);
+			break;
+		case MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD:
+		case MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE:
+		case MAIL_INDEX_SYNC_TYPE_KEYWORD_RESET:
+			if (!array_is_created(&mail->keywords)) {
+				/* no existing keywords */
+				if (syncs[i].type !=
+				    MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD)
+					break;
+
+				/* adding, create the array */
+				ARRAY_CREATE(&mail->keywords,
+					     sync_ctx->mail_keyword_pool,
+					     unsigned int,
+					     I_MIN(10, count - i));
+			}
+			if (mail_index_sync_keywords_apply(&syncs[i],
+							   &mail->keywords))
+				*keywords_changed_r = TRUE;
+			break;
+		default:
+			break;
+		}
 	}
 }
 
@@ -335,12 +362,27 @@
 	return 0;
 }
 
-static int mbox_sync_update_index(struct mbox_sync_context *sync_ctx,
-                                  struct mbox_sync_mail_context *mail_ctx,
+static void
+mbox_sync_update_index_keywords(struct mbox_sync_mail_context *mail_ctx)
+{
+        struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
+	struct mail_keywords *keywords;
+
+	keywords = !array_is_created(&mail_ctx->mail.keywords) ?
+		mail_index_keywords_create(sync_ctx->t, NULL) :
+		mail_index_keywords_create_from_indexes(sync_ctx->t,
+						&mail_ctx->mail.keywords);
+	mail_index_update_keywords(sync_ctx->t, sync_ctx->idx_seq,
+				   MODIFY_REPLACE, keywords);
+	mail_index_keywords_free(keywords);
+}
+
+static int mbox_sync_update_index(struct mbox_sync_mail_context *mail_ctx,
 				  const struct mail_index_record *rec)
 {
+        struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx;
 	struct mbox_sync_mail *mail = &mail_ctx->mail;
-	uint8_t idx_flags, mbox_flags;
+	uint8_t mbox_flags;
 
 	mbox_flags = mail->flags & MAIL_FLAGS_MASK;
 
@@ -354,6 +396,7 @@
 		mail_index_append(sync_ctx->t, mail->uid, &sync_ctx->idx_seq);
 		mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
 					MODIFY_REPLACE, mbox_flags);
+		mbox_sync_update_index_keywords(mail_ctx);
 
 		if (sync_ctx->ibox->mbox_save_md5 != 0) {
 			mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq,
@@ -364,24 +407,41 @@
 		/* see if we need to update flags in index file. the flags in
 		   sync records are automatically applied to rec->flags at the
 		   end of index syncing, so calculate those new flags first */
-		idx_flags = rec->flags;
-		mbox_sync_apply_index_syncs(&sync_ctx->syncs, &idx_flags);
+		struct mbox_sync_mail idx_mail;
+		int keywords_changed;
 
-		if ((idx_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
+		memset(&idx_mail, 0, sizeof(idx_mail));
+		idx_mail.flags = rec->flags;
+
+		/* get old keywords */
+		t_push();
+		ARRAY_CREATE(&idx_mail.keywords, pool_datastack_create(),
+			     unsigned int, 32);
+		if (mail_index_lookup_keywords(sync_ctx->sync_view,
+					       sync_ctx->idx_seq,
+					       &idx_mail.keywords) < 0) {
+			mail_storage_set_index_error(sync_ctx->ibox);
+			t_pop();
+			return -1;
+		}
+		mbox_sync_apply_index_syncs(sync_ctx, &idx_mail,
+					    &keywords_changed);
+
+		if ((idx_mail.flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) {
 			/* flags are dirty. ignore whatever was in the mbox,
 			   but update recent flag state if needed. */
 			mbox_flags &= MAIL_RECENT;
-			mbox_flags |= idx_flags & ~MAIL_RECENT;
+			mbox_flags |= idx_mail.flags & ~MAIL_RECENT;
 		} else {
 			/* keep index's internal flags */
 			mbox_flags &= MAIL_FLAGS_MASK;
-			mbox_flags |= idx_flags & ~MAIL_FLAGS_MASK;
+			mbox_flags |= idx_mail.flags & ~MAIL_FLAGS_MASK;
 		}
 
-		if ((idx_flags & ~MAIL_INDEX_MAIL_FLAG_DIRTY) ==
+		if ((idx_mail.flags & ~MAIL_INDEX_MAIL_FLAG_DIRTY) ==
 		    (mbox_flags & ~MAIL_INDEX_MAIL_FLAG_DIRTY)) {
 			/* all flags are same, except possibly dirty flag */
-			if (idx_flags != mbox_flags) {
+			if (idx_mail.flags != mbox_flags) {
 				/* dirty flag state changed */
 				int dirty = (mbox_flags &
 					     MAIL_INDEX_MAIL_FLAG_DIRTY) != 0;
@@ -390,18 +450,21 @@
 					dirty ? MODIFY_ADD : MODIFY_REMOVE,
 					MAIL_INDEX_MAIL_FLAG_DIRTY);
 			}
-		} else if ((idx_flags & ~MAIL_RECENT) !=
+		} else if ((idx_mail.flags & ~MAIL_RECENT) !=
 			   (mbox_flags & ~MAIL_RECENT)) {
 			/* flags other than MAIL_RECENT have changed */
 			mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
 						MODIFY_REPLACE, mbox_flags);
-		} else if (((idx_flags ^ mbox_flags) & MAIL_RECENT) != 0) {
+		} else if (((idx_mail.flags ^ mbox_flags) & MAIL_RECENT) != 0) {
 			/* drop recent flag */
 			mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq,
 						MODIFY_REMOVE, MAIL_RECENT);
 		}
 
-		// FIXME: keywords
+		if ((idx_mail.flags & MAIL_INDEX_MAIL_FLAG_DIRTY) == 0 &&
+		    !array_cmp(&idx_mail.keywords, &mail_ctx->mail.keywords))
+			mbox_sync_update_index_keywords(mail_ctx);
+		t_pop();
 	}
 
 	if (mail_ctx->recent &&
@@ -523,7 +586,7 @@
 		if (mbox_read_from_line(mail_ctx) < 0)
 			return -1;
 
-		mbox_sync_update_header(mail_ctx, &sync_ctx->syncs);
+		mbox_sync_update_header(mail_ctx);
 		ret = mbox_sync_try_rewrite(mail_ctx, move_diff);
 		if (ret < 0)
 			return -1;
@@ -546,7 +609,7 @@
 		   array_count(&sync_ctx->syncs) != 0 ||
 		   (mail_ctx->seq == 1 &&
 		    sync_ctx->update_base_uid_last != 0)) {
-		mbox_sync_update_header(mail_ctx, &sync_ctx->syncs);
+		mbox_sync_update_header(mail_ctx);
 		if (sync_ctx->delay_writes) {
 			/* mark it dirty and do it later */
 			mail_ctx->dirty = TRUE;
@@ -887,8 +950,7 @@
 
 		if (!mail_ctx->pseudo) {
 			if (!expunged) {
-				if (mbox_sync_update_index(sync_ctx, mail_ctx,
-							   rec) < 0)
+				if (mbox_sync_update_index(mail_ctx, rec) < 0)
 					return -1;
 			}
 			sync_ctx->idx_seq++;
@@ -1424,6 +1486,10 @@
 	sync_ctx.index_sync_ctx = index_sync_ctx;
 	sync_ctx.sync_view = sync_view;
 	sync_ctx.t = mail_index_transaction_begin(sync_view, FALSE, TRUE);
+	sync_ctx.mail_keyword_pool = pool_alloconly_create("keywords", 4096);
+
+	/* make sure we've read the latest keywords in index */
+	(void)mail_index_get_keywords(ibox->index);
 
 	ARRAY_CREATE(&sync_ctx.mails, default_pool,
 		     struct mbox_sync_mail, 64);
@@ -1523,6 +1589,7 @@
 			ret = -1;
 	}
 
+	pool_unref(sync_ctx.mail_keyword_pool);
 	str_free(sync_ctx.header);
 	str_free(sync_ctx.from_line);
 	array_free(&sync_ctx.mails);

More information about the dovecot-cvs mailing list