[dovecot-cvs] dovecot/src/lib-storage/index/maildir maildir-mail.c, 1.1, 1.2 maildir-storage.c, 1.67, 1.68 maildir-sync.c, 1.4, 1.5 maildir-uidlist.c, 1.4, 1.5 maildir-uidlist.h, 1.1, 1.2 maildir-util.c, 1.1, 1.2

[cras at procontrol.fi]

Update of /home/cvs/dovecot/src/lib-storage/index/maildir
In directory talvi:/tmp/cvs-serv20974/lib-storage/index/maildir

Modified Files:
	maildir-mail.c maildir-storage.c maildir-sync.c 
	maildir-uidlist.c maildir-uidlist.h maildir-util.c 
Log Message:
Recent-flag should work now



Index: maildir-mail.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/maildir/maildir-mail.c,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- maildir-mail.c	27 Apr 2004 20:25:54 -0000	1.1
+++ maildir-mail.c	1 May 2004 18:30:53 -0000	1.2
@@ -64,6 +64,19 @@
 	}
 }
 
+static const struct mail_full_flags *maildir_mail_get_flags(struct mail *_mail)
+{
+	struct index_mail *mail = (struct index_mail *)_mail;
+	struct index_mail_data *data = &mail->data;
+        const struct mail_full_flags *flags;
+
+	flags = index_mail_get_flags(_mail);
+
+	if (maildir_uidlist_is_recent(mail->ibox->uidlist, _mail->uid))
+		data->flags.flags |= MAIL_RECENT;
+	return &data->flags;
+}
+
 static time_t maildir_mail_get_received_date(struct mail *_mail)
 {
 	struct index_mail *mail = (struct index_mail *) _mail;
@@ -107,7 +120,7 @@
 	struct index_mail_data *data = &mail->data;
 	const char *fname, *p;
 	uoff_t virtual_size;
-	int new_dir;
+        enum maildir_uidlist_rec_flag flags;
 
 	if (data->size != (uoff_t)-1)
 		return data->size;
@@ -119,7 +132,7 @@
 	}
 
 	fname = maildir_uidlist_lookup(mail->ibox->uidlist,
-				       mail->mail.uid, &new_dir);
+				       mail->mail.uid, &flags);
 	if (fname == NULL)
 		return (uoff_t)-1;
 
@@ -165,7 +178,7 @@
 struct mail maildir_mail = {
 	0, 0, 0, 0, 0, 0,
 
-	index_mail_get_flags,
+	maildir_mail_get_flags,
 	index_mail_get_parts,
 	maildir_mail_get_received_date,
 	index_mail_get_date,

Index: maildir-storage.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/maildir/maildir-storage.c,v
retrieving revision 1.67
retrieving revision 1.68
diff -u -d -r1.67 -r1.68
--- maildir-storage.c	27 Apr 2004 20:25:54 -0000	1.67
+++ maildir-storage.c	1 May 2004 18:30:53 -0000	1.68
@@ -393,6 +393,11 @@
 	return 0;
 }
 
+static uint32_t maildir_get_recent_count(struct index_mailbox *ibox)
+{
+	return maildir_uidlist_get_recent_count(ibox->uidlist);
+}
+
 static struct mailbox *
 maildir_open(struct index_storage *storage, const char *name,
 	     enum mailbox_open_flags flags)
@@ -416,6 +421,7 @@
 	ibox->path = i_strdup(path);
 	ibox->control_dir = i_strdup(control_dir);
 
+	ibox->get_recent_count = maildir_get_recent_count;
 	ibox->mail_interface = &maildir_mail;
 	ibox->uidlist = maildir_uidlist_init(ibox);
 

Index: maildir-sync.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/maildir/maildir-sync.c,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -d -r1.4 -r1.5
--- maildir-sync.c	1 May 2004 16:30:58 -0000	1.4
+++ maildir-sync.c	1 May 2004 18:30:53 -0000	1.5
@@ -187,7 +187,8 @@
 	DIR *dirp;
 	string_t *src, *dest;
 	struct dirent *dp;
-	int move_new, this_new, ret = 1;
+        enum maildir_uidlist_rec_flag flags;
+	int move_new, ret = 1;
 
 	src = t_str_new(1024);
 	dest = t_str_new(1024);
@@ -200,34 +201,43 @@
 		return -1;
 	}
 
-	move_new = new_dir;
+	move_new = new_dir && !mailbox_is_readonly(&ctx->ibox->box);
 	while ((dp = readdir(dirp)) != NULL) {
 		if (dp->d_name[0] == '.')
 			continue;
 
-		this_new = new_dir;
+		flags = 0;
 		if (move_new) {
 			str_truncate(src, 0);
 			str_truncate(dest, 0);
 			str_printfa(src, "%s/%s", ctx->new_dir, dp->d_name);
 			str_printfa(dest, "%s/%s", ctx->cur_dir, dp->d_name);
-			if (rename(str_c(src), str_c(dest)) == 0 ||
-			    ENOTFOUND(errno)) {
-				/* moved - we'll look at it later in cur/ dir */
-				this_new = FALSE;
-				continue;
+			if (rename(str_c(src), str_c(dest)) == 0) {
+				/* we moved it - it's \Recent for use */
+				flags |= MAILDIR_UIDLIST_REC_FLAG_MOVED |
+					MAILDIR_UIDLIST_REC_FLAG_RECENT;
+			} else if (ENOTFOUND(errno)) {
+				/* someone else moved it already */
+				flags |= MAILDIR_UIDLIST_REC_FLAG_MOVED;
 			} else if (ENOSPACE(errno)) {
 				/* not enough disk space, leave here */
+				flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
+					MAILDIR_UIDLIST_REC_FLAG_RECENT;
 				move_new = FALSE;
 			} else {
+				flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
+					MAILDIR_UIDLIST_REC_FLAG_RECENT;
 				mail_storage_set_critical(storage,
 					"rename(%s, %s) failed: %m",
 					str_c(src), str_c(dest));
 			}
+		} else if (new_dir) {
+			flags |= MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
+				MAILDIR_UIDLIST_REC_FLAG_RECENT;
 		}
 
 		ret = maildir_uidlist_sync_next(ctx->uidlist_sync_ctx,
-						dp->d_name, this_new);
+						dp->d_name, flags);
 		if (ret <= 0) {
 			if (ret < 0)
 				break;
@@ -353,8 +363,6 @@
 		}
 
 		maildir_filename_get_flags(filename, &flags, custom_flags);
-		if (rec->flags & MAIL_RECENT)
-			flags |= MAIL_RECENT;
 		if ((uint8_t)flags != (rec->flags & MAIL_FLAGS_MASK) ||
 		    memcmp(custom_flags, rec->custom_flags,
 			   INDEX_CUSTOM_FLAGS_BYTE_COUNT) != 0) {
@@ -398,14 +406,17 @@
 	return ret;
 }
 
-static int maildir_sync_context(struct maildir_sync_context *ctx,
-				int *changes_r)
+static int maildir_sync_context(struct maildir_sync_context *ctx)
 {
 	int ret, new_changed, cur_changed;
 
 	if (maildir_sync_quick_check(ctx, &new_changed, &cur_changed) < 0)
 		return -1;
 
+	if (!new_changed && !cur_changed)
+		return 0;
+
+	// FIXME: don't sync cur/ directory if not needed
 	ctx->uidlist_sync_ctx = maildir_uidlist_sync_init(ctx->ibox->uidlist);
 
 	if (maildir_scan_dir(ctx, TRUE) < 0)
@@ -453,14 +464,14 @@
 {
 	struct index_mailbox *ibox = (struct index_mailbox *)box;
 	struct maildir_sync_context *ctx;
-	int changes, ret;
+	int ret;
 
 	if ((flags & MAILBOX_SYNC_FLAG_FAST) == 0 ||
 	    ibox->sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <= ioloop_time) {
 		ibox->sync_last_check = ioloop_time;
 
 		ctx = maildir_sync_context_new(ibox);
-		ret = maildir_sync_context(ctx, &changes);
+		ret = maildir_sync_context(ctx);
 		maildir_sync_deinit(ctx);
 
 		if (ret < 0)

Index: maildir-uidlist.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/maildir/maildir-uidlist.c,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -d -r1.4 -r1.5
--- maildir-uidlist.c	1 May 2004 14:30:25 -0000	1.4
+++ maildir-uidlist.c	1 May 2004 18:30:53 -0000	1.5
@@ -22,8 +22,6 @@
 #define UIDLIST_IS_LOCKED(uidlist) \
 	((uidlist)->lock_fd != -1)
 
-#define MAILDIR_UIDLIST_REC_FLAG_NEW_DIR 0x01
-
 struct maildir_uidlist_rec {
 	uint32_t uid;
 	uint32_t flags;
@@ -43,6 +41,7 @@
 
 	unsigned int version;
 	unsigned int uid_validity, next_uid, last_read_uid;
+	uint32_t first_recent_uid;
 };
 
 struct maildir_uidlist_sync_ctx {
@@ -50,8 +49,8 @@
 
 	pool_t filename_pool;
 	struct hash_table *files;
+	buffer_t *new_record_buf;
 
-	struct maildir_uidlist_rec new_rec, cur_rec;
 	unsigned int new_files:1;
 	unsigned int synced:1;
 	unsigned int failed:1;
@@ -134,8 +133,16 @@
 	i_free(uidlist);
 }
 
+static void
+maildir_uidlist_mark_recent(struct maildir_uidlist *uidlist, uint32_t uid)
+{
+	if (uidlist->first_recent_uid == 0)
+		uidlist->first_recent_uid = uid;
+	i_assert(uid >= uidlist->first_recent_uid);
+}
+
 static int maildir_uidlist_next(struct maildir_uidlist *uidlist,
-				const char *line)
+				const char *line, uint32_t last_uid)
 {
         struct maildir_uidlist_rec *rec;
 	uint32_t uid, flags;
@@ -146,6 +153,11 @@
 		line++;
 	}
 
+	if (uid <= last_uid) {
+		/* we already have this */
+		return 1;
+	}
+
 	if (uid == 0 || *line != ' ') {
 		/* invalid file */
                 mail_storage_set_critical(uidlist->ibox->box.storage,
@@ -202,9 +214,12 @@
 int maildir_uidlist_update(struct maildir_uidlist *uidlist)
 {
 	struct mail_storage *storage = uidlist->ibox->box.storage;
+	const struct maildir_uidlist_rec *rec;
 	const char *line;
 	struct istream *input;
 	struct stat st;
+	uint32_t last_uid;
+	size_t size;
 	int fd, ret;
 
 	if (uidlist->last_mtime != 0) {
@@ -239,17 +254,16 @@
 		return -1;
 	}
 
-	hash_clear(uidlist->files, FALSE);
-	if (uidlist->filename_pool != NULL)
-		p_clear(uidlist->filename_pool);
-	else {
+	if (uidlist->filename_pool == NULL) {
 		uidlist->filename_pool =
 			pool_alloconly_create("uidlist filename_pool",
 					      nearest_power(st.st_size -
 							    st.st_size/8));
 	}
 
-	buffer_set_used_size(uidlist->record_buf, 0);
+	rec = buffer_get_data(uidlist->record_buf, &size);
+	last_uid = size == 0 ? 0 : rec[(size / sizeof(*rec))-1].uid;
+
 	uidlist->version = 0;
 
 	input = i_stream_create_file(fd, default_pool, 4096, TRUE);
@@ -268,7 +282,7 @@
 	} else {
 		ret = 1;
 		while ((line = i_stream_read_next_line(input)) != NULL) {
-			if (!maildir_uidlist_next(uidlist, line)) {
+			if (!maildir_uidlist_next(uidlist, line, last_uid)) {
 				ret = 0;
 				break;
 			}
@@ -286,8 +300,9 @@
 	return ret;
 }
 
-const char *maildir_uidlist_lookup(struct maildir_uidlist *uidlist,
-				   uint32_t uid, int *new_dir_r)
+static const struct maildir_uidlist_rec *
+maildir_uidlist_lookup_rec(struct maildir_uidlist *uidlist, uint32_t uid,
+			   unsigned int *idx_r)
 {
 	const struct maildir_uidlist_rec *rec;
 	unsigned int idx, left_idx, right_idx;
@@ -310,15 +325,67 @@
 		else if (rec[idx].uid > uid)
 			right_idx = idx;
 		else {
-			*new_dir_r = (rec[idx].flags &
-				      MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) != 0;
-			return rec[idx].filename;
+			*idx_r = idx;
+			return &rec[idx];
 		}
 	}
 
+	if (idx > 0) idx--;
+	*idx_r = idx;
 	return NULL;
 }
 
+const char *
+maildir_uidlist_lookup(struct maildir_uidlist *uidlist, uint32_t uid,
+		       enum maildir_uidlist_rec_flag *flags_r)
+{
+	const struct maildir_uidlist_rec *rec;
+	unsigned int idx;
+
+	rec = maildir_uidlist_lookup_rec(uidlist, uid, &idx);
+	if (rec == NULL)
+		return NULL;
+
+	*flags_r = rec->flags;
+	return rec->filename;
+}
+
+int maildir_uidlist_is_recent(struct maildir_uidlist *uidlist, uint32_t uid)
+{
+	enum maildir_uidlist_rec_flag flags;
+
+	if (uidlist->first_recent_uid == 0 || uid < uidlist->first_recent_uid)
+		return FALSE;
+
+	if (maildir_uidlist_lookup(uidlist, uid, &flags) == NULL)
+		return FALSE;
+
+	i_assert(uidlist->first_recent_uid != uid ||
+		 (flags & MAILDIR_UIDLIST_REC_FLAG_RECENT) != 0);
+	return (flags & MAILDIR_UIDLIST_REC_FLAG_RECENT) != 0;
+}
+
+uint32_t maildir_uidlist_get_recent_count(struct maildir_uidlist *uidlist)
+{
+	const struct maildir_uidlist_rec *rec;
+	unsigned int idx;
+	size_t size;
+	uint32_t count;
+
+	if (uidlist->first_recent_uid == 0)
+		return 0;
+
+	rec = buffer_get_data(uidlist->record_buf, &size);
+	size /= sizeof(*rec);
+
+	maildir_uidlist_lookup(uidlist, uidlist->first_recent_uid, &idx);
+	for (count = 0; idx < size; idx++) {
+		if ((rec[idx].flags & MAILDIR_UIDLIST_REC_FLAG_RECENT) != 0)
+			count++;
+	}
+	return count;
+}
+
 static int maildir_uidlist_rewrite_fd(struct maildir_uidlist *uidlist,
 				      const char *temp_path)
 {
@@ -427,6 +494,8 @@
 	ctx->uidlist = uidlist;
 	ctx->filename_pool =
 		pool_alloconly_create("maildir_uidlist_sync", 16384);
+	ctx->new_record_buf =
+		buffer_create_dynamic(default_pool, 512, (size_t)-1);
 	ctx->files = hash_create(default_pool, ctx->filename_pool, 4096,
 				 maildir_hash, maildir_cmp);
 
@@ -439,7 +508,8 @@
 }
 
 int maildir_uidlist_sync_next(struct maildir_uidlist_sync_ctx *ctx,
-			      const char *filename, int new_dir)
+			      const char *filename,
+			      enum maildir_uidlist_rec_flag flags)
 {
 	struct maildir_uidlist_rec *rec;
 	char *fname;
@@ -450,12 +520,14 @@
 
 	rec = hash_lookup(ctx->files, filename);
 	if (rec != NULL) {
-		if ((rec->flags & MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) == 0) {
+		if ((rec->flags & (MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
+				   MAILDIR_UIDLIST_REC_FLAG_MOVED)) == 0) {
 			/* possibly duplicate */
 			return 0;
 		}
 
-		rec->flags &= ~MAILDIR_UIDLIST_REC_FLAG_NEW_DIR;
+		rec->flags &= ~(MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
+				MAILDIR_UIDLIST_REC_FLAG_MOVED);
 	} else {
 		rec = hash_lookup(ctx->uidlist->files, filename);
 		if (rec == NULL && !ctx->synced) {
@@ -479,11 +551,17 @@
 
 		if (rec == NULL) {
 			ctx->new_files = TRUE;
-			rec = new_dir ? &ctx->new_rec : &ctx->cur_rec;
+			rec = buffer_append_space_unsafe(ctx->new_record_buf,
+							 sizeof(*rec));
+			memset(rec, 0, sizeof(*rec));
 		}
 	}
 
+	rec->flags |= flags;
+
 	fname = p_strdup(ctx->filename_pool, filename);
+	if (rec->filename == NULL)
+		rec->filename = fname;
 	hash_insert(ctx->files, fname, rec);
 	return 1;
 }
@@ -513,48 +591,50 @@
 {
 	struct maildir_uidlist *uidlist = ctx->uidlist;
 	struct maildir_uidlist_rec *rec;
-	struct hash_iterate_context *iter;
 	void *key, *value;
 	size_t size;
 	unsigned int src, dest;
 
+	/* @UNSAFE */
+
 	rec = buffer_get_modifyable_data(uidlist->record_buf, &size);
 	size /= sizeof(*rec);
 
 	/* update filename pointers, skip deleted messages */
 	for (dest = src = 0; src < size; src++) {
-		if (hash_lookup_full(ctx->files, rec[src].filename,
-				     &key, &value)) {
-			rec[dest].uid = rec[src].uid;
-			rec[dest].flags = rec[src].flags;
-			rec[dest].filename = key;
-			dest++;
+		if (!hash_lookup_full(ctx->files, rec[src].filename,
+				      &key, &value))
+			continue;
+
+		rec[dest].uid = rec[src].uid;
+		rec[dest].flags =
+			rec[src].flags & ~MAILDIR_UIDLIST_REC_FLAG_MOVED;
+		rec[dest].filename = key;
+		if ((rec[dest].flags & MAILDIR_UIDLIST_REC_FLAG_RECENT) != 0) {
+			maildir_uidlist_mark_recent(ctx->uidlist,
+						    rec[dest].uid);
 		}
+		dest++;
 	}
 	buffer_set_used_size(uidlist->record_buf, dest * sizeof(*rec));
 
-	/* append new files */
-	iter = hash_iterate_init(ctx->files);
-	while (hash_iterate(iter, &key, &value)) {
-		if (value == &ctx->new_rec ||
-		    value == &ctx->cur_rec) {
-			rec = buffer_append_space_unsafe(uidlist->record_buf,
-							 sizeof(*rec));
-			rec->flags = value == &ctx->cur_rec ?
-				0 : MAILDIR_UIDLIST_REC_FLAG_NEW_DIR;
-			rec->filename = key;
-			hash_update(ctx->files, key, rec);
-		}
-	}
-	hash_iterate_deinit(iter);
+	buffer_append_buf(uidlist->record_buf, ctx->new_record_buf,
+			  0, (size_t)-1);
 
 	rec = buffer_get_modifyable_data(uidlist->record_buf, &size);
 	size /= sizeof(*rec);
 
 	/* sort new files and assign UIDs for them */
 	qsort(rec + dest, size - dest, sizeof(*rec), maildir_time_cmp);
-	for (; dest < size; dest++)
+	for (; dest < size; dest++) {
 		rec[dest].uid = uidlist->next_uid++;
+		rec[dest].flags &= ~MAILDIR_UIDLIST_REC_FLAG_MOVED;
+
+		if ((rec[dest].flags & MAILDIR_UIDLIST_REC_FLAG_RECENT) != 0) {
+			maildir_uidlist_mark_recent(ctx->uidlist,
+						    rec[dest].uid);
+		}
+	}
 
 	hash_destroy(uidlist->files);
 	uidlist->files = ctx->files;
@@ -587,6 +667,7 @@
 		hash_destroy(ctx->files);
 	if (ctx->filename_pool != NULL)
 		pool_unref(ctx->filename_pool);
+	buffer_free(ctx->new_record_buf);
 	i_free(ctx);
 	return ret;
 }

Index: maildir-uidlist.h
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/maildir/maildir-uidlist.h,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- maildir-uidlist.h	27 Apr 2004 20:25:54 -0000	1.1
+++ maildir-uidlist.h	1 May 2004 18:30:53 -0000	1.2
@@ -3,6 +3,12 @@
 
 #define MAILDIR_UIDLIST_NAME "dovecot-uidlist"
 
+enum maildir_uidlist_rec_flag {
+	MAILDIR_UIDLIST_REC_FLAG_NEW_DIR	= 0x01,
+	MAILDIR_UIDLIST_REC_FLAG_MOVED		= 0x02,
+	MAILDIR_UIDLIST_REC_FLAG_RECENT		= 0x04
+};
+
 int maildir_uidlist_try_lock(struct maildir_uidlist *uidlist);
 void maildir_uidlist_unlock(struct maildir_uidlist *uidlist);
 
@@ -13,14 +19,20 @@
 int maildir_uidlist_update(struct maildir_uidlist *uidlist);
 
 /* Returns uidlist record for given filename, or NULL if not found. */
-const char *maildir_uidlist_lookup(struct maildir_uidlist *uidlist,
-				   uint32_t uid, int *new_dir_r);
+const char *
+maildir_uidlist_lookup(struct maildir_uidlist *uidlist, uint32_t uid,
+		       enum maildir_uidlist_rec_flag *flags_r);
+/* Returns TRUE if mail with given UID is recent. */
+int maildir_uidlist_is_recent(struct maildir_uidlist *uidlist, uint32_t uid);
+/* Returns number of recent messages. */
+uint32_t maildir_uidlist_get_recent_count(struct maildir_uidlist *uidlist);
 
 /* Sync uidlist with what's actually on maildir. */
 struct maildir_uidlist_sync_ctx *
 maildir_uidlist_sync_init(struct maildir_uidlist *uidlist);
 int maildir_uidlist_sync_next(struct maildir_uidlist_sync_ctx *ctx,
-			      const char *filename, int new_dir);
+			      const char *filename,
+			      enum maildir_uidlist_rec_flag flags);
 int maildir_uidlist_sync_deinit(struct maildir_uidlist_sync_ctx *ctx);
 
 /* List all maildir files. */

Index: maildir-util.c
===================================================================
RCS file: /home/cvs/dovecot/src/lib-storage/index/maildir/maildir-util.c,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -d -r1.1 -r1.2
--- maildir-util.c	27 Apr 2004 20:25:54 -0000	1.1
+++ maildir-util.c	1 May 2004 18:30:53 -0000	1.2
@@ -15,13 +15,14 @@
 			       maildir_file_do_func *func, void *context)
 {
 	const char *fname, *path;
-	int ret, new_dir;
+        enum maildir_uidlist_rec_flag flags;
+	int ret;
 
-	fname = maildir_uidlist_lookup(ibox->uidlist, uid, &new_dir);
+	fname = maildir_uidlist_lookup(ibox->uidlist, uid, &flags);
 	if (fname == NULL)
 		return -2; /* expunged */
 
-	if (new_dir) {
+	if ((flags & MAILDIR_UIDLIST_REC_FLAG_NEW_DIR) != 0) {
 		/* probably in new/ dir */
 		path = t_strconcat(ibox->path, "/new/", fname, NULL);
 		ret = func(ibox, path, context);