dovecot-2.1: imapc: Redesigned remote sequence <-> index file re...
dovecot at dovecot.org
dovecot at dovecot.org
Sun Sep 4 18:29:50 EEST 2011
details: http://hg.dovecot.org/dovecot-2.1/rev/26c38da7b7fc
changeset: 13387:26c38da7b7fc
user: Timo Sirainen <tss at iki.fi>
date: Sun Sep 04 17:59:58 2011 +0300
description:
imapc: Redesigned remote sequence <-> index file record mapping is done.
The previous code didn't work when multiple connections modified the same
index files.
diffstat:
src/lib-storage/index/imapc/Makefile.am | 23 +--
src/lib-storage/index/imapc/imapc-client-private.h | 2 +-
src/lib-storage/index/imapc/imapc-client.c | 12 +-
src/lib-storage/index/imapc/imapc-client.h | 4 +-
src/lib-storage/index/imapc/imapc-connection.c | 1 -
src/lib-storage/index/imapc/imapc-mail.c | 10 +-
src/lib-storage/index/imapc/imapc-mailbox.c | 162 +++++++++-----------
src/lib-storage/index/imapc/imapc-msgmap.c | 85 +++++++++++
src/lib-storage/index/imapc/imapc-msgmap.h | 17 ++
src/lib-storage/index/imapc/imapc-seqmap.c | 124 ----------------
src/lib-storage/index/imapc/imapc-seqmap.h | 27 ---
src/lib-storage/index/imapc/imapc-storage.h | 7 +-
src/lib-storage/index/imapc/imapc-sync.c | 61 ++++---
src/lib-storage/index/imapc/test-imapc-seqmap.c | 111 --------------
14 files changed, 232 insertions(+), 414 deletions(-)
diffs (truncated from 968 to 300 lines):
diff -r 1da4bc23d4dc -r 26c38da7b7fc src/lib-storage/index/imapc/Makefile.am
--- a/src/lib-storage/index/imapc/Makefile.am Sun Sep 04 16:34:23 2011 +0300
+++ b/src/lib-storage/index/imapc/Makefile.am Sun Sep 04 17:59:58 2011 +0300
@@ -20,8 +20,8 @@
imapc-mail.c \
imapc-mail-fetch.c \
imapc-mailbox.c \
+ imapc-msgmap.c \
imapc-save.c \
- imapc-seqmap.c \
imapc-settings.c \
imapc-sync.c \
imapc-storage.c
@@ -32,29 +32,10 @@
imapc-connection.h \
imapc-list.h \
imapc-mail.h \
- imapc-seqmap.h \
+ imapc-msgmap.h \
imapc-settings.h \
imapc-storage.h \
imapc-sync.h
pkginc_libdir=$(pkgincludedir)
pkginc_lib_HEADERS = $(headers)
-
-test_programs = \
- test-imapc-seqmap
-
-noinst_PROGRAMS = $(test_programs)
-
-test_libs = \
- ../../../lib-test/libtest.la \
- ../../../lib/liblib.la
-
-test_imapc_seqmap_SOURCES = test-imapc-seqmap.c
-test_imapc_seqmap_LDADD = imapc-seqmap.lo $(test_libs)
-test_imapc_seqmap_DEPENDENCIES = imapc-seqmap.lo $(test_libs)
-
-check: check-am check-test
-check-test: all-am
- for bin in $(test_programs); do \
- if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \
- done
diff -r 1da4bc23d4dc -r 26c38da7b7fc src/lib-storage/index/imapc/imapc-client-private.h
--- a/src/lib-storage/index/imapc/imapc-client-private.h Sun Sep 04 16:34:23 2011 +0300
+++ b/src/lib-storage/index/imapc/imapc-client-private.h Sun Sep 04 17:59:58 2011 +0300
@@ -26,7 +26,7 @@
struct imapc_client_mailbox {
struct imapc_client *client;
struct imapc_connection *conn;
- struct imapc_seqmap *seqmap;
+ struct imapc_msgmap *msgmap;
void *untagged_box_context;
unsigned int pending_box_command_count;
diff -r 1da4bc23d4dc -r 26c38da7b7fc src/lib-storage/index/imapc/imapc-client.c
--- a/src/lib-storage/index/imapc/imapc-client.c Sun Sep 04 16:34:23 2011 +0300
+++ b/src/lib-storage/index/imapc/imapc-client.c Sun Sep 04 17:59:58 2011 +0300
@@ -6,7 +6,7 @@
#include "ioloop.h"
#include "safe-mkstemp.h"
#include "iostream-ssl.h"
-#include "imapc-seqmap.h"
+#include "imapc-msgmap.h"
#include "imapc-connection.h"
#include "imapc-client-private.h"
@@ -219,7 +219,7 @@
conn = imapc_client_get_unboxed_connection(client);
conn->box = box;
box->conn = conn->conn;
- box->seqmap = imapc_seqmap_init();
+ box->msgmap = imapc_msgmap_init();
imapc_connection_select(box, name, examine, callback, context);
return box;
@@ -247,7 +247,7 @@
if (box->conn != NULL)
imapc_connection_unselect(box);
- imapc_seqmap_deinit(&box->seqmap);
+ imapc_msgmap_deinit(&box->msgmap);
i_free(box);
}
@@ -340,10 +340,10 @@
va_end(args);
}
-struct imapc_seqmap *
-imapc_client_mailbox_get_seqmap(struct imapc_client_mailbox *box)
+struct imapc_msgmap *
+imapc_client_mailbox_get_msgmap(struct imapc_client_mailbox *box)
{
- return box->seqmap;
+ return box->msgmap;
}
void imapc_client_mailbox_idle(struct imapc_client_mailbox *box)
diff -r 1da4bc23d4dc -r 26c38da7b7fc src/lib-storage/index/imapc/imapc-client.h
--- a/src/lib-storage/index/imapc/imapc-client.h Sun Sep 04 16:34:23 2011 +0300
+++ b/src/lib-storage/index/imapc/imapc-client.h Sun Sep 04 17:59:58 2011 +0300
@@ -135,8 +135,8 @@
imapc_command_callback_t *callback,
void *context, const char *cmd_fmt, ...)
ATTR_FORMAT(4, 5);
-struct imapc_seqmap *
-imapc_client_mailbox_get_seqmap(struct imapc_client_mailbox *box);
+struct imapc_msgmap *
+imapc_client_mailbox_get_msgmap(struct imapc_client_mailbox *box);
void imapc_client_mailbox_idle(struct imapc_client_mailbox *box);
bool imapc_client_mailbox_is_connected(struct imapc_client_mailbox *box);
diff -r 1da4bc23d4dc -r 26c38da7b7fc src/lib-storage/index/imapc/imapc-connection.c
--- a/src/lib-storage/index/imapc/imapc-connection.c Sun Sep 04 16:34:23 2011 +0300
+++ b/src/lib-storage/index/imapc/imapc-connection.c Sun Sep 04 17:59:58 2011 +0300
@@ -14,7 +14,6 @@
#include "imap-util.h"
#include "imap-parser.h"
#include "imapc-client-private.h"
-#include "imapc-seqmap.h"
#include "imapc-connection.h"
#include <unistd.h>
diff -r 1da4bc23d4dc -r 26c38da7b7fc src/lib-storage/index/imapc/imapc-mail.c
--- a/src/lib-storage/index/imapc/imapc-mail.c Sun Sep 04 16:34:23 2011 +0300
+++ b/src/lib-storage/index/imapc/imapc-mail.c Sun Sep 04 17:59:58 2011 +0300
@@ -4,7 +4,7 @@
#include "str.h"
#include "istream.h"
#include "imap-envelope.h"
-#include "imapc-seqmap.h"
+#include "imapc-msgmap.h"
#include "imapc-mail.h"
#include "imapc-client.h"
#include "imapc-storage.h"
@@ -101,8 +101,8 @@
static bool imapc_mail_is_expunged(struct mail *_mail)
{
struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box;
- struct imapc_seqmap *seqmap;
- uint32_t lseq;
+ struct imapc_msgmap *msgmap;
+ uint32_t lseq, rseq;
/* first we'll need to convert the mail's sequence to sync_view's
sequence. if there's no sync_view, then no mails have been
@@ -113,8 +113,8 @@
if (!mail_index_lookup_seq(mbox->sync_view, _mail->uid, &lseq))
return TRUE;
- seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box);
- return imapc_seqmap_lseq_to_rseq(seqmap, lseq) == 0;
+ msgmap = imapc_client_mailbox_get_msgmap(mbox->client_box);
+ return !imapc_msgmap_uid_to_rseq(msgmap, _mail->uid, &rseq);
}
static int
diff -r 1da4bc23d4dc -r 26c38da7b7fc src/lib-storage/index/imapc/imapc-mailbox.c
--- a/src/lib-storage/index/imapc/imapc-mailbox.c Sun Sep 04 16:34:23 2011 +0300
+++ b/src/lib-storage/index/imapc/imapc-mailbox.c Sun Sep 04 17:59:58 2011 +0300
@@ -5,8 +5,8 @@
#include "imap-arg.h"
#include "imap-util.h"
#include "imapc-client.h"
-#include "imapc-seqmap.h"
#include "imapc-mail.h"
+#include "imapc-msgmap.h"
#include "imapc-sync.h"
#include "imapc-storage.h"
@@ -22,6 +22,8 @@
mbox->box.name, t_strdup_vprintf(reason, va));
va_end(va);
+ sleep(3600);
+
mail_index_mark_corrupted(mbox->box.index);
imapc_client_mailbox_disconnect(mbox->client_box);
}
@@ -44,8 +46,6 @@
MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
mbox->delayed_sync_view =
mail_index_transaction_open_updated_view(mbox->delayed_sync_trans);
- mbox->min_append_uid =
- mail_index_get_header(mbox->delayed_sync_view)->next_uid;
}
int imapc_mailbox_commit_delayed_trans(struct imapc_mailbox *mbox,
@@ -73,10 +73,7 @@
struct imapc_mailbox *mbox)
{
struct mail_index_view *view = mbox->delayed_sync_view;
- uint32_t rcount = reply->num;
const struct mail_index_header *hdr;
- struct imapc_seqmap *seqmap;
- uint32_t next_lseq, next_rseq;
if (mbox == NULL)
return;
@@ -84,26 +81,10 @@
if (view == NULL)
view = imapc_mailbox_get_sync_view(mbox);
- if (rcount == 0) {
- /* nothing in this mailbox */
- return;
- }
if (mbox->opening) {
/* We don't know the latest flags, refresh them. */
mbox->sync_fetch_first_uid = 1;
- } else {
- seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box);
- next_lseq = mail_index_view_get_messages_count(view) + 1;
- next_rseq = imapc_seqmap_lseq_to_rseq(seqmap, next_lseq);
- if (rcount < next_rseq) {
- if (rcount == next_rseq-1) {
- /* duplicate EXISTS - ignore */
- return;
- }
- imapc_mailbox_set_corrupted(mbox,
- "EXISTS reply shrank mailbox size");
- return;
- }
+ } else if (mbox->sync_fetch_first_uid != 1) {
hdr = mail_index_get_header(view);
mbox->sync_fetch_first_uid = hdr->next_uid;
}
@@ -152,13 +133,13 @@
struct imapc_mailbox *mbox)
{
uint32_t lseq, rseq = reply->num;
- struct imapc_seqmap *seqmap;
struct imapc_mail *const *mailp;
const struct imap_arg *list, *flags_list;
const char *atom;
const struct mail_index_record *rec = NULL;
+ struct imapc_msgmap *msgmap;
enum mail_flags flags;
- uint32_t uid, cur_count;
+ uint32_t fetch_uid, uid, msg_count;
unsigned int i, j;
ARRAY_TYPE(const_string) keywords = ARRAY_INIT;
bool seen_flags = FALSE;
@@ -166,14 +147,14 @@
if (mbox == NULL || rseq == 0 || !imap_arg_get_list(reply->args, &list))
return;
- uid = 0; flags = 0;
+ fetch_uid = 0; flags = 0;
for (i = 0; list[i].type != IMAP_ARG_EOL; i += 2) {
if (!imap_arg_get_atom(&list[i], &atom))
return;
if (strcasecmp(atom, "UID") == 0) {
if (!imap_arg_get_atom(&list[i+1], &atom) ||
- str_to_uint32(atom, &uid) < 0)
+ str_to_uint32(atom, &fetch_uid) < 0)
return;
} else if (strcasecmp(atom, "FLAGS") == 0) {
if (!imap_arg_get_list(&list[i+1], &flags_list))
@@ -196,12 +177,52 @@
/* FIXME: need to do something about recent flags */
flags &= ~MAIL_RECENT;
- seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box);
- lseq = imapc_seqmap_rseq_to_lseq(seqmap, rseq);
+ imapc_mailbox_init_delayed_trans(mbox);
- /* fetch_mails' view is different from sync_view, so we can't compare
- their sequences directly. that is why this code supports only
- UID FETCH commands which are guaranteed to have UID in the reply. */
+ msgmap = imapc_client_mailbox_get_msgmap(mbox->client_box);
+ msg_count = imapc_msgmap_count(msgmap);
+ if (rseq > msg_count) {
+ /* newly seen message */
+ if (!mbox->syncing || fetch_uid == 0 || rseq != msg_count+1)
+ return;
+ uid = fetch_uid;
+
+ if (uid < imapc_msgmap_uidnext(msgmap)) {
+ imapc_mailbox_set_corrupted(mbox,
+ "Expunged message reappeared "
+ "(uid=%u < next_uid=%u)",
+ uid, imapc_msgmap_uidnext(msgmap));
+ return;
+ }
+
+ imapc_msgmap_append(msgmap, rseq, uid);
+ if (uid < mbox->min_append_uid) {
+ /* message is already added to index */
+ lseq = 0;
+ } else {
+ mail_index_append(mbox->delayed_sync_trans, uid, &lseq);
+ mbox->min_append_uid = uid + 1;
+ }
+ } else {
+ uid = imapc_msgmap_rseq_to_uid(msgmap, rseq);
+ if (uid != fetch_uid && fetch_uid != 0) {
More information about the dovecot-cvs
mailing list