From dovecot at dovecot.org Thu Jul 2 09:08:57 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 02 Jul 2015 09:08:57 +0000 Subject: dovecot-2.2: mdbox: Make sure rebuilding doesn't try to add a co... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ae19beafa55b changeset: 18899:ae19beafa55b user: Timo Sirainen date: Thu Jul 02 11:06:04 2015 +0200 description: mdbox: Make sure rebuilding doesn't try to add a copy of the message too many times. Currently it just silently overflowed the 16bit refcount, which caused problems later. diffstat: src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-) diffs (30 lines): diff -r 7a6452869981 -r ae19beafa55b src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c --- a/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Tue Jun 30 14:26:00 2015 +0300 +++ b/src/lib-storage/index/dbox-multi/mdbox-storage-rebuild.c Thu Jul 02 11:06:04 2015 +0200 @@ -20,6 +20,8 @@ #include #include +#define REBUILD_MAX_REFCOUNT 32768 + struct mdbox_rebuild_msg { struct mdbox_rebuild_msg *guid_hash_next; @@ -458,7 +460,8 @@ GUID exists multiple times */ } - if (rec != NULL) T_BEGIN { + if (rec != NULL && + rec->refcount < REBUILD_MAX_REFCOUNT) T_BEGIN { /* keep this message. add it to mailbox index. */ i_assert(map_uid != 0); rec->refcount++; @@ -758,6 +761,7 @@ mail_index_update_ext(ctx->prev_msg.trans, seq, mbox->guid_ext_id, msg->guid_128, NULL); + i_assert(msg->refcount == 0); msg->refcount++; return 0; } From dovecot at dovecot.org Fri Jul 3 10:57:13 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 03 Jul 2015 10:57:13 +0000 Subject: dovecot-2.2: quota: Fixed error handling in quota counting code. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f3847ac93623 changeset: 18900:f3847ac93623 user: Timo Sirainen date: Fri Jul 03 13:54:18 2015 +0300 description: quota: Fixed error handling in quota counting code. Errors weren't logged and some error checking was missing. diffstat: src/plugins/quota/quota-count.c | 27 +++++++++++++++++++++++---- 1 files changed, 23 insertions(+), 4 deletions(-) diffs (64 lines): diff -r ae19beafa55b -r f3847ac93623 src/plugins/quota/quota-count.c --- a/src/plugins/quota/quota-count.c Thu Jul 02 11:06:04 2015 +0200 +++ b/src/plugins/quota/quota-count.c Fri Jul 03 13:54:18 2015 +0300 @@ -19,6 +19,7 @@ struct mail *mail; struct mail_search_args *search_args; enum mail_error error; + const char *errstr; uoff_t size; int ret = 0; @@ -30,10 +31,13 @@ box = mailbox_alloc(ns->list, vname, MAILBOX_FLAG_READONLY); if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) { - error = mailbox_get_last_mail_error(box); + errstr = mailbox_get_last_error(box, &error); mailbox_free(&box); - if (error == MAIL_ERROR_TEMP) + if (error == MAIL_ERROR_TEMP) { + i_error("quota: Couldn't sync mailbox %s: %s", + vname, errstr); return -1; + } /* non-temporary error, e.g. ACLs denied access. */ return 0; } @@ -49,10 +53,22 @@ while (mailbox_search_next(ctx, &mail)) { if (mail_get_physical_size(mail, &size) == 0) *bytes_r += size; + else { + errstr = mailbox_get_last_error(box, &error); + if (error != MAIL_ERROR_EXPUNGED) { + i_error("quota: Couldn't get size of mail UID %u in %s: %s", + mail->uid, vname, mailbox_get_last_error(box, NULL)); + ret = -1; + break; + } + } *count_r += 1; } - if (mailbox_search_deinit(&ctx) < 0) + if (mailbox_search_deinit(&ctx) < 0) { + i_error("quota: Listing mails in %s failed: %s", + vname, mailbox_get_last_error(box, NULL)); ret = -1; + } if (ret < 0) mailbox_transaction_rollback(&trans); @@ -83,8 +99,11 @@ break; } } - if (mailbox_list_iter_deinit(&ctx) < 0) + if (mailbox_list_iter_deinit(&ctx) < 0) { + i_error("quota: Listing namespace '%s' failed: %s", + ns->prefix, mailbox_list_get_last_error(ns->list, NULL)); ret = -1; + } if (ns->prefix_len > 0 && ret == 0 && (ns->prefix_len != 6 || strncasecmp(ns->prefix, "INBOX", 5) != 0)) { /* if the namespace prefix itself exists, count it also */ From dovecot at dovecot.org Fri Jul 3 10:58:09 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 03 Jul 2015 10:58:09 +0000 Subject: dovecot-2.2: quota: Even if quota counting fails, commit the mai... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/02088971322e changeset: 18901:02088971322e user: Timo Sirainen date: Fri Jul 03 13:55:17 2015 +0300 description: quota: Even if quota counting fails, commit the mailbox transaction. The only changes in the transaction are changes to dovecot.index.cache file and we don't want to rollback those. diffstat: src/plugins/quota/quota-count.c | 6 +----- 1 files changed, 1 insertions(+), 5 deletions(-) diffs (16 lines): diff -r f3847ac93623 -r 02088971322e src/plugins/quota/quota-count.c --- a/src/plugins/quota/quota-count.c Fri Jul 03 13:54:18 2015 +0300 +++ b/src/plugins/quota/quota-count.c Fri Jul 03 13:55:17 2015 +0300 @@ -69,11 +69,7 @@ vname, mailbox_get_last_error(box, NULL)); ret = -1; } - - if (ret < 0) - mailbox_transaction_rollback(&trans); - else - (void)mailbox_transaction_commit(&trans); + (void)mailbox_transaction_commit(&trans); mailbox_free(&box); return ret; From pigeonhole at rename-it.nl Mon Jul 6 21:48:45 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 06 Jul 2015 23:48:45 +0200 Subject: dovecot-2.2-pigeonhole: doveadm-sieve: Fixed one memory leak. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/1d5a1d4f7bb7 changeset: 2080:1d5a1d4f7bb7 user: Stephan Bosch date: Mon Jul 06 23:47:05 2015 +0200 description: doveadm-sieve: Fixed one memory leak. diffstat: src/plugins/doveadm-sieve/doveadm-sieve-sync.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r add4db882327 -r 1d5a1d4f7bb7 src/plugins/doveadm-sieve/doveadm-sieve-sync.c --- a/src/plugins/doveadm-sieve/doveadm-sieve-sync.c Fri May 22 23:38:58 2015 +0200 +++ b/src/plugins/doveadm-sieve/doveadm-sieve-sync.c Mon Jul 06 23:47:05 2015 +0200 @@ -439,6 +439,7 @@ inputs[1] = input; inputs[2] = NULL; value_r->value_stream = i_stream_create_concat(inputs); + i_stream_unref(&inputs[0]); } sieve_script_unref(&script); return 1; From dovecot at dovecot.org Sat Jul 11 09:20:12 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 11 Jul 2015 09:20:12 +0000 Subject: dovecot-2.2: lib-storage: Moved mailbox vsize calculation code t... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/819649386f93 changeset: 18902:819649386f93 user: Timo Sirainen date: Sat Jul 11 12:00:38 2015 +0300 description: lib-storage: Moved mailbox vsize calculation code to its own file. No functional differences. diffstat: src/lib-storage/index/Makefile.am | 1 + src/lib-storage/index/index-mailbox-size.c | 114 +++++++++++++++++++++++++++++ src/lib-storage/index/index-status.c | 112 +---------------------------- src/lib-storage/index/index-storage.h | 2 + 4 files changed, 118 insertions(+), 111 deletions(-) diffs (270 lines): diff -r 02088971322e -r 819649386f93 src/lib-storage/index/Makefile.am --- a/src/lib-storage/index/Makefile.am Fri Jul 03 13:55:17 2015 +0300 +++ b/src/lib-storage/index/Makefile.am Sat Jul 11 12:00:38 2015 +0300 @@ -20,6 +20,7 @@ index-mail-binary.c \ index-mail-headers.c \ index-mailbox-check.c \ + index-mailbox-size.c \ index-rebuild.c \ index-search.c \ index-search-result.c \ diff -r 02088971322e -r 819649386f93 src/lib-storage/index/index-mailbox-size.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/index/index-mailbox-size.c Sat Jul 11 12:00:38 2015 +0300 @@ -0,0 +1,114 @@ +/* Copyright (c) 2002-2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "mail-search-build.h" +#include "index-storage.h" + +static int +virtual_size_add_new(struct mailbox *box, + struct index_vsize_header *vsize_hdr) +{ + struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); + const struct mail_index_header *hdr; + struct mailbox_transaction_context *trans; + struct mail_search_context *search_ctx; + struct mail_search_args *search_args; + struct mail *mail; + uint32_t seq1, seq2; + uoff_t vsize; + int ret = 0; + + hdr = mail_index_get_header(box->view); + if (vsize_hdr->highest_uid == 0) + seq2 = 0; + else if (!mail_index_lookup_seq_range(box->view, 1, + vsize_hdr->highest_uid, + &seq1, &seq2)) + seq2 = 0; + + if (vsize_hdr->message_count != seq2) { + if (vsize_hdr->message_count < seq2) { + mail_storage_set_critical(box->storage, + "vsize-hdr has invalid message-count (%u < %u)", + vsize_hdr->message_count, seq2); + } else { + /* some messages have been expunged, rescan */ + } + memset(vsize_hdr, 0, sizeof(*vsize_hdr)); + seq2 = 0; + } + + search_args = mail_search_build_init(); + mail_search_build_add_seqset(search_args, seq2 + 1, + hdr->messages_count); + + trans = mailbox_transaction_begin(box, 0); + search_ctx = mailbox_search_init(trans, search_args, NULL, + MAIL_FETCH_VIRTUAL_SIZE, NULL); + while (mailbox_search_next(search_ctx, &mail)) { + if (mail_get_virtual_size(mail, &vsize) < 0) { + if (mail->expunged) + continue; + ret = -1; + break; + } + vsize_hdr->vsize += vsize; + vsize_hdr->highest_uid = mail->uid; + vsize_hdr->message_count++; + } + if (mailbox_search_deinit(&search_ctx) < 0) + ret = -1; + mail_search_args_unref(&search_args); + + if (ret == 0) { + /* success, cache all */ + vsize_hdr->highest_uid = hdr->next_uid - 1; + } else { + /* search failed, cache only up to highest seen uid */ + } + mail_index_update_header_ext(trans->itrans, ibox->vsize_hdr_ext_id, + 0, vsize_hdr, sizeof(*vsize_hdr)); + (void)mailbox_transaction_commit(&trans); + return ret; +} + +int index_mailbox_get_virtual_size(struct mailbox *box, + struct mailbox_metadata *metadata_r) +{ + struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); + struct index_vsize_header vsize_hdr; + struct mailbox_status status; + const void *data; + size_t size; + int ret; + + mailbox_get_open_status(box, STATUS_MESSAGES | STATUS_UIDNEXT, &status); + mail_index_get_header_ext(box->view, ibox->vsize_hdr_ext_id, + &data, &size); + if (size == sizeof(vsize_hdr)) + memcpy(&vsize_hdr, data, sizeof(vsize_hdr)); + else { + if (size != 0) { + mail_storage_set_critical(box->storage, + "vsize-hdr has invalid size: %"PRIuSIZE_T, + size); + } + memset(&vsize_hdr, 0, sizeof(vsize_hdr)); + } + + if (vsize_hdr.highest_uid + 1 == status.uidnext && + vsize_hdr.message_count == status.messages) { + /* up to date */ + metadata_r->virtual_size = vsize_hdr.vsize; + return 0; + } + if (vsize_hdr.highest_uid >= status.uidnext) { + mail_storage_set_critical(box->storage, + "vsize-hdr has invalid highest-uid (%u >= %u)", + vsize_hdr.highest_uid, status.uidnext); + memset(&vsize_hdr, 0, sizeof(vsize_hdr)); + } + ret = virtual_size_add_new(box, &vsize_hdr); + metadata_r->virtual_size = vsize_hdr.vsize; + return ret; +} diff -r 02088971322e -r 819649386f93 src/lib-storage/index/index-status.c --- a/src/lib-storage/index/index-status.c Fri Jul 03 13:55:17 2015 +0300 +++ b/src/lib-storage/index/index-status.c Sat Jul 11 12:00:38 2015 +0300 @@ -261,116 +261,6 @@ metadata_r->precache_fields = cache; } -static int -virtual_size_add_new(struct mailbox *box, - struct index_vsize_header *vsize_hdr) -{ - struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); - const struct mail_index_header *hdr; - struct mailbox_transaction_context *trans; - struct mail_search_context *search_ctx; - struct mail_search_args *search_args; - struct mail *mail; - uint32_t seq1, seq2; - uoff_t vsize; - int ret = 0; - - hdr = mail_index_get_header(box->view); - if (vsize_hdr->highest_uid == 0) - seq2 = 0; - else if (!mail_index_lookup_seq_range(box->view, 1, - vsize_hdr->highest_uid, - &seq1, &seq2)) - seq2 = 0; - - if (vsize_hdr->message_count != seq2) { - if (vsize_hdr->message_count < seq2) { - mail_storage_set_critical(box->storage, - "vsize-hdr has invalid message-count (%u < %u)", - vsize_hdr->message_count, seq2); - } else { - /* some messages have been expunged, rescan */ - } - memset(vsize_hdr, 0, sizeof(*vsize_hdr)); - seq2 = 0; - } - - search_args = mail_search_build_init(); - mail_search_build_add_seqset(search_args, seq2 + 1, - hdr->messages_count); - - trans = mailbox_transaction_begin(box, 0); - search_ctx = mailbox_search_init(trans, search_args, NULL, - MAIL_FETCH_VIRTUAL_SIZE, NULL); - while (mailbox_search_next(search_ctx, &mail)) { - if (mail_get_virtual_size(mail, &vsize) < 0) { - if (mail->expunged) - continue; - ret = -1; - break; - } - vsize_hdr->vsize += vsize; - vsize_hdr->highest_uid = mail->uid; - vsize_hdr->message_count++; - } - if (mailbox_search_deinit(&search_ctx) < 0) - ret = -1; - mail_search_args_unref(&search_args); - - if (ret == 0) { - /* success, cache all */ - vsize_hdr->highest_uid = hdr->next_uid - 1; - } else { - /* search failed, cache only up to highest seen uid */ - } - mail_index_update_header_ext(trans->itrans, ibox->vsize_hdr_ext_id, - 0, vsize_hdr, sizeof(*vsize_hdr)); - (void)mailbox_transaction_commit(&trans); - return ret; -} - -static int -get_metadata_virtual_size(struct mailbox *box, - struct mailbox_metadata *metadata_r) -{ - struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); - struct index_vsize_header vsize_hdr; - struct mailbox_status status; - const void *data; - size_t size; - int ret; - - mailbox_get_open_status(box, STATUS_MESSAGES | STATUS_UIDNEXT, &status); - mail_index_get_header_ext(box->view, ibox->vsize_hdr_ext_id, - &data, &size); - if (size == sizeof(vsize_hdr)) - memcpy(&vsize_hdr, data, sizeof(vsize_hdr)); - else { - if (size != 0) { - mail_storage_set_critical(box->storage, - "vsize-hdr has invalid size: %"PRIuSIZE_T, - size); - } - memset(&vsize_hdr, 0, sizeof(vsize_hdr)); - } - - if (vsize_hdr.highest_uid + 1 == status.uidnext && - vsize_hdr.message_count == status.messages) { - /* up to date */ - metadata_r->virtual_size = vsize_hdr.vsize; - return 0; - } - if (vsize_hdr.highest_uid >= status.uidnext) { - mail_storage_set_critical(box->storage, - "vsize-hdr has invalid highest-uid (%u >= %u)", - vsize_hdr.highest_uid, status.uidnext); - memset(&vsize_hdr, 0, sizeof(vsize_hdr)); - } - ret = virtual_size_add_new(box, &vsize_hdr); - metadata_r->virtual_size = vsize_hdr.vsize; - return ret; -} - int index_mailbox_get_metadata(struct mailbox *box, enum mailbox_metadata_items items, struct mailbox_metadata *metadata_r) @@ -396,7 +286,7 @@ } if ((items & MAILBOX_METADATA_VIRTUAL_SIZE) != 0) { - if (get_metadata_virtual_size(box, metadata_r) < 0) + if (index_mailbox_get_virtual_size(box, metadata_r) < 0) return -1; } if ((items & MAILBOX_METADATA_CACHE_FIELDS) != 0) diff -r 02088971322e -r 819649386f93 src/lib-storage/index/index-storage.h --- a/src/lib-storage/index/index-storage.h Fri Jul 03 13:55:17 2015 +0300 +++ b/src/lib-storage/index/index-storage.h Sat Jul 11 12:00:38 2015 +0300 @@ -117,6 +117,8 @@ int index_mailbox_get_metadata(struct mailbox *box, enum mailbox_metadata_items items, struct mailbox_metadata *metadata_r); +int index_mailbox_get_virtual_size(struct mailbox *box, + struct mailbox_metadata *metadata_r); int index_storage_attribute_set(struct mailbox_transaction_context *t, enum mail_attribute_type type, const char *key, From dovecot at dovecot.org Sat Jul 11 09:20:12 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 11 Jul 2015 09:20:12 +0000 Subject: dovecot-2.2: lib-storage: Renamed struct index_vsize_header to s... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7cdef26d857d changeset: 18903:7cdef26d857d user: Timo Sirainen date: Sat Jul 11 12:03:56 2015 +0300 description: lib-storage: Renamed struct index_vsize_header to struct mailbox_index_vsize Also moved the struct to mail-storage-private.h for more global access. diffstat: src/lib-storage/index/index-mailbox-size.c | 4 ++-- src/lib-storage/index/index-storage.c | 2 +- src/lib-storage/index/index-storage.h | 6 ------ src/lib-storage/mail-storage-private.h | 6 ++++++ 4 files changed, 9 insertions(+), 9 deletions(-) diffs (65 lines): diff -r 819649386f93 -r 7cdef26d857d src/lib-storage/index/index-mailbox-size.c --- a/src/lib-storage/index/index-mailbox-size.c Sat Jul 11 12:00:38 2015 +0300 +++ b/src/lib-storage/index/index-mailbox-size.c Sat Jul 11 12:03:56 2015 +0300 @@ -6,7 +6,7 @@ static int virtual_size_add_new(struct mailbox *box, - struct index_vsize_header *vsize_hdr) + struct mailbox_index_vsize *vsize_hdr) { struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); const struct mail_index_header *hdr; @@ -76,7 +76,7 @@ struct mailbox_metadata *metadata_r) { struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); - struct index_vsize_header vsize_hdr; + struct mailbox_index_vsize vsize_hdr; struct mailbox_status status; const void *data; size_t size; diff -r 819649386f93 -r 7cdef26d857d src/lib-storage/index/index-storage.c --- a/src/lib-storage/index/index-storage.c Sat Jul 11 12:00:38 2015 +0300 +++ b/src/lib-storage/index/index-storage.c Sat Jul 11 12:03:56 2015 +0300 @@ -291,7 +291,7 @@ ibox->keyword_names = mail_index_get_keywords(box->index); ibox->vsize_hdr_ext_id = mail_index_ext_register(box->index, "hdr-vsize", - sizeof(struct index_vsize_header), 0, + sizeof(struct mailbox_index_vsize), 0, sizeof(uint64_t)); box->opened = TRUE; diff -r 819649386f93 -r 7cdef26d857d src/lib-storage/index/index-storage.h --- a/src/lib-storage/index/index-storage.h Sat Jul 11 12:00:38 2015 +0300 +++ b/src/lib-storage/index/index-storage.h Sat Jul 11 12:03:56 2015 +0300 @@ -16,12 +16,6 @@ MAILBOX_LOCK_NOTIFY_MAILBOX_OVERRIDE }; -struct index_vsize_header { - uint64_t vsize; - uint32_t highest_uid; - uint32_t message_count; -}; - struct index_mailbox_context { union mailbox_module_context module_ctx; enum mail_index_open_flags index_flags; diff -r 819649386f93 -r 7cdef26d857d src/lib-storage/mail-storage-private.h --- a/src/lib-storage/mail-storage-private.h Sat Jul 11 12:00:38 2015 +0300 +++ b/src/lib-storage/mail-storage-private.h Sat Jul 11 12:03:56 2015 +0300 @@ -265,6 +265,12 @@ uoff_t physical_pos, virtual_pos; }; +struct mailbox_index_vsize { + uint64_t vsize; + uint32_t highest_uid; + uint32_t message_count; +}; + struct mailbox { const char *name; /* mailbox's virtual name (from mail_namespace_get_vname()) */ From dovecot at dovecot.org Sat Jul 11 09:20:12 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 11 Jul 2015 09:20:12 +0000 Subject: dovecot-2.2: lib-storage: Moved vsize_hdr_ext_id to struct mailb... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8e47bb182a42 changeset: 18904:8e47bb182a42 user: Timo Sirainen date: Sat Jul 11 12:06:44 2015 +0300 description: lib-storage: Moved vsize_hdr_ext_id to struct mailbox for more global access. diffstat: src/lib-storage/index/index-mailbox-size.c | 6 ++---- src/lib-storage/index/index-storage.c | 2 +- src/lib-storage/index/index-storage.h | 1 - src/lib-storage/mail-storage-private.h | 1 + 4 files changed, 4 insertions(+), 6 deletions(-) diffs (71 lines): diff -r 7cdef26d857d -r 8e47bb182a42 src/lib-storage/index/index-mailbox-size.c --- a/src/lib-storage/index/index-mailbox-size.c Sat Jul 11 12:03:56 2015 +0300 +++ b/src/lib-storage/index/index-mailbox-size.c Sat Jul 11 12:06:44 2015 +0300 @@ -8,7 +8,6 @@ virtual_size_add_new(struct mailbox *box, struct mailbox_index_vsize *vsize_hdr) { - struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); const struct mail_index_header *hdr; struct mailbox_transaction_context *trans; struct mail_search_context *search_ctx; @@ -66,7 +65,7 @@ } else { /* search failed, cache only up to highest seen uid */ } - mail_index_update_header_ext(trans->itrans, ibox->vsize_hdr_ext_id, + mail_index_update_header_ext(trans->itrans, box->vsize_hdr_ext_id, 0, vsize_hdr, sizeof(*vsize_hdr)); (void)mailbox_transaction_commit(&trans); return ret; @@ -75,7 +74,6 @@ int index_mailbox_get_virtual_size(struct mailbox *box, struct mailbox_metadata *metadata_r) { - struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); struct mailbox_index_vsize vsize_hdr; struct mailbox_status status; const void *data; @@ -83,7 +81,7 @@ int ret; mailbox_get_open_status(box, STATUS_MESSAGES | STATUS_UIDNEXT, &status); - mail_index_get_header_ext(box->view, ibox->vsize_hdr_ext_id, + mail_index_get_header_ext(box->view, box->vsize_hdr_ext_id, &data, &size); if (size == sizeof(vsize_hdr)) memcpy(&vsize_hdr, data, sizeof(vsize_hdr)); diff -r 7cdef26d857d -r 8e47bb182a42 src/lib-storage/index/index-storage.c --- a/src/lib-storage/index/index-storage.c Sat Jul 11 12:03:56 2015 +0300 +++ b/src/lib-storage/index/index-storage.c Sat Jul 11 12:06:44 2015 +0300 @@ -289,7 +289,7 @@ index_cache_register_defaults(box); box->view = mail_index_view_open(box->index); ibox->keyword_names = mail_index_get_keywords(box->index); - ibox->vsize_hdr_ext_id = + box->vsize_hdr_ext_id = mail_index_ext_register(box->index, "hdr-vsize", sizeof(struct mailbox_index_vsize), 0, sizeof(uint64_t)); diff -r 7cdef26d857d -r 8e47bb182a42 src/lib-storage/index/index-storage.h --- a/src/lib-storage/index/index-storage.h Sat Jul 11 12:03:56 2015 +0300 +++ b/src/lib-storage/index/index-storage.h Sat Jul 11 12:06:44 2015 +0300 @@ -33,7 +33,6 @@ ARRAY_TYPE(seq_range) recent_flags; uint32_t recent_flags_prev_uid, recent_flags_last_check_nextuid; uint32_t recent_flags_count; - uint32_t vsize_hdr_ext_id; time_t sync_last_check; uint32_t list_index_sync_ext_id; diff -r 7cdef26d857d -r 8e47bb182a42 src/lib-storage/mail-storage-private.h --- a/src/lib-storage/mail-storage-private.h Sat Jul 11 12:03:56 2015 +0300 +++ b/src/lib-storage/mail-storage-private.h Sat Jul 11 12:06:44 2015 +0300 @@ -317,6 +317,7 @@ unsigned int transaction_count; enum mailbox_feature enabled_features; struct mail_msgpart_partial_cache partial_cache; + uint32_t vsize_hdr_ext_id; struct mail_index_view *tmp_sync_view; From dovecot at dovecot.org Sat Jul 11 09:20:12 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 11 Jul 2015 09:20:12 +0000 Subject: dovecot-2.2: lib-storage: Added MAILBOX_METADATA_PHYSICAL_SIZE Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/84392ca460ab changeset: 18905:84392ca460ab user: Timo Sirainen date: Sat Jul 11 12:11:48 2015 +0300 description: lib-storage: Added MAILBOX_METADATA_PHYSICAL_SIZE If backend always uses the same virtual and physical sizes, this is implemented via the MAILBOX_METADATA_VIRTUAL_SIZE code. Otherwise it searches all the messages and sums up their physical sizes. diffstat: src/lib-storage/index/index-mailbox-size.c | 58 ++++++++++++++++++++++++++++++ src/lib-storage/index/index-status.c | 5 ++- src/lib-storage/index/index-storage.h | 2 + src/lib-storage/mail-storage.h | 7 ++- 4 files changed, 69 insertions(+), 3 deletions(-) diffs (125 lines): diff -r 8e47bb182a42 -r 84392ca460ab src/lib-storage/index/index-mailbox-size.c --- a/src/lib-storage/index/index-mailbox-size.c Sat Jul 11 12:06:44 2015 +0300 +++ b/src/lib-storage/index/index-mailbox-size.c Sat Jul 11 12:11:48 2015 +0300 @@ -110,3 +110,61 @@ metadata_r->virtual_size = vsize_hdr.vsize; return ret; } + +int index_mailbox_get_physical_size(struct mailbox *box, + struct mailbox_metadata *metadata_r) +{ + struct mailbox_transaction_context *trans; + struct mail_search_context *ctx; + struct mail *mail; + struct mail_search_args *search_args; + uoff_t size; + int ret = 0; + + /* if physical size = virtual size always for the storage, we can + use the optimized vsize code for this */ + if (box->mail_vfuncs->get_physical_size == + box->mail_vfuncs->get_virtual_size) { + if (index_mailbox_get_virtual_size(box, metadata_r) < 0) + return -1; + metadata_r->physical_size = metadata_r->virtual_size; + return 0; + } + /* do it the slow way (we could implement similar logic as for vsize, + but for now it's not really needed) */ + if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) + return -1; + + trans = mailbox_transaction_begin(box, 0); + + search_args = mail_search_build_init(); + mail_search_build_add_all(search_args); + ctx = mailbox_search_init(trans, search_args, NULL, + MAIL_FETCH_PHYSICAL_SIZE, NULL); + mail_search_args_unref(&search_args); + + metadata_r->physical_size = 0; + while (mailbox_search_next(ctx, &mail)) { + if (mail_get_physical_size(mail, &size) == 0) + metadata_r->physical_size += size; + else { + const char *errstr; + enum mail_error error; + + errstr = mailbox_get_last_error(box, &error); + if (error != MAIL_ERROR_EXPUNGED) { + i_error("Couldn't get size of mail UID %u in %s: %s", + mail->uid, box->vname, errstr); + ret = -1; + break; + } + } + } + if (mailbox_search_deinit(&ctx) < 0) { + i_error("Listing mails in %s failed: %s", + box->vname, mailbox_get_last_error(box, NULL)); + ret = -1; + } + (void)mailbox_transaction_commit(&trans); + return ret; +} diff -r 8e47bb182a42 -r 84392ca460ab src/lib-storage/index/index-status.c --- a/src/lib-storage/index/index-status.c Sat Jul 11 12:06:44 2015 +0300 +++ b/src/lib-storage/index/index-status.c Sat Jul 11 12:11:48 2015 +0300 @@ -3,7 +3,6 @@ #include "lib.h" #include "array.h" #include "mail-cache.h" -#include "mail-search-build.h" #include "mail-index-modseq.h" #include "index-storage.h" @@ -289,6 +288,10 @@ if (index_mailbox_get_virtual_size(box, metadata_r) < 0) return -1; } + if ((items & MAILBOX_METADATA_PHYSICAL_SIZE) != 0) { + if (index_mailbox_get_physical_size(box, metadata_r) < 0) + return -1; + } if ((items & MAILBOX_METADATA_CACHE_FIELDS) != 0) get_metadata_cache_fields(box, metadata_r); if ((items & MAILBOX_METADATA_PRECACHE_FIELDS) != 0) diff -r 8e47bb182a42 -r 84392ca460ab src/lib-storage/index/index-storage.h --- a/src/lib-storage/index/index-storage.h Sat Jul 11 12:06:44 2015 +0300 +++ b/src/lib-storage/index/index-storage.h Sat Jul 11 12:11:48 2015 +0300 @@ -112,6 +112,8 @@ struct mailbox_metadata *metadata_r); int index_mailbox_get_virtual_size(struct mailbox *box, struct mailbox_metadata *metadata_r); +int index_mailbox_get_physical_size(struct mailbox *box, + struct mailbox_metadata *metadata_r); int index_storage_attribute_set(struct mailbox_transaction_context *t, enum mail_attribute_type type, const char *key, diff -r 8e47bb182a42 -r 84392ca460ab src/lib-storage/mail-storage.h --- a/src/lib-storage/mail-storage.h Sat Jul 11 12:06:44 2015 +0300 +++ b/src/lib-storage/mail-storage.h Sat Jul 11 12:11:48 2015 +0300 @@ -90,10 +90,11 @@ MAILBOX_METADATA_VIRTUAL_SIZE = 0x02, MAILBOX_METADATA_CACHE_FIELDS = 0x04, MAILBOX_METADATA_PRECACHE_FIELDS = 0x08, - MAILBOX_METADATA_BACKEND_NAMESPACE = 0x10 + MAILBOX_METADATA_BACKEND_NAMESPACE = 0x10, + MAILBOX_METADATA_PHYSICAL_SIZE = 0x20 /* metadata items that require mailbox to be synced at least once. */ #define MAILBOX_METADATA_SYNC_ITEMS \ - (MAILBOX_METADATA_VIRTUAL_SIZE) + (MAILBOX_METADATA_VIRTUAL_SIZE | MAILBOX_METADATA_PHYSICAL_SIZE) }; enum mailbox_search_result_flags { @@ -267,6 +268,8 @@ guid_128_t guid; /* sum of virtual size of all messages in mailbox */ uint64_t virtual_size; + /* sum of physical size of all messages in mailbox */ + uint64_t physical_size; /* Fields that have "temp" or "yes" caching decision. */ const ARRAY_TYPE(mailbox_cache_field) *cache_fields; /* Fields that should be precached */ From dovecot at dovecot.org Sat Jul 11 09:20:17 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 11 Jul 2015 09:20:17 +0000 Subject: dovecot-2.2: quota: Use MAILBOX_METADATA_PHYSICAL_SIZE for recal... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/421f595a0e93 changeset: 18906:421f595a0e93 user: Timo Sirainen date: Sat Jul 11 12:14:48 2015 +0300 description: quota: Use MAILBOX_METADATA_PHYSICAL_SIZE for recalculating mailbox's size. diffstat: src/plugins/quota/quota-count.c | 63 +++++++++++----------------------------- 1 files changed, 17 insertions(+), 46 deletions(-) diffs (96 lines): diff -r 84392ca460ab -r 421f595a0e93 src/plugins/quota/quota-count.c --- a/src/plugins/quota/quota-count.c Sat Jul 11 12:11:48 2015 +0300 +++ b/src/plugins/quota/quota-count.c Sat Jul 11 12:14:48 2015 +0300 @@ -1,27 +1,22 @@ /* Copyright (c) 2006-2015 Dovecot authors, see the included COPYING file */ #include "lib.h" -#include "array.h" -#include "mail-search-build.h" -#include "mail-storage.h" -#include "mail-namespace.h" #include "mailbox-list-iter.h" #include "quota-private.h" +extern struct quota_backend quota_backend_count; + static int quota_count_mailbox(struct quota_root *root, struct mail_namespace *ns, const char *vname, uint64_t *bytes_r, uint64_t *count_r) { struct quota_rule *rule; struct mailbox *box; - struct mailbox_transaction_context *trans; - struct mail_search_context *ctx; - struct mail *mail; - struct mail_search_args *search_args; + struct mailbox_metadata metadata; + struct mailbox_status status; enum mail_error error; const char *errstr; - uoff_t size; - int ret = 0; + int ret; rule = quota_root_rule_find(root->set, vname); if (rule != NULL && rule->ignore) { @@ -30,47 +25,23 @@ } box = mailbox_alloc(ns->list, vname, MAILBOX_FLAG_READONLY); - if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) { + if (mailbox_get_metadata(box, MAILBOX_METADATA_PHYSICAL_SIZE, + &metadata) < 0 || + mailbox_get_status(box, STATUS_MESSAGES, &status) < 0) { errstr = mailbox_get_last_error(box, &error); - mailbox_free(&box); if (error == MAIL_ERROR_TEMP) { - i_error("quota: Couldn't sync mailbox %s: %s", + i_error("quota: Couldn't get physical size of mailbox %s: %s", vname, errstr); - return -1; + ret = -1; + } else { + /* non-temporary error, e.g. ACLs denied access. */ + ret = 0; } - /* non-temporary error, e.g. ACLs denied access. */ - return 0; + } else { + ret = 1; + *bytes_r = metadata.physical_size; + *count_r = status.messages; } - - trans = mailbox_transaction_begin(box, 0); - - search_args = mail_search_build_init(); - mail_search_build_add_all(search_args); - ctx = mailbox_search_init(trans, search_args, NULL, - MAIL_FETCH_PHYSICAL_SIZE, NULL); - mail_search_args_unref(&search_args); - - while (mailbox_search_next(ctx, &mail)) { - if (mail_get_physical_size(mail, &size) == 0) - *bytes_r += size; - else { - errstr = mailbox_get_last_error(box, &error); - if (error != MAIL_ERROR_EXPUNGED) { - i_error("quota: Couldn't get size of mail UID %u in %s: %s", - mail->uid, vname, mailbox_get_last_error(box, NULL)); - ret = -1; - break; - } - } - *count_r += 1; - } - if (mailbox_search_deinit(&ctx) < 0) { - i_error("quota: Listing mails in %s failed: %s", - vname, mailbox_get_last_error(box, NULL)); - ret = -1; - } - (void)mailbox_transaction_commit(&trans); - mailbox_free(&box); return ret; } From dovecot at dovecot.org Sat Jul 11 09:20:18 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 11 Jul 2015 09:20:18 +0000 Subject: dovecot-2.2: lib-storage: If mailboxes' vsizes are used, keep th... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/46cbde67f50b changeset: 18907:46cbde67f50b user: Timo Sirainen date: Sat Jul 11 12:16:56 2015 +0300 description: lib-storage: If mailboxes' vsizes are used, keep them updated also in mailbox list index. This allows looking them up quickly without opening the actual mailbox indexes. diffstat: src/lib-storage/list/mailbox-list-index-backend.c | 2 +- src/lib-storage/list/mailbox-list-index-notify.c | 2 +- src/lib-storage/list/mailbox-list-index-status.c | 131 +++++++++++++++++++-- src/lib-storage/list/mailbox-list-index.h | 5 +- src/lib-storage/list/mailbox-list-notify-tree.c | 2 +- 5 files changed, 122 insertions(+), 20 deletions(-) diffs (truncated from 309 to 300 lines): diff -r 421f595a0e93 -r 46cbde67f50b src/lib-storage/list/mailbox-list-index-backend.c --- a/src/lib-storage/list/mailbox-list-index-backend.c Sat Jul 11 12:14:48 2015 +0300 +++ b/src/lib-storage/list/mailbox-list-index-backend.c Sat Jul 11 12:16:56 2015 +0300 @@ -175,7 +175,7 @@ T_MAIL_ERR_MAILBOX_NOT_FOUND(name)); ret = -1; } else if (!mailbox_list_index_status(_list, view, seq, 0, - &status, mailbox_guid) || + &status, mailbox_guid, NULL) || guid_128_is_empty(mailbox_guid)) { mailbox_list_set_error(_list, MAIL_ERROR_NOTFOUND, T_MAIL_ERR_MAILBOX_NOT_FOUND(name)); diff -r 421f595a0e93 -r 46cbde67f50b src/lib-storage/list/mailbox-list-index-notify.c --- a/src/lib-storage/list/mailbox-list-index-notify.c Sat Jul 11 12:14:48 2015 +0300 +++ b/src/lib-storage/list/mailbox-list-index-notify.c Sat Jul 11 12:16:56 2015 +0300 @@ -160,7 +160,7 @@ memset(status_r, 0, sizeof(*status_r)); memset(guid_r, 0, GUID_128_SIZE); (void)mailbox_list_index_status(inotify->notify.list, view, seq, - items, status_r, guid_r); + items, status_r, guid_r, NULL); return index_node; } diff -r 421f595a0e93 -r 46cbde67f50b src/lib-storage/list/mailbox-list-index-status.c --- a/src/lib-storage/list/mailbox-list-index-status.c Sat Jul 11 12:14:48 2015 +0300 +++ b/src/lib-storage/list/mailbox-list-index-status.c Sat Jul 11 12:16:56 2015 +0300 @@ -14,10 +14,12 @@ struct mailbox_status status; guid_128_t guid; uint32_t seq; + struct mailbox_index_vsize vsize; bool rec_changed; bool msgs_changed; bool hmodseq_changed; + bool vsize_changed; }; struct index_list_storage_module index_list_storage_module = @@ -108,7 +110,8 @@ struct mail_index_view *view, uint32_t seq, enum mailbox_status_items items, struct mailbox_status *status_r, - uint8_t *mailbox_guid) + uint8_t *mailbox_guid, + struct mailbox_index_vsize *vsize_r) { struct mailbox_list_index *ilist = INDEX_LIST_CONTEXT(list); const void *data; @@ -161,6 +164,14 @@ else status_r->highest_modseq = *rec; } + if (vsize_r != NULL) { + mail_index_lookup_ext(view, seq, ilist->vsize_ext_id, + &data, &expunged); + if (data == NULL) + ret = FALSE; + else + memcpy(vsize_r, data, sizeof(*vsize_r)); + } return ret; } @@ -184,7 +195,7 @@ return ret; ret = mailbox_list_index_status(box->list, view, seq, items, - status_r, NULL) ? 1 : 0; + status_r, NULL, NULL) ? 1 : 0; mail_index_view_close(&view); return ret; } @@ -221,13 +232,81 @@ return ret; ret = mailbox_list_index_status(box->list, view, seq, 0, - &status, guid_r) ? 1 : 0; + &status, guid_r, NULL) ? 1 : 0; if (ret > 0 && guid_128_is_empty(guid_r)) ret = 0; mail_index_view_close(&view); return ret; } +static int index_list_get_cached_vsize(struct mailbox *box, uoff_t *vsize_r) +{ + struct mailbox_list_index *ilist = INDEX_LIST_CONTEXT(box->list); + struct mailbox_status status; + struct mailbox_index_vsize vsize; + struct mail_index_view *view; + uint32_t seq; + int ret; + + i_assert(!ilist->syncing); + + if ((ret = index_list_open_view(box, &view, &seq)) <= 0) + return ret; + + ret = mailbox_list_index_status(box->list, view, seq, + STATUS_MESSAGES | STATUS_UIDNEXT, + &status, NULL, &vsize) ? 1 : 0; + if (ret > 0 && (vsize.highest_uid + 1 != status.uidnext || + vsize.message_count != status.messages)) { + /* out of date vsize info */ + ret = 0; + } + if (ret > 0) + *vsize_r = vsize.vsize; + mail_index_view_close(&view); + return ret; +} + +static int +index_list_try_get_metadata(struct mailbox *box, + enum mailbox_metadata_items items, + struct mailbox_metadata *metadata_r) +{ + enum mailbox_metadata_items noncached_items; + int ret; + + if (box->opened) { + /* if mailbox is already opened, don't bother using the values + in mailbox list index. they have a higher chance of being + wrong. */ + return 0; + } + /* see if we have a chance of fulfilling this without opening + the mailbox. */ + noncached_items = items & ~(MAILBOX_METADATA_GUID | + MAILBOX_METADATA_VIRTUAL_SIZE); + if ((noncached_items & MAILBOX_METADATA_PHYSICAL_SIZE) != 0 && + box->mail_vfuncs->get_physical_size == + box->mail_vfuncs->get_virtual_size) + noncached_items = items & ~MAILBOX_METADATA_PHYSICAL_SIZE; + + if (noncached_items != 0) + return 0; + + if ((items & MAILBOX_METADATA_GUID) != 0) { + if ((ret = index_list_get_cached_guid(box, metadata_r->guid)) <= 0) + return ret; + } + if ((items & (MAILBOX_METADATA_VIRTUAL_SIZE | + MAILBOX_METADATA_PHYSICAL_SIZE)) != 0) { + if ((ret = index_list_get_cached_vsize(box, &metadata_r->virtual_size)) <= 0) + return ret; + if ((items & MAILBOX_METADATA_PHYSICAL_SIZE) != 0) + metadata_r->physical_size = metadata_r->virtual_size; + } + return 1; +} + static int index_list_get_metadata(struct mailbox *box, enum mailbox_metadata_items items, @@ -235,14 +314,25 @@ { struct index_list_mailbox *ibox = INDEX_LIST_STORAGE_CONTEXT(box); - if (items == MAILBOX_METADATA_GUID && !box->opened) { - if (index_list_get_cached_guid(box, metadata_r->guid) > 0) - return 0; - /* nonsynced / error, fallback to doing it the slow way */ - } + if (index_list_try_get_metadata(box, items, metadata_r) != 0) + return 0; return ibox->module_ctx.super.get_metadata(box, items, metadata_r); } +static void +index_list_update_fill_vsize(struct mailbox *box, + struct mail_index_view *view, + struct index_list_changes *changes_r) +{ + const void *data; + size_t size; + + mail_index_get_header_ext(view, box->vsize_hdr_ext_id, + &data, &size); + if (size == sizeof(changes_r->vsize)) + memcpy(&changes_r->vsize, data, sizeof(changes_r->vsize)); +} + static bool index_list_update_fill_changes(struct mailbox *box, struct mail_index_view *list_view, @@ -286,6 +376,7 @@ /* modseqs not enabled yet, but we can't return 0 */ changes_r->status.highest_modseq = 1; } + index_list_update_fill_vsize(box, view, changes_r); mail_index_view_close(&view); hdr = NULL; if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, &metadata) == 0) @@ -299,13 +390,15 @@ { struct mailbox_list_index *ilist = INDEX_LIST_CONTEXT(box->list); struct mailbox_status old_status; + struct mailbox_index_vsize old_vsize; guid_128_t old_guid; memset(&old_status, 0, sizeof(old_status)); + memset(&old_vsize, 0, sizeof(old_vsize)); memset(old_guid, 0, sizeof(old_guid)); (void)mailbox_list_index_status(box->list, list_view, changes->seq, CACHED_STATUS_ITEMS, - &old_status, old_guid); + &old_status, old_guid, &old_vsize); changes->rec_changed = old_status.uidvalidity != changes->status.uidvalidity && @@ -331,13 +424,11 @@ ilist->hmodseq_ext_id, &data, &expunged); changes->hmodseq_changed = data != NULL; } - - if (changes->hmodseq_changed && - old_status.highest_modseq != changes->status.highest_modseq) - changes->hmodseq_changed = TRUE; + if (memcmp(&old_vsize, &changes->vsize, sizeof(old_vsize)) != 0) + changes->vsize_changed = TRUE; return changes->rec_changed || changes->msgs_changed || - changes->hmodseq_changed; + changes->hmodseq_changed || changes->vsize_changed; } static void @@ -382,6 +473,11 @@ ilist->hmodseq_ext_id, &changes->status.highest_modseq, NULL); } + if (changes->vsize_changed) { + mail_index_update_ext(list_trans, changes->seq, + ilist->vsize_ext_id, + &changes->vsize, NULL); + } } static int index_list_update_mailbox(struct mailbox *box) @@ -482,7 +578,7 @@ (void)mailbox_list_index_status(box->list, list_view, changes.seq, CACHED_STATUS_ITEMS, &status, - mailbox_guid); + mailbox_guid, NULL); if (update->uid_validity != 0) { changes.rec_changed = TRUE; changes.status.uidvalidity = update->uid_validity; @@ -575,7 +671,7 @@ status.recent = 0; (void)mailbox_list_index_status(box->list, view, seq, STATUS_RECENT, - &status, NULL); + &status, NULL, NULL); mail_index_view_close(&view); if (status.recent != 0) @@ -604,4 +700,7 @@ ilist->hmodseq_ext_id = mail_index_ext_register(ilist->index, "hmodseq", 0, sizeof(uint64_t), sizeof(uint64_t)); + ilist->vsize_ext_id = + mail_index_ext_register(ilist->index, "vsize", 0, + sizeof(struct mailbox_index_vsize), sizeof(uint64_t)); } diff -r 421f595a0e93 -r 46cbde67f50b src/lib-storage/list/mailbox-list-index.h --- a/src/lib-storage/list/mailbox-list-index.h Sat Jul 11 12:14:48 2015 +0300 +++ b/src/lib-storage/list/mailbox-list-index.h Sat Jul 11 12:16:56 2015 +0300 @@ -36,6 +36,7 @@ MODULE_CONTEXT(obj, mailbox_list_index_module) struct mail_index_view; +struct mailbox_index_vsize; /* stored in mail_index_record.flags: */ enum mailbox_list_index_flags { @@ -88,6 +89,7 @@ const char *path; struct mail_index *index; uint32_t ext_id, msgs_ext_id, hmodseq_ext_id, subs_hdr_ext_id; + uint32_t vsize_ext_id; struct timeval last_refresh_timeval; pool_t mailbox_pool; @@ -170,7 +172,8 @@ struct mail_index_view *view, uint32_t seq, enum mailbox_status_items items, struct mailbox_status *status_r, - uint8_t *mailbox_guid); + uint8_t *mailbox_guid, + struct mailbox_index_vsize *vsize_r); void mailbox_list_index_status_set_info_flags(struct mailbox *box, uint32_t uid, enum mailbox_info_flags *flags); void mailbox_list_index_update_mailbox_index(struct mailbox *box, diff -r 421f595a0e93 -r 46cbde67f50b src/lib-storage/list/mailbox-list-notify-tree.c --- a/src/lib-storage/list/mailbox-list-notify-tree.c Sat Jul 11 12:14:48 2015 +0300 +++ b/src/lib-storage/list/mailbox-list-notify-tree.c Sat Jul 11 12:16:56 2015 +0300 From dovecot at dovecot.org Tue Jul 14 13:08:36 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 14 Jul 2015 13:08:36 +0000 Subject: dovecot-2.2: pop3-migration: Show the first message's number and... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b51dfee18fd2 changeset: 18908:b51dfee18fd2 user: Timo Sirainen date: Tue Jul 14 15:08:24 2015 +0200 description: pop3-migration: Show the first message's number and UIDL which wasn't found from IMAP. diffstat: src/plugins/pop3-migration/pop3-migration-plugin.c | 11 +++++++++-- 1 files changed, 9 insertions(+), 2 deletions(-) diffs (36 lines): diff -r 46cbde67f50b -r b51dfee18fd2 src/plugins/pop3-migration/pop3-migration-plugin.c --- a/src/plugins/pop3-migration/pop3-migration-plugin.c Sat Jul 11 12:16:56 2015 +0300 +++ b/src/plugins/pop3-migration/pop3-migration-plugin.c Tue Jul 14 15:08:24 2015 +0200 @@ -470,6 +470,7 @@ struct imap_msg_map *imap_map; unsigned int pop3_idx, imap_idx, pop3_count, imap_count; unsigned int first_seq, missing_uids_count; + uint32_t first_missing_idx = (uint32_t)-1; int ret; first_seq = mbox->first_unfound_idx+1; @@ -512,8 +513,11 @@ } missing_uids_count = 0; for (pop3_idx = 0; pop3_idx < pop3_count; pop3_idx++) { - if (pop3_map[pop3_idx].imap_uid == 0) + if (pop3_map[pop3_idx].imap_uid == 0) { + if (first_missing_idx == (uint32_t)-1) + first_missing_idx = pop3_map[pop3_idx].pop3_seq; missing_uids_count++; + } } if (missing_uids_count > 0 && !mstorage->all_mailboxes) { if (!mstorage->ignore_missing_uidls) { @@ -524,7 +528,10 @@ return -1; } i_warning("pop3_migration: %u POP3 messages have no " - "matching IMAP messages", missing_uids_count); + "matching IMAP messages (first POP3 msg %u UIDL %s)", + missing_uids_count, + pop3_map[first_missing_idx].pop3_seq, + pop3_map[first_missing_idx].pop3_uidl); } else if (box->storage->user->mail_debug) { i_debug("pop3_migration: %u mails matched by headers", pop3_count); } From dovecot at dovecot.org Thu Jul 16 15:08:53 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 16 Jul 2015 15:08:53 +0000 Subject: dovecot-2.2: pop3-migration: Truncate header if there's line con... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ab441df52e86 changeset: 18909:ab441df52e86 user: Timo Sirainen date: Thu Jul 16 18:08:40 2015 +0300 description: pop3-migration: Truncate header if there's line containing only CR(s). This fixes matching IMAP <-> POP3 messages when the servers behave differently. diffstat: src/plugins/pop3-migration/Makefile.am | 22 ++++++ src/plugins/pop3-migration/pop3-migration-plugin.c | 58 ++++++++++++---- src/plugins/pop3-migration/pop3-migration-plugin.h | 7 ++ src/plugins/pop3-migration/test-pop3-migration-plugin.c | 50 ++++++++++++++ 4 files changed, 123 insertions(+), 14 deletions(-) diffs (226 lines): diff -r b51dfee18fd2 -r ab441df52e86 src/plugins/pop3-migration/Makefile.am --- a/src/plugins/pop3-migration/Makefile.am Tue Jul 14 15:08:24 2015 +0200 +++ b/src/plugins/pop3-migration/Makefile.am Thu Jul 16 18:08:40 2015 +0300 @@ -1,5 +1,6 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ -I$(top_srcdir)/src/lib-mail \ -I$(top_srcdir)/src/lib-index \ -I$(top_srcdir)/src/lib-storage @@ -15,3 +16,24 @@ noinst_HEADERS = \ pop3-migration-plugin.h + +noinst_PROGRAMS = $(test_programs) + +test_programs = \ + test-pop3-migration-plugin + +test_libs = \ + ../../lib-storage/libstorage.la \ + ../../lib-test/libtest.la \ + ../../lib/liblib.la +test_deps = $(module_LTLIBRARIES) $(test_libs) + +test_pop3_migration_plugin_SOURCES = test-pop3-migration-plugin.c +test_pop3_migration_plugin_LDADD = pop3-migration-plugin.lo $(test_libs) +test_pop3_migration_plugin_DEPENDENCIES = $(test_deps) + +check: check-am check-test +check-test: all-am + for bin in $(test_programs); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done diff -r b51dfee18fd2 -r ab441df52e86 src/plugins/pop3-migration/pop3-migration-plugin.c --- a/src/plugins/pop3-migration/pop3-migration-plugin.c Tue Jul 14 15:08:24 2015 +0200 +++ b/src/plugins/pop3-migration/pop3-migration-plugin.c Thu Jul 16 18:08:40 2015 +0300 @@ -112,33 +112,56 @@ return memcmp(map1->hdr_sha1, map2->hdr_sha1, sizeof(map1->hdr_sha1)); } +struct pop3_hdr_context { + bool have_eoh; + bool stop; +}; + static void pop3_header_filter_callback(struct header_filter_istream *input ATTR_UNUSED, struct message_header_line *hdr, - bool *matched ATTR_UNUSED, bool *have_eoh) + bool *matched, struct pop3_hdr_context *ctx) { - if (hdr != NULL && hdr->eoh) - *have_eoh = TRUE; + if (hdr == NULL) + return; + if (hdr->eoh) { + ctx->have_eoh = TRUE; + if (ctx->stop) { + /* matched is handled differently for eoh by + istream-header-filter. a design bug I guess.. */ + *matched = FALSE; + } + } else { + if (strspn(hdr->name, "\r") == hdr->name_len) { + /* CR+CR+LF - some servers stop the header processing + here while others don't. To make sure they can be + matched correctly we want to stop here entirely. */ + ctx->stop = TRUE; + } + if (ctx->stop) + *matched = TRUE; + } } -static int -get_hdr_sha1_stream(struct mail *mail, struct istream *input, uoff_t hdr_size, - unsigned char sha1_r[SHA1_RESULTLEN], bool *have_eoh_r) +int pop3_migration_get_hdr_sha1(uint32_t mail_seq, struct istream *input, + uoff_t hdr_size, + unsigned char sha1_r[SHA1_RESULTLEN], + bool *have_eoh_r) { struct istream *input2; const unsigned char *data, *p; size_t size, idx; struct sha1_ctxt sha1_ctx; + struct pop3_hdr_context hdr_ctx; - *have_eoh_r = FALSE; - + memset(&hdr_ctx, 0, sizeof(hdr_ctx)); input2 = i_stream_create_limit(input, hdr_size); /* hide headers that might change or be different in IMAP vs. POP3 */ input = i_stream_create_header_filter(input2, HEADER_FILTER_EXCLUDE | HEADER_FILTER_NO_CR, hdr_hash_skip_headers, N_ELEMENTS(hdr_hash_skip_headers), - pop3_header_filter_callback, have_eoh_r); + pop3_header_filter_callback, &hdr_ctx); i_stream_unref(&input2); sha1_init(&sha1_ctx); @@ -159,12 +182,14 @@ } if (input->stream_errno != 0) { i_error("pop3_migration: Failed to read header for msg %u: %s", - mail->seq, i_stream_get_error(input)); + mail_seq, i_stream_get_error(input)); i_stream_unref(&input); return -1; } sha1_result(&sha1_ctx, sha1_r); i_stream_unref(&input); + + *have_eoh_r = hdr_ctx.have_eoh; return 0; } @@ -180,8 +205,9 @@ mail->seq, mailbox_get_last_error(mail->box, NULL)); return -1; } - if (get_hdr_sha1_stream(mail, input, hdr_size.physical_size, - sha1_r, &have_eoh) < 0) + if (pop3_migration_get_hdr_sha1(mail->seq, input, + hdr_size.physical_size, + sha1_r, &have_eoh) < 0) return -1; if (have_eoh) return 0; @@ -199,6 +225,9 @@ truncating the rest. POP3 TOP instead returns the entire header. This causes the IMAP and POP3 hashes not to match. + If there's LF+CR+CR+LF in the middle of headers, Courier IMAP's + FETCH BODY[HEADER] stops after that, but Courier POP3's TOP doesn't. + So we'll try to avoid this by falling back to full FETCH BODY[] (and/or RETR) and we'll parse the header ourself from it. This should work around any similar bugs in all IMAP/POP3 servers. */ @@ -207,8 +236,9 @@ mail->seq, mailbox_get_last_error(mail->box, NULL)); return -1; } - return get_hdr_sha1_stream(mail, input, hdr_size.physical_size, - sha1_r, &have_eoh); + return pop3_migration_get_hdr_sha1(mail->seq, input, + hdr_size.physical_size, + sha1_r, &have_eoh); } diff -r b51dfee18fd2 -r ab441df52e86 src/plugins/pop3-migration/pop3-migration-plugin.h --- a/src/plugins/pop3-migration/pop3-migration-plugin.h Tue Jul 14 15:08:24 2015 +0200 +++ b/src/plugins/pop3-migration/pop3-migration-plugin.h Thu Jul 16 18:08:40 2015 +0300 @@ -1,7 +1,14 @@ #ifndef POP3_MIGRATION_PLUGIN_H #define POP3_MIGRATION_PLUGIN_H +struct module; + void pop3_migration_plugin_init(struct module *module); void pop3_migration_plugin_deinit(void); +int pop3_migration_get_hdr_sha1(uint32_t mail_seq, struct istream *input, + uoff_t hdr_size, + unsigned char sha1_r[SHA1_RESULTLEN], + bool *have_eoh_r); + #endif diff -r b51dfee18fd2 -r ab441df52e86 src/plugins/pop3-migration/test-pop3-migration-plugin.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/plugins/pop3-migration/test-pop3-migration-plugin.c Thu Jul 16 18:08:40 2015 +0300 @@ -0,0 +1,50 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "sha1.h" +#include "hex-binary.h" +#include "istream.h" +#include "test-common.h" +#include "pop3-migration-plugin.h" + +static void test_pop3_migration_get_hdr_sha1(void) +{ + struct { + const char *input; + const char *sha1; + bool have_eoh; + } tests[] = { + { "", "da39a3ee5e6b4b0d3255bfef95601890afd80709", FALSE }, + { "\n", "adc83b19e793491b1c6ea0fd8b46cd9f32e592fc", TRUE }, + { "a: b\r\n", "3edb5ce145cf1d1e2413e02b8bed70f1ae3ed105", FALSE }, + { "a: b\r\n\r\n", "d14841695e1d9e2de6625d9222abd149ec821b0d", TRUE }, + { "a: b\r\n\r\r\n", "3edb5ce145cf1d1e2413e02b8bed70f1ae3ed105", FALSE }, + { "a: b\r\n\r\r\nc: d\r\n\r\n", "3edb5ce145cf1d1e2413e02b8bed70f1ae3ed105", TRUE } + }; + struct istream *input; + unsigned char digest[SHA1_RESULTLEN]; + unsigned int i; + bool have_eoh; + + test_begin("pop3 migration get hdr sha1"); + + for (i = 0; i < N_ELEMENTS(tests); i++) { + input = i_stream_create_from_data(tests[i].input, + strlen(tests[i].input)); + test_assert_idx(pop3_migration_get_hdr_sha1(1, input, strlen(tests[i].input), + digest, &have_eoh) == 0, i); + test_assert_idx(strcasecmp(binary_to_hex(digest, sizeof(digest)), tests[i].sha1) == 0, i); + test_assert_idx(tests[i].have_eoh == have_eoh, i); + } + + test_end(); +} + +int main(void) +{ + static void (*test_functions[])(void) = { + test_pop3_migration_get_hdr_sha1, + NULL + }; + return test_run(test_functions); +} From dovecot at dovecot.org Thu Jul 16 15:09:28 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 16 Jul 2015 15:09:28 +0000 Subject: dovecot-2.2: lib-mail: Updated test-message-header-parser unit test Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f0148bae3d62 changeset: 18910:f0148bae3d62 user: Timo Sirainen date: Thu Jul 16 18:09:17 2015 +0300 description: lib-mail: Updated test-message-header-parser unit test diffstat: src/lib-mail/test-message-header-parser.c | 24 ++++++++++++++++++++++++ 1 files changed, 24 insertions(+), 0 deletions(-) diffs (40 lines): diff -r ab441df52e86 -r f0148bae3d62 src/lib-mail/test-message-header-parser.c --- a/src/lib-mail/test-message-header-parser.c Thu Jul 16 18:08:40 2015 +0300 +++ b/src/lib-mail/test-message-header-parser.c Thu Jul 16 18:09:17 2015 +0300 @@ -239,12 +239,36 @@ test_end(); } +static void test_message_header_parser_extra_cr_in_eoh(void) +{ + static const char *str = "a:b\n\r\r\n"; + struct message_header_parser_ctx *parser; + struct message_header_line *hdr; + struct istream *input; + + test_begin("message header parser extra CR in EOH"); + + input = test_istream_create(str); + parser = message_parse_header_init(input, NULL, 0); + test_assert(message_parse_header_next(parser, &hdr) > 0 && + strcmp(hdr->name, "a") == 0); + test_assert(message_parse_header_next(parser, &hdr) > 0 && + strcmp(hdr->name, "\r") == 0 && hdr->middle_len == 0 && + hdr->value_len == 0 && !hdr->eoh); + test_assert(message_parse_header_next(parser, &hdr) < 0); + message_parse_header_deinit(&parser); + test_assert(input->stream_errno == 0); + i_stream_unref(&input); + test_end(); +} + int main(void) { static void (*test_functions[])(void) = { test_message_header_parser, test_message_header_parser_partial, test_message_header_parser_long_lines, + test_message_header_parser_extra_cr_in_eoh, NULL }; return test_run(test_functions); From dovecot at dovecot.org Thu Jul 16 15:10:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 16 Jul 2015 15:10:25 +0000 Subject: dovecot-2.2: pop3-migration: Use LIST instead of RETRs to get th... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4bebfbb32410 changeset: 18911:4bebfbb32410 user: Timo Sirainen date: Thu Jul 16 18:10:12 2015 +0300 description: pop3-migration: Use LIST instead of RETRs to get the messages' sizes. diffstat: src/plugins/pop3-migration/pop3-migration-plugin.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (13 lines): diff -r f0148bae3d62 -r 4bebfbb32410 src/plugins/pop3-migration/pop3-migration-plugin.c --- a/src/plugins/pop3-migration/pop3-migration-plugin.c Thu Jul 16 18:09:17 2015 +0300 +++ b/src/plugins/pop3-migration/pop3-migration-plugin.c Thu Jul 16 18:10:12 2015 +0300 @@ -290,6 +290,9 @@ mail_search_args_unref(&search_args); while (mailbox_search_next(ctx, &mail)) { + /* get the size with LIST instead of RETR */ + mail->lookup_abort = MAIL_LOOKUP_ABORT_READ_MAIL; + if (mail_get_virtual_size(mail, &size) < 0) { i_error("pop3_migration: Failed to get size for msg %u: %s", mail->seq, From dovecot at dovecot.org Thu Jul 16 16:37:34 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 16 Jul 2015 16:37:34 +0000 Subject: dovecot-2.2: pop3-migration: Fetch physical sizes instead of vir... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/64568a033fc8 changeset: 18912:64568a033fc8 user: Timo Sirainen date: Thu Jul 16 19:37:22 2015 +0300 description: pop3-migration: Fetch physical sizes instead of virtual sizes so pop3c uses LIST 4bebfbb32410 caused the fetching to break entirely. diffstat: src/plugins/pop3-migration/pop3-migration-plugin.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (20 lines): diff -r 4bebfbb32410 -r 64568a033fc8 src/plugins/pop3-migration/pop3-migration-plugin.c --- a/src/plugins/pop3-migration/pop3-migration-plugin.c Thu Jul 16 18:10:12 2015 +0300 +++ b/src/plugins/pop3-migration/pop3-migration-plugin.c Thu Jul 16 19:37:22 2015 +0300 @@ -286,14 +286,14 @@ search_args = mail_search_build_init(); mail_search_build_add_all(search_args); ctx = mailbox_search_init(t, search_args, NULL, - MAIL_FETCH_VIRTUAL_SIZE, NULL); + MAIL_FETCH_PHYSICAL_SIZE, NULL); mail_search_args_unref(&search_args); while (mailbox_search_next(ctx, &mail)) { /* get the size with LIST instead of RETR */ mail->lookup_abort = MAIL_LOOKUP_ABORT_READ_MAIL; - if (mail_get_virtual_size(mail, &size) < 0) { + if (mail_get_physical_size(mail, &size) < 0) { i_error("pop3_migration: Failed to get size for msg %u: %s", mail->seq, mailbox_get_last_error(pop3_box, NULL)); From dovecot at dovecot.org Sun Jul 19 07:57:41 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 19 Jul 2015 07:57:41 +0000 Subject: dovecot-2.2: lib-storage: Fixed doing multiple changes via mailb... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/501ce127ae05 changeset: 18913:501ce127ae05 user: Timo Sirainen date: Sun Jul 19 10:57:26 2015 +0300 description: lib-storage: Fixed doing multiple changes via mailbox_attribute_set/unset() Only the last change was committed and the earlier changes were just leaking memory. diffstat: src/lib-storage/index/index-attribute.c | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diffs (16 lines): diff -r 64568a033fc8 -r 501ce127ae05 src/lib-storage/index/index-attribute.c --- a/src/lib-storage/index/index-attribute.c Thu Jul 16 19:37:22 2015 +0300 +++ b/src/lib-storage/index/index-attribute.c Sun Jul 19 10:57:26 2015 +0300 @@ -170,6 +170,12 @@ } i_assert(dtransp != NULL); + if (*dtransp != NULL) { + /* transaction already created */ + *dtrans_r = *dtransp; + return 0; + } + if (index_storage_get_dict(t->box, type, &dict, mailbox_prefix_r) < 0) return -1; *dtransp = *dtrans_r = dict_transaction_begin(dict); From dovecot at dovecot.org Sun Jul 19 13:21:18 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 19 Jul 2015 13:21:18 +0000 Subject: dovecot-2.2: pop3-migration: Compiling fix in some systems Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a83cc1411205 changeset: 18914:a83cc1411205 user: Timo Sirainen date: Sun Jul 19 16:21:04 2015 +0300 description: pop3-migration: Compiling fix in some systems diffstat: src/plugins/pop3-migration/Makefile.am | 10 ++++++---- 1 files changed, 6 insertions(+), 4 deletions(-) diffs (20 lines): diff -r 501ce127ae05 -r a83cc1411205 src/plugins/pop3-migration/Makefile.am --- a/src/plugins/pop3-migration/Makefile.am Sun Jul 19 10:57:26 2015 +0300 +++ b/src/plugins/pop3-migration/Makefile.am Sun Jul 19 16:21:04 2015 +0300 @@ -23,10 +23,12 @@ test-pop3-migration-plugin test_libs = \ - ../../lib-storage/libstorage.la \ - ../../lib-test/libtest.la \ - ../../lib/liblib.la -test_deps = $(module_LTLIBRARIES) $(test_libs) + $(LIBDOVECOT_STORAGE) \ + $(LIBDOVECOT) +test_deps = \ + $(module_LTLIBRARIES) \ + $(LIBDOVECOT_STORAGE_DEPS) \ + $(LIBDOVECOT_DEPS) test_pop3_migration_plugin_SOURCES = test-pop3-migration-plugin.c test_pop3_migration_plugin_LDADD = pop3-migration-plugin.lo $(test_libs) From dovecot at dovecot.org Sun Jul 19 13:45:38 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 19 Jul 2015 13:45:38 +0000 Subject: dovecot-2.2: lib-mail: Fixed message_part_to_idx() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/012a355f9f8a changeset: 18915:012a355f9f8a user: Timo Sirainen date: Sun Jul 19 16:45:23 2015 +0300 description: lib-mail: Fixed message_part_to_idx() diffstat: src/lib-mail/message-part.c | 39 ++++++++++++++++++++++++++++++--------- src/lib-mail/test-message-part.c | 31 ++++++++++++++++++++++++++----- 2 files changed, 56 insertions(+), 14 deletions(-) diffs (119 lines): diff -r a83cc1411205 -r 012a355f9f8a src/lib-mail/message-part.c --- a/src/lib-mail/message-part.c Sun Jul 19 16:21:04 2015 +0300 +++ b/src/lib-mail/message-part.c Sun Jul 19 16:45:23 2015 +0300 @@ -3,18 +3,39 @@ #include "lib.h" #include "message-part.h" +static const struct message_part * +message_part_root(const struct message_part *part) +{ + while (part->parent != NULL) + part = part->parent; + return part; +} + +static bool message_part_find(const struct message_part *siblings, + const struct message_part *part, + unsigned int *n) +{ + const struct message_part *p; + + for (p = siblings; p != NULL; p = p->next) { + if (p == part) + return TRUE; + *n += 1; + if (message_part_find(p->children, part, n)) + return TRUE; + } + return FALSE; +} + unsigned int message_part_to_idx(const struct message_part *part) { - const struct message_part *p; - unsigned int n; + const struct message_part *root; + unsigned int n = 0; - if (part->parent == NULL) { - /* root */ - return 0; - } - for (n = 0, p = part->parent->children; p != part; p = p->next, n++) - ; - return n + 1 + message_part_to_idx(part->parent); + root = message_part_root(part); + if (!message_part_find(root, part, &n)) + i_unreached(); + return n; } static struct message_part * diff -r a83cc1411205 -r 012a355f9f8a src/lib-mail/test-message-part.c --- a/src/lib-mail/test-message-part.c Sun Jul 19 16:21:04 2015 +0300 +++ b/src/lib-mail/test-message-part.c Sun Jul 19 16:45:23 2015 +0300 @@ -34,13 +34,30 @@ "

Hello world

\n" "\n" "--sub1\n" -"Content-Type: text/plain\n" +"Content-Type: multipart/alternative; boundary=\"sub2\"\n" "\n" -"Hello another world\n" +"--sub2\n" +"Content-Type: multipart/alternative; boundary=\"sub3\"\n" "\n" +"--sub3\n" +"\n" +"sub3 text\n" +"--sub3\n" +"\n" +"sub3 text2\n" +"--sub3--\n" +"\n" +"sub2 text\n" +"--sub2\n" +"\n" +"sub2 text2\n" "--sub1--\n" "Sub MIME epilogue\n" "\n" +"--foo bar\n" +"Content-Type: text/plain\n" +"\n" +"Another part\n" "--foo bar--\n" "Root MIME epilogue\n" "\n"; @@ -52,7 +69,7 @@ struct istream *input; struct message_part *parts, *part, *prev_part; struct message_block block; - unsigned int i; + unsigned int i, prev_idx = 0, part_idx; pool_t pool; int ret; @@ -61,7 +78,11 @@ input = i_stream_create_from_data(test_msg, TEST_MSG_LEN); parser = message_parser_init(pool, input, 0, 0); - while ((ret = message_parser_parse_next_block(parser, &block)) > 0) ; + while ((ret = message_parser_parse_next_block(parser, &block)) > 0) { + part_idx = message_part_to_idx(block.part); + test_assert(part_idx >= prev_idx); + prev_idx = part_idx; + } test_assert(ret < 0); test_assert(message_parser_deinit(&parser, &parts) == 0); @@ -69,7 +90,7 @@ test_assert(part == parts); test_assert(message_part_by_idx(parts, 1) == parts->children); - for (i = 1; i < 6; i++) { + for (i = 1; i < 11; i++) { prev_part = part; part = message_part_by_idx(parts, i); test_assert(part != NULL); From dovecot at dovecot.org Mon Jul 20 07:33:59 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 20 Jul 2015 07:33:59 +0000 Subject: dovecot-2.2: lib-storage: Another fix for doing multiple changes... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/32d72cb26f9e changeset: 18916:32d72cb26f9e user: Timo Sirainen date: Mon Jul 20 10:33:39 2015 +0300 description: lib-storage: Another fix for doing multiple changes via mailbox_attribute_set/unset diffstat: src/lib-storage/index/index-attribute.c | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diffs (22 lines): diff -r 012a355f9f8a -r 32d72cb26f9e src/lib-storage/index/index-attribute.c --- a/src/lib-storage/index/index-attribute.c Sun Jul 19 16:45:23 2015 +0300 +++ b/src/lib-storage/index/index-attribute.c Mon Jul 20 10:33:39 2015 +0300 @@ -159,6 +159,7 @@ { struct dict_transaction_context **dtransp = NULL; struct dict *dict; + struct mailbox_metadata metadata; switch (type) { case MAIL_ATTRIBUTE_TYPE_PRIVATE: @@ -172,6 +173,10 @@ if (*dtransp != NULL) { /* transaction already created */ + if (mailbox_get_metadata(t->box, MAILBOX_METADATA_GUID, + &metadata) < 0) + return -1; + *mailbox_prefix_r = guid_128_to_string(metadata.guid); *dtrans_r = *dtransp; return 0; } From pigeonhole at rename-it.nl Fri Jul 24 21:26:06 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Fri, 24 Jul 2015 23:26:06 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: util: Fixed RFC5322 header fo... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/e97b533c0be6 changeset: 2081:e97b533c0be6 user: Stephan Bosch date: Fri Jul 24 23:24:33 2015 +0200 description: lib-sieve: util: Fixed RFC5322 header folding. diffstat: src/lib-sieve/util/rfc2822.c | 30 +++++++---- tests/extensions/vacation/message.svtest | 78 +++++++++++++++++++++++++++++++- 2 files changed, 96 insertions(+), 12 deletions(-) diffs (158 lines): diff -r 1d5a1d4f7bb7 -r e97b533c0be6 src/lib-sieve/util/rfc2822.c --- a/src/lib-sieve/util/rfc2822.c Mon Jul 06 23:47:05 2015 +0200 +++ b/src/lib-sieve/util/rfc2822.c Fri Jul 24 23:24:33 2015 +0200 @@ -158,34 +158,42 @@ if ( *bp == '\0' ) break; /* Existing newline ? */ - if ( nlp != NULL ) { + if ( nlp != NULL ) { /* Replace any sort of newline for consistency */ while ( *bp == '\r' || *bp == '\n' ) bp++; str_append_n(header, sp, nlp-sp); - if ( *bp != '\0' && *bp != ' ' && *bp != '\t' ) { - if ( crlf ) - str_append_n(header, "\r\n\t", 3); - else - str_append_n(header, "\n\t", 2); - } else { - if ( crlf ) - str_append_n(header, "\r\n", 2); - else - str_append_n(header, "\n", 1); + if ( crlf ) + str_append_n(header, "\r\n", 2); + else + str_append_n(header, "\n", 1); + + if ( *bp != '\0' && (*bp == ' ' || *bp == '\t') ) { + /* Continued line; replace leading whitespace with single TAB */ + str_append_c(header, '\t'); + while ( *bp == ' ' || *bp == '\t' ) + bp++; } sp = bp; } else { /* Insert newline at last whitespace within the max_line limit */ str_append_n(header, sp, wp-sp); + + /* Force continued line; drop any existing whitespace */ + while ( *wp == ' ' || *wp == '\t' ) + wp++; + if ( crlf ) str_append_n(header, "\r\n", 2); else str_append_n(header, "\n", 1); + /* Insert single TAB instead of the original whitespace */ + str_append_c(header, '\t'); + sp = wp; } diff -r 1d5a1d4f7bb7 -r e97b533c0be6 tests/extensions/vacation/message.svtest --- a/tests/extensions/vacation/message.svtest Mon Jul 06 23:47:05 2015 +0200 +++ b/tests/extensions/vacation/message.svtest Fri Jul 24 23:24:33 2015 +0200 @@ -49,15 +49,92 @@ } /* + * References - long IDs + */ + +test_result_reset; + +test_set "message" text: +Date: Fri, 21 Jul 2013 10:34:14 +0200 (CEST) +From: Test +To: User Two +Message-ID: <1294794880.187.416268f9-b907-4566-af85-c77155eb7d96.farce at fresno.local> +In-Reply-To: <1813483923.1202.aa78bea5-b5bc-4ab9-a64f-af96521e3af3.frobnitzm at dev.frobnitzm.com> +References: + <500510465.1519.d2ac1c0c-08f7-44fd-97aa-dd711411aacf.frobnitzm at dev.frobnitzm.com> + <717028309.1200.aa78bea5-b5bc-4ab9-a64f-af96521e3af3.frobnitzm at dev.frobnitzm.com> + <1813483923.1202.aa78bea5-b5bc-4ab9-a64f-af96521e3af3.frobnitzm at dev.frobnitzm.com> +Subject: Re: Fwd: My mail +MIME-Version: 1.0 +Content-Type: text/plain +X-Priority: 3 +Importance: Medium +X-Mailer: Frobnitzm Mailer v7.8.0-Rev0 + +Frop +. +; + +test "References - long IDs" { + vacation "I am not in today!"; + + if not test_result_execute { + test_fail "execution of result failed"; + } + + test_message :smtp 0; + + if not header :contains "references" "1294794880.187.416268f9-b907-4566-af85-c77155eb7d96.farce at fresno.local" { + test_fail "references header does not contain new id"; + } + + if anyof ( + not header :contains "references" "d660a7d1-43c9-47ea-a59a-0b29abc861d2 at frop.xi.local", + not header :contains "references" "500510465.1519.d2ac1c0c-08f7-44fd-97aa-dd711411aacf.frobnitzm at dev.frobnitzm.com", + not header :contains "references" "717028309.1200.aa78bea5-b5bc-4ab9-a64f-af96521e3af3.frobnitzm at dev.frobnitzm.com", + not header :contains "references" "1813483923.1202.aa78bea5-b5bc-4ab9-a64f-af96521e3af3.frobnitzm at dev.frobnitzm.com" + ) { + test_fail "references header does not contain all existing ids"; + } + + if header :contains "references" "hutsefluts" { + test_fail "references header contains nonsense"; + } +} + +/* * In-Reply-To */ +test_result_reset; + +test_set "message" text: +From: stephan at example.org +Subject: frop +References: <1234 at local.machine.example> <3456 at example.net> + <435444 at ttms.example.org> <4223 at froop.example.net> +Message-ID: <432df324 at example.org> +To: nico at frop.example.org + +Frop +. +; + test "In-Reply-To" { + vacation "I am not in today!"; + + if not test_result_execute { + test_fail "execution of result failed"; + } + + test_message :smtp 0; + if not header :is "in-reply-to" "<432df324 at example.org>" { test_fail "in-reply-to header set incorrectly"; } } + /* * Variables */ @@ -177,4 +254,3 @@ test_fail "envelope sender not set properly"; } } - From pigeonhole at rename-it.nl Fri Jul 24 21:54:30 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Fri, 24 Jul 2015 23:54:30 +0200 Subject: dovecot-2.2-pigeonhole: testsuite: Fixed default envelope extrac... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/4359d879bd1d changeset: 2082:4359d879bd1d user: Stephan Bosch date: Fri Jul 24 23:54:25 2015 +0200 description: testsuite: Fixed default envelope extraction fron test message. diffstat: src/testsuite/testsuite-message.c | 46 +++++++++++++++++++++++++++++++------- 1 files changed, 37 insertions(+), 9 deletions(-) diffs (95 lines): diff -r e97b533c0be6 -r 4359d879bd1d src/testsuite/testsuite-message.c --- a/src/testsuite/testsuite-message.c Fri Jul 24 23:24:33 2015 +0200 +++ b/src/testsuite/testsuite-message.c Fri Jul 24 23:54:25 2015 +0200 @@ -4,6 +4,7 @@ #include "lib.h" #include "str.h" #include "istream.h" +#include "message-address.h" #include "mail-storage.h" #include "master-service.h" @@ -38,6 +39,23 @@ pool_t message_pool; +static const char * +testsuite_message_get_address(struct mail *mail, const char *header) +{ + struct message_address *addr; + const char *str; + + if (mail_get_first_header(mail, header, &str) <= 0) + return NULL; + addr = message_address_parse(pool_datastack_create(), + (const unsigned char *)str, + strlen(str), 1, FALSE); + return (addr == NULL || + addr->mailbox == NULL || *addr->mailbox == '\0' ? NULL : + ( addr->domain == NULL || *addr->domain == '\0' ? addr->mailbox : + t_strconcat(addr->mailbox, "@", addr->domain, NULL))); +} + static void testsuite_message_set_data(struct mail *mail) { const char *recipient = NULL, *sender = NULL; @@ -47,27 +65,36 @@ */ /* Get recipient address */ - (void)mail_get_first_header(mail, "Envelope-To", &recipient); + recipient = testsuite_message_get_address(mail, "Envelope-To"); if ( recipient == NULL ) - (void)mail_get_first_header(mail, "To", &recipient); + recipient = testsuite_message_get_address(mail, "To"); if ( recipient == NULL ) recipient = "recipient at example.com"; /* Get sender address */ - (void)mail_get_first_header(mail, "Return-path", &sender); + sender = testsuite_message_get_address(mail, "Return-path"); if ( sender == NULL ) - (void)mail_get_first_header(mail, "Sender", &sender); + sender = testsuite_message_get_address(mail, "Sender"); if ( sender == NULL ) - (void)mail_get_first_header(mail, "From", &sender); + sender = testsuite_message_get_address(mail, "From"); if ( sender == NULL ) sender = "sender at example.com"; memset(&testsuite_msgdata, 0, sizeof(testsuite_msgdata)); testsuite_msgdata.mail = mail; testsuite_msgdata.auth_user = sieve_tool_get_username(sieve_tool); - testsuite_msgdata.return_path = sender; - testsuite_msgdata.orig_envelope_to = recipient; - testsuite_msgdata.final_envelope_to = recipient; + + str_truncate(envelope_from, 0); + str_append(envelope_from, sender); + testsuite_msgdata.return_path = str_c(envelope_from); + + str_truncate(envelope_to, 0); + str_append(envelope_to, recipient); + testsuite_msgdata.final_envelope_to = str_c(envelope_to); + + str_truncate(envelope_orig_to, 0); + str_append(envelope_orig_to, recipient); + testsuite_msgdata.orig_envelope_to = str_c(envelope_orig_to); (void)mail_get_first_header(mail, "Message-ID", &testsuite_msgdata.id); } @@ -80,12 +107,13 @@ str_append(default_message, _default_message_data); testsuite_mail = sieve_tool_open_data_as_mail(sieve_tool, default_message); - testsuite_message_set_data(testsuite_mail); envelope_to = str_new(message_pool, 256); envelope_orig_to = str_new(message_pool, 256); envelope_from = str_new(message_pool, 256); envelope_auth = str_new(message_pool, 256); + + testsuite_message_set_data(testsuite_mail); } void testsuite_message_set_string From pigeonhole at rename-it.nl Fri Jul 24 22:53:57 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sat, 25 Jul 2015 00:53:57 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: Substituted inappropriate use... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/f8374f7c6e92 changeset: 2083:f8374f7c6e92 user: Stephan Bosch date: Sat Jul 25 00:53:51 2015 +0200 description: lib-sieve: Substituted inappropriate use of str_append_n() with normal str_append(). diffstat: src/lib-sieve/util/rfc2822.c | 14 +++++++------- 1 files changed, 7 insertions(+), 7 deletions(-) diffs (48 lines): diff -r 4359d879bd1d -r f8374f7c6e92 src/lib-sieve/util/rfc2822.c --- a/src/lib-sieve/util/rfc2822.c Fri Jul 24 23:54:25 2015 +0200 +++ b/src/lib-sieve/util/rfc2822.c Sat Jul 25 00:53:51 2015 +0200 @@ -134,7 +134,7 @@ /* Write header field name first */ str_append_n(header, name, line_len); - str_append_n(header, ": ", 2); + str_append(header, ": "); if ( body_offset_r != NULL ) *body_offset_r = str_len(header); @@ -166,9 +166,9 @@ str_append_n(header, sp, nlp-sp); if ( crlf ) - str_append_n(header, "\r\n", 2); + str_append(header, "\r\n"); else - str_append_n(header, "\n", 1); + str_append(header, "\n"); if ( *bp != '\0' && (*bp == ' ' || *bp == '\t') ) { /* Continued line; replace leading whitespace with single TAB */ @@ -187,9 +187,9 @@ wp++; if ( crlf ) - str_append_n(header, "\r\n", 2); + str_append(header, "\r\n"); else - str_append_n(header, "\n", 1); + str_append(header, "\n"); /* Insert single TAB instead of the original whitespace */ str_append_c(header, '\t'); @@ -207,9 +207,9 @@ if ( bp != sp || lines == 0 ) { str_append_n(header, sp, bp-sp); if ( crlf ) - str_append_n(header, "\r\n", 2); + str_append(header, "\r\n"); else - str_append_n(header, "\n", 1); + str_append(header, "\n"); lines++; }