From dovecot at dovecot.org Tue Mar 4 15:12:29 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 04 Mar 2014 15:12:29 +0000 Subject: dovecot-2.2: lib-storage: Fixed assert-crash in mailbox rename w... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/346981286d20 changeset: 17152:346981286d20 user: Timo Sirainen date: Tue Mar 04 17:11:49 2014 +0200 description: lib-storage: Fixed assert-crash in mailbox rename with layout=index diffstat: src/lib-storage/list/mailbox-list-index-sync.c | 3 +++ src/lib-storage/list/mailbox-list-index.c | 2 +- src/lib-storage/list/mailbox-list-index.h | 1 + 3 files changed, 5 insertions(+), 1 deletions(-) diffs (36 lines): diff -r fad7f879caa1 -r 346981286d20 src/lib-storage/list/mailbox-list-index-sync.c --- a/src/lib-storage/list/mailbox-list-index-sync.c Mon Feb 24 12:30:51 2014 -0600 +++ b/src/lib-storage/list/mailbox-list-index-sync.c Tue Mar 04 17:11:49 2014 +0200 @@ -238,6 +238,9 @@ i_assert(!ilist->syncing); + if (mailbox_list_index_index_open(list) < 0) + return -1; + if (mail_index_sync_begin(ilist->index, &index_sync_ctx, &view, &trans, MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES) < 0) { mailbox_list_index_set_index_error(list); diff -r fad7f879caa1 -r 346981286d20 src/lib-storage/list/mailbox-list-index.c --- a/src/lib-storage/list/mailbox-list-index.c Mon Feb 24 12:30:51 2014 -0600 +++ b/src/lib-storage/list/mailbox-list-index.c Tue Mar 04 17:11:49 2014 +0200 @@ -44,7 +44,7 @@ mailbox_list_index_init_pool(ilist); } -static int mailbox_list_index_index_open(struct mailbox_list *list) +int mailbox_list_index_index_open(struct mailbox_list *list) { struct mailbox_list_index *ilist = INDEX_LIST_CONTEXT(list); const struct mail_storage_settings *set = list->mail_set; diff -r fad7f879caa1 -r 346981286d20 src/lib-storage/list/mailbox-list-index.h --- a/src/lib-storage/list/mailbox-list-index.h Mon Feb 24 12:30:51 2014 -0600 +++ b/src/lib-storage/list/mailbox-list-index.h Tue Mar 04 17:11:49 2014 +0200 @@ -137,6 +137,7 @@ void mailbox_list_index_node_unlink(struct mailbox_list_index *ilist, struct mailbox_list_index_node *node); +int mailbox_list_index_index_open(struct mailbox_list *list); bool mailbox_list_index_need_refresh(struct mailbox_list_index *ilist, struct mail_index_view *view); int mailbox_list_index_refresh(struct mailbox_list *list); From pigeonhole at rename-it.nl Tue Mar 4 21:29:36 2014 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Tue, 04 Mar 2014 22:29:36 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: Upgraded "vnd.dovecot.duplica... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/8c53243b341f changeset: 1845:8c53243b341f user: Stephan Bosch date: Tue Mar 04 22:28:56 2014 +0100 description: lib-sieve: Upgraded "vnd.dovecot.duplicate" to "duplicate". Backwards compatibility is provided for vnd.dovecot.duplicate. Still need to fix the constraint that it must not track duplicates from failed Sieve executions. diffstat: Makefile.am | 5 +- README | 4 +- configure.ac | 2 +- doc/extensions/duplicate.txt | 47 + doc/extensions/vnd.dovecot.duplicate.txt | 57 - doc/rfc/draft-ietf-appsawg-sieve-duplicate-03.txt | 728 ++++++++++ src/lib-sieve/Makefile.am | 2 +- src/lib-sieve/plugins/Makefile.am | 1 + src/lib-sieve/plugins/duplicate/Makefile.am | 20 + src/lib-sieve/plugins/duplicate/ext-duplicate-common.c | 262 +++ src/lib-sieve/plugins/duplicate/ext-duplicate-common.h | 46 + src/lib-sieve/plugins/duplicate/ext-duplicate.c | 107 + src/lib-sieve/plugins/duplicate/tst-duplicate.c | 416 +++++ src/lib-sieve/plugins/vnd.dovecot/Makefile.am | 3 +- src/lib-sieve/plugins/vnd.dovecot/debug/cmd-debug-log.c | 2 +- src/lib-sieve/plugins/vnd.dovecot/debug/ext-debug-common.h | 2 +- src/lib-sieve/plugins/vnd.dovecot/debug/ext-debug.c | 2 +- src/lib-sieve/plugins/vnd.dovecot/duplicate/Makefile.am | 20 - src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.c | 231 --- src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate-common.h | 45 - src/lib-sieve/plugins/vnd.dovecot/duplicate/ext-duplicate.c | 50 - src/lib-sieve/plugins/vnd.dovecot/duplicate/tst-duplicate.c | 364 ----- src/lib-sieve/sieve-extensions.c | 18 +- src/plugins/sieve-extprograms/cmd-execute.c | 2 +- src/plugins/sieve-extprograms/cmd-filter.c | 2 +- src/plugins/sieve-extprograms/cmd-pipe.c | 2 +- src/plugins/sieve-extprograms/ext-execute.c | 2 +- src/plugins/sieve-extprograms/ext-filter.c | 2 +- src/plugins/sieve-extprograms/ext-pipe.c | 4 +- src/plugins/sieve-extprograms/sieve-extprograms-common.c | 4 +- src/plugins/sieve-extprograms/sieve-extprograms-common.h | 6 +- src/plugins/sieve-extprograms/sieve-extprograms-plugin.c | 6 +- tests/extensions/duplicate/errors.svtest | 54 + tests/extensions/duplicate/errors/conflict-vnd.sieve | 4 + tests/extensions/duplicate/errors/conflict.sieve | 4 + tests/extensions/duplicate/errors/syntax-vnd.sieve | 19 + tests/extensions/duplicate/errors/syntax.sieve | 54 + tests/extensions/duplicate/execute-vnd.svtest | 20 + tests/extensions/duplicate/execute.svtest | 41 + tests/extensions/vnd.dovecot/duplicate/errors.svtest | 18 - tests/extensions/vnd.dovecot/duplicate/errors/syntax.sieve | 19 - tests/extensions/vnd.dovecot/duplicate/execute.svtest | 20 - 42 files changed, 1858 insertions(+), 859 deletions(-) diffs (truncated from 3053 to 300 lines): diff -r cf5eba41f6bd -r 8c53243b341f Makefile.am --- a/Makefile.am Tue Feb 25 00:49:08 2014 +0100 +++ b/Makefile.am Tue Mar 04 22:28:56 2014 +0100 @@ -136,9 +136,10 @@ tests/extensions/editheader/utf8.svtest \ tests/extensions/editheader/protected.svtest \ tests/extensions/editheader/errors.svtest \ + tests/extensions/duplicate/errors.svtest \ + tests/extensions/duplicate/execute.svtest \ + tests/extensions/duplicate/execute-vnd.svtest \ tests/extensions/vnd.dovecot/debug/execute.svtest \ - tests/extensions/vnd.dovecot/duplicate/errors.svtest \ - tests/extensions/vnd.dovecot/duplicate/execute.svtest \ tests/deprecated/notify/basic.svtest \ tests/deprecated/notify/mailto.svtest \ tests/deprecated/notify/errors.svtest \ diff -r cf5eba41f6bd -r 8c53243b341f README --- a/README Tue Feb 25 00:49:08 2014 +0100 +++ b/README Tue Mar 04 22:28:56 2014 +0100 @@ -120,6 +120,7 @@ mailbox (RFC 5490; Section 3): fully supported (v0.1.10+), but ACL permissions are not verified for mailboxexists. include (RFC 6609): fully supported (v0.4.0+) + duplicate (draft v03): fully supported (v0.4.3+). regex (draft v08; not latest version): almost fully supported, but UTF-8 is not supported. @@ -135,9 +136,6 @@ vnd.dovecot.debug (v0.3.0+): Allows logging debug messages - vnd.dovecot.duplicate (v0.3.1+): - Allows detecting duplicate message deliveries based on message ID and - other criteria. vnd.dovecot.pipe (v0.4.0+; sieve_extprograms plugin): Implements piping messages to a pre-defined set of external programs vnd.dovecot.filter (v0.4.0+; sieve_extprograms plugin): diff -r cf5eba41f6bd -r 8c53243b341f configure.ac --- a/configure.ac Tue Feb 25 00:49:08 2014 +0100 +++ b/configure.ac Tue Mar 04 22:28:56 2014 +0100 @@ -122,9 +122,9 @@ src/lib-sieve/plugins/ihave/Makefile src/lib-sieve/plugins/editheader/Makefile src/lib-sieve/plugins/metadata/Makefile +src/lib-sieve/plugins/duplicate/Makefile src/lib-sieve/plugins/vnd.dovecot/Makefile src/lib-sieve/plugins/vnd.dovecot/debug/Makefile -src/lib-sieve/plugins/vnd.dovecot/duplicate/Makefile src/lib-sieve-tool/Makefile src/lib-sievestorage/Makefile src/lib-managesieve/Makefile diff -r cf5eba41f6bd -r 8c53243b341f doc/extensions/duplicate.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/doc/extensions/duplicate.txt Tue Mar 04 22:28:56 2014 +0100 @@ -0,0 +1,47 @@ +Duplicate Extension + +Relevant specifications +======================= + + doc/rfc/draft-ietf-appsawg-sieve-duplicate-03.txt + +Description +=========== + +The duplicate extension augments the Sieve filtering implementation with a test +to verify whether the evaluated string value was seen before in an earlier +execution of the Sieve script. The main application for this new test is +detecting and handling duplicate message deliveries, e.g. as caused by +mailinglists when people reply both to the mailinglist and the user directly. + +Refer to doc/rfc/draft-ietf-appsawg-sieve-duplicate-03.txt for a specification +of the Sieve language extension. Previously, this extension was Dovecot-specific +and available under the name "vnd.dovecot.duplicate". That implementation +differs significantly from what is now published as an internet draft, but +for backwards compatibility the original extension is still supported. + +Configuration +============= + +The "duplicate" extension is not enabled by default. + +The following configuration settings are used: + +sieve_duplicate_default_period = 14d +sieve_duplicate_max_period = 7d + These options respectively specify the default and the maximum value for the + period after which tracked values are purged from the duplicate tracking + database. The period is specified in s(econds), unless followed by a d(ay), + h(our) or m(inute) specifier character. + +Example +======= + +plugin { + sieve = ~/.dovecot.sieve + + sieve_extensions = +vnd.dovecot.duplicate + + sieve_duplicate_default_period = 1h + sieve_duplicate_max_period = 1d +}d diff -r cf5eba41f6bd -r 8c53243b341f doc/extensions/vnd.dovecot.duplicate.txt --- a/doc/extensions/vnd.dovecot.duplicate.txt Tue Feb 25 00:49:08 2014 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,58 +0,0 @@ -Vnd.dovecot.duplicate Extension - -Relevant specifications -======================= - - doc/rfc/spec-bosch-sieve-duplicate.txt - -Description -=========== - -The vnd.dovecot.duplicate extension augments the Sieve filtering implementation -with a test to verify whether the evaluated string value was seen before in an -earlier execution of the Sieve script. The main application for this new test is -detecting and handling duplicate message deliveries, e.g. as caused by -mailinglists when people reply both to the mailinglist and the user directly. - -This extension is specific to the Pigeonhole Sieve implementation for the -Dovecot Secure IMAP server. It will therefore most likely not be supported by -web interfaces or GUI-based Sieve editors. - -Refer to doc/rfc/spec-bosch-sieve-duplicate.txt for a specification of the Sieve -language extension. - -Implementation Status ---------------------- - -The "vnd.dovecot.duplicate" Sieve language extension is vendor-specific with -draft status and its implementation for Pigeonhole is experimental, which means -that the language extensions are still subject to change and that the current -implementation is not thoroughly tested. - -Configuration -============= - -The "vnd.dovecot.duplicate" extension is not enabled by default and thus it -needs to be enabled explicitly by adding it to the `sieve_extensions' or the -`sieve_global_extensions' setting. - -The following configuration settings are used: - -sieve_duplicate_default_period = 12h -sieve_duplicate_max_period = 2d - These options respectively specify the default and the maximum value for the - period after which tracked values are purged from the duplicate tracking - database. The period is specified in s(econds), unless followed by a d(ay), - h(our) or m(inute) specifier character. - -Example -======= - -plugin { - sieve = ~/.dovecot.sieve - - sieve_extensions = +vnd.dovecot.duplicate - - sieve_duplicate_default_period = 1h - sieve_duplicate_max_period = 1d -} diff -r cf5eba41f6bd -r 8c53243b341f doc/rfc/draft-ietf-appsawg-sieve-duplicate-03.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/doc/rfc/draft-ietf-appsawg-sieve-duplicate-03.txt Tue Mar 04 22:28:56 2014 +0100 @@ -0,0 +1,728 @@ + + + +APPSAWG S. Bosch +Internet-Draft March 3, 2014 +Intended status: Standards Track +Expires: September 4, 2014 + + + Sieve Email Filtering: Detecting Duplicate Deliveries + draft-ietf-appsawg-sieve-duplicate-03 + +Abstract + + This document defines a new test command "duplicate" for the "Sieve" + email filtering language. This test adds the ability to detect + duplications. The main application for this new test is handling + duplicate deliveries commonly caused by mailing list subscriptions or + redirected mail addresses. The detection is normally performed by + matching the message ID to an internal list of message IDs from + previously delivered messages. For more complex applications, the + "duplicate" test can also use the content of a specific header or + other parts of the message. + +Status of this Memo + + This Internet-Draft is submitted in full conformance with the + provisions of BCP 78 and BCP 79. + + Internet-Drafts are working documents of the Internet Engineering + Task Force (IETF). Note that other groups may also distribute + working documents as Internet-Drafts. The list of current Internet- + Drafts is at http://datatracker.ietf.org/drafts/current/. + + Internet-Drafts are draft documents valid for a maximum of six months + and may be updated, replaced, or obsoleted by other documents at any + time. It is inappropriate to use Internet-Drafts as reference + material or to cite them other than as "work in progress." + + This Internet-Draft will expire on September 4, 2014. + +Copyright Notice + + Copyright (c) 2014 IETF Trust and the persons identified as the + document authors. All rights reserved. + + This document is subject to BCP 78 and the IETF Trust's Legal + Provisions Relating to IETF Documents + (http://trustee.ietf.org/license-info) in effect on the date of + publication of this document. Please review these documents + carefully, as they describe your rights and restrictions with respect + + + +Bosch Expires September 4, 2014 [Page 1] + +Internet-Draft Sieve: Detecting Duplicate Deliveries March 2014 + + + to this document. Code Components extracted from this document must + include Simplified BSD License text as described in Section 4.e of + the Trust Legal Provisions and are provided without warranty as + described in the Simplified BSD License. + + +Table of Contents + + 1. Introduction . . . . . . . . . . . . . . . . . . . . . . . . . 3 + 2. Conventions Used in This Document . . . . . . . . . . . . . . 3 + 3. Test "duplicate" . . . . . . . . . . . . . . . . . . . . . . . 3 + 3.1. Interaction with Other Sieve Extensions . . . . . . . . . 8 + 4. Sieve Capability Strings . . . . . . . . . . . . . . . . . . . 8 + 5. Examples . . . . . . . . . . . . . . . . . . . . . . . . . . . 8 + 5.1. Example 1 . . . . . . . . . . . . . . . . . . . . . . . . 8 + 5.2. Example 2 . . . . . . . . . . . . . . . . . . . . . . . . 8 + 5.3. Example 3 . . . . . . . . . . . . . . . . . . . . . . . . 9 + 5.4. Example 4 . . . . . . . . . . . . . . . . . . . . . . . . 10 + 6. Security Considerations . . . . . . . . . . . . . . . . . . . 11 + 7. IANA Considerations . . . . . . . . . . . . . . . . . . . . . 11 + 8. Acknowledgements . . . . . . . . . . . . . . . . . . . . . . . 11 + 9. References . . . . . . . . . . . . . . . . . . . . . . . . . . 12 + 9.1. Normative References . . . . . . . . . . . . . . . . . . . 12 + 9.2. Informative References . . . . . . . . . . . . . . . . . . 12 + Author's Address . . . . . . . . . . . . . . . . . . . . . . . . . 13 + + + + + + + + + + + + + + + + + + + + + + + + + + +Bosch Expires September 4, 2014 [Page 2] + +Internet-Draft Sieve: Detecting Duplicate Deliveries March 2014 + + +1. Introduction + + This document specifies an extension to the Sieve filtering language + defined by RFC 5228 [SIEVE]. It adds a test to track whether or not + a text string was seen before by the delivery agent in an earlier + execution of the Sieve script. This can be used to detect and handle + duplicate message deliveries. + + Duplicate deliveries are a common side-effect of being subscribed to + a mailing list. For example, if a member of the list decides to + reply to both the user and the mailing list itself, the user will + often get one copy of the message directly and another through the + mailing list. Also, if someone cross-posts over several mailing + lists to which the user is subscribed, the user will likely receive a + copy from each of those lists. In another scenario, the user has + several redirected mail addresses all pointing to his main mail + account. If one of the user's contacts sends the message to more From dovecot at dovecot.org Mon Mar 10 22:39:39 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 10 Mar 2014 22:39:39 +0000 Subject: dovecot-2.2: mdbox: Added mdbox_purge_preserve_alt setting to ke... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a94e1b560a80 changeset: 17153:a94e1b560a80 user: Timo Sirainen date: Tue Mar 11 00:39:27 2014 +0200 description: mdbox: Added mdbox_purge_preserve_alt setting to keep the file within alt storage during purge. Perhaps it would be better for this setting to be the default in v2.3 and maybe be removed entirely? Not a good idea to change during v2.2.x anyway, so for now this is a setting. diffstat: src/lib-storage/index/dbox-multi/mdbox-purge.c | 9 +++++++-- src/lib-storage/index/dbox-multi/mdbox-settings.c | 2 ++ src/lib-storage/index/dbox-multi/mdbox-settings.h | 1 + 3 files changed, 10 insertions(+), 2 deletions(-) diffs (60 lines): diff -r 346981286d20 -r a94e1b560a80 src/lib-storage/index/dbox-multi/mdbox-purge.c --- a/src/lib-storage/index/dbox-multi/mdbox-purge.c Tue Mar 04 17:11:49 2014 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-purge.c Tue Mar 11 00:39:27 2014 +0200 @@ -165,11 +165,16 @@ } static bool -mdbox_purge_want_altpath(struct mdbox_purge_context *ctx, uint32_t map_uid) +mdbox_purge_want_altpath(struct mdbox_purge_context *ctx, + struct dbox_file *file, uint32_t map_uid) { enum mdbox_msg_action action; void *value; + if (dbox_file_is_in_alt(file) && + ctx->storage->set->mdbox_purge_preserve_alt) + return TRUE; + if (!ctx->have_altmoves) return FALSE; @@ -193,7 +198,7 @@ if (ctx->append_ctx == NULL) ctx->append_ctx = mdbox_map_append_begin(ctx->atomic); - append_flags = !mdbox_purge_want_altpath(ctx, msg->map_uid) ? 0 : + append_flags = !mdbox_purge_want_altpath(ctx, file, msg->map_uid) ? 0 : DBOX_MAP_APPEND_FLAG_ALT; msg_size = file->msg_header_size + file->cur_physical_size; if (mdbox_map_append_next(ctx->append_ctx, file->cur_physical_size, diff -r 346981286d20 -r a94e1b560a80 src/lib-storage/index/dbox-multi/mdbox-settings.c --- a/src/lib-storage/index/dbox-multi/mdbox-settings.c Tue Mar 04 17:11:49 2014 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-settings.c Tue Mar 11 00:39:27 2014 +0200 @@ -13,6 +13,7 @@ static const struct setting_define mdbox_setting_defines[] = { DEF(SET_BOOL, mdbox_preallocate_space), + DEF(SET_BOOL, mdbox_purge_preserve_alt), DEF(SET_SIZE, mdbox_rotate_size), DEF(SET_TIME, mdbox_rotate_interval), @@ -21,6 +22,7 @@ static const struct mdbox_settings mdbox_default_settings = { .mdbox_preallocate_space = FALSE, + .mdbox_purge_preserve_alt = FALSE, .mdbox_rotate_size = 2*1024*1024, .mdbox_rotate_interval = 0 }; diff -r 346981286d20 -r a94e1b560a80 src/lib-storage/index/dbox-multi/mdbox-settings.h --- a/src/lib-storage/index/dbox-multi/mdbox-settings.h Tue Mar 04 17:11:49 2014 +0200 +++ b/src/lib-storage/index/dbox-multi/mdbox-settings.h Tue Mar 11 00:39:27 2014 +0200 @@ -3,6 +3,7 @@ struct mdbox_settings { bool mdbox_preallocate_space; + bool mdbox_purge_preserve_alt; uoff_t mdbox_rotate_size; unsigned int mdbox_rotate_interval; }; From dovecot at dovecot.org Tue Mar 11 11:26:49 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 11 Mar 2014 11:26:49 +0000 Subject: dovecot-2.2: fts-lucene: Skip returned duplicate results. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d63b209737be changeset: 17154:d63b209737be user: Timo Sirainen date: Tue Mar 11 13:26:33 2014 +0200 description: fts-lucene: Skip returned duplicate results. This caused assert-crash at least when searching from virtual mailbox because the score array was larger than the actual number of results. diffstat: src/plugins/fts-lucene/lucene-wrapper.cc | 16 ++++++++++------ 1 files changed, 10 insertions(+), 6 deletions(-) diffs (40 lines): diff -r a94e1b560a80 -r d63b209737be src/plugins/fts-lucene/lucene-wrapper.cc --- a/src/plugins/fts-lucene/lucene-wrapper.cc Tue Mar 11 00:39:27 2014 +0200 +++ b/src/plugins/fts-lucene/lucene-wrapper.cc Tue Mar 11 13:26:33 2014 +0200 @@ -1271,7 +1271,9 @@ break; } - if (result != NULL) { + if (seq_range_array_add(uids_r, uid)) { + /* duplicate result */ + } else if (result != NULL) { if (uid < last_uid) result->scores_sorted = false; last_uid = uid; @@ -1280,7 +1282,6 @@ score->uid = uid; score->score = hits->score(i); } - seq_range_array_add(uids_r, uid); } _CLDELETE(hits); return ret; @@ -1390,10 +1391,13 @@ p_array_init(&br->definite_uids, result->pool, 32); p_array_init(&br->scores, result->pool, 32); } - seq_range_array_add(&br->definite_uids, uid); - score = array_append_space(&br->scores); - score->uid = uid; - score->score = hits->score(i); + if (seq_range_array_add(&br->definite_uids, uid)) { + /* duplicate result */ + } else { + score = array_append_space(&br->scores); + score->uid = uid; + score->score = hits->score(i); + } } _CLDELETE(hits); return ret; From dovecot at dovecot.org Tue Mar 11 11:26:50 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 11 Mar 2014 11:26:50 +0000 Subject: dovecot-2.1: fts-lucene: Skip returned duplicate results. Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/79bae66e666c changeset: 15012:79bae66e666c user: Timo Sirainen date: Tue Mar 11 13:24:09 2014 +0200 description: fts-lucene: Skip returned duplicate results. This caused assert-crash at least when searching from virtual mailbox because the score array was larger than the actual number of results. diffstat: src/plugins/fts-lucene/lucene-wrapper.cc | 16 ++++++++++------ 1 files changed, 10 insertions(+), 6 deletions(-) diffs (40 lines): diff -r 8e6b8d53c02b -r 79bae66e666c src/plugins/fts-lucene/lucene-wrapper.cc --- a/src/plugins/fts-lucene/lucene-wrapper.cc Wed Feb 05 00:47:55 2014 +0200 +++ b/src/plugins/fts-lucene/lucene-wrapper.cc Tue Mar 11 13:24:09 2014 +0200 @@ -1240,7 +1240,9 @@ break; } - if (result != NULL) { + if (seq_range_array_add(uids_r, 0, uid)) { + /* duplicate result */ + } else if (result != NULL) { if (uid < last_uid) result->scores_sorted = false; last_uid = uid; @@ -1249,7 +1251,6 @@ score->uid = uid; score->score = hits->score(i); } - seq_range_array_add(uids_r, 0, uid); } _CLDELETE(hits); return ret; @@ -1358,10 +1359,13 @@ p_array_init(&br->definite_uids, result->pool, 32); p_array_init(&br->scores, result->pool, 32); } - seq_range_array_add(&br->definite_uids, 0, uid); - score = array_append_space(&br->scores); - score->uid = uid; - score->score = hits->score(i); + if (seq_range_array_add(&br->definite_uids, 0, uid)) { + /* duplicate result */ + } else { + score = array_append_space(&br->scores); + score->uid = uid; + score->score = hits->score(i); + } } _CLDELETE(hits); return ret; From dovecot at dovecot.org Wed Mar 12 09:54:06 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 12 Mar 2014 09:54:06 +0000 Subject: dovecot-2.2: auth: If auth-worker lookup times out, log a bit mo... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8df5fc361103 changeset: 17155:8df5fc361103 user: Timo Sirainen date: Wed Mar 12 11:53:49 2014 +0200 description: auth: If auth-worker lookup times out, log a bit more details about it. diffstat: src/auth/auth-worker-server.c | 8 ++++++-- src/auth/auth-worker-server.h | 2 +- src/auth/passdb-blocking.c | 6 +++--- src/auth/userdb-blocking.c | 6 ++++-- 4 files changed, 14 insertions(+), 8 deletions(-) diffs (103 lines): diff -r d63b209737be -r 8df5fc361103 src/auth/auth-worker-server.c --- a/src/auth/auth-worker-server.c Tue Mar 11 13:26:33 2014 +0200 +++ b/src/auth/auth-worker-server.c Wed Mar 12 11:53:49 2014 +0200 @@ -26,6 +26,7 @@ struct auth_worker_request { unsigned int id; time_t created; + const char *username; const char *data; auth_worker_callback_t *callback; void *context; @@ -223,7 +224,9 @@ idle_count--; if (conn->request != NULL) { - i_error("auth worker: Aborted request: %s", reason); + i_error("auth worker: Aborted %s request for %s: %s", + t_strcut(conn->request->data, '\t'), + conn->request->username, reason); conn->request->callback(t_strdup_printf( "FAIL\t%d", PASSDB_RESULT_INTERNAL_FAILURE), conn->request->context); @@ -399,7 +402,7 @@ } struct auth_worker_connection * -auth_worker_call(pool_t pool, const char *data, +auth_worker_call(pool_t pool, const char *username, const char *data, auth_worker_callback_t *callback, void *context) { struct auth_worker_connection *conn; @@ -407,6 +410,7 @@ request = p_new(pool, struct auth_worker_request, 1); request->created = ioloop_time; + request->username = p_strdup(pool, username); request->data = p_strdup(pool, data); request->callback = callback; request->context = context; diff -r d63b209737be -r 8df5fc361103 src/auth/auth-worker-server.h --- a/src/auth/auth-worker-server.h Tue Mar 11 13:26:33 2014 +0200 +++ b/src/auth/auth-worker-server.h Wed Mar 12 11:53:49 2014 +0200 @@ -7,7 +7,7 @@ typedef bool auth_worker_callback_t(const char *reply, void *context); struct auth_worker_connection * ATTR_NOWARN_UNUSED_RESULT -auth_worker_call(pool_t pool, const char *data, +auth_worker_call(pool_t pool, const char *username, const char *data, auth_worker_callback_t *callback, void *context); void auth_worker_server_resume_input(struct auth_worker_connection *conn); diff -r d63b209737be -r 8df5fc361103 src/auth/passdb-blocking.c --- a/src/auth/passdb-blocking.c Tue Mar 11 13:26:33 2014 +0200 +++ b/src/auth/passdb-blocking.c Wed Mar 12 11:53:49 2014 +0200 @@ -83,7 +83,7 @@ auth_request_export(request, str); auth_request_ref(request); - auth_worker_call(request->pool, str_c(str), + auth_worker_call(request->pool, request->user, str_c(str), verify_plain_callback, request); } @@ -123,7 +123,7 @@ auth_request_export(request, str); auth_request_ref(request); - auth_worker_call(request->pool, str_c(str), + auth_worker_call(request->pool, request->user, str_c(str), lookup_credentials_callback, request); } @@ -151,6 +151,6 @@ auth_request_export(request, str); auth_request_ref(request); - auth_worker_call(request->pool, str_c(str), + auth_worker_call(request->pool, request->user, str_c(str), set_credentials_callback, request); } diff -r d63b209737be -r 8df5fc361103 src/auth/userdb-blocking.c --- a/src/auth/userdb-blocking.c Tue Mar 11 13:26:33 2014 +0200 +++ b/src/auth/userdb-blocking.c Wed Mar 12 11:53:49 2014 +0200 @@ -57,7 +57,8 @@ auth_request_export(request, str); auth_request_ref(request); - auth_worker_call(request->pool, str_c(str), user_callback, request); + auth_worker_call(request->pool, request->user, + str_c(str), user_callback, request); } static bool iter_callback(const char *reply, void *context) @@ -95,7 +96,8 @@ ctx->ctx.context = context; auth_request_ref(request); - ctx->conn = auth_worker_call(request->pool, str_c(str), iter_callback, ctx); + ctx->conn = auth_worker_call(request->pool, "*", + str_c(str), iter_callback, ctx); return &ctx->ctx; } From dovecot at dovecot.org Wed Mar 12 10:26:27 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 12 Mar 2014 10:26:27 +0000 Subject: dovecot-2.2: auth worker: Log a warning if master disconnects us... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4856ff35f7be changeset: 17156:4856ff35f7be user: Timo Sirainen date: Wed Mar 12 12:19:09 2014 +0200 description: auth worker: Log a warning if master disconnects us while handling a long request. diffstat: src/auth/auth-worker-client.c | 24 +++++++++++++++++++----- 1 files changed, 19 insertions(+), 5 deletions(-) diffs (95 lines): diff -r 8df5fc361103 -r 4856ff35f7be src/auth/auth-worker-client.c --- a/src/auth/auth-worker-client.c Wed Mar 12 11:53:49 2014 +0200 +++ b/src/auth/auth-worker-client.c Wed Mar 12 12:19:09 2014 +0200 @@ -16,6 +16,7 @@ #include +#define AUTH_WORKER_WARN_DISCONNECTED_LONG_CMD_SECS 30 #define OUTBUF_THROTTLE_SIZE (1024*10) #define CLIENT_STATE_HANDSHAKE "handshaking" @@ -32,6 +33,7 @@ struct istream *input; struct ostream *output; struct timeout *to_idle; + time_t cmd_start; unsigned int version_received:1; unsigned int dbhash_received:1; @@ -103,11 +105,20 @@ } static void auth_worker_send_reply(struct auth_worker_client *client, + struct auth_request *request, string_t *str) { + time_t cmd_duration = time(NULL) - client->cmd_start; + if (worker_restart_request) o_stream_nsend_str(client->output, "RESTART\n"); o_stream_nsend(client->output, str_data(str), str_len(str)); + if (o_stream_nfinish(client->output) < 0 && request != NULL && + cmd_duration > AUTH_WORKER_WARN_DISCONNECTED_LONG_CMD_SECS) { + i_warning("Auth master disconnected us while handling " + "request for %s for %ld secs", + request->user, (long)cmd_duration); + } } static void @@ -150,7 +161,7 @@ reply_append_extra_fields(str, request); } str_append_c(str, '\n'); - auth_worker_send_reply(client, str); + auth_worker_send_reply(client, request, str); auth_request_unref(&request); auth_worker_client_check_throttle(client); @@ -231,7 +242,7 @@ reply_append_extra_fields(str, request); } str_append_c(str, '\n'); - auth_worker_send_reply(client, str); + auth_worker_send_reply(client, request, str); auth_request_unref(&request); auth_worker_client_check_throttle(client); @@ -293,7 +304,7 @@ str = t_str_new(64); str_printfa(str, "%u\t%s\n", request->id, success ? "OK" : "FAIL"); - auth_worker_send_reply(client, str); + auth_worker_send_reply(client, request, str); auth_request_unref(&request); auth_worker_client_check_throttle(client); @@ -361,7 +372,7 @@ } str_append_c(str, '\n'); - auth_worker_send_reply(client, str); + auth_worker_send_reply(client, auth_request, str); auth_request_unref(&auth_request); auth_worker_client_check_throttle(client); @@ -449,7 +460,7 @@ str_printfa(str, "%u\tFAIL\n", ctx->auth_request->id); else str_printfa(str, "%u\tOK\n", ctx->auth_request->id); - auth_worker_send_reply(client, str); + auth_worker_send_reply(client, NULL, str); client->io = io_add(client->fd, IO_READ, auth_worker_input, client); auth_worker_client_set_idle_timeout(client); @@ -584,6 +595,9 @@ return FALSE; } + io_loop_time_refresh(); + client->cmd_start = ioloop_time; + auth_worker_refresh_proctitle(args[1]); if (strcmp(args[1], "PASSV") == 0) ret = auth_worker_handle_passv(client, id, args + 2); From dovecot at dovecot.org Wed Mar 12 10:26:27 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 12 Mar 2014 10:26:27 +0000 Subject: dovecot-2.2: auth worker: Log a warning if master disconnects us... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9ddb41a2c6c7 changeset: 17157:9ddb41a2c6c7 user: Timo Sirainen date: Wed Mar 12 12:26:09 2014 +0200 description: auth worker: Log a warning if master disconnects us while handling a long request. diffstat: src/auth/auth-worker-client.c | 8 ++++++-- 1 files changed, 6 insertions(+), 2 deletions(-) diffs (25 lines): diff -r 4856ff35f7be -r 9ddb41a2c6c7 src/auth/auth-worker-client.c --- a/src/auth/auth-worker-client.c Wed Mar 12 12:19:09 2014 +0200 +++ b/src/auth/auth-worker-client.c Wed Mar 12 12:26:09 2014 +0200 @@ -109,15 +109,19 @@ string_t *str) { time_t cmd_duration = time(NULL) - client->cmd_start; + const char *p; if (worker_restart_request) o_stream_nsend_str(client->output, "RESTART\n"); o_stream_nsend(client->output, str_data(str), str_len(str)); if (o_stream_nfinish(client->output) < 0 && request != NULL && cmd_duration > AUTH_WORKER_WARN_DISCONNECTED_LONG_CMD_SECS) { + p = strchr(str_c(str), '\t'); + p = p == NULL ? "BUG" : t_strcut(p+1, '\t'); + i_warning("Auth master disconnected us while handling " - "request for %s for %ld secs", - request->user, (long)cmd_duration); + "request for %s for %ld secs (result=%s)", + request->user, (long)cmd_duration, p); } } From dovecot at dovecot.org Wed Mar 12 12:09:27 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 12 Mar 2014 12:09:27 +0000 Subject: dovecot-2.2: auth: Don't disconnect auth-worker during long-runn... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/20a25e17bc3e changeset: 17158:20a25e17bc3e user: Timo Sirainen date: Wed Mar 12 14:09:14 2014 +0200 description: auth: Don't disconnect auth-worker during long-running user iterations. diffstat: src/auth/auth-worker-server.c | 29 ++++++++++++++++++++++++----- 1 files changed, 24 insertions(+), 5 deletions(-) diffs (82 lines): diff -r 9ddb41a2c6c7 -r 20a25e17bc3e src/auth/auth-worker-server.c --- a/src/auth/auth-worker-server.c Wed Mar 12 12:26:09 2014 +0200 +++ b/src/auth/auth-worker-server.c Wed Mar 12 14:09:14 2014 +0200 @@ -84,6 +84,8 @@ struct const_iovec iov[3]; unsigned int age_secs = ioloop_time - request->created; + i_assert(conn->to != NULL); + if (age_secs >= AUTH_WORKER_ABORT_SECS) { i_error("Aborting auth request that was queued for %d secs, " "%d left in queue", @@ -236,7 +238,8 @@ io_remove(&conn->io); i_stream_destroy(&conn->input); o_stream_destroy(&conn->output); - timeout_remove(&conn->to); + if (conn->to != NULL) + timeout_remove(&conn->to); if (close(conn->fd) < 0) i_error("close(auth worker) failed: %m"); @@ -266,7 +269,7 @@ return NULL; } -static void auth_worker_request_handle(struct auth_worker_connection *conn, +static bool auth_worker_request_handle(struct auth_worker_connection *conn, struct auth_worker_request *request, const char *line) { @@ -281,8 +284,12 @@ idle_count++; } - if (!request->callback(line, request->context) && conn->io != NULL) + if (!request->callback(line, request->context) && conn->io != NULL) { + timeout_remove(&conn->to); io_remove(&conn->io); + return FALSE; + } + return TRUE; } static bool auth_worker_error(struct auth_worker_connection *conn) @@ -376,8 +383,9 @@ continue; if (conn->request != NULL && id == conn->request->id) { - auth_worker_request_handle(conn, conn->request, - line + 1); + if (!auth_worker_request_handle(conn, conn->request, + line + 1)) + break; } else { if (conn->request != NULL) { i_error("BUG: Worker sent reply with id %u, " @@ -401,6 +409,14 @@ auth_worker_request_send_next(conn); } +static void worker_input_resume(struct auth_worker_connection *conn) +{ + timeout_remove(&conn->to); + conn->to = timeout_add(AUTH_WORKER_LOOKUP_TIMEOUT_SECS * 1000, + auth_worker_call_timeout, conn); + worker_input(conn); +} + struct auth_worker_connection * auth_worker_call(pool_t pool, const char *username, const char *data, auth_worker_callback_t *callback, void *context) @@ -440,6 +456,9 @@ { if (conn->io == NULL) conn->io = io_add(conn->fd, IO_READ, worker_input, conn); + if (conn->to != NULL) + timeout_remove(&conn->to); + conn->to = timeout_add_short(0, worker_input_resume, conn); } void auth_worker_server_init(void) From dovecot at dovecot.org Mon Mar 17 14:57:56 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 17 Mar 2014 14:57:56 +0000 Subject: dovecot-2.2: liblib: Added negative %variable lengths to count b... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/97d31cbbfe01 changeset: 17159:97d31cbbfe01 user: Timo Sirainen date: Mon Mar 17 16:57:21 2014 +0200 description: liblib: Added negative %variable lengths to count backwards from the end of the value. For example %0.-2 for "12345" returns "123". Patch by Norbert Weinhold / Open-Xchange. diffstat: src/lib/var-expand.c | 20 ++++++++++++++------ 1 files changed, 14 insertions(+), 6 deletions(-) diffs (65 lines): diff -r 20a25e17bc3e -r 97d31cbbfe01 src/lib/var-expand.c --- a/src/lib/var-expand.c Wed Mar 12 14:09:14 2014 +0200 +++ b/src/lib/var-expand.c Mon Mar 17 16:57:21 2014 +0200 @@ -19,7 +19,7 @@ struct var_expand_context { int offset; - unsigned int width; + int width; bool zero_padding; }; @@ -268,7 +268,8 @@ } if (*str == '.') { - ctx.offset = sign * (int)ctx.width; + ctx.offset = sign * ctx.width; + sign = 1; ctx.width = 0; str++; @@ -280,11 +281,16 @@ ctx.zero_padding = TRUE; str++; } + if (*str == '-') { + sign = -1; + str++; + } while (*str >= '0' && *str <= '9') { ctx.width = ctx.width*10 + (*str - '0'); str++; } + ctx.width = sign * ctx.width; } modifier_count = 0; @@ -350,11 +356,13 @@ } if (ctx.width == 0) str_append(dest, var); - else if (!ctx.zero_padding) + else if (!ctx.zero_padding) { + if (ctx.width < 0) + ctx.width = strlen(var) - (-ctx.width); str_append_n(dest, var, ctx.width); - else { + } else { /* %05d -like padding. no truncation. */ - size_t len = strlen(var); + int len = strlen(var); while (len < ctx.width) { str_append_c(dest, '0'); ctx.width--; @@ -392,7 +400,7 @@ if (str[i] == '.') { i++; - while (str[i] >= '0' && str[i] <= '9') + while ((str[i] >= '0' && str[i] <= '9') || str[i] == '-') i++; } From dovecot at dovecot.org Mon Mar 17 14:57:57 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 17 Mar 2014 14:57:57 +0000 Subject: dovecot-2.2: test-var-expand: Added more unit tests for %variabl... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cb2607ffa93a changeset: 17160:cb2607ffa93a user: Timo Sirainen date: Mon Mar 17 16:57:36 2014 +0200 description: test-var-expand: Added more unit tests for %variable offsets/lengths. diffstat: src/lib/test-var-expand.c | 28 ++++++++++++++++++++++++++++ 1 files changed, 28 insertions(+), 0 deletions(-) diffs (45 lines): diff -r 97d31cbbfe01 -r cb2607ffa93a src/lib/test-var-expand.c --- a/src/lib/test-var-expand.c Mon Mar 17 16:57:21 2014 +0200 +++ b/src/lib/test-var-expand.c Mon Mar 17 16:57:36 2014 +0200 @@ -16,6 +16,33 @@ unsigned int idx, size; }; +static void test_var_expand_ranges(void) +{ + static struct var_expand_test tests[] = { + { "%v", "value1234" }, + { "%3v", "val" }, + { "%3.2v", "ue" }, + { "%3.-2v", "ue12" }, + { "%-3.2v", "23" }, + { "%0.-1v", "value123" }, + { "%-4.-1v", "123" } + }; + static struct var_expand_table table[] = { + { 'v', "value1234", NULL }, + { '\0', NULL, NULL } + }; + string_t *str = t_str_new(128); + unsigned int i; + + test_begin("var_expand"); + for (i = 0; i < N_ELEMENTS(tests); i++) { + str_truncate(str, 0); + var_expand(str, tests[i].in, table); + test_assert(strcmp(tests[i].out, str_c(str)) == 0); + } + test_end(); +} + static void test_var_expand_builtin(void) { static struct var_expand_test tests[] = { @@ -75,6 +102,7 @@ void test_var_expand(void) { + test_var_expand_ranges(); test_var_expand_builtin(); test_var_get_key_range(); } From dovecot at dovecot.org Tue Mar 18 12:39:35 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Mar 2014 12:39:35 +0000 Subject: dovecot-2.2: auth: Added login_user extra field to change the us... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d7e09442b666 changeset: 17161:d7e09442b666 user: Timo Sirainen date: Tue Mar 18 14:39:15 2014 +0200 description: auth: Added login_user extra field to change the username in master passdb. diffstat: src/auth/auth-request.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r cb2607ffa93a -r d7e09442b666 src/auth/auth-request.c --- a/src/auth/auth-request.c Mon Mar 17 16:57:36 2014 +0200 +++ b/src/auth/auth-request.c Tue Mar 18 14:39:15 2014 +0200 @@ -1424,6 +1424,8 @@ if (auth_request_try_update_username(request, name, value)) { /* don't change the original value so it gets saved correctly to cache. */ + } else if (strcmp(name, "login_user") == 0) { + request->requested_login_user = p_strdup(request->pool, value); } else if (strcmp(name, "allow_nets") == 0) { auth_request_validate_networks(request, value); } else if (strncmp(name, "userdb_", 7) == 0) { From dovecot at dovecot.org Tue Mar 18 14:49:41 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Mar 2014 14:49:41 +0000 Subject: dovecot-2.2: maildir: Fixed looking up physical size for a mail ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/30fb6e95b110 changeset: 17162:30fb6e95b110 user: Timo Sirainen date: Tue Mar 18 16:49:10 2014 +0200 description: maildir: Fixed looking up physical size for a mail while it's being saved. Based on patch by Teemu Huovila. diffstat: src/lib-storage/index/maildir/maildir-mail.c | 4 ++++ src/lib-storage/index/maildir/maildir-save.c | 23 +++++++++++++++++++++-- src/lib-storage/index/maildir/maildir-storage.h | 3 +++ 3 files changed, 28 insertions(+), 2 deletions(-) diffs (67 lines): diff -r d7e09442b666 -r 30fb6e95b110 src/lib-storage/index/maildir/maildir-mail.c --- a/src/lib-storage/index/maildir/maildir-mail.c Tue Mar 18 14:39:15 2014 +0200 +++ b/src/lib-storage/index/maildir/maildir-mail.c Tue Mar 18 16:49:10 2014 +0200 @@ -288,6 +288,10 @@ if (maildir_mail_get_fname(mbox, _mail, &fname) <= 0) return -1; } else { + if (maildir_save_file_get_size(_mail->transaction, _mail->seq, + vsize, size_r) == 0) + return 1; + path = maildir_save_file_get_path(_mail->transaction, _mail->seq); fname = strrchr(path, '/'); diff -r d7e09442b666 -r 30fb6e95b110 src/lib-storage/index/maildir/maildir-save.c --- a/src/lib-storage/index/maildir/maildir-save.c Tue Mar 18 14:39:15 2014 +0200 +++ b/src/lib-storage/index/maildir/maildir-save.c Tue Mar 18 16:49:10 2014 +0200 @@ -304,8 +304,9 @@ return t_strdup_printf("%s/%s", dir, fname); } -const char *maildir_save_file_get_path(struct mailbox_transaction_context *t, - uint32_t seq) + +static struct maildir_filename * +maildir_save_get_mf(struct mailbox_transaction_context *t, uint32_t seq) { struct maildir_save_context *save_ctx = (struct maildir_save_context *)t->save_ctx; @@ -320,6 +321,24 @@ i_assert(mf != NULL); seq--; } + return mf; +} + +int maildir_save_file_get_size(struct mailbox_transaction_context *t, + uint32_t seq, bool vsize, uoff_t *size_r) +{ + struct maildir_filename *mf = maildir_save_get_mf(t, seq); + + *size_r = vsize ? mf->vsize : mf->size; + return *size_r == (uoff_t)-1 ? -1 : 0; +} + +const char *maildir_save_file_get_path(struct mailbox_transaction_context *t, + uint32_t seq) +{ + struct maildir_save_context *save_ctx = + (struct maildir_save_context *)t->save_ctx; + struct maildir_filename *mf = maildir_save_get_mf(t, seq); return maildir_mf_get_path(save_ctx, mf); } diff -r d7e09442b666 -r 30fb6e95b110 src/lib-storage/index/maildir/maildir-storage.h --- a/src/lib-storage/index/maildir/maildir-storage.h Tue Mar 18 14:39:15 2014 +0200 +++ b/src/lib-storage/index/maildir/maildir-storage.h Tue Mar 18 16:49:10 2014 +0200 @@ -126,6 +126,9 @@ const char *basename); void maildir_save_set_sizes(struct maildir_filename *mf, uoff_t size, uoff_t vsize); + +int maildir_save_file_get_size(struct mailbox_transaction_context *t, + uint32_t seq, bool vsize, uoff_t *size_r); const char *maildir_save_file_get_path(struct mailbox_transaction_context *t, uint32_t seq); From dovecot at dovecot.org Tue Mar 18 16:23:29 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Mar 2014 16:23:29 +0000 Subject: dovecot-2.1: maildir: Fixed looking up physical size for a mail ... Message-ID: details: http://hg.dovecot.org/dovecot-2.1/rev/2799cc960c0d changeset: 15013:2799cc960c0d user: Timo Sirainen date: Tue Mar 18 16:49:10 2014 +0200 description: maildir: Fixed looking up physical size for a mail while it's being saved. Based on patch by Teemu Huovila. diffstat: src/lib-storage/index/maildir/maildir-mail.c | 4 ++++ src/lib-storage/index/maildir/maildir-save.c | 23 +++++++++++++++++++++-- src/lib-storage/index/maildir/maildir-storage.h | 3 +++ 3 files changed, 28 insertions(+), 2 deletions(-) diffs (67 lines): diff -r 79bae66e666c -r 2799cc960c0d src/lib-storage/index/maildir/maildir-mail.c --- a/src/lib-storage/index/maildir/maildir-mail.c Tue Mar 11 13:24:09 2014 +0200 +++ b/src/lib-storage/index/maildir/maildir-mail.c Tue Mar 18 16:49:10 2014 +0200 @@ -289,6 +289,10 @@ if (maildir_mail_get_fname(mbox, _mail, &fname) <= 0) return -1; } else { + if (maildir_save_file_get_size(_mail->transaction, _mail->seq, + vsize, size_r) == 0) + return 1; + path = maildir_save_file_get_path(_mail->transaction, _mail->seq); fname = strrchr(path, '/'); diff -r 79bae66e666c -r 2799cc960c0d src/lib-storage/index/maildir/maildir-save.c --- a/src/lib-storage/index/maildir/maildir-save.c Tue Mar 11 13:24:09 2014 +0200 +++ b/src/lib-storage/index/maildir/maildir-save.c Tue Mar 18 16:49:10 2014 +0200 @@ -302,8 +302,9 @@ return t_strdup_printf("%s/%s", dir, fname); } -const char *maildir_save_file_get_path(struct mailbox_transaction_context *t, - uint32_t seq) + +static struct maildir_filename * +maildir_save_get_mf(struct mailbox_transaction_context *t, uint32_t seq) { struct maildir_save_context *save_ctx = (struct maildir_save_context *)t->save_ctx; @@ -318,6 +319,24 @@ i_assert(mf != NULL); seq--; } + return mf; +} + +int maildir_save_file_get_size(struct mailbox_transaction_context *t, + uint32_t seq, bool vsize, uoff_t *size_r) +{ + struct maildir_filename *mf = maildir_save_get_mf(t, seq); + + *size_r = vsize ? mf->vsize : mf->size; + return *size_r == (uoff_t)-1 ? -1 : 0; +} + +const char *maildir_save_file_get_path(struct mailbox_transaction_context *t, + uint32_t seq) +{ + struct maildir_save_context *save_ctx = + (struct maildir_save_context *)t->save_ctx; + struct maildir_filename *mf = maildir_save_get_mf(t, seq); return maildir_mf_get_path(save_ctx, mf); } diff -r 79bae66e666c -r 2799cc960c0d src/lib-storage/index/maildir/maildir-storage.h --- a/src/lib-storage/index/maildir/maildir-storage.h Tue Mar 11 13:24:09 2014 +0200 +++ b/src/lib-storage/index/maildir/maildir-storage.h Tue Mar 18 16:49:10 2014 +0200 @@ -132,6 +132,9 @@ const char *basename); void maildir_save_set_sizes(struct maildir_filename *mf, uoff_t size, uoff_t vsize); + +int maildir_save_file_get_size(struct mailbox_transaction_context *t, + uint32_t seq, bool vsize, uoff_t *size_r); const char *maildir_save_file_get_path(struct mailbox_transaction_context *t, uint32_t seq); From dovecot at dovecot.org Wed Mar 19 07:45:45 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 19 Mar 2014 07:45:45 +0000 Subject: dovecot-2.2: lib-dns: Fixed handling failed DNS lookups. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b4ab1a27b66e changeset: 17163:b4ab1a27b66e user: Timo Sirainen date: Wed Mar 19 09:45:15 2014 +0200 description: lib-dns: Fixed handling failed DNS lookups. diffstat: src/lib-dns/dns-lookup.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (15 lines): diff -r 30fb6e95b110 -r b4ab1a27b66e src/lib-dns/dns-lookup.c --- a/src/lib-dns/dns-lookup.c Tue Mar 18 16:49:10 2014 +0200 +++ b/src/lib-dns/dns-lookup.c Wed Mar 19 09:45:15 2014 +0200 @@ -100,9 +100,9 @@ } return 1; } - /* first line: */ + /* first line: [] */ if (sscanf(line, "%d %u", &result->ret, - &result->ips_count) != 2) + &result->ips_count) < 1) return -1; if (result->ret != 0) { result->error = net_gethosterror(result->ret); From dovecot at dovecot.org Wed Mar 19 13:43:02 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 19 Mar 2014 13:43:02 +0000 Subject: dovecot-2.2: passdb-imap: Fixed reference counting. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/87e6c9706756 changeset: 17164:87e6c9706756 user: Timo Sirainen date: Wed Mar 19 15:42:42 2014 +0200 description: passdb-imap: Fixed reference counting. diffstat: src/auth/passdb-imap.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (19 lines): diff -r b4ab1a27b66e -r 87e6c9706756 src/auth/passdb-imap.c --- a/src/auth/passdb-imap.c Wed Mar 19 09:45:15 2014 +0200 +++ b/src/auth/passdb-imap.c Wed Mar 19 15:42:42 2014 +0200 @@ -64,6 +64,7 @@ } request->verify_callback(result, request->auth_request); imapc_client_deinit(&client); + auth_request_unref(&request->auth_request); } static void @@ -105,6 +106,7 @@ request->auth_request = auth_request; request->verify_callback = callback; + auth_request_ref(auth_request); imapc_client_login(request->client, passdb_imap_login_callback, request); } From dovecot at dovecot.org Thu Mar 20 14:15:47 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 20 Mar 2014 14:15:47 +0000 Subject: dovecot-2.2: imapc: Fixed crash when FETCH command immediately f... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/610c79565959 changeset: 17165:610c79565959 user: Timo Sirainen date: Thu Mar 20 16:15:24 2014 +0200 description: imapc: Fixed crash when FETCH command immediately failed due to connection error. diffstat: src/lib-storage/index/imapc/imapc-mail-fetch.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (15 lines): diff -r 87e6c9706756 -r 610c79565959 src/lib-storage/index/imapc/imapc-mail-fetch.c --- a/src/lib-storage/index/imapc/imapc-mail-fetch.c Wed Mar 19 15:42:42 2014 +0200 +++ b/src/lib-storage/index/imapc/imapc-mail-fetch.c Thu Mar 20 16:15:24 2014 +0200 @@ -385,9 +385,10 @@ imapc_mail_fetch_callback, mbox->pending_fetch_request); imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE); + array_append(&mbox->fetch_requests, &mbox->pending_fetch_request, 1); + imapc_command_send(cmd, str_c(mbox->pending_fetch_cmd)); - array_append(&mbox->fetch_requests, &mbox->pending_fetch_request, 1); mbox->pending_fetch_request = NULL; timeout_remove(&mbox->to_pending_fetch_send); str_truncate(mbox->pending_fetch_cmd, 0); From dovecot at dovecot.org Thu Mar 20 15:01:05 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 20 Mar 2014 15:01:05 +0000 Subject: dovecot-2.2: imapc: Avoid hanging when running into server probl... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/af7a606dc15c changeset: 17166:af7a606dc15c user: Timo Sirainen date: Thu Mar 20 17:00:31 2014 +0200 description: imapc: Avoid hanging when running into server problems. imapc_mail_fetch_flush() caused imail->fetch_count to drop to 0, but imapc_mailbox_run() still started imapc_client_run() which was doing nothing. diffstat: src/lib-storage/index/imapc/imapc-mail-fetch.c | 3 ++- src/lib-storage/index/imapc/imapc-mail.c | 7 +++++-- src/lib-storage/index/imapc/imapc-storage.c | 5 +++++ src/lib-storage/index/imapc/imapc-storage.h | 1 + 4 files changed, 13 insertions(+), 3 deletions(-) diffs (59 lines): diff -r 610c79565959 -r af7a606dc15c src/lib-storage/index/imapc/imapc-mail-fetch.c --- a/src/lib-storage/index/imapc/imapc-mail-fetch.c Thu Mar 20 16:15:24 2014 +0200 +++ b/src/lib-storage/index/imapc/imapc-mail-fetch.c Thu Mar 20 17:00:31 2014 +0200 @@ -365,10 +365,11 @@ /* we'll continue waiting until we've got all the fields we wanted, or until all FETCH replies have been received (i.e. some FETCHes failed) */ + imapc_mail_fetch_flush(mbox); while (imail->fetch_count > 0 && (!imapc_mail_have_fields(imail, fields) || !imail->header_list_fetched)) - imapc_mailbox_run(mbox); + imapc_mailbox_run_nofetch(mbox); return 0; } diff -r 610c79565959 -r af7a606dc15c src/lib-storage/index/imapc/imapc-mail.c --- a/src/lib-storage/index/imapc/imapc-mail.c Thu Mar 20 16:15:24 2014 +0200 +++ b/src/lib-storage/index/imapc/imapc-mail.c Thu Mar 20 17:00:31 2014 +0200 @@ -357,8 +357,11 @@ struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box; struct imapc_mail_cache *cache = &mbox->prev_mail_cache; - while (mail->fetch_count > 0) - imapc_mailbox_run(mbox); + if (mail->fetch_count > 0) { + imapc_mail_fetch_flush(mbox); + while (mail->fetch_count > 0) + imapc_mailbox_run_nofetch(mbox); + } index_mail_close(_mail); diff -r 610c79565959 -r af7a606dc15c src/lib-storage/index/imapc/imapc-storage.c --- a/src/lib-storage/index/imapc/imapc-storage.c Thu Mar 20 16:15:24 2014 +0200 +++ b/src/lib-storage/index/imapc/imapc-storage.c Thu Mar 20 17:00:31 2014 +0200 @@ -116,6 +116,11 @@ void imapc_mailbox_run(struct imapc_mailbox *mbox) { imapc_mail_fetch_flush(mbox); + imapc_mailbox_run_nofetch(mbox); +} + +void imapc_mailbox_run_nofetch(struct imapc_mailbox *mbox) +{ do { imapc_client_run(mbox->storage->client->client); } while (mbox->storage->reopen_count > 0); diff -r 610c79565959 -r af7a606dc15c src/lib-storage/index/imapc/imapc-storage.h --- a/src/lib-storage/index/imapc/imapc-storage.h Thu Mar 20 16:15:24 2014 +0200 +++ b/src/lib-storage/index/imapc/imapc-storage.h Thu Mar 20 17:00:31 2014 +0200 @@ -153,6 +153,7 @@ void imapc_transaction_save_rollback(struct mail_save_context *ctx); void imapc_mailbox_run(struct imapc_mailbox *mbox); +void imapc_mailbox_run_nofetch(struct imapc_mailbox *mbox); void imapc_mail_cache_free(struct imapc_mail_cache *cache); int imapc_mailbox_select(struct imapc_mailbox *mbox); From dovecot at dovecot.org Thu Mar 20 15:07:53 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 20 Mar 2014 15:07:53 +0000 Subject: dovecot-2.2: dsync: Avoid unnecessary data stack memory usage. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8fe564dd4018 changeset: 17167:8fe564dd4018 user: Timo Sirainen date: Thu Mar 20 17:07:32 2014 +0200 description: dsync: Avoid unnecessary data stack memory usage. diffstat: src/doveadm/dsync/dsync-mailbox-export.c | 10 ++++++---- 1 files changed, 6 insertions(+), 4 deletions(-) diffs (20 lines): diff -r af7a606dc15c -r 8fe564dd4018 src/doveadm/dsync/dsync-mailbox-export.c --- a/src/doveadm/dsync/dsync-mailbox-export.c Thu Mar 20 17:00:31 2014 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-export.c Thu Mar 20 17:07:32 2014 +0200 @@ -374,10 +374,12 @@ mail_search_args_unref(&search_args); while (mailbox_search_next(search_ctx, &mail)) { - if (mail->uid <= exporter->last_common_uid) - ret = search_update_flag_change_guid(exporter, mail); - else - ret = search_add_save(exporter, mail); + T_BEGIN { + if (mail->uid <= exporter->last_common_uid) + ret = search_update_flag_change_guid(exporter, mail); + else + ret = search_add_save(exporter, mail); + } T_END; if (ret < 0) break; } From dovecot at dovecot.org Fri Mar 21 15:53:39 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 21 Mar 2014 15:53:39 +0000 Subject: dovecot-2.2: imapc: Cache LIST replies for a while to be able to... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d9ebebdb931b changeset: 17168:d9ebebdb931b user: Timo Sirainen date: Fri Mar 21 17:52:50 2014 +0200 description: imapc: Cache LIST replies for a while to be able to immediately ask for mailbox flags/existence. Besides giving better performance, this will fix bugs in some servers where LISTing a single mailbox doesn't necessarily return it. (In this case a mailbox named "in" wasn't returned, but I think Dovecot has had bugs related to this also.) diffstat: src/lib-storage/index/imapc/imapc-list.c | 18 ++++++------------ src/lib-storage/index/imapc/imapc-list.h | 7 +++++++ src/lib-storage/index/imapc/imapc-sync.c | 8 ++++++++ 3 files changed, 21 insertions(+), 12 deletions(-) diffs (98 lines): diff -r 8fe564dd4018 -r d9ebebdb931b src/lib-storage/index/imapc/imapc-list.c --- a/src/lib-storage/index/imapc/imapc-list.c Thu Mar 20 17:07:32 2014 +0200 +++ b/src/lib-storage/index/imapc/imapc-list.c Fri Mar 21 17:52:50 2014 +0200 @@ -1,6 +1,7 @@ /* Copyright (c) 2011-2014 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "ioloop.h" #include "str.h" #include "imap-arg.h" #include "imap-match.h" @@ -565,6 +566,8 @@ if (ctx.ret == 0) { list->refreshed_mailboxes = TRUE; + list->refreshed_mailboxes_recently = TRUE; + list->last_refreshed_mailboxes = ioloop_time; imapc_list_delete_unused_indexes(list); } return ctx.ret; @@ -867,23 +870,14 @@ enum mailbox_info_flags *flags_r) { struct imapc_mailbox_list *list = (struct imapc_mailbox_list *)_list; - struct imapc_command *cmd; - struct imapc_simple_context sctx; struct mailbox_node *node; const char *vname; vname = mailbox_list_get_vname(_list, name); - if (!list->refreshed_mailboxes) { - node = mailbox_tree_lookup(list->mailboxes, vname); - if (node != NULL) - node->flags |= MAILBOX_NONEXISTENT; - - /* refresh the mailbox flags */ - cmd = imapc_list_simple_context_init(&sctx, list); - imapc_command_sendf(cmd, "LIST \"\" %s", name); - imapc_simple_run(&sctx); - if (sctx.ret < 0) + if (!list->refreshed_mailboxes_recently) { + if (imapc_list_refresh(list) < 0) return -1; + i_assert(list->refreshed_mailboxes_recently); } node = mailbox_tree_lookup(list->mailboxes, vname); diff -r 8fe564dd4018 -r d9ebebdb931b src/lib-storage/index/imapc/imapc-list.h --- a/src/lib-storage/index/imapc/imapc-list.h Thu Mar 20 17:07:32 2014 +0200 +++ b/src/lib-storage/index/imapc/imapc-list.h Fri Mar 21 17:52:50 2014 +0200 @@ -15,11 +15,18 @@ struct mailbox_tree_context *mailboxes, *tmp_subscriptions; char root_sep; + time_t last_refreshed_mailboxes; unsigned int iter_count; + /* mailboxes/subscriptions are fully refreshed only during + mailbox list iteration. */ unsigned int refreshed_subscriptions:1; unsigned int refreshed_mailboxes:1; + /* mailbox list's "recently refreshed" state is reset by syncing a + mailbox. mainly we use this to cache mailboxes' existence to avoid + issuing a LIST command every time. */ + unsigned int refreshed_mailboxes_recently:1; unsigned int index_list_failed:1; unsigned int root_sep_pending:1; }; diff -r 8fe564dd4018 -r d9ebebdb931b src/lib-storage/index/imapc/imapc-sync.c --- a/src/lib-storage/index/imapc/imapc-sync.c Thu Mar 20 17:07:32 2014 +0200 +++ b/src/lib-storage/index/imapc/imapc-sync.c Fri Mar 21 17:52:50 2014 +0200 @@ -7,6 +7,7 @@ #include "index-sync-private.h" #include "imapc-client.h" #include "imapc-msgmap.h" +#include "imapc-list.h" #include "imapc-storage.h" #include "imapc-sync.h" @@ -448,10 +449,17 @@ imapc_mailbox_sync_init(struct mailbox *box, enum mailbox_sync_flags flags) { struct imapc_mailbox *mbox = (struct imapc_mailbox *)box; + struct imapc_mailbox_list *list = mbox->storage->client->_list; enum imapc_capability capabilities; bool changes; int ret = 0; + if (list != NULL) { + if (!list->refreshed_mailboxes && + list->last_refreshed_mailboxes < ioloop_time) + list->refreshed_mailboxes_recently = FALSE; + } + if (!box->opened) { if (mailbox_open(box) < 0) ret = -1; From dovecot at dovecot.org Mon Mar 24 18:07:53 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 24 Mar 2014 18:07:53 +0000 Subject: dovecot-2.2: dsync: Fixed matching namespace for the namespace p... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d972b95720db changeset: 17169:d972b95720db user: Timo Sirainen date: Mon Mar 24 19:07:03 2014 +0100 description: dsync: Fixed matching namespace for the namespace prefix itself. diffstat: src/doveadm/dsync/dsync-brain-mailbox-tree.c | 10 +++++++++- 1 files changed, 9 insertions(+), 1 deletions(-) diffs (20 lines): diff -r d9ebebdb931b -r d972b95720db src/doveadm/dsync/dsync-brain-mailbox-tree.c --- a/src/doveadm/dsync/dsync-brain-mailbox-tree.c Fri Mar 21 17:52:50 2014 +0200 +++ b/src/doveadm/dsync/dsync-brain-mailbox-tree.c Mon Mar 24 19:07:03 2014 +0100 @@ -135,7 +135,15 @@ return FALSE; prefix += part_len + 1; } - return *name_parts != NULL; + if (*name_parts != NULL) { + /* namespace prefix found with a mailbox */ + return TRUE; + } + if (*prefix == '\0') { + /* namespace prefix itself matched */ + return TRUE; + } + return FALSE; } static struct mail_namespace * From dovecot at dovecot.org Tue Mar 25 18:00:06 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 25 Mar 2014 18:00:06 +0000 Subject: dovecot-2.2: dsync: Support multiple -n parameters. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/08f1c7af0ac0 changeset: 17170:08f1c7af0ac0 user: Timo Sirainen date: Tue Mar 25 19:59:24 2014 +0200 description: dsync: Support multiple -n parameters. diffstat: src/doveadm/dsync/doveadm-dsync.c | 21 +++++++++-- src/doveadm/dsync/dsync-brain-mailbox-tree.c | 5 -- src/doveadm/dsync/dsync-brain-mailbox.c | 6 --- src/doveadm/dsync/dsync-brain-private.h | 2 +- src/doveadm/dsync/dsync-brain.c | 48 ++++++++++++++++++++++----- src/doveadm/dsync/dsync-brain.h | 4 +- src/doveadm/dsync/dsync-ibc-pipe.c | 3 +- src/doveadm/dsync/dsync-ibc-stream.c | 6 +- src/doveadm/dsync/dsync-ibc.h | 4 +- 9 files changed, 65 insertions(+), 34 deletions(-) diffs (263 lines): diff -r d972b95720db -r 08f1c7af0ac0 src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Mon Mar 24 19:07:03 2014 +0100 +++ b/src/doveadm/dsync/doveadm-dsync.c Tue Mar 25 19:59:24 2014 +0200 @@ -54,10 +54,11 @@ struct dsync_cmd_context { struct doveadm_mail_cmd_context ctx; enum dsync_brain_sync_type sync_type; - const char *mailbox, *namespace_prefix; + const char *mailbox; guid_128_t mailbox_guid; const char *state_input, *rawlog_path; ARRAY_TYPE(const_string) exclude_mailboxes; + ARRAY_TYPE(const_string) namespace_prefixes; const char *remote_name; const char *local_location; @@ -510,6 +511,8 @@ struct dsync_ibc *ibc, *ibc2 = NULL; struct dsync_brain *brain; struct dsync_brain_settings set; + struct mail_namespace *ns; + const char *const *strp; enum dsync_brain_flags brain_flags; bool remote_errors_logged = FALSE; bool changes_during_sync = FALSE; @@ -526,9 +529,15 @@ } doveadm_user_init_dsync(user); - if (ctx->namespace_prefix != NULL) { - set.sync_ns = mail_namespace_find(user->namespaces, - ctx->namespace_prefix); + t_array_init(&set.sync_namespaces, array_count(&ctx->namespace_prefixes)); + array_foreach(&ctx->namespace_prefixes, strp) { + ns = mail_namespace_find(user->namespaces, *strp); + if (ns == NULL) { + i_error("Namespace not found: '%s'", *strp); + ctx->ctx.exit_code = EX_USAGE; + return -1; + } + array_append(&set.sync_namespaces, &ns, 1); } if (ctx->run_type == DSYNC_RUN_TYPE_LOCAL) @@ -903,7 +912,8 @@ array_append(&ctx->exclude_mailboxes, &str, 1); break; case 'n': - ctx->namespace_prefix = optarg; + str = optarg; + array_append(&ctx->namespace_prefixes, &str, 1); break; case 'N': ctx->sync_visible_namespaces = TRUE; @@ -948,6 +958,7 @@ doveadm_print_header("state", "state", DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); p_array_init(&ctx->exclude_mailboxes, ctx->ctx.pool, 4); + p_array_init(&ctx->namespace_prefixes, ctx->ctx.pool, 4); return &ctx->ctx; } diff -r d972b95720db -r 08f1c7af0ac0 src/doveadm/dsync/dsync-brain-mailbox-tree.c --- a/src/doveadm/dsync/dsync-brain-mailbox-tree.c Mon Mar 24 19:07:03 2014 +0100 +++ b/src/doveadm/dsync/dsync-brain-mailbox-tree.c Tue Mar 25 19:59:24 2014 +0200 @@ -17,11 +17,6 @@ i_assert(brain->hierarchy_sep == '\0'); - if (brain->sync_ns != NULL) { - brain->hierarchy_sep = mail_namespace_get_sep(brain->sync_ns); - return; - } - for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) { if (!dsync_brain_want_namespace(brain, ns)) continue; diff -r d972b95720db -r 08f1c7af0ac0 src/doveadm/dsync/dsync-brain-mailbox.c --- a/src/doveadm/dsync/dsync-brain-mailbox.c Mon Mar 24 19:07:03 2014 +0100 +++ b/src/doveadm/dsync/dsync-brain-mailbox.c Tue Mar 25 19:59:24 2014 +0200 @@ -47,12 +47,6 @@ int ret; *box_r = NULL; - if (brain->sync_ns != NULL) { - ret = ns_mailbox_try_alloc(brain->sync_ns, guid, box_r, error_r); - if (ret < 0) - brain->failed = TRUE; - return ret; - } for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) { if (!dsync_brain_want_namespace(brain, ns)) diff -r d972b95720db -r 08f1c7af0ac0 src/doveadm/dsync/dsync-brain-private.h --- a/src/doveadm/dsync/dsync-brain-private.h Mon Mar 24 19:07:03 2014 +0100 +++ b/src/doveadm/dsync/dsync-brain-private.h Tue Mar 25 19:59:24 2014 +0200 @@ -49,7 +49,7 @@ pool_t pool; struct mail_user *user; struct dsync_ibc *ibc; - struct mail_namespace *sync_ns; + ARRAY(struct mail_namespace *) sync_namespaces; const char *sync_box; guid_128_t sync_box_guid; const char *const *exclude_mailboxes; diff -r d972b95720db -r 08f1c7af0ac0 src/doveadm/dsync/dsync-brain.c --- a/src/doveadm/dsync/dsync-brain.c Mon Mar 24 19:07:03 2014 +0100 +++ b/src/doveadm/dsync/dsync-brain.c Tue Mar 25 19:59:24 2014 +0200 @@ -143,6 +143,8 @@ { struct dsync_ibc_settings ibc_set; struct dsync_brain *brain; + struct mail_namespace *const *nsp; + string_t *sync_ns_str = NULL; const char *error; i_assert(sync_type != DSYNC_BRAIN_SYNC_TYPE_UNKNOWN); @@ -152,8 +154,17 @@ brain = dsync_brain_common_init(user, ibc); brain->sync_type = sync_type; - if (set->sync_ns != NULL) - brain->sync_ns = set->sync_ns; + if (array_count(&set->sync_namespaces) > 0) { + sync_ns_str = t_str_new(128); + p_array_init(&brain->sync_namespaces, brain->pool, + array_count(&set->sync_namespaces)); + array_foreach(&set->sync_namespaces, nsp) { + if (str_len(sync_ns_str) > 0) + str_append_c(sync_ns_str, '\n'); + str_append(sync_ns_str, (*nsp)->prefix); + array_append(&brain->sync_namespaces, nsp, 1); + } + } brain->sync_box = p_strdup(brain->pool, set->sync_box); brain->exclude_mailboxes = set->exclude_mailboxes == NULL ? NULL : p_strarray_dup(brain->pool, set->exclude_mailboxes); @@ -175,8 +186,8 @@ memset(&ibc_set, 0, sizeof(ibc_set)); ibc_set.hostname = my_hostdomain(); - ibc_set.sync_ns_prefix = set->sync_ns == NULL ? NULL : - set->sync_ns->prefix; + ibc_set.sync_ns_prefixes = sync_ns_str == NULL ? + NULL : str_c(sync_ns_str); ibc_set.sync_box = set->sync_box; ibc_set.exclude_mailboxes = set->exclude_mailboxes; memcpy(ibc_set.sync_box_guid, set->sync_box_guid, @@ -380,6 +391,8 @@ static bool dsync_brain_slave_recv_handshake(struct dsync_brain *brain) { const struct dsync_ibc_settings *ibc_set; + struct mail_namespace *ns; + const char *const *prefixes; i_assert(!brain->master_brain); @@ -394,9 +407,19 @@ } } - if (ibc_set->sync_ns_prefix != NULL) { - brain->sync_ns = mail_namespace_find(brain->user->namespaces, - ibc_set->sync_ns_prefix); + if (ibc_set->sync_ns_prefixes != NULL) { + p_array_init(&brain->sync_namespaces, brain->pool, 4); + prefixes = t_strsplit(ibc_set->sync_ns_prefixes, "\n"); + for (; *prefixes != NULL; prefixes++) { + ns = mail_namespace_find(brain->user->namespaces, + *prefixes); + if (ns == NULL) { + i_error("Namespace not found: '%s'", *prefixes); + brain->failed = TRUE; + return FALSE; + } + array_append(&brain->sync_namespaces, &ns, 1); + } } brain->sync_box = p_strdup(brain->pool, ibc_set->sync_box); brain->exclude_mailboxes = ibc_set->exclude_mailboxes == NULL ? NULL : @@ -619,8 +642,15 @@ bool dsync_brain_want_namespace(struct dsync_brain *brain, struct mail_namespace *ns) { - if (brain->sync_ns != NULL) - return brain->sync_ns == ns; + struct mail_namespace *const *nsp; + + if (array_is_created(&brain->sync_namespaces)) { + array_foreach(&brain->sync_namespaces, nsp) { + if (ns == *nsp) + return TRUE; + } + return FALSE; + } if (ns->alias_for != NULL) { /* always skip aliases */ return FALSE; diff -r d972b95720db -r 08f1c7af0ac0 src/doveadm/dsync/dsync-brain.h --- a/src/doveadm/dsync/dsync-brain.h Mon Mar 24 19:07:03 2014 +0100 +++ b/src/doveadm/dsync/dsync-brain.h Tue Mar 25 19:59:24 2014 +0200 @@ -38,8 +38,8 @@ }; struct dsync_brain_settings { - /* Sync only this namespace */ - struct mail_namespace *sync_ns; + /* Sync only these namespaces */ + ARRAY(struct mail_namespace *) sync_namespaces; /* Sync only this mailbox name */ const char *sync_box; /* Sync only this mailbox GUID */ diff -r d972b95720db -r 08f1c7af0ac0 src/doveadm/dsync/dsync-ibc-pipe.c --- a/src/doveadm/dsync/dsync-ibc-pipe.c Mon Mar 24 19:07:03 2014 +0100 +++ b/src/doveadm/dsync/dsync-ibc-pipe.c Tue Mar 25 19:59:24 2014 +0200 @@ -164,7 +164,8 @@ item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_HANDSHAKE); item->u.set = *set; - item->u.set.sync_ns_prefix = p_strdup(item->pool, set->sync_ns_prefix); + item->u.set.sync_ns_prefixes = + p_strdup(item->pool, set->sync_ns_prefixes); item->u.set.sync_box = p_strdup(item->pool, set->sync_box); item->u.set.exclude_mailboxes = set->exclude_mailboxes == NULL ? NULL : p_strarray_dup(item->pool, set->exclude_mailboxes); diff -r d972b95720db -r 08f1c7af0ac0 src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Mon Mar 24 19:07:03 2014 +0100 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Tue Mar 25 19:59:24 2014 +0200 @@ -604,9 +604,9 @@ str_append_c(str, items[ITEM_HANDSHAKE].chr); encoder = dsync_serializer_encode_begin(ibc->serializers[ITEM_HANDSHAKE]); dsync_serializer_encode_add(encoder, "hostname", set->hostname); - if (set->sync_ns_prefix != NULL) { + if (set->sync_ns_prefixes != NULL) { dsync_serializer_encode_add(encoder, "sync_ns_prefix", - set->sync_ns_prefix); + set->sync_ns_prefixes); } if (set->sync_box != NULL) dsync_serializer_encode_add(encoder, "sync_box", set->sync_box); @@ -700,7 +700,7 @@ ibc->name = i_strdup(set->hostname); if (dsync_deserializer_decode_try(decoder, "sync_ns_prefix", &value)) - set->sync_ns_prefix = p_strdup(pool, value); + set->sync_ns_prefixes = p_strdup(pool, value); if (dsync_deserializer_decode_try(decoder, "sync_box", &value)) set->sync_box = p_strdup(pool, value); if (dsync_deserializer_decode_try(decoder, "sync_box_guid", &value) && diff -r d972b95720db -r 08f1c7af0ac0 src/doveadm/dsync/dsync-ibc.h --- a/src/doveadm/dsync/dsync-ibc.h Mon Mar 24 19:07:03 2014 +0100 +++ b/src/doveadm/dsync/dsync-ibc.h Tue Mar 25 19:59:24 2014 +0200 @@ -43,8 +43,8 @@ /* Server hostname. Used for determining which server does the locking. */ const char *hostname; - /* if non-NULL, sync only this namespace */ - const char *sync_ns_prefix; + /* if non-NULL, sync only these namespaces (LF-separated) */ + const char *sync_ns_prefixes; /* if non-NULL, sync only this mailbox name */ const char *sync_box; /* if non-empty, sync only this mailbox GUID */ From dovecot at dovecot.org Wed Mar 26 13:41:34 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 26 Mar 2014 13:41:34 +0000 Subject: dovecot-2.2: lib-http: Added debug logging. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/22424f1a32ad changeset: 17171:22424f1a32ad user: Timo Sirainen date: Wed Mar 26 14:41:05 2014 +0100 description: lib-http: Added debug logging. diffstat: src/lib-http/http-client-queue.c | 16 +++++++++++++++- 1 files changed, 15 insertions(+), 1 deletions(-) diffs (41 lines): diff -r 08f1c7af0ac0 -r 22424f1a32ad src/lib-http/http-client-queue.c --- a/src/lib-http/http-client-queue.c Tue Mar 25 19:59:24 2014 +0200 +++ b/src/lib-http/http-client-queue.c Wed Mar 26 14:41:05 2014 +0100 @@ -286,6 +286,14 @@ { struct http_client_host *host = queue->host; + http_client_queue_debug(queue, "Failed to set up connection to %s%s: %s " + "(%u peers pending, %u requests pending)", + http_client_peer_addr2str(addr), + (addr->https_name == NULL ? "" : + t_strdup_printf(" (SSL=%s)", addr->https_name)), reason, + (array_is_created(&queue->pending_peers) ? + array_count(&queue->pending_peers): 0), + array_count(&queue->request_queue)); if (array_is_created(&queue->pending_peers) && array_count(&queue->pending_peers) > 0) { struct http_client_peer *const *peer_idx; @@ -301,8 +309,11 @@ break; } } - if (array_count(&queue->pending_peers) > 0) + if (array_count(&queue->pending_peers) > 0) { + http_client_queue_debug(queue, + "Waiting for remaining pending peers."); return TRUE; + } } /* one of the connections failed. if we're not using soft timeouts, @@ -312,6 +323,9 @@ timeout_remove(&queue->to_connect); if (http_client_queue_is_last_connect_ip(queue)) { + http_client_queue_debug(queue, + "Failed to set up any connection; failing all queued requests"); + /* all IPs failed, but retry all of them again on the next request. */ queue->ips_connect_idx = queue->ips_connect_start_idx = From dovecot at dovecot.org Wed Mar 26 15:18:08 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 26 Mar 2014 15:18:08 +0000 Subject: dovecot-2.2: lib-http: Fixed a hang Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/57638bf8ee3e changeset: 17172:57638bf8ee3e user: Timo Sirainen date: Wed Mar 26 16:17:34 2014 +0100 description: lib-http: Fixed a hang Patch by Stephan Bosch diffstat: src/lib-http/http-client-queue.c | 15 ++++++++++++++- 1 files changed, 14 insertions(+), 1 deletions(-) diffs (29 lines): diff -r 22424f1a32ad -r 57638bf8ee3e src/lib-http/http-client-queue.c --- a/src/lib-http/http-client-queue.c Wed Mar 26 14:41:05 2014 +0100 +++ b/src/lib-http/http-client-queue.c Wed Mar 26 16:17:34 2014 +0100 @@ -228,11 +228,24 @@ if (!http_client_peer_is_connected(peer)) { unsigned int msecs; + bool new_peer = TRUE; /* not already connected, wait for connections */ if (!array_is_created(&queue->pending_peers)) i_array_init(&queue->pending_peers, 8); - array_append(&queue->pending_peers, &peer, 1); + else { + struct http_client_peer *const *peer_idx; + + /* we may be waiting for this peer already */ + array_foreach(&queue->pending_peers, peer_idx) { + if (http_client_peer_addr_cmp(&(*peer_idx)->addr, addr) == 0) { + new_peer = FALSE; + break; + } + } + } + if (new_peer) + array_append(&queue->pending_peers, &peer, 1); /* start soft connect time-out (but only if we have another IP left) */ msecs = host->client->set.soft_connect_timeout_msecs; From dovecot at dovecot.org Thu Mar 27 09:02:42 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 27 Mar 2014 09:02:42 +0000 Subject: dovecot-2.2: doveadm director move/map: Use director_username_ha... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b697a214e122 changeset: 17173:b697a214e122 user: Timo Sirainen date: Thu Mar 27 10:02:08 2014 +0100 description: doveadm director move/map: Use director_username_hash setting for calculating hashes. diffstat: src/doveadm/doveadm-director.c | 16 +++------------- src/doveadm/doveadm-settings.c | 2 ++ src/doveadm/doveadm-settings.h | 1 + 3 files changed, 6 insertions(+), 13 deletions(-) diffs (77 lines): diff -r 57638bf8ee3e -r b697a214e122 src/doveadm/doveadm-director.c --- a/src/doveadm/doveadm-director.c Wed Mar 26 16:17:34 2014 +0100 +++ b/src/doveadm/doveadm-director.c Thu Mar 27 10:02:08 2014 +0100 @@ -9,6 +9,7 @@ #include "write-full.h" #include "master-service.h" #include "auth-master.h" +#include "mail-user-hash.h" #include "doveadm.h" #include "doveadm-print.h" @@ -180,17 +181,6 @@ director_disconnect(ctx); } -static unsigned int director_username_hash(const char *username) -{ - unsigned char md5[MD5_RESULTLEN]; - unsigned int i, hash = 0; - - md5_get_digest(username, strlen(username), md5); - for (i = 0; i < sizeof(hash); i++) - hash = (hash << CHAR_BIT) | md5[i]; - return hash; -} - static void user_list_add(const char *username, pool_t pool, HASH_TABLE_TYPE(user_list) users) @@ -200,7 +190,7 @@ user = p_new(pool, struct user_list, 1); user->name = p_strdup(pool, username); - user_hash = director_username_hash(username); + user_hash = mail_user_hash(username, doveadm_settings->director_username_hash); old_user = hash_table_lookup(users, POINTER_CAST(user_hash)); if (old_user != NULL) @@ -440,7 +430,7 @@ argv[optind+2] != NULL) director_cmd_help(cmd_director_move); - user_hash = director_username_hash(argv[optind++]); + user_hash = mail_user_hash(argv[optind++], doveadm_settings->director_username_hash); host = argv[optind]; director_get_host(host, &ips, &ips_count); diff -r 57638bf8ee3e -r b697a214e122 src/doveadm/doveadm-settings.c --- a/src/doveadm/doveadm-settings.c Wed Mar 26 16:17:34 2014 +0100 +++ b/src/doveadm/doveadm-settings.c Thu Mar 27 10:02:08 2014 +0100 @@ -67,6 +67,7 @@ DEF(SET_STR, dsync_remote_cmd), DEF(SET_STR, ssl_client_ca_dir), DEF(SET_STR, ssl_client_ca_file), + DEF(SET_STR, director_username_hash), { SET_STRLIST, "plugin", offsetof(struct doveadm_settings, plugin_envs), NULL }, @@ -88,6 +89,7 @@ .dsync_remote_cmd = "ssh -l%{login} %{host} doveadm dsync-server -u%u -U", .ssl_client_ca_dir = "", .ssl_client_ca_file = "", + .director_username_hash = "%Lu", .plugin_envs = ARRAY_INIT }; diff -r 57638bf8ee3e -r b697a214e122 src/doveadm/doveadm-settings.h --- a/src/doveadm/doveadm-settings.h Wed Mar 26 16:17:34 2014 +0100 +++ b/src/doveadm/doveadm-settings.h Thu Mar 27 10:02:08 2014 +0100 @@ -16,6 +16,7 @@ const char *dsync_remote_cmd; const char *ssl_client_ca_dir; const char *ssl_client_ca_file; + const char *director_username_hash; ARRAY(const char *) plugin_envs; }; From dovecot at dovecot.org Thu Mar 27 12:25:40 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 27 Mar 2014 12:25:40 +0000 Subject: dovecot-2.2: dsync: Optimization for a large number of mails wit... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ea737947fac8 changeset: 17174:ea737947fac8 user: Timo Sirainen date: Thu Mar 27 13:25:12 2014 +0100 description: dsync: Optimization for a large number of mails with the same GUID. Still not ideal, but better than before. diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 30 ++++++++++++++++++++---------- 1 files changed, 20 insertions(+), 10 deletions(-) diffs (85 lines): diff -r b697a214e122 -r ea737947fac8 src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Thu Mar 27 10:02:08 2014 +0100 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Thu Mar 27 13:25:12 2014 +0100 @@ -616,7 +616,10 @@ } } /* 1) add the newmail to the end of the linked list - 2) find our link */ + 2) find our link + + FIXME: this loop is slow if the same GUID has a ton of instances. + Could it be improved in some way? */ last = &first_mail->next; for (mail = first_mail; mail != NULL; mail = mail->next) { if (mail->final_uid == newmail->final_uid) @@ -1877,20 +1880,22 @@ static int dsync_msg_try_copy(struct dsync_mailbox_importer *importer, struct mail_save_context **save_ctx_p, - struct importer_new_mail *all_newmails) + struct importer_new_mail **all_newmails_forcopy) { struct importer_new_mail *inst; - for (inst = all_newmails; inst != NULL; inst = inst->next) { + for (inst = *all_newmails_forcopy; inst != NULL; inst = inst->next) { if (inst->uid_in_local && !inst->copy_failed && mail_set_uid(importer->mail, inst->local_uid)) { if (mailbox_copy(save_ctx_p, importer->mail) < 0) { inst->copy_failed = TRUE; return -1; } + *all_newmails_forcopy = inst; return 1; } } + *all_newmails_forcopy = NULL; return 0; } @@ -1914,10 +1919,11 @@ return save_ctx; } -static void dsync_mailbox_save_body(struct dsync_mailbox_importer *importer, - const struct dsync_mail *mail, - struct importer_new_mail *newmail, - struct importer_new_mail *all_newmails) +static void +dsync_mailbox_save_body(struct dsync_mailbox_importer *importer, + const struct dsync_mail *mail, + struct importer_new_mail *newmail, + struct importer_new_mail **all_newmails_forcopy) { struct mail_save_context *save_ctx; ssize_t ret; @@ -1925,7 +1931,7 @@ /* try to save the mail by copying an existing mail */ save_ctx = dsync_mailbox_save_init(importer, mail, newmail); - if ((ret = dsync_msg_try_copy(importer, &save_ctx, all_newmails)) < 0) { + if ((ret = dsync_msg_try_copy(importer, &save_ctx, all_newmails_forcopy)) < 0) { if (save_ctx == NULL) save_ctx = dsync_mailbox_save_init(importer, mail, newmail); } @@ -2000,13 +2006,17 @@ const struct dsync_mail *mail, struct importer_new_mail *all_newmails) { - struct importer_new_mail *newmail; + struct importer_new_mail *newmail, *all_newmails_forcopy; + + /* if all_newmails list is large, avoid scanning through the + uninteresting ones for each newmail */ + all_newmails_forcopy = all_newmails; /* save all instances of the message */ for (newmail = all_newmails; newmail != NULL; newmail = newmail->next) { if (!newmail->skip) T_BEGIN { dsync_mailbox_save_body(importer, mail, newmail, - all_newmails); + &all_newmails_forcopy); } T_END; } } From dovecot at dovecot.org Thu Mar 27 13:37:44 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 27 Mar 2014 13:37:44 +0000 Subject: dovecot-2.2: mdbox: Don't hoard file descriptors when saving mai... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7ccee18b5085 changeset: 17175:7ccee18b5085 user: Timo Sirainen date: Thu Mar 27 14:37:10 2014 +0100 description: mdbox: Don't hoard file descriptors when saving mails that are larger than mdbox_rotate_size. diffstat: src/lib-storage/index/dbox-multi/mdbox-map.c | 3 --- 1 files changed, 0 insertions(+), 3 deletions(-) diffs (13 lines): diff -r ea737947fac8 -r 7ccee18b5085 src/lib-storage/index/dbox-multi/mdbox-map.c --- a/src/lib-storage/index/dbox-multi/mdbox-map.c Thu Mar 27 13:25:12 2014 +0100 +++ b/src/lib-storage/index/dbox-multi/mdbox-map.c Thu Mar 27 14:37:10 2014 +0100 @@ -909,9 +909,6 @@ unsigned int i, count; uoff_t append_offset; - if (mail_size >= map->set->mdbox_rotate_size) - return NULL; - /* first try to use files already used in this append */ file_appends = array_get(&ctx->file_appends, &count); for (i = count; i > ctx->files_nonappendable_count; i--) { From dovecot at dovecot.org Thu Mar 27 13:40:01 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 27 Mar 2014 13:40:01 +0000 Subject: dovecot-2.2: lib-fs: Fixes to posix backend's FS_OPEN_MODE_CREAT... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0b4a0361f1aa changeset: 17176:0b4a0361f1aa user: Timo Sirainen date: Thu Mar 27 15:39:33 2014 +0200 description: lib-fs: Fixes to posix backend's FS_OPEN_MODE_CREATE_UNIQUE_128 handling. diffstat: src/lib-fs/fs-posix.c | 26 +++++++++----------------- 1 files changed, 9 insertions(+), 17 deletions(-) diffs (54 lines): diff -r 7ccee18b5085 -r 0b4a0361f1aa src/lib-fs/fs-posix.c --- a/src/lib-fs/fs-posix.c Thu Mar 27 14:37:10 2014 +0100 +++ b/src/lib-fs/fs-posix.c Thu Mar 27 15:39:33 2014 +0200 @@ -183,12 +183,8 @@ i_assert(file->temp_path == NULL); - if (file->open_mode == FS_OPEN_MODE_CREATE_UNIQUE_128) { - str_append(str, file->file.path); - str_append_c(str, '/'); - } else if ((slash = strrchr(file->file.path, '/')) != NULL) { + if ((slash = strrchr(file->file.path, '/')) != NULL) str_append_n(str, file->file.path, slash - file->file.path + 1); - } str_append(str, fs->temp_file_prefix); fd = safe_mkstemp_hostpid(str, fs->mode, (uid_t)-1, (gid_t)-1); @@ -243,10 +239,17 @@ enum fs_open_mode mode, enum fs_open_flags flags) { struct posix_fs_file *file; + guid_128_t guid; file = i_new(struct posix_fs_file, 1); file->file.fs = _fs; - file->file.path = i_strdup(path); + if (mode != FS_OPEN_MODE_CREATE_UNIQUE_128) + file->file.path = i_strdup(path); + else { + guid_128_generate(guid); + file->file.path = i_strdup_printf("%s/%s", path, + guid_128_to_string(guid)); + } file->open_mode = mode; file->open_flags = flags; file->fd = -1; @@ -377,17 +380,6 @@ switch (file->open_mode) { case FS_OPEN_MODE_CREATE_UNIQUE_128: - T_BEGIN { - guid_128_t guid; - char *path; - - guid_128_generate(guid); - path = i_strdup_printf("%s/%s", file->file.path, - guid_128_to_string(guid)); - i_free(file->file.path); - file->file.path = path; - } T_END; - /* fall through */ case FS_OPEN_MODE_CREATE: if ((ret = link(file->temp_path, file->file.path)) < 0) { fs_set_error(file->file.fs, "link(%s, %s) failed: %m", From dovecot at dovecot.org Thu Mar 27 13:58:20 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 27 Mar 2014 13:58:20 +0000 Subject: dovecot-2.2: dovecot.m4: Minor message improvement. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/45dd96eb4139 changeset: 17177:45dd96eb4139 user: Timo Sirainen date: Thu Mar 27 14:57:44 2014 +0100 description: dovecot.m4: Minor message improvement. diffstat: dovecot.m4 | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (21 lines): diff -r 0b4a0361f1aa -r 45dd96eb4139 dovecot.m4 --- a/dovecot.m4 Thu Mar 27 15:39:33 2014 +0200 +++ b/dovecot.m4 Thu Mar 27 14:57:44 2014 +0100 @@ -6,7 +6,7 @@ # unlimited permission to copy and/or distribute it, with or without # modifications, as long as this notice is preserved. -# serial 11 +# serial 12 AC_DEFUN([DC_DOVECOT_MODULEDIR],[ AC_ARG_WITH(moduledir, @@ -56,7 +56,7 @@ fi, use_install_dirs=yes) - AC_MSG_CHECKING([for dovecot-config in "$dovecotdir"]) + AC_MSG_CHECKING([for "$dovecotdir/dovecot-config"]) if test -f "$dovecotdir/dovecot-config"; then AC_MSG_RESULT([$dovecotdir/dovecot-config]) else From dovecot at dovecot.org Thu Mar 27 15:01:20 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 27 Mar 2014 15:01:20 +0000 Subject: dovecot-2.2: lib-fs: ostream-metawrap didn't update the output s... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/234e41394894 changeset: 17178:234e41394894 user: Timo Sirainen date: Thu Mar 27 16:00:49 2014 +0100 description: lib-fs: ostream-metawrap didn't update the output stream's offset diffstat: src/lib-fs/ostream-metawrap.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (21 lines): diff -r 45dd96eb4139 -r 234e41394894 src/lib-fs/ostream-metawrap.c --- a/src/lib-fs/ostream-metawrap.c Thu Mar 27 14:57:44 2014 +0100 +++ b/src/lib-fs/ostream-metawrap.c Thu Mar 27 16:00:49 2014 +0100 @@ -30,6 +30,8 @@ o_stream_metawrap_call_callback(mstream); if ((ret = o_stream_sendv(stream->parent, iov, iov_count)) < 0) o_stream_copy_error_from_parent(stream); + else + stream->ostream.offset += ret; return ret; } @@ -44,6 +46,8 @@ o_stream_metawrap_call_callback(outstream); if ((ret = o_stream_send_istream(_outstream->parent, instream)) < 0) o_stream_copy_error_from_parent(_outstream); + else + _outstream->ostream.offset += ret; return ret; } From dovecot at dovecot.org Fri Mar 28 13:30:24 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 28 Mar 2014 13:30:24 +0000 Subject: dovecot-2.2: lib-fs: ostream-metawrap shouldn't count metadata h... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b64b507c0377 changeset: 17179:b64b507c0377 user: Timo Sirainen date: Fri Mar 28 15:29:53 2014 +0200 description: lib-fs: ostream-metawrap shouldn't count metadata header as part of the stream offsets. diffstat: src/lib-fs/ostream-metawrap.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 234e41394894 -r b64b507c0377 src/lib-fs/ostream-metawrap.c --- a/src/lib-fs/ostream-metawrap.c Thu Mar 27 16:00:49 2014 +0100 +++ b/src/lib-fs/ostream-metawrap.c Fri Mar 28 15:29:53 2014 +0200 @@ -17,6 +17,8 @@ if (write_callback != NULL) { mstream->write_callback = NULL; write_callback(mstream->context); + /* metadata headers aren't counted as part of the offset */ + mstream->ostream.ostream.offset = 0; } } From dovecot at dovecot.org Fri Mar 28 13:31:35 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 28 Mar 2014 13:31:35 +0000 Subject: dovecot-2.2: lib-fs: posix backend now names the istreams with t... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/46d1da1cdfc7 changeset: 17180:46d1da1cdfc7 user: Timo Sirainen date: Fri Mar 28 15:31:06 2014 +0200 description: lib-fs: posix backend now names the istreams with the file's path. diffstat: src/lib-fs/fs-posix.c | 7 +++---- 1 files changed, 3 insertions(+), 4 deletions(-) diffs (19 lines): diff -r b64b507c0377 -r 46d1da1cdfc7 src/lib-fs/fs-posix.c --- a/src/lib-fs/fs-posix.c Fri Mar 28 15:29:53 2014 +0200 +++ b/src/lib-fs/fs-posix.c Fri Mar 28 15:31:06 2014 +0200 @@ -348,12 +348,11 @@ struct posix_fs_file *file = (struct posix_fs_file *)_file; struct istream *input; - if (file->fd == -1 && fs_posix_open(file) < 0) { + if (file->fd == -1 && fs_posix_open(file) < 0) input = i_stream_create_error(errno); - i_stream_set_name(input, _file->path); - } else { + else input = i_stream_create_fd(file->fd, max_buffer_size, FALSE); - } + i_stream_set_name(input, _file->path); i_stream_add_destroy_callback(input, fs_posix_file_close, _file); return input; } From dovecot at dovecot.org Fri Mar 28 14:00:41 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 28 Mar 2014 14:00:41 +0000 Subject: dovecot-2.2: pop3: Fixed off-by-one bug in the sequence number f... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b0359910ec96 changeset: 17181:b0359910ec96 user: Teemu Huovila date: Fri Mar 28 15:59:48 2014 +0200 description: pop3: Fixed off-by-one bug in the sequence number fix of d20059f7d3a1. diffstat: src/pop3/pop3-commands.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 46d1da1cdfc7 -r b0359910ec96 src/pop3/pop3-commands.c --- a/src/pop3/pop3-commands.c Fri Mar 28 15:31:06 2014 +0200 +++ b/src/pop3/pop3-commands.c Fri Mar 28 15:59:48 2014 +0200 @@ -826,7 +826,7 @@ client->messages_count+1); for (msgnum = 0; msgnum < client->messages_count; msgnum++) { client->message_uidls[msgnum] = - seq_uidls[msgnum_to_seq(client, msgnum)]; + seq_uidls[msgnum_to_seq(client, msgnum) - 1]; } i_free(seq_uidls); } From dovecot at dovecot.org Sat Mar 29 20:45:52 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 29 Mar 2014 20:45:52 +0000 Subject: dovecot-2.2: fts-sorl: Skip returned duplicate results. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d55ae5d268a2 changeset: 17182:d55ae5d268a2 user: Timo Sirainen date: Sat Mar 29 22:45:16 2014 +0200 description: fts-sorl: Skip returned duplicate results. Similar to bugfix d63b209737be in fts-lucene. This caused assert-crash at least when searching from virtual mailbox because the score array was larger than the actual number of results. diffstat: src/plugins/fts-solr/solr-connection.c | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diffs (15 lines): diff -r b0359910ec96 -r d55ae5d268a2 src/plugins/fts-solr/solr-connection.c --- a/src/plugins/fts-solr/solr-connection.c Fri Mar 28 15:59:48 2014 +0200 +++ b/src/plugins/fts-solr/solr-connection.c Sat Mar 29 22:45:16 2014 +0200 @@ -260,8 +260,9 @@ } result = solr_result_get(ctx, box_id); - seq_range_array_add(&result->uids, ctx->uid); - if (ctx->score != 0) { + if (seq_range_array_add(&result->uids, ctx->uid)) { + /* duplicate result */ + } else if (ctx->score != 0) { score = array_append_space(&result->scores); score->uid = ctx->uid; score->score = ctx->score;