From dovecot at dovecot.org Wed Sep 3 13:27:34 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Sep 2014 13:27:34 +0000 Subject: dovecot-2.2: fs layout: Absolute paths as mailbox names shouldn'... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d136c0869774 changeset: 17759:d136c0869774 user: Timo Sirainen date: Fri Aug 29 13:22:11 2014 +0900 description: fs layout: Absolute paths as mailbox names shouldn't return index with INDEX=MEMORY diffstat: src/lib-storage/list/mailbox-list-fs.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (13 lines): diff -r 6980c53bf7d7 -r d136c0869774 src/lib-storage/list/mailbox-list-fs.c --- a/src/lib-storage/list/mailbox-list-fs.c Fri Aug 29 02:14:43 2014 +0900 +++ b/src/lib-storage/list/mailbox-list-fs.c Fri Aug 29 13:22:11 2014 +0900 @@ -75,6 +75,9 @@ i_assert(mailbox_list_is_valid_name(_list, name, &error)); if (mailbox_list_try_get_absolute_path(_list, &name)) { + if (type == MAILBOX_LIST_PATH_TYPE_INDEX && + *set->index_dir == '\0') + return 0; *path_r = name; return 1; } From dovecot at dovecot.org Wed Sep 3 13:30:31 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Sep 2014 13:30:31 +0000 Subject: dovecot-2.2: lib-storage: Fixed off-by-one memory allocation in ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3d9f233f5b77 changeset: 17760:3d9f233f5b77 user: Timo Sirainen date: Wed Sep 03 06:29:52 2014 -0700 description: lib-storage: Fixed off-by-one memory allocation in dynamic settings allocation code. This didn't actually affect normal Dovecot builds, because there were always some storages with get_setting_parser_info=NULL that added the necessary padding. Patch by Michael M Slusarz diffstat: src/lib-storage/mail-storage-settings.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (20 lines): diff -r d136c0869774 -r 3d9f233f5b77 src/lib-storage/mail-storage-settings.c --- a/src/lib-storage/mail-storage-settings.c Fri Aug 29 13:22:11 2014 +0900 +++ b/src/lib-storage/mail-storage-settings.c Wed Sep 03 06:29:52 2014 -0700 @@ -322,7 +322,7 @@ unsigned int i, j, count; storages = array_get(&mail_storage_classes, &count); - parsers = p_new(pool, struct dynamic_settings_parser, count + 1); + parsers = p_new(pool, struct dynamic_settings_parser, 1 + count + 1); parsers[0].name = MAIL_STORAGE_SET_DRIVER_NAME; parsers[0].info = &mail_storage_setting_parser_info; @@ -334,6 +334,7 @@ parsers[j].info = storages[i]->v.get_setting_parser_info(); j++; } + parsers[j] = NULL; return parsers; } From dovecot at dovecot.org Wed Sep 3 14:30:02 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 03 Sep 2014 14:30:02 +0000 Subject: dovecot-2.2: lib-storage: Compile fix to previous change Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/58738244dc34 changeset: 17761:58738244dc34 user: Timo Sirainen date: Wed Sep 03 07:29:25 2014 -0700 description: lib-storage: Compile fix to previous change The original patch was correct, my copy&pasting by hand just sucked. :) diffstat: src/lib-storage/mail-storage-settings.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 3d9f233f5b77 -r 58738244dc34 src/lib-storage/mail-storage-settings.c --- a/src/lib-storage/mail-storage-settings.c Wed Sep 03 06:29:52 2014 -0700 +++ b/src/lib-storage/mail-storage-settings.c Wed Sep 03 07:29:25 2014 -0700 @@ -334,7 +334,7 @@ parsers[j].info = storages[i]->v.get_setting_parser_info(); j++; } - parsers[j] = NULL; + parsers[j].name = NULL; return parsers; } From pigeonhole at rename-it.nl Fri Sep 5 12:39:11 2014 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Fri, 05 Sep 2014 14:39:11 +0200 Subject: dovecot-2.2-pigeonhole: Valgrindn needs --trace-children to work... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/c191996854b3 changeset: 1915:c191996854b3 user: Stephan Bosch date: Fri Sep 05 14:38:35 2014 +0200 description: Valgrindn needs --trace-children to work properly these days. diffstat: Makefile.am | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 1c90311178ac -r c191996854b3 Makefile.am --- a/Makefile.am Thu Aug 28 15:38:39 2014 +0200 +++ b/Makefile.am Fri Sep 05 14:38:35 2014 +0200 @@ -28,7 +28,8 @@ TESTSUITE_BIN = $(top_builddir)/src/testsuite/testsuite $(TESTSUITE_OPTIONS) if TESTSUITE_VALGRIND -TEST_BIN = libtool --mode=execute valgrind -q --error-exitcode=1 --show-reachable=yes --leak-check=full $(TESTSUITE_BIN) +TEST_BIN = libtool --mode=execute valgrind -q --error-exitcode=1 \ + --show-reachable=yes --leak-check=full --trace-children=yes $(TESTSUITE_BIN) else TEST_BIN = $(TESTSUITE_BIN) endif From dovecot at dovecot.org Mon Sep 8 07:15:15 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 08 Sep 2014 07:15:15 +0000 Subject: dovecot-2.2: pop3c: If base_dir isn't set, lookup pop3c_host wit... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0953e6c5d931 changeset: 17762:0953e6c5d931 user: Timo Sirainen date: Mon Sep 08 10:14:42 2014 +0300 description: pop3c: If base_dir isn't set, lookup pop3c_host with regular blocking DNS lookup. diffstat: src/lib-storage/index/pop3c/pop3c-client.c | 45 ++++++++++++++++++++++------ src/lib-storage/index/pop3c/pop3c-storage.c | 1 + 2 files changed, 36 insertions(+), 10 deletions(-) diffs (80 lines): diff -r 58738244dc34 -r 0953e6c5d931 src/lib-storage/index/pop3c/pop3c-client.c --- a/src/lib-storage/index/pop3c/pop3c-client.c Wed Sep 03 07:29:25 2014 -0700 +++ b/src/lib-storage/index/pop3c/pop3c-client.c Mon Sep 08 10:14:42 2014 +0300 @@ -67,6 +67,7 @@ static void pop3c_dns_callback(const struct dns_lookup_result *result, struct pop3c_client *client); +static void pop3c_client_connect_ip(struct pop3c_client *client); struct pop3c_client * pop3c_client_init(const struct pop3c_client_settings *set) @@ -196,6 +197,39 @@ pop3c_client_disconnect(client); } +static int pop3c_client_dns_lookup(struct pop3c_client *client) +{ + struct dns_lookup_settings dns_set; + + i_assert(client->state == POP3C_CLIENT_STATE_CONNECTING); + + if (client->set.dns_client_socket_path[0] == '\0') { + struct ip_addr *ips; + unsigned int ips_count; + int ret; + + ret = net_gethostbyname(client->set.host, &ips, &ips_count); + if (ret != 0) { + i_error("pop3c(%s): net_gethostbyname() failed: %s", + client->set.host, net_gethosterror(ret)); + return -1; + } + i_assert(ips_count > 0); + client->ip = ips[0]; + pop3c_client_connect_ip(client); + } else { + memset(&dns_set, 0, sizeof(dns_set)); + dns_set.dns_client_socket_path = + client->set.dns_client_socket_path; + dns_set.timeout_msecs = POP3C_DNS_LOOKUP_TIMEOUT_MSECS; + if (dns_lookup(client->set.host, &dns_set, + pop3c_dns_callback, client, + &client->dns_lookup) < 0) + return -1; + } + return 0; +} + void pop3c_client_run(struct pop3c_client *client) { struct ioloop *ioloop, *prev_ioloop = current_ioloop; @@ -210,16 +244,7 @@ if (client->ip.family == 0) { /* we're connecting, start DNS lookup after our ioloop is created */ - struct dns_lookup_settings dns_set; - - i_assert(client->state == POP3C_CLIENT_STATE_CONNECTING); - memset(&dns_set, 0, sizeof(dns_set)); - dns_set.dns_client_socket_path = - client->set.dns_client_socket_path; - dns_set.timeout_msecs = POP3C_DNS_LOOKUP_TIMEOUT_MSECS; - if (dns_lookup(client->set.host, &dns_set, - pop3c_dns_callback, client, - &client->dns_lookup) < 0) + if (pop3c_client_dns_lookup(client) < 0) failed = TRUE; } else if (client->to == NULL) { client->to = timeout_add(POP3C_COMMAND_TIMEOUT_MSECS, diff -r 58738244dc34 -r 0953e6c5d931 src/lib-storage/index/pop3c/pop3c-storage.c --- a/src/lib-storage/index/pop3c/pop3c-storage.c Wed Sep 03 07:29:25 2014 -0700 +++ b/src/lib-storage/index/pop3c/pop3c-storage.c Mon Sep 08 10:14:42 2014 +0300 @@ -63,6 +63,7 @@ client_set.master_user = set->pop3c_master_user; client_set.password = set->pop3c_password; client_set.dns_client_socket_path = + storage->user->set->base_dir[0] == '\0' ? "" : t_strconcat(storage->user->set->base_dir, "/", DNS_CLIENT_SOCKET_NAME, NULL); str = t_str_new(128); From dovecot at dovecot.org Mon Sep 8 10:39:58 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 08 Sep 2014 10:39:58 +0000 Subject: dovecot-2.2: COPYING.LGPL: Updated FSF mailing address. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a41089e5bd8e changeset: 17763:a41089e5bd8e user: Timo Sirainen date: Mon Sep 08 13:39:25 2014 +0300 description: COPYING.LGPL: Updated FSF mailing address. Also some whitespace and line wrapping changes. Used the same file as http://www.gnu.org/licenses/lgpl-2.1.txt diffstat: COPYING.LGPL | 84 +++++++++++++++++++++++++++-------------------------------- 1 files changed, 38 insertions(+), 46 deletions(-) diffs (213 lines): diff -r 0953e6c5d931 -r a41089e5bd8e COPYING.LGPL --- a/COPYING.LGPL Mon Sep 08 10:14:42 2014 +0300 +++ b/COPYING.LGPL Mon Sep 08 13:39:25 2014 +0300 @@ -1,9 +1,8 @@ - GNU LESSER GENERAL PUBLIC LICENSE Version 2.1, February 1999 Copyright (C) 1991, 1999 Free Software Foundation, Inc. - 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. @@ -23,8 +22,7 @@ Free Software Foundation and other authors who decide to use it. You can use it too, but we suggest you first think carefully about whether this license or the ordinary General Public License is the better -strategy to use in any particular case, based on the explanations -below. +strategy to use in any particular case, based on the explanations below. When we speak of free software, we are referring to freedom of use, not price. Our General Public Licenses are designed to make sure that @@ -57,7 +55,7 @@ that what they have is not the original version, so that the original author's reputation will not be affected by problems that might be introduced by others. -^L + Finally, software patents pose a constant threat to the existence of any free program. We wish to make sure that a company cannot effectively restrict the users of a free program by obtaining a @@ -89,9 +87,9 @@ special circumstances. For example, on rare occasions, there may be a special need to -encourage the widest possible use of a certain library, so that it -becomes a de-facto standard. To achieve this, non-free programs must -be allowed to use the library. A more frequent case is that a free +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free library does the same job as widely used non-free libraries. In this case, there is little to gain by limiting the free library to free software only, so we use the Lesser General Public License. @@ -113,7 +111,7 @@ "work based on the library" and a "work that uses the library". The former contains code derived from the library, whereas the latter must be combined with the library in order to run. -^L + GNU LESSER GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION @@ -138,8 +136,8 @@ "Source code" for a work means the preferred form of the work for making modifications to it. For a library, complete source code means all the source code for all modules it contains, plus any associated -interface definition files, plus the scripts used to control -compilation and installation of the library. +interface definition files, plus the scripts used to control compilation +and installation of the library. Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of @@ -218,7 +216,7 @@ ordinary GNU General Public License has appeared, then you can specify that version instead if you wish.) Do not make any other change in these notices. -^L + Once this change is made in a given copy, it is irreversible for that copy, so the ordinary GNU General Public License applies to all subsequent copies and derivative works made from that copy. @@ -269,7 +267,7 @@ distribute the object code for the work under the terms of Section 6. Any executables containing that work also fall under Section 6, whether or not they are linked directly with the Library itself. -^L + 6. As an exception to the Sections above, you may also combine or link a "work that uses the Library" with the Library to produce a work containing portions of the Library, and distribute that work @@ -305,10 +303,10 @@ the user installs one, as long as the modified version is interface-compatible with the version that the work was made with. - c) Accompany the work with a written offer, valid for at least - three years, to give the same user the materials specified in - Subsection 6a, above, for a charge no more than the cost of - performing this distribution. + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. d) If distribution of the work is made by offering access to copy from a designated place, offer equivalent access to copy the above @@ -331,7 +329,7 @@ accompany the operating system. Such a contradiction means you cannot use both them and the Library together in an executable that you distribute. -^L + 7. You may place library facilities that are a work based on the Library side-by-side in a single library together with other library facilities not covered by this License, and distribute such a combined @@ -372,7 +370,7 @@ restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties with this License. -^L + 11. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or @@ -386,10 +384,9 @@ the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Library. -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply, and the section as a whole is intended to apply in other -circumstances. +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any @@ -407,11 +404,11 @@ 12. If the distribution and/or use of the Library is restricted in certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Library under this License -may add an explicit geographical distribution limitation excluding those -countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. 13. The Free Software Foundation may publish revised and/or new versions of the Lesser General Public License from time to time. @@ -425,7 +422,7 @@ the Free Software Foundation. If the Library does not specify a license version number, you may choose any version ever published by the Free Software Foundation. -^L + 14. If you wish to incorporate parts of the Library into other free programs whose distribution conditions are incompatible with these, write to the author to ask for permission. For software which is @@ -459,21 +456,19 @@ DAMAGES. END OF TERMS AND CONDITIONS -^L + How to Apply These Terms to Your New Libraries If you develop a new library, and you want it to be of the greatest possible use to the public, we recommend making it free software that everyone can redistribute and change. You can do so by permitting -redistribution under these terms (or, alternatively, under the terms -of the ordinary General Public License). +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). - To apply these terms, attach the following notices to the library. -It is safest to attach them to the start of each source file to most -effectively convey the exclusion of warranty; and each file should -have at least the "copyright" line and a pointer to where the full -notice is found. - + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. Copyright (C) @@ -490,21 +485,18 @@ You should have received a copy of the GNU Lesser General Public License along with this library; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA Also add information on how to contact you by electronic and paper mail. -You should also get your employer (if you work as a programmer) or -your school, if any, to sign a "copyright disclaimer" for the library, -if necessary. Here is a sample; alter the names: +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: Yoyodyne, Inc., hereby disclaims all copyright interest in the - library `Frob' (a library for tweaking knobs) written by James - Random Hacker. + library `Frob' (a library for tweaking knobs) written by James Random Hacker. , 1 April 1990 Ty Coon, President of Vice That's all there is to it! - - From dovecot at dovecot.org Mon Sep 8 15:12:05 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 08 Sep 2014 15:12:05 +0000 Subject: dovecot-2.2: pop3c: Added missing support for pop3c_ssl=starttls Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bf1faf8ab847 changeset: 17764:bf1faf8ab847 user: Timo Sirainen date: Mon Sep 08 18:11:31 2014 +0300 description: pop3c: Added missing support for pop3c_ssl=starttls diffstat: src/lib-storage/index/pop3c/pop3c-client.c | 22 +++++++++++++++++++++- 1 files changed, 21 insertions(+), 1 deletions(-) diffs (53 lines): diff -r a41089e5bd8e -r bf1faf8ab847 src/lib-storage/index/pop3c/pop3c-client.c --- a/src/lib-storage/index/pop3c/pop3c-client.c Mon Sep 08 13:39:25 2014 +0300 +++ b/src/lib-storage/index/pop3c/pop3c-client.c Mon Sep 08 18:11:31 2014 +0300 @@ -27,6 +27,7 @@ POP3C_CLIENT_STATE_DISCONNECTED = 0, /* Trying to connect */ POP3C_CLIENT_STATE_CONNECTING, + POP3C_CLIENT_STATE_STARTTLS, /* Connected, trying to authenticate */ POP3C_CLIENT_STATE_USER, POP3C_CLIENT_STATE_AUTH, @@ -68,6 +69,7 @@ pop3c_dns_callback(const struct dns_lookup_result *result, struct pop3c_client *client); static void pop3c_client_connect_ip(struct pop3c_client *client); +static int pop3c_client_ssl_init(struct pop3c_client *client); struct pop3c_client * pop3c_client_init(const struct pop3c_client_settings *set) @@ -267,6 +269,12 @@ io_loop_destroy(&ioloop); } +static void pop3c_client_starttls(struct pop3c_client *client) +{ + o_stream_nsend_str(client->output, "STLS\r\n"); + client->state = POP3C_CLIENT_STATE_STARTTLS; +} + static void pop3c_client_authenticate1(struct pop3c_client *client) { const struct pop3c_client_settings *set = &client->set; @@ -339,7 +347,19 @@ client->set.host, line); return -1; } - pop3c_client_authenticate1(client); + if (client->set.ssl_mode == POP3C_CLIENT_SSL_MODE_STARTTLS) + pop3c_client_starttls(client); + else + pop3c_client_authenticate1(client); + break; + case POP3C_CLIENT_STATE_STARTTLS: + if (!success) { + i_error("pop3c(%s): STLS failed: %s", + client->set.host, line); + return -1; + } + if (pop3c_client_ssl_init(client) < 0) + pop3c_client_disconnect(client); break; case POP3C_CLIENT_STATE_USER: if (!success) { From dovecot at dovecot.org Tue Sep 9 14:10:14 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 09 Sep 2014 14:10:14 +0000 Subject: dovecot-2.2: lib-master: Fixed -c & -i command line parameters w... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8b5664bce4a0 changeset: 17765:8b5664bce4a0 user: Timo Sirainen date: Tue Sep 09 17:09:30 2014 +0300 description: lib-master: Fixed -c & -i command line parameters when config socket was readable. The config socket was always being read, even if another config file was attempted to be used. diffstat: src/lib-master/master-service-settings.c | 9 ++++++++- 1 files changed, 8 insertions(+), 1 deletions(-) diffs (26 lines): diff -r bf1faf8ab847 -r 8b5664bce4a0 src/lib-master/master-service-settings.c --- a/src/lib-master/master-service-settings.c Mon Sep 08 18:11:31 2014 +0300 +++ b/src/lib-master/master-service-settings.c Tue Sep 09 17:09:30 2014 +0300 @@ -177,7 +177,8 @@ *path_r = path = input->config_path != NULL ? input->config_path : master_service_get_config_path(service); - if (service->config_fd != -1 && input->config_path == NULL) { + if (service->config_fd != -1 && input->config_path == NULL && + service->config_path_is_default) { /* use the already opened config socket */ fd = service->config_fd; service->config_fd = -1; @@ -344,6 +345,12 @@ const char *path, *error; int fd; + /* we'll get here before command line parameters have been parsed, + so -O, -c and -i parameters haven't been handled yet at this point. + this means we could end up opening config socket connection + unnecessarily, but this isn't a problem. we'll just have to + ignore it later on. (unfortunately there isn't a master_service_*() + call where this function would be better called.) */ if (getenv("DOVECONF_ENV") != NULL || (service->flags & MASTER_SERVICE_FLAG_NO_CONFIG_SETTINGS) != 0) return; From dovecot at dovecot.org Wed Sep 10 10:43:58 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 10 Sep 2014 10:43:58 +0000 Subject: dovecot-2.2: lib-http: server: Fixed handling of disconnection w... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/992f17769cca changeset: 17766:992f17769cca user: Stephan Bosch date: Wed Sep 10 13:39:36 2014 +0300 description: lib-http: server: Fixed handling of disconnection while request is being handled. This would cause an assert failure. diffstat: src/lib-http/http-server-connection.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (14 lines): diff -r 8b5664bce4a0 -r 992f17769cca src/lib-http/http-server-connection.c --- a/src/lib-http/http-server-connection.c Tue Sep 09 17:09:30 2014 +0300 +++ b/src/lib-http/http-server-connection.c Wed Sep 10 13:39:36 2014 +0300 @@ -303,8 +303,8 @@ http_server_connection_ref(conn); http_server_connection_request_callback(conn, req); http_server_connection_unref(&conn); - if (conn == NULL) { - /* the callback managed to get this connection destroyed */ + if (conn == NULL || conn->closed) { + /* the callback managed to get this connection destroyed/closed */ return FALSE; } From dovecot at dovecot.org Wed Sep 10 10:44:03 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 10 Sep 2014 10:44:03 +0000 Subject: dovecot-2.2: lib-http: server: Fixed segfault occurring in conne... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7ccf5dd98be8 changeset: 17767:7ccf5dd98be8 user: Stephan Bosch date: Wed Sep 10 13:39:37 2014 +0300 description: lib-http: server: Fixed segfault occurring in connection input handler. Request handlers could close and destroy the connection early. Fixed by holding a reference in the input handler. diffstat: src/lib-http/http-server-connection.c | 19 ++++++++++++++++--- 1 files changed, 16 insertions(+), 3 deletions(-) diffs (70 lines): diff -r 992f17769cca -r 7ccf5dd98be8 src/lib-http/http-server-connection.c --- a/src/lib-http/http-server-connection.c Wed Sep 10 13:39:36 2014 +0300 +++ b/src/lib-http/http-server-connection.c Wed Sep 10 13:39:37 2014 +0300 @@ -426,12 +426,15 @@ /* parse requests */ ret = 1; while (!conn->close_indicated && ret != 0) { + http_server_connection_ref(conn); while ((ret = http_request_parse_next (conn->http_parser, req->pool, &req->req, &error_code, &error)) > 0) { if (pending_request != NULL) { /* previous request is now fully read and ready to respond */ http_server_request_ready_to_respond(pending_request); + if (conn->closed) + break; } http_server_connection_debug(conn, @@ -452,6 +455,7 @@ http_server_request_destroy(&req); else http_server_request_unref(&req); + http_server_connection_unref(&conn); return; } if (req->req.connection_close) @@ -460,15 +464,16 @@ http_server_request_destroy(&req); else http_server_request_unref(&req); - + /* client indicated it will close after this request; stop trying - to read more. */ - if (conn->close_indicated) + to read more. */ + if (conn->close_indicated) break; if (conn->request_queue_count >= conn->server->set.max_pipelined_requests) { http_server_connection_input_halt(conn); + http_server_connection_unref(&conn); return; } @@ -476,6 +481,10 @@ req = http_server_request_new(conn); } + http_server_connection_unref(&conn); + if (conn == NULL || conn->closed) + return; + if (ret <= 0 && (conn->conn.input->eof || conn->conn.input->stream_errno != 0)) { int stream_errno = conn->conn.input->stream_errno; @@ -545,7 +554,11 @@ if (ret == 0 && pending_request != NULL && !http_request_parser_pending_payload(conn->http_parser)) { /* previous request is now fully read and ready to respond */ + http_server_connection_ref(conn); http_server_request_ready_to_respond(pending_request); + http_server_connection_unref(&conn); + if (conn == NULL || conn->closed) + return; } } } From dovecot at dovecot.org Wed Sep 10 10:44:18 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 10 Sep 2014 10:44:18 +0000 Subject: dovecot-2.2: lib-http: server: Made sure destroy callback is cal... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4e339d81e1d2 changeset: 17768:4e339d81e1d2 user: Stephan Bosch date: Wed Sep 10 13:39:37 2014 +0300 description: lib-http: server: Made sure destroy callback is called only once. diffstat: src/lib-http/http-server-request.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (15 lines): diff -r 7ccf5dd98be8 -r 4e339d81e1d2 src/lib-http/http-server-request.c --- a/src/lib-http/http-server-request.c Wed Sep 10 13:39:37 2014 +0300 +++ b/src/lib-http/http-server-request.c Wed Sep 10 13:39:37 2014 +0300 @@ -59,8 +59,10 @@ if (req->delay_destroy) { req->destroy_pending = TRUE; } else if (req->destroy_callback != NULL) { - req->destroy_callback(req->destroy_context); + void (*callback)(void *) = req->destroy_callback; + req->destroy_callback = NULL; + callback(req->destroy_context); } http_server_request_unref(_req); } From dovecot at dovecot.org Wed Sep 10 10:44:29 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 10 Sep 2014 10:44:29 +0000 Subject: dovecot-2.2: lib-http: server: Fixed delayed request destruction. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0f1878544d3b changeset: 17769:0f1878544d3b user: Stephan Bosch date: Wed Sep 10 13:39:37 2014 +0300 description: lib-http: server: Fixed delayed request destruction. Destruction was actually delayed indefinitely, because the delay flag was not reset. Obviously, this caused a memory leak. diffstat: src/lib-http/http-server-connection.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 4e339d81e1d2 -r 0f1878544d3b src/lib-http/http-server-connection.c --- a/src/lib-http/http-server-connection.c Wed Sep 10 13:39:37 2014 +0300 +++ b/src/lib-http/http-server-connection.c Wed Sep 10 13:39:37 2014 +0300 @@ -448,6 +448,7 @@ T_BEGIN { cont = http_server_connection_handle_request(conn, req); } T_END; + req->delay_destroy = FALSE; if (!cont) { /* connection closed or request body not read yet. the request may be destroyed now. */ From dovecot at dovecot.org Wed Sep 10 10:44:44 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 10 Sep 2014 10:44:44 +0000 Subject: dovecot-2.2: lib-http: server: Added assert to connect callback ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b7c29e47b0f6 changeset: 17770:b7c29e47b0f6 user: Stephan Bosch date: Wed Sep 10 13:39:37 2014 +0300 description: lib-http: server: Added assert to connect callback which makes sure request is either responded to or referenced. Earlier change added this only to callback for normal requests, but CONNECT should be handled the same in this respect. diffstat: src/lib-http/http-server-connection.c | 16 ++++++++-------- 1 files changed, 8 insertions(+), 8 deletions(-) diffs (34 lines): diff -r 0f1878544d3b -r b7c29e47b0f6 src/lib-http/http-server-connection.c --- a/src/lib-http/http-server-connection.c Wed Sep 10 13:39:37 2014 +0300 +++ b/src/lib-http/http-server-connection.c Wed Sep 10 13:39:37 2014 +0300 @@ -244,6 +244,8 @@ static void http_server_connection_request_callback( struct http_server_connection *conn, struct http_server_request *req) { + unsigned int old_refcount = req->refcount; + /* CONNECT method */ if (strcmp(req->req.method, "CONNECT") == 0) { if (conn->callbacks->handle_connect_request == NULL) { @@ -262,15 +264,13 @@ if (conn->callbacks->handle_request == NULL) { http_server_request_fail(req, 505, "Not Implemented"); return; - } else { - unsigned int old_refcount = req->refcount; + } + conn->callbacks->handle_request(conn->context, req); + } - conn->callbacks->handle_request(conn->context, req); - i_assert((req->response != NULL && - req->response->submitted) || - req->refcount > old_refcount); - } - } + i_assert((req->response != NULL && + req->response->submitted) || + req->refcount > old_refcount); } static bool From dovecot at dovecot.org Wed Sep 10 10:44:54 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 10 Sep 2014 10:44:54 +0000 Subject: dovecot-2.2: lib-http: Implemented HTTP auth (RFC 7235). Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2121057f994b changeset: 17771:2121057f994b user: Stephan Bosch date: Wed Sep 10 13:39:37 2014 +0300 description: lib-http: Implemented HTTP auth (RFC 7235). diffstat: src/lib-http/Makefile.am | 10 + src/lib-http/http-auth.c | 427 ++++++++++++++++++++++++++++++++++++++++++ src/lib-http/http-auth.h | 70 ++++++ src/lib-http/http-parser.c | 73 ++++--- src/lib-http/http-parser.h | 6 + src/lib-http/test-http-auth.c | 274 ++++++++++++++++++++++++++ 6 files changed, 829 insertions(+), 31 deletions(-) diffs (truncated from 963 to 300 lines): diff -r b7c29e47b0f6 -r 2121057f994b src/lib-http/Makefile.am --- a/src/lib-http/Makefile.am Wed Sep 10 13:39:37 2014 +0300 +++ b/src/lib-http/Makefile.am Wed Sep 10 13:39:37 2014 +0300 @@ -14,6 +14,7 @@ http-header.c \ http-header-parser.c \ http-transfer-chunked.c \ + http-auth.c \ http-message-parser.c \ http-request.c \ http-request-parser.c \ @@ -37,6 +38,7 @@ http-header.h \ http-header-parser.h \ http-transfer.h \ + http-auth.h \ http-message-parser.h \ http-request.h \ http-request-parser.h \ @@ -55,6 +57,7 @@ test-http-url \ test-http-header-parser \ test-http-transfer \ + test-http-auth \ test-http-response-parser \ test-http-request-parser @@ -95,6 +98,13 @@ $(test_libs) test_http_transfer_DEPENDENCIES = $(test_deps) +test_http_auth_SOURCES = test-http-auth.c +test_http_auth_LDADD = \ + http-auth.lo \ + http-parser.lo \ + $(test_libs) +test_http_auth_DEPENDENCIES = $(test_deps) + test_http_response_parser_SOURCES = test-http-response-parser.c test_http_response_parser_LDADD = \ http-date.lo \ diff -r b7c29e47b0f6 -r 2121057f994b src/lib-http/http-auth.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-http/http-auth.c Wed Sep 10 13:39:37 2014 +0300 @@ -0,0 +1,427 @@ +/* Copyright (c) 2013-2014 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "array.h" +#include "http-parser.h" + +#include "http-auth.h" + +/* RFC 7235, Section 2.1: + + challenge = auth-scheme [ 1*SP ( token68 / #auth-param ) ] + credentials = auth-scheme [ 1*SP ( token68 / #auth-param ) ] + + auth-scheme = token + auth-param = token BWS "=" BWS ( token / quoted-string ) + token68 = 1*( ALPHA / DIGIT / + "-" / "." / "_" / "~" / "+" / "/" ) *"=" + + OWS = *( SP / HTAB ) + ; optional whitespace + BWS = OWS + ; "bad" whitespace + */ + +/* + * Parsing + */ + +static int +http_parse_token68(struct http_parser *parser, const char **token68_r) +{ + const unsigned char *first; + + /* token68 = 1*( ALPHA / DIGIT / + "-" / "." / "_" / "~" / "+" / "/" ) *"=" + */ + + /* 1*( ALPHA / DIGIT / "-" / "." / "_" / "~" / "+" / "/" ) */ + if (parser->cur >= parser->end || !http_char_is_token68(*parser->cur)) + return 0; + first = parser->cur++; + while (parser->cur < parser->end && http_char_is_token68(*parser->cur)) + parser->cur++; + + /* *"=" */ + while (parser->cur < parser->end && *parser->cur == '=') + parser->cur++; + + *token68_r = t_strndup(first, parser->cur - first); + return 1; +} + +static int +http_parse_auth_param(struct http_parser *parser, + const char **param_r, const char **value_r) +{ + const unsigned char *first = parser->cur, *end_token; + int ret; + + /* auth-param = token BWS "=" BWS ( token / quoted-string ) */ + + /* token */ + if ((ret=http_parser_skip_token(parser)) <= 0) { + parser->cur = first; + return ret; + } + end_token = parser->cur; + + /* BWS "=" BWS */ + http_parse_ows(parser); + if (parser->cur >= parser->end || *parser->cur != '=') { + parser->cur = first; + return 0; + } + parser->cur++; + http_parse_ows(parser); + + /* ( token / quoted-string ) */ + if ((ret=http_parse_token_or_qstring(parser, value_r)) <= 0) { + parser->cur = first; + return ret; + } + + *param_r = t_strndup(first, end_token - first); + return 1; +} + +static int +http_parse_auth_params(struct http_parser *parser, + ARRAY_TYPE(http_auth_param) *params) +{ + const unsigned char *last = parser->cur; + struct http_auth_param param; + unsigned int count = 0; + int ret; + + memset(¶m, 0, sizeof(param)); + while ((ret=http_parse_auth_param + (parser, ¶m.name, ¶m.value)) > 0) { + if (!array_is_created(params)) + t_array_init(params, 4); + array_append(params, ¶m, 1); + count++; + + last = parser->cur; + + /* OWS "," OWS + --> also allow empty elements + */ + for (;;) { + http_parse_ows(parser); + if (parser->cur >= parser->end || *parser->cur != ',') + break; + parser->cur++; + } + } + + parser->cur = last; + if (ret < 0) + return -1; + return (count > 0 ? 1 : 0); +} + +int http_auth_parse_challenges(const unsigned char *data, size_t size, + ARRAY_TYPE(http_auth_challenge) *chlngs) +{ + struct http_parser parser; + int ret; + + http_parser_init(&parser, data, size); + + /* WWW-Authenticate = 1#challenge + Proxy-Authenticate = 1#challenge + + challenge = auth-scheme [ 1*SP ( token68 / #auth-param ) ] + auth-scheme = token + */ + + /* 1#element => *( "," OWS ) ... ; RFC 7230, Section 7 */ + for (;;) { + if (parser.cur >= parser.end || *parser.cur != ',') + break; + parser.cur++; + http_parse_ows(&parser); + } + + for (;;) { + struct http_auth_challenge chlng; + + memset(&chlng, 0, sizeof(chlng)); + + /* auth-scheme */ + if ((ret=http_parse_token(&parser, &chlng.scheme)) <= 0) { + if (ret < 0) + return -1; + break; + } + + /* [ 1*SP ... ] */ + if (parser.cur >= parser.end || *parser.cur != ' ') + return 1; + parser.cur++; + while (parser.cur < parser.end && *parser.cur == ' ') + parser.cur++; + + /* ( token68 / #auth-param ) */ + if ((ret=http_parse_auth_params(&parser, &chlng.params)) <= 0) { + if (ret < 0) + return -1; + if (http_parse_token68(&parser, &chlng.data) < 0) + return -1; + } + + if (!array_is_created(chlngs)) + t_array_init(chlngs, 4); + array_append(chlngs, &chlng, 1); + + /* OWS "," OWS + --> also allow empty elements + */ + for (;;) { + http_parse_ows(&parser); + if (parser.cur >= parser.end || *parser.cur != ',') + break; + parser.cur++; + } + } + + if (parser.cur != parser.end) + return -1; + return 1; +} + +int http_auth_parse_credentials(const unsigned char *data, size_t size, + struct http_auth_credentials *crdts) +{ + struct http_parser parser; + int ret; + + http_parser_init(&parser, data, size); + + /* Authorization = credentials + Proxy-Authorization = credentials + + credentials = auth-scheme [ 1*SP ( token68 / #auth-param ) ] + auth-scheme = token + */ + + memset(crdts, 0, sizeof(*crdts)); + + /* auth-scheme */ + if (http_parse_token(&parser, &crdts->scheme) <= 0) + return -1; + + /* [ 1*SP ... ] */ + if (parser.cur >= parser.end || *parser.cur != ' ') + return 1; + parser.cur++; + while (parser.cur < parser.end && *parser.cur == ' ') + parser.cur++; + + /* ( token68 / #auth-param ) */ + if ((ret=http_parse_auth_params(&parser, &crdts->params)) <= 0) { + if (ret < 0) + return -1; + if (http_parse_token68(&parser, &crdts->data) < 0) + return -1; + } + + if (parser.cur != parser.end) + return -1; + return 1; +} + +/* + * Construction + */ + +static void +http_auth_create_param(string_t *out, const struct http_auth_param *param) +{ + const char *p, *first; + + /* auth-param = token BWS "=" BWS ( token / quoted-string ) */ + + str_append(out, param->name); + str_append_c(out, '='); + + for (p = param->value; *p != '\0' && http_char_is_token(*p); p++); + + if ( *p != '\0' ) { + str_append_c(out, '"'); + p = first = param->value; + while (*p != '\0') { From dovecot at dovecot.org Wed Sep 10 10:45:14 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 10 Sep 2014 10:45:14 +0000 Subject: dovecot-2.2: lib-http: server: Added support for authentication. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5ab0ab678497 changeset: 17772:5ab0ab678497 user: Stephan Bosch date: Wed Sep 10 13:39:37 2014 +0300 description: lib-http: server: Added support for authentication. diffstat: src/lib-http/http-server-private.h | 1 + src/lib-http/http-server-request.c | 59 +++++++++++++++++++++++++++++++++++- src/lib-http/http-server-response.c | 30 ++++++++++++++++++- src/lib-http/http-server.h | 26 ++++++++++++++++ 4 files changed, 112 insertions(+), 4 deletions(-) diffs (213 lines): diff -r 2121057f994b -r 5ab0ab678497 src/lib-http/http-server-private.h --- a/src/lib-http/http-server-private.h Wed Sep 10 13:39:37 2014 +0300 +++ b/src/lib-http/http-server-private.h Wed Sep 10 13:39:37 2014 +0300 @@ -47,6 +47,7 @@ string_t *headers; time_t date; + ARRAY_TYPE(http_auth_challenge) auth_challenges; struct istream *payload_input; uoff_t payload_size, payload_offset; diff -r 2121057f994b -r 5ab0ab678497 src/lib-http/http-server-request.c --- a/src/lib-http/http-server-request.c Wed Sep 10 13:39:37 2014 +0300 +++ b/src/lib-http/http-server-request.c Wed Sep 10 13:39:37 2014 +0300 @@ -111,6 +111,22 @@ return req->response; } +int http_server_request_get_auth(struct http_server_request *req, + struct http_auth_credentials *credentials) +{ + const char *auth; + + auth = http_request_header_get(&req->req, "Authorization"); + if (auth == NULL) + return 0; + + if (http_auth_parse_credentials + ((const unsigned char *)auth, strlen(auth), credentials) < 0) + return -1; + + return 1; +} + bool http_server_request_is_finished(struct http_server_request *req) { return req->response != NULL || @@ -197,9 +213,9 @@ http_server_connection_send_responses(conn); } -static void -http_server_request_fail_full(struct http_server_request *req, - unsigned int status, const char *reason, bool close) +static struct http_server_response * +http_server_request_create_fail_response(struct http_server_request *req, + unsigned int status, const char *reason) { struct http_server_response *resp; @@ -211,6 +227,18 @@ reason = t_strconcat(reason, "\r\n", NULL); http_server_response_set_payload_data (resp, (const unsigned char *)reason, strlen(reason)); + + return resp; +} + +static void +http_server_request_fail_full(struct http_server_request *req, + unsigned int status, const char *reason, bool close) +{ + struct http_server_response *resp; + + req->failed = TRUE; + resp = http_server_request_create_fail_response(req, status, reason); if (close) http_server_response_submit_close(resp); else @@ -229,3 +257,28 @@ { http_server_request_fail_full(req, status, reason, TRUE); } + +void http_server_request_fail_auth(struct http_server_request *req, + const char *reason, const struct http_auth_challenge *chlng) +{ + struct http_server_response *resp; + + req->failed = TRUE; + + if (reason == NULL) + reason = "Unauthenticated"; + + resp = http_server_request_create_fail_response(req, 401, reason); + http_server_response_add_auth(resp, chlng); + http_server_response_submit(resp); +} + +void http_server_request_fail_auth_basic(struct http_server_request *req, + const char *reason, const char *realm) +{ + struct http_auth_challenge chlng; + + http_auth_basic_challenge_init(&chlng, realm); + http_server_request_fail_auth(req, reason, &chlng); +} + diff -r 2121057f994b -r 5ab0ab678497 src/lib-http/http-server-response.c --- a/src/lib-http/http-server-response.c Wed Sep 10 13:39:37 2014 +0300 +++ b/src/lib-http/http-server-response.c Wed Sep 10 13:39:37 2014 +0300 @@ -2,6 +2,7 @@ #include "lib.h" #include "str.h" +#include "array.h" #include "istream.h" #include "ostream.h" #include "http-date.h" @@ -137,6 +138,29 @@ i_stream_unref(&input); } +void http_server_response_add_auth( + struct http_server_response *resp, + const struct http_auth_challenge *chlng) +{ + struct http_auth_challenge *new; + pool_t pool = resp->request->pool; + + if (!array_is_created(&resp->auth_challenges)) + p_array_init(&resp->auth_challenges, pool, 4); + + new = array_append_space(&resp->auth_challenges); + http_auth_challenge_copy(pool, new, chlng); +} + +void http_server_response_add_auth_basic( + struct http_server_response *resp, const char *realm) +{ + struct http_auth_challenge chlng; + + http_auth_basic_challenge_init(&chlng, realm); + http_server_response_add_auth(resp, &chlng); +} + static void http_server_response_do_submit(struct http_server_response *resp, bool close) { @@ -284,6 +308,11 @@ str_append(rtext, http_date_create(resp->date)); str_append(rtext, "\r\n"); } + if (array_is_created(&resp->auth_challenges)) { + str_append(rtext, "WWW-Authenticate: "); + http_auth_create_challenges(rtext, &resp->auth_challenges); + str_append(rtext, "\r\n"); + } if (resp->payload_chunked) { if (http_server_request_version_equals(req, 1, 0)) { /* cannot use Transfer-Encoding */ @@ -382,4 +411,3 @@ } T_END; return ret; } - diff -r 2121057f994b -r 5ab0ab678497 src/lib-http/http-server.h --- a/src/lib-http/http-server.h Wed Sep 10 13:39:37 2014 +0300 +++ b/src/lib-http/http-server.h Wed Sep 10 13:39:37 2014 +0300 @@ -1,6 +1,7 @@ #ifndef HTTP_SERVER_H #define HTTP_SERVER_H +#include "http-auth.h" #include "http-request.h" struct istream; @@ -90,6 +91,12 @@ or because the request was aborted. */ bool http_server_request_is_finished(struct http_server_request *req); +/* Get the authentication credentials provided in this request. Returns 0 if + the Authorization header is absent, returns -1 when that header cannot be + parsed, and returns 1 otherwise */ +int http_server_request_get_auth(struct http_server_request *req, + struct http_auth_credentials *credentials); + /* Send a failure response to the request with given status/reason. */ void http_server_request_fail(struct http_server_request *req, unsigned int status, const char *reason); @@ -97,6 +104,18 @@ and close the connection. */ void http_server_request_fail_close(struct http_server_request *req, unsigned int status, const char *reason); +/* Send an authentication failure response to the request with given reason. + The provided challenge is set in the WWW-Authenticate header of the + response. */ +void http_server_request_fail_auth(struct http_server_request *req, + const char *reason, const struct http_auth_challenge *chlng) + ATTR_NULL(2); +/* Send a authentication failure response to the request with given reason. + The provided realm is used to construct an Basic challenge in the + WWW-Authenticate header of the response. */ +void http_server_request_fail_auth_basic(struct http_server_request *req, + const char *reason, const char *realm) + ATTR_NULL(2); /* Call the specified callback when HTTP request is destroyed. This happens after one of the following: @@ -131,6 +150,13 @@ struct istream *input); void http_server_response_set_payload_data(struct http_server_response *resp, const unsigned char *data, size_t size); + +void http_server_response_add_auth( + struct http_server_response *resp, + const struct http_auth_challenge *chlng); +void http_server_response_add_auth_basic( + struct http_server_response *resp, const char *realm); + void http_server_response_submit(struct http_server_response *resp); /* Submit response and close the connection. */ void http_server_response_submit_close(struct http_server_response *resp); From dovecot at dovecot.org Wed Sep 10 10:45:35 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 10 Sep 2014 10:45:35 +0000 Subject: dovecot-2.2: lib-http: client: Tunnel connection failure would c... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bf371c4fed6e changeset: 17773:bf371c4fed6e user: Stephan Bosch date: Wed Sep 10 13:39:37 2014 +0300 description: lib-http: client: Tunnel connection failure would cause segfault. That happened because it tried to retry waiting requests, which makes no sense for a CONNECT tunnel. diffstat: src/lib-http/http-client-connection.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (13 lines): diff -r 5ab0ab678497 -r bf371c4fed6e src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Wed Sep 10 13:39:37 2014 +0300 +++ b/src/lib-http/http-client-connection.c Wed Sep 10 13:39:37 2014 +0300 @@ -77,6 +77,9 @@ { struct http_client_request **req; + if (!array_is_created(&conn->request_wait_list)) + return; + array_foreach_modifiable(&conn->request_wait_list, req) { if ((*req)->state < HTTP_REQUEST_STATE_FINISHED) http_client_request_retry(*req, status, error); From dovecot at dovecot.org Wed Sep 10 10:45:45 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 10 Sep 2014 10:45:45 +0000 Subject: dovecot-2.2: lib-http: client: Added support for attempting a si... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/650629de6a73 changeset: 17774:650629de6a73 user: Stephan Bosch date: Wed Sep 10 13:39:37 2014 +0300 description: lib-http: client: Added support for attempting a single IP several times. Also limits the number of attempts when there are many IPs. diffstat: src/lib-http/http-client-private.h | 2 ++ src/lib-http/http-client-queue.c | 23 ++++++++++++++++++++--- src/lib-http/http-client.c | 1 + src/lib-http/http-client.h | 11 +++++++++++ 4 files changed, 34 insertions(+), 3 deletions(-) diffs (104 lines): diff -r bf371c4fed6e -r 650629de6a73 src/lib-http/http-client-private.h --- a/src/lib-http/http-client-private.h Wed Sep 10 13:39:37 2014 +0300 +++ b/src/lib-http/http-client-private.h Wed Sep 10 13:39:37 2014 +0300 @@ -182,6 +182,8 @@ connected IP */ unsigned int ips_connect_start_idx; + unsigned int connect_attempts; + /* peers we are trying to connect to; this can be more than one when soft connect timeouts are enabled */ ARRAY_TYPE(http_client_peer) pending_peers; diff -r bf371c4fed6e -r 650629de6a73 src/lib-http/http-client-queue.c --- a/src/lib-http/http-client-queue.c Wed Sep 10 13:39:37 2014 +0300 +++ b/src/lib-http/http-client-queue.c Wed Sep 10 13:39:37 2014 +0300 @@ -158,13 +158,20 @@ static bool http_client_queue_is_last_connect_ip(struct http_client_queue *queue) { + const struct http_client_settings *set = + &queue->client->set; struct http_client_host *host = queue->host; i_assert(queue->ips_connect_idx < host->ips_count); i_assert(queue->ips_connect_start_idx < host->ips_count); - /* we'll always go through all the IPs. we don't necessarily start - connecting from the first IP, so we'll need to treat the IPs as + /* if a maximum connect attempts > 1 is set, enforce it directly */ + if (set->max_connect_attempts > 1) { + return queue->connect_attempts >= set->max_connect_attempts; + } + + /* otherwise, we'll always go through all the IPs. we don't necessarily + start connecting from the first IP, so we'll need to treat the IPs as a ring buffer where we automatically wrap back to the first IP when necessary. */ return (queue->ips_connect_idx + 1) % host->ips_count == @@ -244,8 +251,14 @@ } } } - if (new_peer) + if (new_peer) { + http_client_queue_debug(queue, "Started new connection to %s%s", + http_client_peer_addr2str(addr), (addr->https_name == NULL ? "" : + t_strdup_printf(" (SSL=%s)", addr->https_name))); + array_append(&queue->pending_peers, &peer, 1); + queue->connect_attempts++; + } /* start soft connect time-out (but only if we have another IP left) */ msecs = host->client->set.soft_connect_timeout_msecs; @@ -265,6 +278,9 @@ queue->ips_connect_start_idx = http_client_host_get_ip_idx(queue->host, &addr->ip); + /* reset attempt counter */ + queue->connect_attempts = 0; + /* stop soft connect time-out */ if (queue->to_connect != NULL) timeout_remove(&queue->to_connect); @@ -343,6 +359,7 @@ next request. */ queue->ips_connect_idx = queue->ips_connect_start_idx = (queue->ips_connect_idx + 1) % host->ips_count; + queue->connect_attempts = 0; http_client_queue_fail(queue, HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED, reason); return FALSE; diff -r bf371c4fed6e -r 650629de6a73 src/lib-http/http-client.c --- a/src/lib-http/http-client.c Wed Sep 10 13:39:37 2014 +0300 +++ b/src/lib-http/http-client.c Wed Sep 10 13:39:37 2014 +0300 @@ -120,6 +120,7 @@ client->set.max_pipelined_requests = (set->max_pipelined_requests > 0 ? set->max_pipelined_requests : 1); client->set.max_attempts = set->max_attempts; + client->set.max_connect_attempts = set->max_connect_attempts; client->set.no_auto_redirect = set->no_auto_redirect; client->set.no_ssl_tunnel = set->no_ssl_tunnel; client->set.max_redirects = set->max_redirects; diff -r bf371c4fed6e -r 650629de6a73 src/lib-http/http-client.h --- a/src/lib-http/http-client.h Wed Sep 10 13:39:37 2014 +0300 +++ b/src/lib-http/http-client.h Wed Sep 10 13:39:37 2014 +0300 @@ -81,6 +81,17 @@ /* maximum number of attempts for a request */ unsigned int max_attempts; + /* maximum number of connection attempts to a host before all associated + requests fail. + + if > 1, the maximum will be enforced across all IPs for that host, + meaning that IPs may be tried more than once eventually if the number + of IPs is smaller than the specified maximum attempts. If the number of IPs + is higher than the maximum attempts, not all IPs are tried. If <= 1, all + IPs are tried at most once. + */ + unsigned int max_connect_attempts; + /* response header limits */ struct http_header_limits response_hdr_limits; From dovecot at dovecot.org Wed Sep 10 10:45:56 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 10 Sep 2014 10:45:56 +0000 Subject: dovecot-2.2: lib-http: client: Implemented support for connectio... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4412508cbc59 changeset: 17775:4412508cbc59 user: Stephan Bosch date: Wed Sep 10 13:39:37 2014 +0300 description: lib-http: client: Implemented support for connection failure backoff. diffstat: src/lib-http/http-client-peer.c | 65 +++++++++++++++++++++++++++++++++++-- src/lib-http/http-client-private.h | 9 ++++- src/lib-http/http-client.c | 4 ++ src/lib-http/http-client.h | 3 + 4 files changed, 75 insertions(+), 6 deletions(-) diffs (187 lines): diff -r 650629de6a73 -r 4412508cbc59 src/lib-http/http-client-peer.c --- a/src/lib-http/http-client-peer.c Wed Sep 10 13:39:37 2014 +0300 +++ b/src/lib-http/http-client-peer.c Wed Sep 10 13:39:37 2014 +0300 @@ -2,6 +2,7 @@ #include "lib.h" #include "net.h" +#include "time-util.h" #include "str.h" #include "hash.h" #include "array.h" @@ -78,7 +79,8 @@ */ static void -http_client_peer_connect(struct http_client_peer *peer, unsigned int count) +http_client_peer_do_connect(struct http_client_peer *peer, + unsigned int count) { unsigned int i; @@ -89,6 +91,42 @@ } } + +static void +http_client_peer_connect_backoff(struct http_client_peer *peer) +{ + i_assert(peer->to_backoff != NULL); + + http_client_peer_debug(peer, + "Backoff timer expired"); + + timeout_remove(&peer->to_backoff); + http_client_peer_do_connect(peer, 1); +} + +static void +http_client_peer_connect(struct http_client_peer *peer, unsigned int count) +{ + if (peer->to_backoff != NULL) + return; + + if (peer->last_failure.tv_sec > 0) { + int backoff_time_spent = + timeval_diff_msecs(&ioloop_timeval, &peer->last_failure); + if (backoff_time_spent < (int)peer->backoff_time_msecs) { + http_client_peer_debug(peer, + "Starting backoff timer for %d msecs", + peer->backoff_time_msecs - backoff_time_spent); + peer->to_backoff = timeout_add + ((unsigned int)(peer->backoff_time_msecs - backoff_time_spent), + http_client_peer_connect_backoff, peer); + return; + } + } + + http_client_peer_do_connect(peer, count); +} + bool http_client_peer_is_connected(struct http_client_peer *peer) { struct http_client_connection *const *conn_idx; @@ -228,7 +266,7 @@ i_assert(idle == 0); /* determine how many new connections we can set up */ - if (peer->last_connect_failed && working_conn_count > 0 && + if (peer->last_failure.tv_sec > 0 && working_conn_count > 0 && working_conn_count == connecting) { /* don't create new connections until the existing ones have finished connecting successfully. */ @@ -381,6 +419,8 @@ if (peer->to_req_handling != NULL) timeout_remove(&peer->to_req_handling); + if (peer->to_backoff != NULL) + timeout_remove(&peer->to_backoff); /* make a copy of the connection array; freed connections modify it */ t_array_init(&conns, array_count(&peer->conns)); @@ -471,7 +511,11 @@ { struct http_client_queue *const *queue; - peer->last_connect_failed = FALSE; + peer->last_failure.tv_sec = peer->last_failure.tv_usec = 0; + peer->backoff_time_msecs = 0; + + if (peer->to_backoff != NULL) + timeout_remove(&peer->to_backoff); array_foreach(&peer->queues, queue) { http_client_queue_connection_success(*queue, &peer->addr); @@ -483,6 +527,7 @@ void http_client_peer_connection_failure(struct http_client_peer *peer, const char *reason) { + const struct http_client_settings *set = &peer->client->set; struct http_client_queue *const *queue; unsigned int num_urgent; @@ -490,7 +535,15 @@ http_client_peer_debug(peer, "Failed to make connection"); - peer->last_connect_failed = TRUE; + peer->last_failure = ioloop_timeval; + + if (array_count(&peer->conns) == 1) { + if (peer->backoff_time_msecs == 0) + peer->backoff_time_msecs = set->connect_backoff_time_msecs; + else + peer->backoff_time_msecs *= 2; + } + if (array_count(&peer->conns) > 1) { /* if there are other connections attempting to connect, wait for them before failing the requests. remember that we had @@ -553,5 +606,9 @@ peer->to_req_handling = io_loop_move_timeout(&peer->to_req_handling); } + if (peer->to_backoff != NULL) { + peer->to_backoff = + io_loop_move_timeout(&peer->to_backoff); + } } diff -r 650629de6a73 -r 4412508cbc59 src/lib-http/http-client-private.h --- a/src/lib-http/http-client-private.h Wed Sep 10 13:39:37 2014 +0300 +++ b/src/lib-http/http-client-private.h Wed Sep 10 13:39:37 2014 +0300 @@ -11,8 +11,9 @@ #define HTTP_CLIENT_DNS_LOOKUP_TIMEOUT_MSECS (1000*30) #define HTTP_CLIENT_CONNECT_TIMEOUT_MSECS (1000*30) +#define HTTP_CLIENT_CONTINUE_TIMEOUT_MSECS (1000*2) #define HTTP_CLIENT_DEFAULT_REQUEST_TIMEOUT_MSECS (1000*60*5) -#define HTTP_CLIENT_CONTINUE_TIMEOUT_MSECS (1000*2) +#define HTTP_CLIENT_DEFAULT_BACKOFF_TIME_MSECS (100) enum http_response_payload_type; @@ -159,10 +160,14 @@ /* zero time-out for consolidating request handling */ struct timeout *to_req_handling; + /* connection retry */ + struct timeval last_failure; + struct timeout *to_backoff; + unsigned int backoff_time_msecs; + unsigned int destroyed:1; /* peer is being destroyed */ unsigned int no_payload_sync:1; /* expect: 100-continue failed before */ unsigned int seen_100_response:1;/* expect: 100-continue succeeded before */ - unsigned int last_connect_failed:1; unsigned int allows_pipelining:1;/* peer is known to allow persistent connections */ }; diff -r 650629de6a73 -r 4412508cbc59 src/lib-http/http-client.c --- a/src/lib-http/http-client.c Wed Sep 10 13:39:37 2014 +0300 +++ b/src/lib-http/http-client.c Wed Sep 10 13:39:37 2014 +0300 @@ -121,6 +121,10 @@ (set->max_pipelined_requests > 0 ? set->max_pipelined_requests : 1); client->set.max_attempts = set->max_attempts; client->set.max_connect_attempts = set->max_connect_attempts; + client->set.connect_backoff_time_msecs = + set->connect_backoff_time_msecs == 0 ? + HTTP_CLIENT_DEFAULT_BACKOFF_TIME_MSECS : + set->connect_backoff_time_msecs; client->set.no_auto_redirect = set->no_auto_redirect; client->set.no_ssl_tunnel = set->no_ssl_tunnel; client->set.max_redirects = set->max_redirects; diff -r 650629de6a73 -r 4412508cbc59 src/lib-http/http-client.h --- a/src/lib-http/http-client.h Wed Sep 10 13:39:37 2014 +0300 +++ b/src/lib-http/http-client.h Wed Sep 10 13:39:37 2014 +0300 @@ -92,6 +92,9 @@ */ unsigned int max_connect_attempts; + /* Initial backoff time; doubled at each connection failure */ + unsigned int connect_backoff_time_msecs; + /* response header limits */ struct http_header_limits response_hdr_limits; From dovecot at dovecot.org Wed Sep 10 10:49:38 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 10 Sep 2014 10:49:38 +0000 Subject: dovecot-2.2: pop3: Optimize POP3 UIDL processing when duplicate ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/be21e17813d4 changeset: 17776:be21e17813d4 user: Michael M Slusarz date: Tue Sep 09 17:26:52 2014 -0600 description: pop3: Optimize POP3 UIDL processing when duplicate renaming is not active. diffstat: src/pop3/pop3-commands.c | 10 +++++++--- 1 files changed, 7 insertions(+), 3 deletions(-) diffs (29 lines): diff -r 4412508cbc59 -r be21e17813d4 src/pop3/pop3-commands.c --- a/src/pop3/pop3-commands.c Wed Sep 10 13:39:37 2014 +0300 +++ b/src/pop3/pop3-commands.c Tue Sep 09 17:26:52 2014 -0600 @@ -784,7 +784,9 @@ uidl_duplicates_rename = strcmp(client->set->pop3_uidl_duplicates, "rename") == 0; - hash_table_create(&prev_uidls, default_pool, 0, str_hash, strcmp); + if (uidl_duplicates_rename) + hash_table_create(&prev_uidls, default_pool, 0, str_hash, + strcmp); client->uidl_pool = pool_alloconly_create("message uidls", 1024); /* first read all the UIDLs into a temporary [seq] array */ @@ -804,10 +806,12 @@ mail_update_pop3_uidl(mail, uidl); seq_uidls[mail->seq-1] = uidl; - hash_table_insert(prev_uidls, uidl, POINTER_CAST(1)); + if (uidl_duplicates_rename) + hash_table_insert(prev_uidls, uidl, POINTER_CAST(1)); } (void)mailbox_search_deinit(&search_ctx); - hash_table_destroy(&prev_uidls); + if (uidl_duplicates_rename) + hash_table_destroy(&prev_uidls); if (failed) { pool_unref(&client->uidl_pool); From dovecot at dovecot.org Wed Sep 10 11:00:02 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 10 Sep 2014 11:00:02 +0000 Subject: dovecot-2.2: dsync: Reset I/O timeout every time when receiving ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/647162da8423 changeset: 17777:647162da8423 user: Timo Sirainen date: Wed Sep 10 13:59:31 2014 +0300 description: dsync: Reset I/O timeout every time when receiving input. This could have caused unwanted I/O timeouts when receiving large mails. diffstat: src/doveadm/dsync/dsync-ibc-stream.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r be21e17813d4 -r 647162da8423 src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Tue Sep 09 17:26:52 2014 -0600 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Wed Sep 10 13:59:31 2014 +0300 @@ -190,6 +190,7 @@ static void dsync_ibc_stream_input(struct dsync_ibc_stream *ibc) { + timeout_reset(ibc->to); if (ibc->value_input != NULL) { if (dsync_ibc_stream_read_mail_stream(ibc) == 0) return; From dovecot at dovecot.org Wed Sep 10 11:11:26 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 10 Sep 2014 11:11:26 +0000 Subject: dovecot-2.2: lib: test-istream-concat - clean up our test-istreams Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e5028f632d1c changeset: 17778:e5028f632d1c user: Phil Carmody date: Wed Sep 10 14:08:58 2014 +0300 description: lib: test-istream-concat - clean up our test-istreams This reduces noise when memory leak debugging the unit tests. Signed-off-by: Phil Carmody diffstat: src/lib/test-istream-concat.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (13 lines): diff -r 647162da8423 -r e5028f632d1c src/lib/test-istream-concat.c --- a/src/lib/test-istream-concat.c Wed Sep 10 13:59:31 2014 +0300 +++ b/src/lib/test-istream-concat.c Wed Sep 10 14:08:58 2014 +0300 @@ -47,6 +47,9 @@ } } i_stream_unref(&input); + + for (i = 0; i < STREAM_COUNT; i++) + i_stream_unref(&streams[i]); } static bool test_istream_concat_random(void) From dovecot at dovecot.org Wed Sep 10 11:11:36 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 10 Sep 2014 11:11:36 +0000 Subject: dovecot-2.2: lib: test-aqueue - clean up the test queue each tim... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e1eaf5d06769 changeset: 17779:e1eaf5d06769 user: Phil Carmody date: Wed Sep 10 14:08:58 2014 +0300 description: lib: test-aqueue - clean up the test queue each time round the loop Helps keep noise out of memory leak debugging during unit tests. Signed-off-by: Phil Carmody diffstat: src/lib/test-aqueue.c | 10 ++++++---- 1 files changed, 6 insertions(+), 4 deletions(-) diffs (32 lines): diff -r e5028f632d1c -r e1eaf5d06769 src/lib/test-aqueue.c --- a/src/lib/test-aqueue.c Wed Sep 10 14:08:58 2014 +0300 +++ b/src/lib/test-aqueue.c Wed Sep 10 14:08:58 2014 +0300 @@ -25,10 +25,11 @@ { ARRAY(unsigned int) aqueue_array; unsigned int i, j, k; - struct aqueue *aqueue; for (i = 0; i < N_ELEMENTS(aqueue_input); i++) { for (k = 0; k < N_ELEMENTS(aqueue_input); k++) { + struct aqueue *aqueue; + t_array_init(&aqueue_array, initial_size); aqueue = aqueue_init(&aqueue_array.arr); aqueue->head = aqueue->tail = initial_size - 1; @@ -49,11 +50,12 @@ if (!aqueue_is_ok(aqueue, i)) return "Invalid data after delete"; } + aqueue_clear(aqueue); + if (aqueue_count(aqueue) != 0) + return "aqueue_clear() broken"; + aqueue_deinit(&aqueue); } } - aqueue_clear(aqueue); - if (aqueue_count(aqueue) != 0) - return "aqueue_clear() broken"; return NULL; } From dovecot at dovecot.org Wed Sep 10 11:11:46 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 10 Sep 2014 11:11:46 +0000 Subject: dovecot-2.2: lib: test-seq-range-array - clean up array after use Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c749a40eda0a changeset: 17780:c749a40eda0a user: Phil Carmody date: Wed Sep 10 14:08:58 2014 +0300 description: lib: test-seq-range-array - clean up array after use Removes noise when memory-leak debugging during unit tests. Signed-off-by: Phil Carmody diffstat: src/lib/test-seq-range-array.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r e1eaf5d06769 -r c749a40eda0a src/lib/test-seq-range-array.c --- a/src/lib/test-seq-range-array.c Wed Sep 10 14:08:58 2014 +0300 +++ b/src/lib/test-seq-range-array.c Wed Sep 10 14:08:58 2014 +0300 @@ -151,6 +151,7 @@ test_out_reason("seq_range_array random", FALSE, t_strdup_printf("round %u test %d failed", i, test)); } + array_free(&range); } static void test_seq_range_array_invert(void) From dovecot at dovecot.org Wed Sep 10 11:11:51 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 10 Sep 2014 11:11:51 +0000 Subject: dovecot-2.2: lib-index: test suite memory leak cleanup Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f0701f84ca01 changeset: 17781:f0701f84ca01 user: Phil Carmody date: Wed Sep 10 14:08:58 2014 +0300 description: lib-index: test suite memory leak cleanup So that memory leak checkers can be active while running the test suites, ensure that the suite itself doesn't leak. Signed-off-by: Phil Carmody diffstat: src/lib-index/test-mail-index-transaction-finish.c | 1 + src/lib-index/test-mail-index-transaction-update.c | 32 ++++++++++++++++++++++ src/lib-index/test-mail-transaction-log-append.c | 4 ++ src/lib-index/test-mail-transaction-log-view.c | 18 +++++++++++- 4 files changed, 54 insertions(+), 1 deletions(-) diffs (195 lines): diff -r c749a40eda0a -r f0701f84ca01 src/lib-index/test-mail-index-transaction-finish.c --- a/src/lib-index/test-mail-index-transaction-finish.c Wed Sep 10 14:08:58 2014 +0300 +++ b/src/lib-index/test-mail-index-transaction-finish.c Wed Sep 10 14:08:58 2014 +0300 @@ -152,6 +152,7 @@ test_assert(conflicts[1].seq1 == 8 && conflicts[1].seq2 == 8); test_end(); + array_free(t->conflict_seqs); } static void test_mail_index_transaction_finish_modseq_updates(void) diff -r c749a40eda0a -r f0701f84ca01 src/lib-index/test-mail-index-transaction-update.c --- a/src/lib-index/test-mail-index-transaction-update.c Wed Sep 10 14:08:58 2014 +0300 +++ b/src/lib-index/test-mail-index-transaction-update.c Wed Sep 10 14:08:58 2014 +0300 @@ -70,6 +70,17 @@ t->first_new_seq = hdr.messages_count + 1; return t; } +static void mail_index_transaction_cleanup(struct mail_index_transaction *t) +{ + if (array_is_created(&t->appends)) + array_free(&t->appends); + if (array_is_created(&t->updates)) + array_free(&t->updates); + if (array_is_created(&t->modseq_updates)) + array_free(&t->modseq_updates); + if (array_is_created(&t->expunges)) + array_free(&t->expunges); +} static void test_mail_index_append(void) { @@ -103,6 +114,7 @@ test_assert(appends[1].uid == 124); test_assert(appends[1].flags == 0); test_end(); + mail_index_transaction_cleanup(t); /* test with some uids */ t = mail_index_transaction_new(); @@ -138,6 +150,8 @@ test_assert(appends[3].uid == 131); test_assert(appends[4].uid == 128); test_end(); + + mail_index_transaction_cleanup(t); } static void test_mail_index_flag_update_fastpath(void) @@ -178,6 +192,8 @@ test_assert(updates[1].remove_flags == 0); test_assert(!t->log_updates); test_end(); + + mail_index_transaction_cleanup(t); } static void test_mail_index_flag_update_simple_merges(void) @@ -221,6 +237,8 @@ test_assert(updates[0].uid1 == 4); test_assert(updates[0].uid2 == 12); test_end(); + + mail_index_transaction_cleanup(t); } static void test_mail_index_flag_update_complex_merges(void) @@ -279,6 +297,8 @@ test_assert(updates[6].remove_flags == 0); test_end(); + + mail_index_transaction_cleanup(t); } static void @@ -353,6 +373,8 @@ flags_array_check(t, flags, hdr.messages_count); } test_end(); + + mail_index_transaction_cleanup(t); } static void test_mail_index_cancel_flag_updates(void) @@ -385,6 +407,8 @@ test_assert(updates[1].uid1 == 7 && updates[1].uid2 == 7); test_end(); + + mail_index_transaction_cleanup(t); } static void test_mail_index_flag_update_appends(void) @@ -430,6 +454,8 @@ test_assert(updates[0].uid2 == 4); test_assert(updates[0].add_flags == MAIL_ANSWERED); test_end(); + + mail_index_transaction_cleanup(t); } static bool test_flag_update_pos(struct mail_index_transaction *t, @@ -475,6 +501,8 @@ test_assert(test_flag_update_pos(t, 11, 4)); test_assert(test_flag_update_pos(t, 12, 4)); test_end(); + + mail_index_transaction_cleanup(t); } static void test_mail_index_modseq_update(void) @@ -511,6 +539,8 @@ ups[3].modseq_high32 == 0 && ups[3].modseq_low32 == 2); test_end(); + + mail_index_transaction_cleanup(t); } static void test_mail_index_expunge(void) @@ -553,6 +583,8 @@ test_assert(memcmp(expunges[4].guid_128, empty_guid, sizeof(empty_guid)) == 0); test_end(); + + mail_index_transaction_cleanup(t); } int main(void) diff -r c749a40eda0a -r f0701f84ca01 src/lib-index/test-mail-transaction-log-append.c --- a/src/lib-index/test-mail-transaction-log-append.c Wed Sep 10 14:08:58 2014 +0300 +++ b/src/lib-index/test-mail-transaction-log-append.c Wed Sep 10 14:08:58 2014 +0300 @@ -157,6 +157,10 @@ file->fd = -1; test_end(); + buffer_free(&log->head->buffer); + i_free(log->head); + i_free(log->index); + i_free(log); unlink(tmp_path); } diff -r c749a40eda0a -r f0701f84ca01 src/lib-index/test-mail-transaction-log-view.c --- a/src/lib-index/test-mail-transaction-log-view.c Wed Sep 10 14:08:58 2014 +0300 +++ b/src/lib-index/test-mail-transaction-log-view.c Wed Sep 10 14:08:58 2014 +0300 @@ -71,8 +71,10 @@ /* files must be sorted by file_seq */ for (p = &log->files; *p != NULL; p = &(*p)->next) { - if ((*p)->hdr.file_seq > file->hdr.file_seq) + if ((*p)->hdr.file_seq > file->hdr.file_seq) { + file->next = *p; break; + } } *p = file; log->head = file; @@ -119,6 +121,7 @@ const struct mail_index_record *rec; struct mail_index_record append_rec; const void *data; + void *oldfile; uint32_t seq; uoff_t offset, last_log_size; bool reset; @@ -186,7 +189,10 @@ mail_transaction_log_view_clear(view, 2); test_assert(!view_is_file_refed(1) && view_is_file_refed(2) && view_is_file_refed(3)); + oldfile = log->files; + buffer_free(&log->files->buffer); log->files = log->files->next; + i_free(oldfile); test_assert(log->files->hdr.file_seq == 2); test_end(); @@ -205,6 +211,16 @@ test_assert(mail_transaction_log_view_set(view, 0, 0, (uint32_t)-1, (uoff_t)-1, &reset) == -1); view->log = log; test_end(); + + mail_transaction_log_view_close(&view); + i_free(log->index); + while (log->files != NULL) { + oldfile = log->files; + buffer_free(&log->files->buffer); + log->files = log->files->next; + i_free(oldfile); + } + i_free(log); } int main(void) From dovecot at dovecot.org Wed Sep 10 11:12:06 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 10 Sep 2014 11:12:06 +0000 Subject: dovecot-2.2: lib-test: don't use 0 for old size in realloc call Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cf08c2614979 changeset: 17782:cf08c2614979 user: Phil Carmody date: Wed Sep 10 14:08:58 2014 +0300 description: lib-test: don't use 0 for old size in realloc call We have the real size, use it. Signed-off-by: Phil Carmody diffstat: src/lib-test/test-common.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r f0701f84ca01 -r cf08c2614979 src/lib-test/test-common.c --- a/src/lib-test/test-common.c Wed Sep 10 14:08:58 2014 +0300 +++ b/src/lib-test/test-common.c Wed Sep 10 14:08:58 2014 +0300 @@ -59,7 +59,8 @@ /* use exactly correct buffer size so valgrind can catch read overflows */ if (stream->buffer_size != cur_max && cur_max > 0) { - stream->w_buffer = i_realloc(stream->w_buffer, 0, + stream->w_buffer = i_realloc(stream->w_buffer, + stream->buffer_size, cur_max); stream->buffer = stream->w_buffer; stream->buffer_size = cur_max; From dovecot at dovecot.org Wed Sep 10 11:49:43 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 10 Sep 2014 11:49:43 +0000 Subject: dovecot-2.2: man: doveadm-dump man page updated Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6848068151bf changeset: 17783:6848068151bf user: Pascal Volk date: Wed Sep 10 14:37:50 2014 +0300 description: man: doveadm-dump man page updated diffstat: doc/man/doveadm-dump.1.in | 18 ++++++++++++++++-- 1 files changed, 16 insertions(+), 2 deletions(-) diffs (32 lines): diff -r cf08c2614979 -r 6848068151bf doc/man/doveadm-dump.1.in --- a/doc/man/doveadm-dump.1.in Wed Sep 10 14:08:58 2014 +0300 +++ b/doc/man/doveadm-dump.1.in Wed Sep 10 14:37:50 2014 +0300 @@ -1,5 +1,5 @@ -.\" Copyright (c) 2010-2012 Dovecot authors, see the included COPYING file -.TH DOVEADM\-DUMP 1 "2012-02-21" "Dovecot v2.2" "Dovecot" +.\" Copyright (c) 2010-2014 Dovecot authors, see the included COPYING file +.TH DOVEADM\-DUMP 1 "2014-08-24" "Dovecot v2.2" "Dovecot" .SH NAME doveadm\-dump \- Dump the content of Dovecot\(aqs binary mailbox index/log .\"------------------------------------------------------------------------ @@ -41,6 +41,20 @@ .I n (sdbox or mdbox mailbox file) .TP +.B fts\-expunge\-log +Dump the list of expunged mails in +.IR dovecot\-expunges.log . +Currently used only by fts\-lucene. +.TP +.B fts\-lucene +Dump the list of indexed mails in +.I lucene\-indexes +directory +.TP +.B imapzlib +Uncompress an IMAP traffic log, which contains data compressed using the +IMAP COMPRESSION extension. +.TP .B index \(rA dovecot.index, dovecot.map.index .TP From dovecot at dovecot.org Wed Sep 10 11:49:48 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 10 Sep 2014 11:49:48 +0000 Subject: dovecot-2.2: man: doveadm-auth man page updated Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/03653943b6ad changeset: 17784:03653943b6ad user: Pascal Volk date: Wed Sep 10 14:42:34 2014 +0300 description: man: doveadm-auth man page updated diffstat: doc/man/doveadm-auth.1.in | 134 ++++++++++++++++++++++++++++++++++++--------- 1 files changed, 105 insertions(+), 29 deletions(-) diffs (176 lines): diff -r 6848068151bf -r 03653943b6ad doc/man/doveadm-auth.1.in --- a/doc/man/doveadm-auth.1.in Wed Sep 10 14:37:50 2014 +0300 +++ b/doc/man/doveadm-auth.1.in Wed Sep 10 14:42:34 2014 +0300 @@ -1,47 +1,32 @@ -.\" Copyright (c) 2010 Dovecot authors, see the included COPYING file -.TH DOVEADM\-AUTH 1 "2010-06-09" "Dovecot v2.2" "Dovecot" +.\" Copyright (c) 2010-2014 Dovecot authors, see the included COPYING file +.TH DOVEADM\-AUTH 1 "2014-08-30" "Dovecot v2.2" "Dovecot" .SH NAME -doveadm\-auth \- Test authentication for a user +doveadm\-auth \- Flush/lookup/test authentication data .\"------------------------------------------------------------------------ .SH SYNOPSIS -.BR doveadm " [" \-Dv "] " auth -[\fB\-a\fP \fIauth_socket_path\fP] -[\fB\-x\fP \fIauth_info\fP] -.I user -[\fIpassword\fP] +.BR doveadm " [" \-Dv ] +[\fB\-f\fP \fIformatter\fP] +.BI auth \ command +.RI [ OPTIONS ]\ [ ARGUMENTS ] .\"------------------------------------------------------------------------ .SH DESCRIPTION The -.B auth -command can be used to test the authentication for the given -.IR user . +.B doveadm \ auth +.I COMMANDS +can be used to perform various authentication related actions. .\"------------------------------------------------------------------------ - at INCLUDE:global-options@ + at INCLUDE:global-options-formatter@ .\" --- command specific options --- "/. .PP Command specific .IR options : .\"------------------------------------- .TP -.BI \-a\ auth_socket_path -This option is used to specify an absolute path to an alternative UNIX -domain socket. -.sp -By default -.BR doveadm (1) -will use the socket -.IR @rundir@/auth\-client . -The socket may be located in another directory, when the default -.I base_dir -setting was overridden in -.IR @pkgsysconfdir@/dovecot.conf . -.\"------------------------------------- -.TP .BI \-x\ auth_info .I auth_info specifies additional conditions for the -.B auth -command. +.BR "auth lookup" " and " "auth test" +commands. The .I auth_info option string has to be given as @@ -93,13 +78,103 @@ .BR doveadm (1) will prompt for the password, if none was given. .\"------------------------------------------------------------------------ +.SH COMMANDS +.SS auth cache flush +.B doveadm auth cache flush +.RB [ \-a +.IR master_socket_path ] +.RI [ user " ...]" +.PP +Flush the authentication cache. +By default the cache is flushed for all the users (which can also be done +by sending SIGHUP to the auth process). +You can also flush the cache for one or more users by providing their +usernames. +.PP +.TP +.BI \-a \ master_socket_path +This option is used to specify an absolute path to an alternative UNIX +domain socket. +.sp +By default +.BR doveadm (1) +will use the socket +.IR @rundir@/auth\-master . +The socket may be located in another directory, when the default +.I base_dir +setting was overridden in +.IR @pkgsysconfdir@/dovecot.conf . +.\"------------------------------------- +.SS auth lookup +.B doveadm auth lookup +.RB [ \-a +.IR userdb_socket_path ] +.RB [ \-x +.IR auth_info ] +.RB [ \-f +.IR field ] \ user \ [...] +.PP +Similar to +.BR doveadm\-user (1) +command, except it performs a +.I passdb +lookup (without authentication) instead of a +.I userdb +lookup. +.PP +.TP +.BI \-a \ userdb_socket_path +This option is used to specify an absolute path to an alternative UNIX +domain socket. +.sp +By default +.BR doveadm (1) +will use the socket +.IR @rundir@/auth\-userdb . +The socket may be located in another directory, when the default +.I base_dir +setting was overridden in +.IR @pkgsysconfdir@/dovecot.conf . +.\"----------------- +.TP +.BI \-f \ field +When this option and the name of a userdb field is given, +.BR doveadm (1) +will show only the value of the specified field. +.\"------------------------------------- +.SS auth test +.B doveadm auth test +.RB [ \-a +.IR auth_socket_path ] +.RB [ \-x +.IR auth_info ] +.IR user \ [ password ] +.PP +Test authentication for the given user. +.\"------------------------------------- +.TP +.BI \-a\ auth_socket_path +This option is used to specify an absolute path to an alternative UNIX +domain socket. +.sp +By default +.BR doveadm (1) +will use the socket +.IR @rundir@/auth\-client . +The socket may be located in another directory, when the default +.I base_dir +setting was overridden in +.IR @pkgsysconfdir@/dovecot.conf . + +.\"------------------------------------------------------------------------ .SH EXAMPLE This example demonstrates an imap authentication test for user john, assuming the user is connected from the host with the IP address 192.0.2.143. .PP .nf -.B doveadm auth \-x service=imap \-x rip=192.0.2.143 john johns_password +.B doveadm auth test \-x service=imap \-x rip=192.0.2.143 john +Password: passdb: john auth succeeded extra fields: user=john @@ -109,4 +184,5 @@ .\"------------------------------------------------------------------------ .SH SEE ALSO .BR doveadm (1), +.BR doveadm\-user (1), .BR doveconf (1) \ No newline at end of file From dovecot at dovecot.org Wed Sep 10 11:50:13 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 10 Sep 2014 11:50:13 +0000 Subject: dovecot-2.2: man: doveadm-director man page updated Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ee3db95744f5 changeset: 17785:ee3db95744f5 user: Pascal Volk date: Wed Sep 10 14:48:58 2014 +0300 description: man: doveadm-director man page updated diffstat: doc/man/doveadm-director.1.in | 111 ++++++++++++++++++++++++++++------------- 1 files changed, 75 insertions(+), 36 deletions(-) diffs (175 lines): diff -r 03653943b6ad -r ee3db95744f5 doc/man/doveadm-director.1.in --- a/doc/man/doveadm-director.1.in Wed Sep 10 14:42:34 2014 +0300 +++ b/doc/man/doveadm-director.1.in Wed Sep 10 14:48:58 2014 +0300 @@ -1,44 +1,20 @@ -.\" Copyright (c) 2013 Dovecot authors, see the included COPYING file -.TH DOVEADM\-DIRECTOR 1 "2013-07-12" "Dovecot v2.2" "Dovecot" +.\" Copyright (c) 2013-2014 Dovecot authors, see the included COPYING file +.TH DOVEADM\-DIRECTOR 1 "2014-08-30" "Dovecot v2.2" "Dovecot" .SH NAME doveadm\-director \- Manage Dovecot directors .\"------------------------------------------------------------------------ .SH SYNOPSIS -.BR doveadm " [" \-Dv "] " "director add" -[\fB\-a\fP \fIdirector_socket_path\fP] -.IR host " [" vhost_count ] -.\"------------------------------------- -.br -.BR doveadm " [" \-Dv "] " "director flush" -[\fB\-a\fP \fIdirector_socket_path\fP] -\fIhost\fP|\fBall\fP -.\"------------------------------------- -.br -.BR doveadm " [" \-Dv "] " "director map" -[\fB\-a\fP \fIdirector_socket_path\fP] -[\fB\-f\fP \fIusers_file\fP] -.RI [ host ] -.\"------------------------------------- -.br -.BR doveadm " [" \-Dv "] " "director remove" -[\fB\-a\fP \fIdirector_socket_path\fP] -.I host -.\"------------------------------------- -.br -.BR doveadm " [" \-Dv "] " "director dump" -[\fB\-a\fP \fIdirector_socket_path\fP] -.\"------------------------------------- -.br -.BR doveadm " [" \-Dv "] " "director status" -[\fB\-a\fP \fIdirector_socket_path\fP] -.RI [ user ] +.BR doveadm " [" \-Dv ] +[\fB\-f\fP \fIformatter\fP] +.BI director \ command +.RI [ OPTIONS ]\ [ ARGUMENTS ] .\"------------------------------------------------------------------------ .SH DESCRIPTION .B doveadm director can be used to manage and query the status of the list of backend mail servers where Dovecot proxy can redirect connections to. .\"------------------------------------------------------------------------ - at INCLUDE:global-options@ + at INCLUDE:global-options-formatter@ .\" --- command specific options --- "/. .PP Command specific @@ -67,6 +43,15 @@ A mail server\(aqs hostname or IP address. .\"------------------------------------- .TP +.I ip +A director\(aqs IP address. +.\"------------------------------------- +.TP +.I port +The TCP port, on which the director server is listening for connections. +The default port is the same as what the local director is listening in. +.\"------------------------------------- +.TP .I user Is a .IR user \(aqs @@ -98,6 +83,14 @@ of an already assigned server. .PP .\"------------------------------------- +.SS director dump +.B doveadm director dump +[\fB\-a\fP \fIdirector_socket_path\fP] +.PP +Dump the current host configuration as doveadm commands. These commands can +be easily run after a full director cluster restart to get back to the +dumped state. +.\"------------------------------------- .SS director flush .B doveadm director flush [\fB\-a\fP \fIdirector_socket_path\fP] @@ -111,10 +104,22 @@ hosts. This command is intended mainly for testing purposes. .\"------------------------------------- +.SS director kick +.B doveadm director kick +[\fB\-a\fP \fIdirector_socket_path\fP] +.I user +.PP +Kick the specified +.I user +from the entire Dovecot cluster. +This is similar to doveadm proxy kick, but this command needs to be run +only once instead of in each director server. +.\"------------------------------------- .SS director map .B doveadm director map [\fB\-a\fP \fIdirector_socket_path\fP] [\fB\-f\fP \fIusers_file\fP] +[\fB\-h\fP|\fB\-u\fP] .RI [ host ] .PP The command @@ -133,12 +138,34 @@ lookup will be performed. This may be a helpful alternative when for example the network connection to the LDAP or SQL server is slow. +.\"----------------- +.TP +.B \-h +XXX Output all usernames, which match the given hash. +.\"----------------- +.TP +.B \-u +XXX Output hash for the given username. +.\"----------------- .TP .I host Specify a server\(aqs IP address or hostname, to list only mappings of the given .IR host . .\"------------------------------------- +.SS director move +.B doveadm director move +[\fB\-a\fP \fIdirector_socket_path\fP] +.I user host +.PP +Move the +.I user +to the specified backend +.IR host . +If the +.I user +has any existing connections they will be killed. +.\"------------------------------------- .SS director remove .B doveadm director remove [\fB\-a\fP \fIdirector_socket_path\fP] @@ -148,13 +175,25 @@ .I host from the director. .\"------------------------------------- -.SS director dump -.B doveadm director dump +.SS director ring add +.B doveadm director ring add +[\fB\-a\fP \fIdirector_socket_path\fP] +.IR ip \ [ port ] +.PP +Add a new director to the ring. +.\"------------------------------------- +.SS director ring remove +.B doveadm director ring remove +[\fB\-a\fP \fIdirector_socket_path\fP] +.IR ip \ [ port ] +.PP +Remove a director from the ring. +.\"------------------------------------- +.SS director ring status +.B doveadm director ring status [\fB\-a\fP \fIdirector_socket_path\fP] .PP -Dump the current host configuration as doveadm commands. These commands can -be easily run after a full director cluster restart to get back to the -dumped state. +Show the status of all the directors currently in the ring. .\"------------------------------------- .SS director status .B doveadm director status From dovecot at dovecot.org Wed Sep 10 12:55:35 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 10 Sep 2014 12:55:35 +0000 Subject: dovecot-2.2: replication plugin: Added replication_disabled sett... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/95c2f5916958 changeset: 17786:95c2f5916958 user: Timo Sirainen date: Wed Sep 10 15:54:58 2014 +0300 description: replication plugin: Added replication_disabled setting. diffstat: src/plugins/replication/replication-plugin.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (13 lines): diff -r ee3db95744f5 -r 95c2f5916958 src/plugins/replication/replication-plugin.c --- a/src/plugins/replication/replication-plugin.c Wed Sep 10 14:48:58 2014 +0300 +++ b/src/plugins/replication/replication-plugin.c Wed Sep 10 15:54:58 2014 +0300 @@ -307,6 +307,9 @@ struct replication_user *ruser; const char *value; + if (mail_user_plugin_getenv(user, "replication_disabled") != NULL) + return; + ruser = p_new(user->pool, struct replication_user, 1); ruser->module_ctx.super = *v; user->vlast = &ruser->module_ctx.super; From dovecot at dovecot.org Wed Sep 10 13:06:12 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 10 Sep 2014 13:06:12 +0000 Subject: dovecot-2.2: replication plugin: Actually, use empty mail_replic... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c1c67bdc8752 changeset: 17787:c1c67bdc8752 user: Timo Sirainen date: Wed Sep 10 16:05:32 2014 +0300 description: replication plugin: Actually, use empty mail_replica as "replication is disabled". diffstat: src/plugins/replication/replication-plugin.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 95c2f5916958 -r c1c67bdc8752 src/plugins/replication/replication-plugin.c --- a/src/plugins/replication/replication-plugin.c Wed Sep 10 15:54:58 2014 +0300 +++ b/src/plugins/replication/replication-plugin.c Wed Sep 10 16:05:32 2014 +0300 @@ -307,7 +307,8 @@ struct replication_user *ruser; const char *value; - if (mail_user_plugin_getenv(user, "replication_disabled") != NULL) + value = mail_user_plugin_getenv(user, "mail_replica"); + if (value == NULL || value[0] == '\0') return; ruser = p_new(user->pool, struct replication_user, 1); From dovecot at dovecot.org Wed Sep 10 15:42:27 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 10 Sep 2014 15:42:27 +0000 Subject: dovecot-2.2: man: Forgot to remove XXX prefixes from doveadm-dir... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4016a491e958 changeset: 17788:4016a491e958 user: Timo Sirainen date: Wed Sep 10 18:41:39 2014 +0300 description: man: Forgot to remove XXX prefixes from doveadm-director.1.in diffstat: doc/man/doveadm-director.1.in | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (17 lines): diff -r c1c67bdc8752 -r 4016a491e958 doc/man/doveadm-director.1.in --- a/doc/man/doveadm-director.1.in Wed Sep 10 16:05:32 2014 +0300 +++ b/doc/man/doveadm-director.1.in Wed Sep 10 18:41:39 2014 +0300 @@ -141,11 +141,11 @@ .\"----------------- .TP .B \-h -XXX Output all usernames, which match the given hash. +Output all usernames, which match the given hash. .\"----------------- .TP .B \-u -XXX Output hash for the given username. +Output hash for the given username. .\"----------------- .TP .I host From dovecot at dovecot.org Thu Sep 11 13:12:06 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 11 Sep 2014 13:12:06 +0000 Subject: dovecot-2.2: fts-lucene: Log a warning when rebuilding index bec... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f3df2eb2ce8e changeset: 17789:f3df2eb2ce8e user: Timo Sirainen date: Thu Sep 11 16:11:37 2014 +0300 description: fts-lucene: Log a warning when rebuilding index because of settings changes diffstat: src/plugins/fts-lucene/fts-backend-lucene.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 4016a491e958 -r f3df2eb2ce8e src/plugins/fts-lucene/fts-backend-lucene.c --- a/src/plugins/fts-lucene/fts-backend-lucene.c Wed Sep 10 18:41:39 2014 +0300 +++ b/src/plugins/fts-lucene/fts-backend-lucene.c Thu Sep 11 16:11:37 2014 +0300 @@ -197,6 +197,7 @@ if (!fts_index_have_compatible_settings(_backend->ns->list, set_checksum)) { /* need to rebuild the index */ + i_warning("fts-lucene: Settings have changed, rebuilding index"); *last_uid_r = 0; } else { *last_uid_r = hdr.last_indexed_uid; From dovecot at dovecot.org Thu Sep 11 13:13:08 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 11 Sep 2014 13:13:08 +0000 Subject: dovecot-2.2: fts-lucene: Include the mailbox name also in the re... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/73eb8be089a5 changeset: 17790:73eb8be089a5 user: Timo Sirainen date: Thu Sep 11 16:12:38 2014 +0300 description: fts-lucene: Include the mailbox name also in the rebuilding warning. diffstat: src/plugins/fts-lucene/fts-backend-lucene.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r f3df2eb2ce8e -r 73eb8be089a5 src/plugins/fts-lucene/fts-backend-lucene.c --- a/src/plugins/fts-lucene/fts-backend-lucene.c Thu Sep 11 16:11:37 2014 +0300 +++ b/src/plugins/fts-lucene/fts-backend-lucene.c Thu Sep 11 16:12:38 2014 +0300 @@ -197,7 +197,8 @@ if (!fts_index_have_compatible_settings(_backend->ns->list, set_checksum)) { /* need to rebuild the index */ - i_warning("fts-lucene: Settings have changed, rebuilding index"); + i_warning("fts-lucene: Settings have changed, rebuilding index for mailbox %s", + mailbox_get_vname(box)); *last_uid_r = 0; } else { *last_uid_r = hdr.last_indexed_uid; From dovecot at dovecot.org Thu Sep 11 13:15:09 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 11 Sep 2014 13:15:09 +0000 Subject: dovecot-2.2: fts-lucene: No .. the rebuilding warning was still ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ef55b00eef2a changeset: 17791:ef55b00eef2a user: Timo Sirainen date: Thu Sep 11 16:14:24 2014 +0300 description: fts-lucene: No .. the rebuilding warning was still in the wrong place. diffstat: src/plugins/fts-lucene/fts-backend-lucene.c | 2 -- src/plugins/fts-lucene/lucene-wrapper.cc | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diffs (24 lines): diff -r 73eb8be089a5 -r ef55b00eef2a src/plugins/fts-lucene/fts-backend-lucene.c --- a/src/plugins/fts-lucene/fts-backend-lucene.c Thu Sep 11 16:12:38 2014 +0300 +++ b/src/plugins/fts-lucene/fts-backend-lucene.c Thu Sep 11 16:14:24 2014 +0300 @@ -197,8 +197,6 @@ if (!fts_index_have_compatible_settings(_backend->ns->list, set_checksum)) { /* need to rebuild the index */ - i_warning("fts-lucene: Settings have changed, rebuilding index for mailbox %s", - mailbox_get_vname(box)); *last_uid_r = 0; } else { *last_uid_r = hdr.last_indexed_uid; diff -r 73eb8be089a5 -r ef55b00eef2a src/plugins/fts-lucene/lucene-wrapper.cc --- a/src/plugins/fts-lucene/lucene-wrapper.cc Thu Sep 11 16:12:38 2014 +0300 +++ b/src/plugins/fts-lucene/lucene-wrapper.cc Thu Sep 11 16:14:24 2014 +0300 @@ -416,6 +416,8 @@ if (ret != 0) return ret; + i_warning("fts-lucene: Settings have changed, rebuilding index for mailbox"); + /* settings changed, rebuild index */ if (unlink_directory(index->path, (enum unlink_directory_flags)0) < 0) { i_error("unlink_directory(%s) failed: %m", index->path); From dovecot at dovecot.org Thu Sep 11 13:40:35 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 11 Sep 2014 13:40:35 +0000 Subject: dovecot-2.2: lib-mail: Message decoder now runs normalizer also ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bbf760c25a9e changeset: 17792:bbf760c25a9e user: Timo Sirainen date: Thu Sep 11 16:40:00 2014 +0300 description: lib-mail: Message decoder now runs normalizer also for unknown charsets. diffstat: src/lib-mail/message-decoder.c | 14 +++----------- 1 files changed, 3 insertions(+), 11 deletions(-) diffs (31 lines): diff -r ef55b00eef2a -r bbf760c25a9e src/lib-mail/message-decoder.c --- a/src/lib-mail/message-decoder.c Thu Sep 11 16:14:24 2014 +0300 +++ b/src/lib-mail/message-decoder.c Thu Sep 11 16:40:00 2014 +0300 @@ -334,7 +334,9 @@ if (ctx->binary_input) { output->data = data; output->size = size; - } else if (ctx->charset_utf8) { + } else if (ctx->charset_utf8 || ctx->charset_trans == NULL) { + /* handle unknown charsets the same as UTF-8. it might find + usable ASCII text. */ buffer_set_used_size(ctx->buf2, 0); if (ctx->normalizer != NULL) { (void)ctx->normalizer(data, size, ctx->buf2); @@ -347,16 +349,6 @@ output->data = ctx->buf2->data; output->size = ctx->buf2->used; } - } else if (ctx->charset_trans == NULL) { - /* unknown charset */ - buffer_set_used_size(ctx->buf2, 0); - if (uni_utf8_get_valid_data(data, size, ctx->buf2)) { - output->data = data; - output->size = size; - } else { - output->data = ctx->buf2->data; - output->size = ctx->buf2->used; - } } else { buffer_set_used_size(ctx->buf2, 0); if (ctx->translation_size != 0) From dovecot at dovecot.org Thu Sep 11 15:15:41 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 11 Sep 2014 15:15:41 +0000 Subject: dovecot-2.2: virtual: Recent optimizations had broken fast mailb... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2171b0e47055 changeset: 17793:2171b0e47055 user: Timo Sirainen date: Thu Sep 11 18:15:04 2014 +0300 description: virtual: Recent optimizations had broken fast mailbox syncing. We wrongly assumed that all_mails array could have been accessed using vseqs. Broken by 8abf7eea2966 diffstat: src/plugins/virtual/virtual-sync.c | 46 ++++++++++++++++++++----------------- 1 files changed, 25 insertions(+), 21 deletions(-) diffs (117 lines): diff -r bbf760c25a9e -r 2171b0e47055 src/plugins/virtual/virtual-sync.c --- a/src/plugins/virtual/virtual-sync.c Thu Sep 11 16:40:00 2014 +0300 +++ b/src/plugins/virtual/virtual-sync.c Thu Sep 11 18:15:04 2014 +0300 @@ -37,8 +37,9 @@ ARRAY(struct virtual_add_record) all_adds; - /* all messages in this sync */ - ARRAY(const struct virtual_mail_index_record) all_mails; + /* all messages in this sync, sorted by mailbox_id + (but unsorted inside it for now, since it doesn't matter) */ + ARRAY(struct virtual_sync_mail) all_mails; uint32_t all_mails_idx, all_mails_prev_mailbox_id; enum mailbox_sync_flags flags; @@ -85,7 +86,7 @@ mail_index_keywords_unref(&keywords); } -static int virtual_sync_mail_cmp(const void *p1, const void *p2) +static int virtual_sync_mail_uid_cmp(const void *p1, const void *p2) { const struct virtual_sync_mail *m1 = p1, *m2 = p2; @@ -664,9 +665,10 @@ struct virtual_backend_box *bbox, struct mail_search_result *result) { - const struct virtual_mail_index_record *vrec, *vrecs; + const struct virtual_mail_index_record *vrec; + const struct virtual_sync_mail *sync_mail, *sync_mails; const void *data; - uint32_t vseq, messages; + uint32_t i, vseq, messages; /* find the messages that currently exist in virtual index and add them to the backend mailbox's list of uids. */ @@ -674,11 +676,11 @@ if (array_is_created(&ctx->all_mails)) { i_assert(ctx->all_mails_prev_mailbox_id < bbox->mailbox_id); - vrecs = array_get(&ctx->all_mails, &messages); - for (vseq = ctx->all_mails_idx + 1; vseq <= messages; vseq++) { - vrec = &vrecs[vseq - 1]; - if (vrec->mailbox_id != bbox->mailbox_id) { - if (vrec->mailbox_id < bbox->mailbox_id) { + sync_mails = array_get(&ctx->all_mails, &messages); + for (i = ctx->all_mails_idx; i < messages; i++) { + sync_mail = &sync_mails[i]; + if (sync_mail->vrec.mailbox_id != bbox->mailbox_id) { + if (sync_mail->vrec.mailbox_id < bbox->mailbox_id) { /* stale mailbox_id, ignore */ continue; } @@ -687,10 +689,10 @@ break; } - virtual_sync_backend_add_vmsgs_results(ctx, - bbox, vrec->real_uid, result, vseq); + virtual_sync_backend_add_vmsgs_results(ctx, bbox, + sync_mail->vrec.real_uid, result, sync_mail->vseq); } - ctx->all_mails_idx = vseq - 1; + ctx->all_mails_idx = i; ctx->all_mails_prev_mailbox_id = bbox->mailbox_id; } else { /* there should be only a single backend mailbox, but in the @@ -1198,7 +1200,7 @@ vmails[vseq-1].vseq = vseq; vmails[vseq-1].vrec = *vrec; } - qsort(vmails, messages, sizeof(*vmails), virtual_sync_mail_cmp); + qsort(vmails, messages, sizeof(*vmails), virtual_sync_mail_uid_cmp); /* create real mailbox uid -> virtual uid mapping and expunge messages no longer matching the search rule */ @@ -1501,13 +1503,12 @@ } } -static int -virtual_sync_mails_mailbox_cmp(const struct virtual_mail_index_record *v1, - const struct virtual_mail_index_record *v2) +static int virtual_sync_mail_mailbox_cmp(const struct virtual_sync_mail *m1, + const struct virtual_sync_mail *m2) { - if (v1->mailbox_id < v2->mailbox_id) + if (m1->vrec.mailbox_id < m2->vrec.mailbox_id) return -1; - if (v1->mailbox_id > v2->mailbox_id) + if (m1->vrec.mailbox_id > m2->vrec.mailbox_id) return 1; return 0; } @@ -1517,6 +1518,7 @@ uint32_t messages, vseq; const void *mail_data; const struct virtual_mail_index_record *vrec; + struct virtual_sync_mail *sync_mail; messages = mail_index_view_get_messages_count(ctx->sync_view); i_array_init(&ctx->all_mails, messages); @@ -1524,9 +1526,11 @@ mail_index_lookup_ext(ctx->sync_view, vseq, ctx->mbox->virtual_ext_id, &mail_data, NULL); vrec = mail_data; - array_append(&ctx->all_mails, vrec, 1); + sync_mail = array_append_space(&ctx->all_mails); + sync_mail->vseq = vseq; + sync_mail->vrec = *vrec; } - array_sort(&ctx->all_mails, virtual_sync_mails_mailbox_cmp); + array_sort(&ctx->all_mails, virtual_sync_mail_mailbox_cmp); } static int virtual_sync_backend_boxes(struct virtual_sync_context *ctx) From dovecot at dovecot.org Fri Sep 12 09:55:51 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 12 Sep 2014 09:55:51 +0000 Subject: dovecot-2.2: lib-master: Earlier config file path fix broke reus... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ccc5701dae72 changeset: 17794:ccc5701dae72 user: Timo Sirainen date: Fri Sep 12 12:55:12 2014 +0300 description: lib-master: Earlier config file path fix broke reusing existing config socket lookups. diffstat: src/lib-master/master-service-private.h | 3 ++- src/lib-master/master-service-settings.c | 6 ++++-- src/lib-master/master-service.c | 10 +++++----- 3 files changed, 11 insertions(+), 8 deletions(-) diffs (70 lines): diff -r 2171b0e47055 -r ccc5701dae72 src/lib-master/master-service-private.h --- a/src/lib-master/master-service-private.h Thu Sep 11 18:15:04 2014 +0300 +++ b/src/lib-master/master-service-private.h Fri Sep 12 12:55:12 2014 +0300 @@ -70,9 +70,10 @@ unsigned int initial_status_sent:1; unsigned int die_with_master:1; unsigned int call_avail_overflow:1; - unsigned int config_path_is_default:1; + unsigned int config_path_changed_with_param:1; unsigned int want_ssl_settings:1; unsigned int ssl_ctx_initialized:1; + unsigned int config_path_from_master:1; }; void master_service_io_listeners_add(struct master_service *service); diff -r 2171b0e47055 -r ccc5701dae72 src/lib-master/master-service-settings.c --- a/src/lib-master/master-service-settings.c Thu Sep 11 18:15:04 2014 +0300 +++ b/src/lib-master/master-service-settings.c Fri Sep 12 12:55:12 2014 +0300 @@ -178,14 +178,16 @@ master_service_get_config_path(service); if (service->config_fd != -1 && input->config_path == NULL && - service->config_path_is_default) { + !service->config_path_changed_with_param) { /* use the already opened config socket */ fd = service->config_fd; service->config_fd = -1; return fd; } - if (service->config_path_is_default && input->config_path == NULL) { + if (!service->config_path_from_master && + !service->config_path_changed_with_param && + input->config_path == NULL) { /* first try to connect to the default config socket. configuration may contain secrets, so in default config this fails because the socket is 0600. it's useful for diff -r 2171b0e47055 -r ccc5701dae72 src/lib-master/master-service.c --- a/src/lib-master/master-service.c Thu Sep 11 18:15:04 2014 +0300 +++ b/src/lib-master/master-service.c Fri Sep 12 12:55:12 2014 +0300 @@ -160,10 +160,10 @@ service->config_fd = -1; service->config_path = i_strdup(getenv(MASTER_CONFIG_FILE_ENV)); - if (service->config_path == NULL) { + if (service->config_path == NULL) service->config_path = i_strdup(DEFAULT_CONFIG_FILE_PATH); - service->config_path_is_default = TRUE; - } + else + service->config_path_from_master = TRUE; if ((flags & MASTER_SERVICE_FLAG_STANDALONE) == 0) { service->version_string = getenv(MASTER_DOVECOT_VERSION_ENV); @@ -370,13 +370,13 @@ switch (opt) { case 'c': service->config_path = i_strdup(arg); - service->config_path_is_default = FALSE; + service->config_path_changed_with_param = TRUE; break; case 'i': if (!get_instance_config(arg, &path)) i_fatal("Unknown instance name: %s", arg); service->config_path = i_strdup(path); - service->config_path_is_default = FALSE; + service->config_path_changed_with_param = TRUE; break; case 'k': service->keep_environment = TRUE; From pigeonhole at rename-it.nl Fri Sep 12 10:17:49 2014 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Fri, 12 Sep 2014 12:17:49 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: duplicate extension: Fixed er... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/ce0657ede24e changeset: 1916:ce0657ede24e user: Stephan Bosch date: Fri Sep 12 12:17:38 2014 +0200 description: lib-sieve: duplicate extension: Fixed erroneous compile error about conflicting tags when ":handle" argument was used last. diffstat: src/lib-sieve/plugins/duplicate/tst-duplicate.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r c191996854b3 -r ce0657ede24e src/lib-sieve/plugins/duplicate/tst-duplicate.c --- a/src/lib-sieve/plugins/duplicate/tst-duplicate.c Fri Sep 05 14:38:35 2014 +0200 +++ b/src/lib-sieve/plugins/duplicate/tst-duplicate.c Fri Sep 12 12:17:38 2014 +0200 @@ -179,7 +179,7 @@ return FALSE; } - if ( (bool)cmd->data ) { + if ( !sieve_argument_is(tag, duplicate_handle_tag) && (bool)cmd->data ) { sieve_argument_validate_error(valdtr, *arg, "conflicting :header and %s arguments specified " "for the duplicate test", From dovecot at dovecot.org Sat Sep 13 19:16:23 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 13 Sep 2014 19:16:23 +0000 Subject: dovecot-2.2: fs-posix: Fixed memory leak in fs_iter_deinit() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/73f806838456 changeset: 17795:73f806838456 user: Timo Sirainen date: Sat Sep 13 22:14:09 2014 +0300 description: fs-posix: Fixed memory leak in fs_iter_deinit() diffstat: src/lib-fs/fs-posix.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (14 lines): diff -r ccc5701dae72 -r 73f806838456 src/lib-fs/fs-posix.c --- a/src/lib-fs/fs-posix.c Fri Sep 12 12:55:12 2014 +0300 +++ b/src/lib-fs/fs-posix.c Sat Sep 13 22:14:09 2014 +0300 @@ -734,6 +734,10 @@ struct posix_fs_iter *iter = (struct posix_fs_iter *)_iter; int ret = 0; + if (closedir(iter->dir) < 0 && iter->err == 0) { + iter->err = errno; + fs_set_error(_iter->fs, "closedir(%s) failed: %m", iter->path); + } if (iter->err != 0) { errno = iter->err; ret = -1; From dovecot at dovecot.org Sat Sep 13 19:17:08 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 13 Sep 2014 19:17:08 +0000 Subject: dovecot-2.2: fs-metawrap: Add FS_PROPERTY_COPY_METADATA property... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1daf1e2fefb9 changeset: 17796:1daf1e2fefb9 user: Timo Sirainen date: Sat Sep 13 22:15:35 2014 +0300 description: fs-metawrap: Add FS_PROPERTY_COPY_METADATA property for wrapped fs backends. This is because if we simply use the parent copy the metadata gets copied as well. diffstat: src/lib-fs/fs-metawrap.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 73f806838456 -r 1daf1e2fefb9 src/lib-fs/fs-metawrap.c --- a/src/lib-fs/fs-metawrap.c Sat Sep 13 22:14:09 2014 +0300 +++ b/src/lib-fs/fs-metawrap.c Sat Sep 13 22:15:35 2014 +0300 @@ -85,6 +85,8 @@ /* we don't have a quick stat() to see the file's size, because of the metadata header */ props &= ~FS_PROPERTY_STAT; + /* Copying can copy the whole metadata. */ + props |= FS_PROPERTY_COPY_METADATA; } return props; } From dovecot at dovecot.org Sat Sep 13 20:42:56 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 13 Sep 2014 20:42:56 +0000 Subject: dovecot-2.2: lib-http server: Keep output stream referenced to a... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/736a3f6bd5a8 changeset: 17797:736a3f6bd5a8 user: Timo Sirainen date: Sat Sep 13 23:42:17 2014 +0300 description: lib-http server: Keep output stream referenced to avoid accessing it after destroy. diffstat: src/lib-http/http-server-response.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (19 lines): diff -r 1daf1e2fefb9 -r 736a3f6bd5a8 src/lib-http/http-server-response.c --- a/src/lib-http/http-server-response.c Sat Sep 13 22:15:35 2014 +0300 +++ b/src/lib-http/http-server-response.c Sat Sep 13 23:42:17 2014 +0300 @@ -379,6 +379,7 @@ iov[2].iov_len = 2; req->state = HTTP_SERVER_REQUEST_STATE_PAYLOAD_OUT; + o_stream_ref(output); o_stream_cork(output); if (o_stream_sendv(output, iov, N_ELEMENTS(iov)) < 0) { if (errno != EPIPE && errno != ECONNRESET) { @@ -398,6 +399,7 @@ http_server_request_finished(resp->request); } o_stream_uncork(output); + o_stream_unref(&output); return ret; } From dovecot at dovecot.org Sun Sep 14 08:29:43 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 14 Sep 2014 08:29:43 +0000 Subject: dovecot-2.2: fs-posix: Fixed fs_iter_deinit() when opendir() had... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7d609a1c0a3e changeset: 17798:7d609a1c0a3e user: Timo Sirainen date: Sun Sep 14 11:29:35 2014 +0300 description: fs-posix: Fixed fs_iter_deinit() when opendir() had failed. diffstat: src/lib-fs/fs-posix.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 736a3f6bd5a8 -r 7d609a1c0a3e src/lib-fs/fs-posix.c --- a/src/lib-fs/fs-posix.c Sat Sep 13 23:42:17 2014 +0300 +++ b/src/lib-fs/fs-posix.c Sun Sep 14 11:29:35 2014 +0300 @@ -734,7 +734,7 @@ struct posix_fs_iter *iter = (struct posix_fs_iter *)_iter; int ret = 0; - if (closedir(iter->dir) < 0 && iter->err == 0) { + if (iter->dir != NULL && closedir(iter->dir) < 0 && iter->err == 0) { iter->err = errno; fs_set_error(_iter->fs, "closedir(%s) failed: %m", iter->path); } From dovecot at dovecot.org Mon Sep 15 08:19:58 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 15 Sep 2014 08:19:58 +0000 Subject: dovecot-2.2: lib-http: server: Fixed connection reference counting. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/25fd54c05522 changeset: 17799:25fd54c05522 user: Stephan Bosch date: Mon Sep 15 11:19:50 2014 +0300 description: lib-http: server: Fixed connection reference counting. Connection often still got destroyed too early. Particularly submitting responses would potentially destroy the connection, which is often unexpected. Sending responses is now postponed until handled by the stream output handler, which is explicitly triggered when necessary. diffstat: src/lib-http/http-server-connection.c | 89 ++++++++++++++++++++++------------ src/lib-http/http-server-private.h | 4 +- src/lib-http/http-server-request.c | 6 +- 3 files changed, 63 insertions(+), 36 deletions(-) diffs (285 lines): diff -r 7d609a1c0a3e -r 25fd54c05522 src/lib-http/http-server-connection.c --- a/src/lib-http/http-server-connection.c Sun Sep 14 11:29:35 2014 +0300 +++ b/src/lib-http/http-server-connection.c Mon Sep 15 11:19:50 2014 +0300 @@ -110,7 +110,8 @@ static void http_server_connection_input_resume(struct http_server_connection *conn) { - if (conn->conn.io == NULL && !conn->input_broken && !conn->close_indicated) { + if (conn->conn.io == NULL && !conn->closed && + !conn->input_broken && !conn->close_indicated) { conn->conn.io = io_add(conn->conn.fd_in, IO_READ, http_server_connection_input, &conn->conn); } @@ -300,10 +301,8 @@ http_server_connection_input_halt(conn); } - http_server_connection_ref(conn); http_server_connection_request_callback(conn, req); - http_server_connection_unref(&conn); - if (conn == NULL || conn->closed) { + if (conn->closed) { /* the callback managed to get this connection destroyed/closed */ return FALSE; } @@ -312,7 +311,7 @@ /* send 100 Continue when appropriate */ if (req->req.expect_100_continue && !req->payload_halted && req->response == NULL) { - http_server_connection_send_responses(conn); + http_server_connection_trigger_responses(conn); } /* delegate payload handling to request handler */ @@ -387,6 +386,7 @@ if (conn->ssl && conn->ssl_iostream == NULL) { if (http_server_connection_ssl_init(conn) < 0) { + /* ssl failed */ http_server_connection_close(&conn, "SSL Initialization failed"); return; } @@ -404,14 +404,18 @@ /* create request object if none was created already */ if (conn->request_queue_tail != NULL && conn->request_queue_tail->state == HTTP_SERVER_REQUEST_STATE_NEW) { - if (conn->request_queue_count > conn->server->set.max_pipelined_requests) { + if (conn->request_queue_count > + conn->server->set.max_pipelined_requests) { + /* pipeline full */ http_server_connection_input_halt(conn); return; } /* continue last unfinished request*/ req = conn->request_queue_tail; } else { - if (conn->request_queue_count >= conn->server->set.max_pipelined_requests) { + if (conn->request_queue_count >= + conn->server->set.max_pipelined_requests) { + /* pipeline full */ http_server_connection_input_halt(conn); return; } @@ -433,8 +437,6 @@ if (pending_request != NULL) { /* previous request is now fully read and ready to respond */ http_server_request_ready_to_respond(pending_request); - if (conn->closed) - break; } http_server_connection_debug(conn, @@ -466,13 +468,20 @@ else http_server_request_unref(&req); - /* client indicated it will close after this request; stop trying - to read more. */ - if (conn->close_indicated) + if (conn->closed) { + /* connection got closed in destroy callback */ break; + } + + if (conn->close_indicated) { + /* client indicated it will close after this request; stop trying + to read more. */ + break; + } if (conn->request_queue_count >= conn->server->set.max_pipelined_requests) { + /* pipeline full */ http_server_connection_input_halt(conn); http_server_connection_unref(&conn); return; @@ -483,12 +492,16 @@ } http_server_connection_unref(&conn); - if (conn == NULL || conn->closed) + if (conn == NULL || conn->closed) { + /* connection got closed */ return; + } if (ret <= 0 && (conn->conn.input->eof || conn->conn.input->stream_errno != 0)) { int stream_errno = conn->conn.input->stream_errno; + + /* connection input broken; output may still be intact */ if (stream_errno != 0 && stream_errno != EPIPE && stream_errno != ECONNRESET) { http_server_connection_client_error(conn, @@ -508,7 +521,8 @@ http_server_connection_close(&conn, "Remote closed connection unexpectedly"); } else { - /* a request is still processing; only drop input io for now */ + /* a request is still processing; only drop input io for now. + the other end may only have shutdown one direction */ conn->input_broken = TRUE; http_server_connection_input_halt(conn); } @@ -517,6 +531,8 @@ } if (ret < 0) { + http_server_connection_ref(conn); + http_server_connection_client_error(conn, "Client sent invalid request: %s", error); @@ -545,6 +561,12 @@ default: i_unreached(); } + + http_server_connection_unref(&conn); + if (conn == NULL || conn->closed) { + /* connection got closed */ + return; + } } if (conn->input_broken || conn->close_indicated) { @@ -555,11 +577,7 @@ if (ret == 0 && pending_request != NULL && !http_request_parser_pending_payload(conn->http_parser)) { /* previous request is now fully read and ready to respond */ - http_server_connection_ref(conn); http_server_request_ready_to_respond(pending_request); - http_server_connection_unref(&conn); - if (conn == NULL || conn->closed) - return; } } } @@ -638,27 +656,26 @@ return TRUE; } -void http_server_connection_send_responses(struct http_server_connection *conn) +static int http_server_connection_send_responses( + struct http_server_connection *conn) { - /* prevent recursion (we're also called from http_server_request_finish) */ - if (conn->sending_responses) - return; - http_server_connection_ref(conn); - conn->sending_responses = TRUE; - + /* send more responses until no more responses remain, the output blocks again, or the connection is closed */ while (!conn->closed && http_server_connection_next_response(conn)); - - conn->sending_responses = FALSE; + http_server_connection_unref(&conn); + if (conn == NULL || conn->closed) + return -1; /* accept more requests if possible */ - if (conn != NULL && !conn->closed && conn->incoming_payload == NULL && + if (conn->incoming_payload == NULL && conn->request_queue_count < conn->server->set.max_pipelined_requests) { http_server_connection_input_resume(conn); + return 1; } + return 0; } int http_server_connection_output(struct http_server_connection *conn) @@ -681,12 +698,15 @@ "Remote closed connection unexpectedly"); } } - return ret; + return -1; } http_server_connection_timeout_reset(conn); - if (conn->request_queue_head != NULL && conn->output_locked) { + if (!conn->output_locked) { + if (http_server_connection_send_responses(conn) < 0) + return -1; + } else if (conn->request_queue_head != NULL) { struct http_server_request *req = conn->request_queue_head; struct http_server_response *resp = req->response; @@ -707,7 +727,8 @@ if (!conn->output_locked) { /* room for more responses */ - http_server_connection_send_responses(conn); + if (http_server_connection_send_responses(conn) < 0) + return -1; } else if (conn->io_resp_payload != NULL) { /* server is causing idle time */ http_server_connection_timeout_stop(conn); @@ -719,6 +740,12 @@ return 1; } +void http_server_connection_trigger_responses( + struct http_server_connection *conn) +{ + o_stream_set_flush_pending(conn->conn.output, TRUE); +} + bool http_server_connection_pending_payload(struct http_server_connection *conn) { diff -r 7d609a1c0a3e -r 25fd54c05522 src/lib-http/http-server-private.h --- a/src/lib-http/http-server-private.h Sun Sep 14 11:29:35 2014 +0300 +++ b/src/lib-http/http-server-private.h Mon Sep 15 11:19:50 2014 +0300 @@ -118,7 +118,6 @@ unsigned int close_indicated:1; unsigned int input_broken:1; unsigned int output_locked:1; - unsigned int sending_responses:1; }; struct http_server { @@ -197,7 +196,8 @@ struct connection_list *http_server_connection_list_init(void); void http_server_connection_switch_ioloop(struct http_server_connection *conn); -void http_server_connection_send_responses(struct http_server_connection *conn); +void http_server_connection_trigger_responses( + struct http_server_connection *conn); int http_server_connection_output(struct http_server_connection *conn); void http_server_connection_tunnel(struct http_server_connection **_conn, http_server_tunnel_callback_t callback, void *context); diff -r 7d609a1c0a3e -r 25fd54c05522 src/lib-http/http-server-request.c --- a/src/lib-http/http-server-request.c Sun Sep 14 11:29:35 2014 +0300 +++ b/src/lib-http/http-server-request.c Mon Sep 15 11:19:50 2014 +0300 @@ -144,13 +144,13 @@ i_assert(req->state <= HTTP_SERVER_REQUEST_STATE_QUEUED); req->payload_halted = FALSE; if (req->req.expect_100_continue && !req->sent_100_continue) - http_server_connection_send_responses(req->conn); + http_server_connection_trigger_responses(req->conn); } void http_server_request_ready_to_respond(struct http_server_request *req) { req->state = HTTP_SERVER_REQUEST_STATE_READY_TO_RESPOND; - http_server_connection_send_responses(req->conn); + http_server_connection_trigger_responses(req->conn); } void http_server_request_submit_response(struct http_server_request *req) @@ -210,7 +210,7 @@ return; } - http_server_connection_send_responses(conn); + http_server_connection_trigger_responses(conn); } static struct http_server_response * From dovecot at dovecot.org Mon Sep 15 15:02:16 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 15 Sep 2014 15:02:16 +0000 Subject: dovecot-2.2: dsync: Moved all doveadm-specific code to doveadm-d... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/eff79a80e0c9 changeset: 17800:eff79a80e0c9 user: Timo Sirainen date: Mon Sep 15 18:02:02 2014 +0300 description: dsync: Moved all doveadm-specific code to doveadm-dsync.c diffstat: src/doveadm/dsync/doveadm-dsync.c | 1 + src/doveadm/dsync/dsync-brain-mailbox-tree.c | 13 +++++-------- src/doveadm/dsync/dsync-brain-private.h | 1 + src/doveadm/dsync/dsync-brain.c | 2 ++ src/doveadm/dsync/dsync-brain.h | 3 +++ src/doveadm/dsync/dsync-mailbox-tree.c | 1 - 6 files changed, 12 insertions(+), 9 deletions(-) diffs (109 lines): diff -r 25fd54c05522 -r eff79a80e0c9 src/doveadm/dsync/doveadm-dsync.c --- a/src/doveadm/dsync/doveadm-dsync.c Mon Sep 15 11:19:50 2014 +0300 +++ b/src/doveadm/dsync/doveadm-dsync.c Mon Sep 15 18:02:02 2014 +0300 @@ -541,6 +541,7 @@ memcpy(set.sync_box_guid, ctx->mailbox_guid, sizeof(set.sync_box_guid)); set.lock_timeout_secs = ctx->lock_timeout; set.state = ctx->state_input; + set.mailbox_alt_char = doveadm_settings->dsync_alt_char[0]; if (array_count(&ctx->exclude_mailboxes) > 0) { /* array is NULL-terminated in init() */ set.exclude_mailboxes = array_idx(&ctx->exclude_mailboxes, 0); diff -r 25fd54c05522 -r eff79a80e0c9 src/doveadm/dsync/dsync-brain-mailbox-tree.c --- a/src/doveadm/dsync/dsync-brain-mailbox-tree.c Mon Sep 15 11:19:50 2014 +0300 +++ b/src/doveadm/dsync/dsync-brain-mailbox-tree.c Mon Sep 15 18:02:02 2014 +0300 @@ -3,7 +3,6 @@ #include "lib.h" #include "str.h" #include "mail-namespace.h" -#include "doveadm-settings.h" #include "dsync-ibc.h" #include "dsync-mailbox-tree.h" #include "dsync-brain-private.h" @@ -47,12 +46,10 @@ dsync_brain_check_namespaces(brain); brain->local_mailbox_tree = - dsync_mailbox_tree_init(brain->hierarchy_sep, - doveadm_settings->dsync_alt_char[0]); + dsync_mailbox_tree_init(brain->hierarchy_sep, brain->alt_char); /* we'll convert remote mailbox names to use our own separator */ brain->remote_mailbox_tree = - dsync_mailbox_tree_init(brain->hierarchy_sep, - doveadm_settings->dsync_alt_char[0]); + dsync_mailbox_tree_init(brain->hierarchy_sep, brain->alt_char); /* fill the local mailbox tree */ for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) { @@ -238,7 +235,7 @@ struct mail_namespace *ns; const char *p; string_t *vname; - char ns_sep, alt_char = doveadm_settings->dsync_alt_char[0]; + char ns_sep; i_assert(*name_parts != NULL); @@ -254,13 +251,13 @@ if (*p != ns_sep) str_append_c(vname, *p); else - str_append_c(vname, alt_char); + str_append_c(vname, brain->alt_char); } str_append_c(vname, ns_sep); } str_truncate(vname, str_len(vname)-1); - dsync_fix_mailbox_name(ns, vname, alt_char); + dsync_fix_mailbox_name(ns, vname, brain->alt_char); *name_r = str_c(vname); *ns_r = ns; return 0; diff -r 25fd54c05522 -r eff79a80e0c9 src/doveadm/dsync/dsync-brain-private.h --- a/src/doveadm/dsync/dsync-brain-private.h Mon Sep 15 11:19:50 2014 +0300 +++ b/src/doveadm/dsync/dsync-brain-private.h Mon Sep 15 18:02:02 2014 +0300 @@ -55,6 +55,7 @@ guid_128_t sync_box_guid; const char *const *exclude_mailboxes; enum dsync_brain_sync_type sync_type; + char alt_char; unsigned int lock_timeout; int lock_fd; diff -r 25fd54c05522 -r eff79a80e0c9 src/doveadm/dsync/dsync-brain.c --- a/src/doveadm/dsync/dsync-brain.c Mon Sep 15 11:19:50 2014 +0300 +++ b/src/doveadm/dsync/dsync-brain.c Mon Sep 15 18:02:02 2014 +0300 @@ -171,6 +171,8 @@ } str_delete(sync_ns_str, str_len(sync_ns_str)-1, 1); } + brain->alt_char = set->mailbox_alt_char == '\0' ? '_' : + set->mailbox_alt_char; brain->sync_box = p_strdup(brain->pool, set->sync_box); brain->exclude_mailboxes = set->exclude_mailboxes == NULL ? NULL : p_strarray_dup(brain->pool, set->exclude_mailboxes); diff -r 25fd54c05522 -r eff79a80e0c9 src/doveadm/dsync/dsync-brain.h --- a/src/doveadm/dsync/dsync-brain.h Mon Sep 15 11:19:50 2014 +0300 +++ b/src/doveadm/dsync/dsync-brain.h Mon Sep 15 18:02:02 2014 +0300 @@ -48,6 +48,9 @@ /* Exclude these mailboxes from the sync. They can contain '*' wildcards and be \special-use flags. */ const char *const *exclude_mailboxes; + /* Alternative character to use in mailbox names where the original + character cannot be used. */ + char mailbox_alt_char; /* If non-zero, use dsync lock file for this user */ unsigned int lock_timeout_secs; diff -r 25fd54c05522 -r eff79a80e0c9 src/doveadm/dsync/dsync-mailbox-tree.c --- a/src/doveadm/dsync/dsync-mailbox-tree.c Mon Sep 15 11:19:50 2014 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-tree.c Mon Sep 15 18:02:02 2014 +0300 @@ -4,7 +4,6 @@ #include "array.h" #include "hash.h" #include "str.h" -#include "doveadm-settings.h" #include "mailbox-list-private.h" #include "dsync-mailbox-tree-private.h" From dovecot at dovecot.org Mon Sep 15 15:04:34 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 15 Sep 2014 15:04:34 +0000 Subject: dovecot-2.2: Install libdovecot-dsync.so library, which can be u... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3675d156b309 changeset: 17801:3675d156b309 user: Timo Sirainen date: Mon Sep 15 18:04:20 2014 +0300 description: Install libdovecot-dsync.so library, which can be used separately. diffstat: Makefile.am | 1 + configure.ac | 2 ++ dovecot-config.in.in | 3 +++ dovecot.m4 | 8 ++++---- src/doveadm/Makefile.am | 2 +- src/doveadm/dsync/Makefile.am | 22 +++++++++++++++++----- 6 files changed, 28 insertions(+), 10 deletions(-) diffs (151 lines): diff -r eff79a80e0c9 -r 3675d156b309 Makefile.am --- a/Makefile.am Mon Sep 15 18:02:02 2014 +0300 +++ b/Makefile.am Mon Sep 15 18:04:20 2014 +0300 @@ -68,6 +68,7 @@ -e "s|^\(LIBDOVECOT_LOGIN\)=.*$$|\1='-ldovecot-login $(SSL_LIBS)'|" \ -e "s|^\(LIBDOVECOT_SQL\)=.*$$|\1=-ldovecot-sql|" \ -e "s|^\(LIBDOVECOT_COMPRESS\)=.*$$|\1=-ldovecot-compression|" \ + -e "s|^\(LIBDOVECOT_DSYNC\)=.*$$|\1=-ldovecot-dsync|" \ -e "s|^\(LIBDOVECOT_LDA\)=.*$$|\1=-ldovecot-lda|" \ -e "s|^\(LIBDOVECOT_STORAGE\)=.*$$|\1='-ldovecot-storage $(LINKED_STORAGE_LDADD)'|" \ -e "s|^\(LIBDOVECOT_INCLUDE\)=.*$$|\1=-I$(pkgincludedir)|" \ diff -r eff79a80e0c9 -r 3675d156b309 configure.ac --- a/configure.ac Mon Sep 15 18:02:02 2014 +0300 +++ b/configure.ac Mon Sep 15 18:04:20 2014 +0300 @@ -2559,6 +2559,7 @@ LIBDOVECOT_LDA='$(top_builddir)/src/lib-lda/liblda.la' fi LIBDOVECOT_STORAGE="$LIBDOVECOT_STORAGE_DEPS $LINKED_STORAGE_LDADD" +LIBDOVECOT_DSYNC='$(top_builddir)/src/doveadm/dsync/libdovecot-dsync.la' LIBDOVECOT_SQL='$(top_builddir)/src/lib-sql/libsql.la' AC_SUBST(LIBDOVECOT) AC_SUBST(LIBDOVECOT_DEPS) @@ -2567,6 +2568,7 @@ AC_SUBST(LIBDOVECOT_LOGIN) AC_SUBST(LIBDOVECOT_SQL) AC_SUBST(LIBDOVECOT_COMPRESS) +AC_SUBST(LIBDOVECOT_DSYNC) AC_SUBST(LIBDOVECOT_LDA) dnl ** diff -r eff79a80e0c9 -r 3675d156b309 dovecot-config.in.in --- a/dovecot-config.in.in Mon Sep 15 18:02:02 2014 +0300 +++ b/dovecot-config.in.in Mon Sep 15 18:04:20 2014 +0300 @@ -10,6 +10,7 @@ LIBDOVECOT_COMPRESS="@LIBDOVECOT_COMPRESS@" LIBDOVECOT_LDA="@LIBDOVECOT_LDA@" LIBDOVECOT_STORAGE="@LIBDOVECOT_STORAGE@" +LIBDOVECOT_DSYNC="@LIBDOVECOT_DSYNC@" LIBDOVECOT_DEPS="@LIBDOVECOT_DEPS@" LIBDOVECOT_LOGIN_DEPS="@LIBDOVECOT_LOGIN@" @@ -17,11 +18,13 @@ LIBDOVECOT_COMPRESS_DEPS="@LIBDOVECOT_COMPRESS@" LIBDOVECOT_LDA_DEPS="@LIBDOVECOT_LDA@" LIBDOVECOT_STORAGE_DEPS="@LIBDOVECOT_STORAGE_DEPS@" +LIBDOVECOT_DSYNC_DEPS="@LIBDOVECOT_DSYNC@" LIBDOVECOT_INCLUDE="-I$(incdir) -I$(incdir)/src/lib -I$(incdir)/src/lib-dict -I$(incdir)/src/lib-dns -I$(incdir)/src/lib-http -I$(incdir)/src/lib-mail -I$(incdir)/src/lib-imap -I$(incdir)/src/lib-fs -I$(incdir)/src/lib-charset -I$(incdir)/src/lib-auth -I$(incdir)/src/lib-master -I$(incdir)/src/lib-ssl-iostream -I$(incdir)/src/lib-compression -I$(incdir)/src/lib-settings -I$(incdir)/src/lib-test" LIBDOVECOT_LDA_INCLUDE="-I$(incdir)/src/lib-lda -I$(incdir)/src/lda" LIBDOVECOT_DOVEADM_INCLUDE="-I$(incdir)/src/doveadm" LIBDOVECOT_STORAGE_INCLUDE="-I$(incdir)/src/lib-index -I$(incdir)/src/lib-storage -I$(incdir)/src/lib-storage/list -I$(incdir)/src/lib-storage/index -I$(incdir)/src/lib-storage/index/raw -I$(incdir)/src/plugins/quota" +LIBDOVECOT_DSYNC_INCLUDE="-I$(incdir)/src/doveadm/dsync" LIBDOVECOT_LOGIN_INCLUDE="-I$(incdir)/src/login-common" LIBDOVECOT_IMAP_INCLUDE="-I$(incdir)/src/imap" LIBDOVECOT_CONFIG_INCLUDE="-I$(incdir)/src/config" diff -r eff79a80e0c9 -r 3675d156b309 dovecot.m4 --- a/dovecot.m4 Mon Sep 15 18:02:02 2014 +0300 +++ b/dovecot.m4 Mon Sep 15 18:04:20 2014 +0300 @@ -6,7 +6,7 @@ # unlimited permission to copy and/or distribute it, with or without # modifications, as long as this notice is preserved. -# serial 12 +# serial 13 AC_DEFUN([DC_DOVECOT_MODULEDIR],[ AC_ARG_WITH(moduledir, @@ -87,9 +87,9 @@ AX_SUBST_L([DISTCHECK_CONFIGURE_FLAGS], [dovecotdir], [dovecot_moduledir], [dovecot_pkgincludedir], [dovecot_pkglibexecdir], [dovecot_pkglibdir], [dovecot_docdir]) AX_SUBST_L([DOVECOT_CFLAGS], [DOVECOT_LIBS], [DOVECOT_SSL_LIBS], [DOVECOT_SQL_LIBS], [DOVECOT_COMPRESS_LIBS]) - AX_SUBST_L([LIBDOVECOT], [LIBDOVECOT_LOGIN], [LIBDOVECOT_SQL], [LIBDOVECOT_SSL], [LIBDOVECOT_COMPRESS], [LIBDOVECOT_LDA], [LIBDOVECOT_STORAGE]) - AX_SUBST_L([LIBDOVECOT_DEPS], [LIBDOVECOT_LOGIN_DEPS], [LIBDOVECOT_SQL_DEPS], [LIBDOVECOT_SSL_DEPS], [LIBDOVECOT_COMPRESS_DEPS], [LIBDOVECOT_LDA_DEPS], [LIBDOVECOT_STORAGE_DEPS]) - AX_SUBST_L([LIBDOVECOT_INCLUDE], [LIBDOVECOT_LDA_INCLUDE], [LIBDOVECOT_DOVEADM_INCLUDE], [LIBDOVECOT_SERVICE_INCLUDE], [LIBDOVECOT_STORAGE_INCLUDE], [LIBDOVECOT_LOGIN_INCLUDE], [LIBDOVECOT_CONFIG_INCLUDE], [LIBDOVECOT_IMAP_INCLUDE]) + AX_SUBST_L([LIBDOVECOT], [LIBDOVECOT_LOGIN], [LIBDOVECOT_SQL], [LIBDOVECOT_SSL], [LIBDOVECOT_COMPRESS], [LIBDOVECOT_LDA], [LIBDOVECOT_STORAGE], [LIBDOVECOT_DSYNC]) + AX_SUBST_L([LIBDOVECOT_DEPS], [LIBDOVECOT_LOGIN_DEPS], [LIBDOVECOT_SQL_DEPS], [LIBDOVECOT_SSL_DEPS], [LIBDOVECOT_COMPRESS_DEPS], [LIBDOVECOT_LDA_DEPS], [LIBDOVECOT_STORAGE_DEPS], [LIBDOVECOT_DSYNC_DEPS]) + AX_SUBST_L([LIBDOVECOT_INCLUDE], [LIBDOVECOT_LDA_INCLUDE], [LIBDOVECOT_DOVEADM_INCLUDE], [LIBDOVECOT_SERVICE_INCLUDE], [LIBDOVECOT_STORAGE_INCLUDE], [LIBDOVECOT_LOGIN_INCLUDE], [LIBDOVECOT_CONFIG_INCLUDE], [LIBDOVECOT_IMAP_INCLUDE], [LIBDOVECOT_DSYNC_INCLUDE]) DC_PLUGIN_DEPS ]) diff -r eff79a80e0c9 -r 3675d156b309 src/doveadm/Makefile.am --- a/src/doveadm/Makefile.am Mon Sep 15 18:02:02 2014 +0300 +++ b/src/doveadm/Makefile.am Mon Sep 15 18:04:20 2014 +0300 @@ -34,7 +34,7 @@ ../lib-otp/libotp.a libs = \ - dsync/libdsync.la \ + dsync/libdoveadm_dsync.la \ ../lib-compression/libcompression.la doveadm_LDADD = \ diff -r eff79a80e0c9 -r 3675d156b309 src/doveadm/dsync/Makefile.am --- a/src/doveadm/dsync/Makefile.am Mon Sep 15 18:02:02 2014 +0300 +++ b/src/doveadm/dsync/Makefile.am Mon Sep 15 18:04:20 2014 +0300 @@ -1,4 +1,5 @@ -noinst_LTLIBRARIES = libdsync.la +pkglib_LTLIBRARIES = libdovecot-dsync.la +noinst_LTLIBRARIES = libdsync.la libdoveadm_dsync.la AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ @@ -13,7 +14,6 @@ -I$(top_srcdir)/src/doveadm libdsync_la_SOURCES = \ - doveadm-dsync.c \ dsync-brain.c \ dsync-brain-mailbox.c \ dsync-brain-mailbox-tree.c \ @@ -34,9 +34,22 @@ dsync-ibc-pipe.c \ dsync-transaction-log-scan.c +libdoveadm_dsync_la_SOURCES = doveadm-dsync.c +libdoveadm_dsync_la_LIBADD = libdsync.la +libdoveadm_dsync_la_DEPENDENCIES = libdsync.la + +libdovecot_dsync_la_SOURCES = +libdovecot_dsync_la_LIBADD = libdsync.la +libdovecot_dsync_la_DEPENDENCIES = libdsync.la +libdovecot_dsync_la_LDFLAGS = -export-dynamic + +pkginc_libdir = $(pkgincludedir) +pkginc_lib_HEADERS = \ + dsync-brain.h \ + dsync-ibc.h + noinst_HEADERS = \ doveadm-dsync.h \ - dsync-brain.h \ dsync-brain-private.h \ dsync-mail.h \ dsync-mailbox.h \ @@ -47,7 +60,6 @@ dsync-mailbox-tree-private.h \ dsync-serializer.h \ dsync-deserializer.h \ - dsync-ibc.h \ dsync-ibc-private.h \ dsync-transaction-log-scan.h @@ -62,7 +74,7 @@ test_dsync_mailbox_tree_sync_SOURCES = test-dsync-mailbox-tree-sync.c test_dsync_mailbox_tree_sync_LDADD = dsync-mailbox-tree-sync.lo dsync-mailbox-tree.lo $(test_libs) -test_dsync_mailbox_tree_sync_DEPENDENCIES = $(noinst_LTLIBRARIES) $(test_libs) +test_dsync_mailbox_tree_sync_DEPENDENCIES = $(pkglib_LTLIBRARIES) $(test_libs) check: check-am check-test check-test: all-am From dovecot at dovecot.org Mon Sep 15 21:01:49 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 15 Sep 2014 21:01:49 +0000 Subject: dovecot-2.2: lib-fs: Added wait_ioloop/prev_ioloop helpers to st... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/784883cc0be6 changeset: 17802:784883cc0be6 user: Timo Sirainen date: Mon Sep 15 23:57:45 2014 +0300 description: lib-fs: Added wait_ioloop/prev_ioloop helpers to struct fs. diffstat: src/lib-fs/fs-api-private.h | 3 +++ src/lib-fs/fs-api.c | 8 ++++++++ 2 files changed, 11 insertions(+), 0 deletions(-) diffs (39 lines): diff -r 3675d156b309 -r 784883cc0be6 src/lib-fs/fs-api-private.h --- a/src/lib-fs/fs-api-private.h Mon Sep 15 18:04:20 2014 +0300 +++ b/src/lib-fs/fs-api-private.h Mon Sep 15 23:57:45 2014 +0300 @@ -64,6 +64,9 @@ struct fs_settings set; string_t *last_error; + /* may be used by fs_wait_async() to do the waiting */ + struct ioloop *wait_ioloop, *prev_ioloop; + unsigned int files_open_count; struct fs_file *files; }; diff -r 3675d156b309 -r 784883cc0be6 src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Mon Sep 15 18:04:20 2014 +0300 +++ b/src/lib-fs/fs-api.c Mon Sep 15 23:57:45 2014 +0300 @@ -529,14 +529,22 @@ { int ret; + /* recursion not allowed */ + i_assert(fs->prev_ioloop == NULL); + if (fs->v.wait_async == NULL) ret = 0; else T_BEGIN { + fs->prev_ioloop = current_ioloop; ret = fs->v.wait_async(fs); + i_assert(current_ioloop == fs->prev_ioloop); + fs->prev_ioloop = NULL; } T_END; return ret; } + + int fs_lock(struct fs_file *file, unsigned int secs, struct fs_lock **lock_r) { int ret; From dovecot at dovecot.org Mon Sep 15 21:01:54 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 15 Sep 2014 21:01:54 +0000 Subject: dovecot-2.2: lib-fs: Keep track of all fs_iters in fs. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/59b920848699 changeset: 17803:59b920848699 user: Timo Sirainen date: Tue Sep 16 00:01:33 2014 +0300 description: lib-fs: Keep track of all fs_iters in fs. diffstat: src/lib-fs/fs-api-private.h | 6 +++++- src/lib-fs/fs-api.c | 2 ++ 2 files changed, 7 insertions(+), 1 deletions(-) diffs (45 lines): diff -r 784883cc0be6 -r 59b920848699 src/lib-fs/fs-api-private.h --- a/src/lib-fs/fs-api-private.h Mon Sep 15 23:57:45 2014 +0300 +++ b/src/lib-fs/fs-api-private.h Tue Sep 16 00:01:33 2014 +0300 @@ -69,10 +69,11 @@ unsigned int files_open_count; struct fs_file *files; + struct fs_iter *iters; }; struct fs_file { - /* linked list of all files (mainly for debugging leaks) */ + /* linked list of all files */ struct fs_file *prev, *next; struct fs *fs; @@ -102,6 +103,9 @@ }; struct fs_iter { + /* linked list of all iters */ + struct fs_iter *prev, *next; + struct fs *fs; enum fs_iter_flags flags; diff -r 784883cc0be6 -r 59b920848699 src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Mon Sep 15 23:57:45 2014 +0300 +++ b/src/lib-fs/fs-api.c Tue Sep 16 00:01:33 2014 +0300 @@ -693,6 +693,7 @@ T_BEGIN { iter = fs->v.iter_init(fs, path, flags); } T_END; + DLLIST_PREPEND(&fs->iters, iter); return iter; } @@ -702,6 +703,7 @@ int ret; *_iter = NULL; + DLLIST_REMOVE(&iter->fs->iters, iter); T_BEGIN { ret = iter->fs->v.iter_deinit(iter); } T_END; From dovecot at dovecot.org Mon Sep 15 21:22:12 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 15 Sep 2014 21:22:12 +0000 Subject: dovecot-2.2: lib-storage: Added mail_user_dup() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9c673534ae93 changeset: 17804:9c673534ae93 user: Timo Sirainen date: Tue Sep 16 00:21:13 2014 +0300 description: lib-storage: Added mail_user_dup() diffstat: src/lib-storage/mail-storage-service.c | 2 ++ src/lib-storage/mail-user.c | 20 ++++++++++++++++++++ src/lib-storage/mail-user.h | 4 ++++ 3 files changed, 26 insertions(+), 0 deletions(-) diffs (53 lines): diff -r 59b920848699 -r 9c673534ae93 src/lib-storage/mail-storage-service.c --- a/src/lib-storage/mail-storage-service.c Tue Sep 16 00:01:33 2014 +0300 +++ b/src/lib-storage/mail-storage-service.c Tue Sep 16 00:21:13 2014 +0300 @@ -633,6 +633,8 @@ const char *home = priv->home; struct mail_user *mail_user; + /* NOTE: if more user initialization is added, add it also to + mail_user_dup() */ mail_user = mail_user_alloc(user->input.username, user->user_info, user->user_set); mail_user_set_home(mail_user, *home == '\0' ? NULL : home); diff -r 59b920848699 -r 9c673534ae93 src/lib-storage/mail-user.c --- a/src/lib-storage/mail-user.c Tue Sep 16 00:01:33 2014 +0300 +++ b/src/lib-storage/mail-user.c Tue Sep 16 00:21:13 2014 +0300 @@ -514,3 +514,23 @@ } return storage; } + +struct mail_user *mail_user_dup(struct mail_user *user) +{ + struct mail_user *user2; + + user2 = mail_user_alloc(user->username, user->set_info, + user->unexpanded_set); + if (user->_home != NULL) + mail_user_set_home(user2, user->_home); + mail_user_set_vars(user2, user->service, + user->local_ip, user->remote_ip); + user2->uid = user->uid; + user2->gid = user->gid; + user2->anonymous = user->anonymous; + user2->admin = user->admin; + user2->auth_token = p_strdup(user2->pool, user->auth_token); + user2->auth_user = p_strdup(user2->pool, user->auth_user); + user2->session_id = p_strdup(user2->pool, user->session_id); + return user2; +} diff -r 59b920848699 -r 9c673534ae93 src/lib-storage/mail-user.h --- a/src/lib-storage/mail-user.h Tue Sep 16 00:01:33 2014 +0300 +++ b/src/lib-storage/mail-user.h Tue Sep 16 00:21:13 2014 +0300 @@ -95,6 +95,10 @@ void mail_user_ref(struct mail_user *user); void mail_user_unref(struct mail_user **user); +/* Duplicate a mail_user. mail_user_init() and mail_namespaces_init() need to + be called before the user is usable. */ +struct mail_user *mail_user_dup(struct mail_user *user); + /* Find another user from the given user's namespaces. */ struct mail_user *mail_user_find(struct mail_user *user, const char *name); From dovecot at dovecot.org Mon Sep 15 21:22:13 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 15 Sep 2014 21:22:13 +0000 Subject: dovecot-2.2: lib-storage: Added mail_namespaces_init_add/finish(... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/98f937eed2e5 changeset: 17805:98f937eed2e5 user: Timo Sirainen date: Tue Sep 16 00:21:52 2014 +0300 description: lib-storage: Added mail_namespaces_init_add/finish() to manually initializing namespaces. diffstat: src/lib-storage/mail-namespace.c | 94 ++++++++++++++++++++++----------------- src/lib-storage/mail-namespace.h | 8 +++ 2 files changed, 61 insertions(+), 41 deletions(-) diffs (155 lines): diff -r 9c673534ae93 -r 98f937eed2e5 src/lib-storage/mail-namespace.c --- a/src/lib-storage/mail-namespace.c Tue Sep 16 00:21:13 2014 +0300 +++ b/src/lib-storage/mail-namespace.c Tue Sep 16 00:21:52 2014 +0300 @@ -81,13 +81,13 @@ return FALSE; } -static int -namespace_add(struct mail_user *user, - struct mail_namespace_settings *ns_set, - struct mail_namespace_settings *unexpanded_ns_set, - const struct mail_storage_settings *mail_set, - struct mail_namespace **ns_p, const char **error_r) +int mail_namespaces_init_add(struct mail_user *user, + struct mail_namespace_settings *ns_set, + struct mail_namespace_settings *unexpanded_ns_set, + struct mail_namespace **ns_p, const char **error_r) { + const struct mail_storage_settings *mail_set = + mail_user_set_get_storage_set(user); struct mail_namespace *ns; const char *driver, *error; @@ -314,6 +314,48 @@ return TRUE; } +int mail_namespaces_init_finish(struct mail_namespace *namespaces, + const char **error_r) +{ + struct mail_namespace *ns; + bool prefixless_found = FALSE; + + for (ns = namespaces; ns != NULL; ns = ns->next) { + if (ns->prefix_len == 0) + prefixless_found = TRUE; + } + if (!prefixless_found) { + prefixless_ns_set = prefixless_ns_unexpanded_set; + /* a pretty evil way to expand the values */ + prefixless_ns_set.prefix++; + prefixless_ns_set.location++; + + if (mail_namespaces_init_add(namespaces->user, + &prefixless_ns_set, + &prefixless_ns_unexpanded_set, + &ns, error_r) < 0) + i_unreached(); + ns->next = namespaces; + namespaces = ns; + } + if (!namespaces_check(namespaces, error_r)) { + *error_r = t_strconcat("namespace configuration error: ", + *error_r, NULL); + while (namespaces != NULL) { + ns = namespaces; + namespaces = ns->next; + mail_namespace_free(ns); + } + return -1; + } + mail_user_add_namespace(namespaces->user, &namespaces); + + T_BEGIN { + hook_mail_namespaces_created(namespaces); + } T_END; + return 0; +} + int mail_namespaces_init(struct mail_user *user, const char **error_r) { const struct mail_storage_settings *mail_set; @@ -321,7 +363,6 @@ struct mail_namespace_settings *const *unexpanded_ns_set; struct mail_namespace *namespaces, *ns, **ns_p; unsigned int i, count, count2; - bool prefixless_found = FALSE; i_assert(user->initialized); @@ -341,8 +382,9 @@ if (ns_set[i]->disabled) continue; - if (namespace_add(user, ns_set[i], unexpanded_ns_set[i], - mail_set, ns_p, error_r) < 0) { + if (mail_namespaces_init_add(user, ns_set[i], + unexpanded_ns_set[i], + ns_p, error_r) < 0) { if (!ns_set[i]->ignore_on_failure) return -1; if (mail_set->mail_debug) { @@ -350,42 +392,12 @@ ns_set[i]->prefix, *error_r); } } else { - if ((*ns_p)->prefix_len == 0) - prefixless_found = TRUE; ns_p = &(*ns_p)->next; } } - if (namespaces != NULL) { - if (!prefixless_found) { - prefixless_ns_set = prefixless_ns_unexpanded_set; - /* a pretty evil way to expand the values */ - prefixless_ns_set.prefix++; - prefixless_ns_set.location++; - - if (namespace_add(user, &prefixless_ns_set, - &prefixless_ns_unexpanded_set, - mail_set, ns_p, - error_r) < 0) - i_unreached(); - } - if (!namespaces_check(namespaces, error_r)) { - *error_r = t_strconcat("namespace configuration error: ", - *error_r, NULL); - while (namespaces != NULL) { - ns = namespaces; - namespaces = ns->next; - mail_namespace_free(ns); - } - return -1; - } - mail_user_add_namespace(user, &namespaces); - - T_BEGIN { - hook_mail_namespaces_created(namespaces); - } T_END; - return 0; - } + if (namespaces != NULL) + return mail_namespaces_init_finish(namespaces, error_r); /* no namespaces defined, create a default one */ return mail_namespaces_init_location(user, NULL, error_r); diff -r 9c673534ae93 -r 98f937eed2e5 src/lib-storage/mail-namespace.h --- a/src/lib-storage/mail-namespace.h Tue Sep 16 00:21:13 2014 +0300 +++ b/src/lib-storage/mail-namespace.h Tue Sep 16 00:21:52 2014 +0300 @@ -85,6 +85,14 @@ for user's namespaces. */ void mail_namespaces_deinit(struct mail_namespace **namespaces); +/* Manually initialize namespaces one by one. */ +int mail_namespaces_init_add(struct mail_user *user, + struct mail_namespace_settings *ns_set, + struct mail_namespace_settings *unexpanded_ns_set, + struct mail_namespace **ns_p, const char **error_r); +int mail_namespaces_init_finish(struct mail_namespace *namespaces, + const char **error_r); + void mail_namespace_ref(struct mail_namespace *ns); void mail_namespace_unref(struct mail_namespace **ns); From dovecot at dovecot.org Mon Sep 15 21:43:51 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 15 Sep 2014 21:43:51 +0000 Subject: dovecot-2.2: lib-fs: fs-metawrap now fully wraps fs_iter_* to fi... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1010f2ba9f2d changeset: 17806:1010f2ba9f2d user: Timo Sirainen date: Tue Sep 16 00:43:36 2014 +0300 description: lib-fs: fs-metawrap now fully wraps fs_iter_* to fix tracking struct fs.iters diffstat: src/lib-fs/fs-metawrap.c | 36 ++++++++++++++++++++++++++++++++---- 1 files changed, 32 insertions(+), 4 deletions(-) diffs (60 lines): diff -r 98f937eed2e5 -r 1010f2ba9f2d src/lib-fs/fs-metawrap.c --- a/src/lib-fs/fs-metawrap.c Tue Sep 16 00:21:52 2014 +0300 +++ b/src/lib-fs/fs-metawrap.c Tue Sep 16 00:43:36 2014 +0300 @@ -28,6 +28,11 @@ bool metadata_read; }; +struct metawrap_fs_iter { + struct fs_iter iter; + struct fs_iter *super; +}; + static struct fs *fs_metawrap_alloc(void) { struct metawrap_fs *fs; @@ -406,9 +411,32 @@ static struct fs_iter * fs_metawrap_iter_init(struct fs *_fs, const char *path, - enum fs_iter_flags flags) + enum fs_iter_flags flags) { - return fs_iter_init(_fs->parent, path, flags); + struct metawrap_fs_iter *iter; + + iter = i_new(struct metawrap_fs_iter, 1); + iter->iter.fs = _fs; + iter->iter.flags = flags; + iter->super = fs_iter_init(_fs->parent, path, flags); + return &iter->iter; +} + +static const char *fs_metawrap_iter_next(struct fs_iter *_iter) +{ + struct metawrap_fs_iter *iter = (struct metawrap_fs_iter *)_iter; + + return fs_iter_next(iter->super); +} + +static int fs_metawrap_iter_deinit(struct fs_iter *_iter) +{ + struct metawrap_fs_iter *iter = (struct metawrap_fs_iter *)_iter; + int ret; + + ret = fs_iter_deinit(&iter->super); + i_free(iter); + return ret; } const struct fs fs_class_metawrap = { @@ -440,7 +468,7 @@ fs_metawrap_rename, fs_metawrap_delete, fs_metawrap_iter_init, - NULL, - NULL + fs_metawrap_iter_next, + fs_metawrap_iter_deinit } }; From dovecot at dovecot.org Mon Sep 15 22:10:07 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 15 Sep 2014 22:10:07 +0000 Subject: dovecot-2.2: Compiler warning fix Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0ce514f5f899 changeset: 17807:0ce514f5f899 user: Timo Sirainen date: Tue Sep 16 01:09:53 2014 +0300 description: Compiler warning fix diffstat: src/lib-storage/mail-namespace.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 1010f2ba9f2d -r 0ce514f5f899 src/lib-storage/mail-namespace.c --- a/src/lib-storage/mail-namespace.c Tue Sep 16 00:43:36 2014 +0300 +++ b/src/lib-storage/mail-namespace.c Tue Sep 16 01:09:53 2014 +0300 @@ -361,7 +361,7 @@ const struct mail_storage_settings *mail_set; struct mail_namespace_settings *const *ns_set; struct mail_namespace_settings *const *unexpanded_ns_set; - struct mail_namespace *namespaces, *ns, **ns_p; + struct mail_namespace *namespaces, **ns_p; unsigned int i, count, count2; i_assert(user->initialized); From dovecot at dovecot.org Mon Sep 15 22:12:26 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 15 Sep 2014 22:12:26 +0000 Subject: dovecot-2.2: Makefile: Fixed linking libdovecot-dsync Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a49c4e0d351a changeset: 17808:a49c4e0d351a user: Timo Sirainen date: Tue Sep 16 01:12:21 2014 +0300 description: Makefile: Fixed linking libdovecot-dsync diffstat: src/doveadm/dsync/Makefile.am | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 0ce514f5f899 -r a49c4e0d351a src/doveadm/dsync/Makefile.am --- a/src/doveadm/dsync/Makefile.am Tue Sep 16 01:09:53 2014 +0300 +++ b/src/doveadm/dsync/Makefile.am Tue Sep 16 01:12:21 2014 +0300 @@ -39,7 +39,7 @@ libdoveadm_dsync_la_DEPENDENCIES = libdsync.la libdovecot_dsync_la_SOURCES = -libdovecot_dsync_la_LIBADD = libdsync.la +libdovecot_dsync_la_LIBADD = libdsync.la ../../lib-storage/libdovecot-storage.la libdovecot_dsync_la_DEPENDENCIES = libdsync.la libdovecot_dsync_la_LDFLAGS = -export-dynamic From dovecot at dovecot.org Tue Sep 16 09:04:16 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 16 Sep 2014 09:04:16 +0000 Subject: dovecot-2.2: fs-metawrap: Fixed iteration wrapping, broken by ea... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d080dc2e1280 changeset: 17809:d080dc2e1280 user: Timo Sirainen date: Tue Sep 16 12:04:07 2014 +0300 description: fs-metawrap: Fixed iteration wrapping, broken by earlier changes. diffstat: src/lib-fs/fs-metawrap.c | 8 +++++++- 1 files changed, 7 insertions(+), 1 deletions(-) diffs (19 lines): diff -r a49c4e0d351a -r d080dc2e1280 src/lib-fs/fs-metawrap.c --- a/src/lib-fs/fs-metawrap.c Tue Sep 16 01:12:21 2014 +0300 +++ b/src/lib-fs/fs-metawrap.c Tue Sep 16 12:04:07 2014 +0300 @@ -425,8 +425,14 @@ static const char *fs_metawrap_iter_next(struct fs_iter *_iter) { struct metawrap_fs_iter *iter = (struct metawrap_fs_iter *)_iter; + const char *fname; - return fs_iter_next(iter->super); + iter->super->async_callback = _iter->async_callback; + iter->super->async_context = _iter->async_context; + + fname = fs_iter_next(iter->super); + _iter->async_have_more = iter->super->async_have_more; + return fname; } static int fs_metawrap_iter_deinit(struct fs_iter *_iter) From dovecot at dovecot.org Tue Sep 16 09:32:19 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 16 Sep 2014 09:32:19 +0000 Subject: dovecot-2.2: dsync: Moved doveadm-specific code to doveadm direc... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fc79ac9cc378 changeset: 17810:fc79ac9cc378 user: Timo Sirainen date: Tue Sep 16 12:32:09 2014 +0300 description: dsync: Moved doveadm-specific code to doveadm directory. diffstat: src/doveadm/Makefile.am | 4 +- src/doveadm/doveadm-dsync.c | 1258 +++++++++++++++++++++++++++++++++++++ src/doveadm/doveadm-dsync.h | 10 + src/doveadm/doveadm-mail.c | 2 +- src/doveadm/doveadm.c | 2 +- src/doveadm/dsync/Makefile.am | 10 +- src/doveadm/dsync/doveadm-dsync.c | 1258 ------------------------------------- src/doveadm/dsync/doveadm-dsync.h | 10 - 8 files changed, 1275 insertions(+), 1279 deletions(-) diffs (truncated from 2643 to 300 lines): diff -r d080dc2e1280 -r fc79ac9cc378 src/doveadm/Makefile.am --- a/src/doveadm/Makefile.am Tue Sep 16 12:04:07 2014 +0300 +++ b/src/doveadm/Makefile.am Tue Sep 16 12:32:09 2014 +0300 @@ -34,7 +34,7 @@ ../lib-otp/libotp.a libs = \ - dsync/libdoveadm_dsync.la \ + dsync/libdsync.la \ ../lib-compression/libcompression.la doveadm_LDADD = \ @@ -59,6 +59,7 @@ $(LIBDOVECOT_DEPS) common = \ + doveadm-dsync.c \ doveadm-mail.c \ doveadm-mail-altmove.c \ doveadm-mail-batch.c \ @@ -121,6 +122,7 @@ pkginc_libdir = $(pkgincludedir) pkginc_lib_HEADERS = \ doveadm.h \ + doveadm-dsync.h \ doveadm-dump.h \ doveadm-mail.h \ doveadm-mail-iter.h \ diff -r d080dc2e1280 -r fc79ac9cc378 src/doveadm/doveadm-dsync.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/doveadm-dsync.c Tue Sep 16 12:32:09 2014 +0300 @@ -0,0 +1,1258 @@ +/* Copyright (c) 2009-2014 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "lib-signals.h" +#include "array.h" +#include "execv-const.h" +#include "fd-set-nonblock.h" +#include "istream.h" +#include "ostream.h" +#include "iostream-ssl.h" +#include "iostream-rawlog.h" +#include "write-full.h" +#include "str.h" +#include "strescape.h" +#include "var-expand.h" +#include "settings-parser.h" +#include "master-service.h" +#include "master-service-ssl-settings.h" +#include "mail-storage-service.h" +#include "mail-user.h" +#include "mail-namespace.h" +#include "mailbox-list-private.h" +#include "doveadm-settings.h" +#include "doveadm-mail.h" +#include "doveadm-print.h" +#include "doveadm-server.h" +#include "client-connection.h" +#include "server-connection.h" +#include "dsync/dsync-brain.h" +#include "dsync/dsync-ibc.h" +#include "doveadm-dsync.h" + +#include +#include +#include +#include +#include + +#define DSYNC_COMMON_GETOPT_ARGS "+1dEfg:l:m:n:NPr:Rs:Ux:" +#define DSYNC_REMOTE_CMD_EXIT_WAIT_SECS 30 +/* The broken_char is mainly set to get a proper error message when trying to + convert a mailbox with a name that can't be used properly translated between + vname/storage_name and would otherwise be mixed up with a normal "mailbox + doesn't exist" error message. This could be any control character, since + none of them are allowed to be created in regular mailbox names. */ +#define DSYNC_LIST_BROKEN_CHAR '\003' + +enum dsync_run_type { + DSYNC_RUN_TYPE_LOCAL, + DSYNC_RUN_TYPE_STREAM, + DSYNC_RUN_TYPE_CMD +}; + +struct dsync_cmd_context { + struct doveadm_mail_cmd_context ctx; + enum dsync_brain_sync_type sync_type; + const char *mailbox; + guid_128_t mailbox_guid; + const char *state_input, *rawlog_path; + ARRAY_TYPE(const_string) exclude_mailboxes; + ARRAY_TYPE(const_string) namespace_prefixes; + + const char *remote_name; + const char *local_location; + pid_t remote_pid; + const char *const *remote_cmd_args; + + int fd_in, fd_out, fd_err; + struct io *io_err; + struct istream *input, *err_stream; + struct ostream *output; + size_t input_orig_bufsize, output_orig_bufsize; + + struct ssl_iostream_context *ssl_ctx; + struct ssl_iostream *ssl_iostream; + + enum dsync_run_type run_type; + struct server_connection *tcp_conn; + const char *error; + + unsigned int lock_timeout; + + unsigned int lock:1; + unsigned int purge_remote:1; + unsigned int sync_visible_namespaces:1; + unsigned int default_replica_location:1; + unsigned int oneway:1; + unsigned int backup:1; + unsigned int reverse_backup:1; + unsigned int remote_user_prefix:1; + unsigned int no_mail_sync:1; + unsigned int local_location_from_arg:1; + unsigned int replicator_notify:1; +}; + +static bool legacy_dsync = FALSE; + +static void remote_error_input(struct dsync_cmd_context *ctx) +{ + const unsigned char *data; + size_t size; + const char *line; + + switch (i_stream_read(ctx->err_stream)) { + case -2: + data = i_stream_get_data(ctx->err_stream, &size); + fprintf(stderr, "%.*s", (int)size, data); + i_stream_skip(ctx->err_stream, size); + break; + case -1: + if (ctx->io_err != NULL) + io_remove(&ctx->io_err); + break; + default: + while ((line = i_stream_next_line(ctx->err_stream)) != NULL) + fprintf(stderr, "%s\n", line); + break; + } +} + +static void +run_cmd(struct dsync_cmd_context *ctx, const char *const *args) +{ + int fd_in[2], fd_out[2], fd_err[2]; + + ctx->remote_cmd_args = p_strarray_dup(ctx->ctx.pool, args); + + if (pipe(fd_in) < 0 || pipe(fd_out) < 0 || pipe(fd_err) < 0) + i_fatal("pipe() failed: %m"); + + ctx->remote_pid = fork(); + switch (ctx->remote_pid) { + case -1: + i_fatal("fork() failed: %m"); + case 0: + /* child, which will execute the proxy server. stdin/stdout + goes to pipes which we'll pass to proxy client. */ + if (dup2(fd_in[0], STDIN_FILENO) < 0 || + dup2(fd_out[1], STDOUT_FILENO) < 0 || + dup2(fd_err[1], STDERR_FILENO) < 0) + i_fatal("dup2() failed: %m"); + + i_close_fd(&fd_in[0]); + i_close_fd(&fd_in[1]); + i_close_fd(&fd_out[0]); + i_close_fd(&fd_out[1]); + i_close_fd(&fd_err[0]); + i_close_fd(&fd_err[1]); + + execvp_const(args[0], args); + default: + /* parent */ + break; + } + + i_close_fd(&fd_in[0]); + i_close_fd(&fd_out[1]); + i_close_fd(&fd_err[1]); + ctx->fd_in = fd_out[0]; + ctx->fd_out = fd_in[1]; + ctx->fd_err = fd_err[0]; + + if (ctx->remote_user_prefix) { + const char *prefix = + t_strdup_printf("%s\n", ctx->ctx.cur_username); + if (write_full(ctx->fd_out, prefix, strlen(prefix)) < 0) + i_fatal("write(remote out) failed: %m"); + } + + fd_set_nonblock(ctx->fd_err, TRUE); + ctx->err_stream = i_stream_create_fd(ctx->fd_err, IO_BLOCK_SIZE, FALSE); + i_stream_set_return_partial_line(ctx->err_stream, TRUE); +} + +static void +mirror_get_remote_cmd_line(const char *const *argv, + const char *const **cmd_args_r) +{ + ARRAY_TYPE(const_string) cmd_args; + unsigned int i; + const char *p; + + i_assert(argv[0] != NULL); + + t_array_init(&cmd_args, 16); + for (i = 0; argv[i] != NULL; i++) { + p = argv[i]; + array_append(&cmd_args, &p, 1); + } + + if (legacy_dsync) { + /* we're executing dsync */ + p = "server"; + } else { + /* we're executing doveadm */ + p = "dsync-server"; + } + array_append(&cmd_args, &p, 1); + array_append_zero(&cmd_args); + *cmd_args_r = array_idx(&cmd_args, 0); +} + +static const char *const * +get_ssh_cmd_args(const char *host, const char *login, const char *mail_user) +{ + static struct var_expand_table static_tab[] = { + { 'u', NULL, "user" }, + { '\0', NULL, "login" }, + { '\0', NULL, "host" }, + { '\0', NULL, NULL } + }; + struct var_expand_table *tab; + ARRAY_TYPE(const_string) cmd_args; + string_t *str, *str2; + const char *value, *const *args; + + tab = t_malloc(sizeof(static_tab)); + memcpy(tab, static_tab, sizeof(static_tab)); + + tab[0].value = mail_user; + tab[1].value = login; + tab[2].value = host; + + t_array_init(&cmd_args, 8); + str = t_str_new(128); + str2 = t_str_new(128); + args = t_strsplit(doveadm_settings->dsync_remote_cmd, " "); + for (; *args != NULL; args++) { + if (strchr(*args, '%') == NULL) + value = *args; + else { + /* some automation: if parameter's all %variables + expand to empty, but the %variable isn't the only + text in the parameter, skip it. */ + str_truncate(str, 0); + str_truncate(str2, 0); + var_expand(str, *args, tab); + var_expand(str2, *args, static_tab); + if (strcmp(str_c(str), str_c(str2)) == 0 && + str_len(str) > 0) + continue; + value = t_strdup(str_c(str)); + } + array_append(&cmd_args, &value, 1); + } + array_append_zero(&cmd_args); + return array_idx(&cmd_args, 0); +} + +static bool mirror_get_remote_cmd(struct dsync_cmd_context *ctx, + const char *user, + const char *const **cmd_args_r) +{ + const char *p, *host, *const *argv = ctx->ctx.args; + + if (argv[1] != NULL) { + /* more than one parameter, so it contains a full command + (e.g. ssh host dsync) */ + mirror_get_remote_cmd_line(argv, cmd_args_r); + return TRUE; + } + + /* if it begins with /[a-z0-9]+:/, it's a mail location + (e.g. mdbox:~/mail) */ + for (p = argv[0]; *p != '\0'; p++) { + if (!i_isalnum(*p)) { + if (*p == ':') + return FALSE; From dovecot at dovecot.org Tue Sep 16 10:38:47 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 16 Sep 2014 10:38:47 +0000 Subject: dovecot-2.2: lib: strnum - ensure str_to_uintmax doesn't return ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9899cc178d0d changeset: 17811:9899cc178d0d user: Phil Carmody date: Tue Sep 16 13:38:30 2014 +0300 description: lib: strnum - ensure str_to_uintmax doesn't return a value on error It was clobbering the _r parameter even though the spec says it shouldn't. Signed-off-by: Phil Carmody diffstat: src/lib/strnum.c | 10 ++++++---- 1 files changed, 6 insertions(+), 4 deletions(-) diffs (20 lines): diff -r fc79ac9cc378 -r 9899cc178d0d src/lib/strnum.c --- a/src/lib/strnum.c Tue Sep 16 12:32:09 2014 +0300 +++ b/src/lib/strnum.c Tue Sep 16 13:38:30 2014 +0300 @@ -106,10 +106,12 @@ int str_to_uintmax(const char *str, uintmax_t *num_r) { const char *endp; - int ret = str_parse_uintmax(str, num_r, &endp); - if ((ret == 0) && (*endp != '\0')) - ret = -1; - return ret; + uintmax_t n; + int ret = str_parse_uintmax(str, &n, &endp); + if ((ret != 0) || (*endp != '\0')) + return -1; + *num_r = n; + return 0; } #define STR_TO_U__TEMPLATE(name, type) \ From dovecot at dovecot.org Tue Sep 16 10:38:48 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 16 Sep 2014 10:38:48 +0000 Subject: dovecot-2.2: lib: remove uintmax tests from uint64 tests Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/138fd3ccc9da changeset: 17812:138fd3ccc9da user: Phil Carmody date: Tue Sep 16 13:38:30 2014 +0300 description: lib: remove uintmax tests from uint64 tests These log a warning on OSX with clang. Signed-off-by: Phil Carmody diffstat: src/lib/test-strnum.c | 16 +++------------- 1 files changed, 3 insertions(+), 13 deletions(-) diffs (57 lines): diff -r 9899cc178d0d -r 138fd3ccc9da src/lib/test-strnum.c --- a/src/lib/test-strnum.c Tue Sep 16 13:38:30 2014 +0300 +++ b/src/lib/test-strnum.c Tue Sep 16 13:38:30 2014 +0300 @@ -7,14 +7,13 @@ #define INVALID(n) { #n, -1, 0 } #define VALID(n) { #n, 0, n } -/* Assumes uintmax_t is 64 bit */ static void test_str_to_u64(void) { unsigned int i; const struct { const char *input; int ret; - uintmax_t val; + uint64_t val; } u64tests[] = { INVALID(-1), INVALID(foo), @@ -29,7 +28,7 @@ }; test_begin("str_to_uint64"); for (i = 0; i < N_ELEMENTS(u64tests); ++i) { - uintmax_t val = 0xBADBEEF15BADF00D; + uint64_t val = 0xBADBEEF15BADF00D; int ret = str_to_uint64(u64tests[i].input, &val); test_assert_idx(ret == u64tests[i].ret, i); if (ret == 0) @@ -37,17 +36,11 @@ else test_assert_idx(val == 0xBADBEEF15BADF00D, i); - /* This stanza totally assumes that uintmax_t == uint64_t */ if (ret == 0) T_BEGIN { const char *longer = t_strconcat(u64tests[i].input, "x", NULL); - const char *endp; - ret = str_to_uintmax(longer, &val); + ret = str_to_uint64(longer, &val); test_assert_idx(ret < 0, i); - ret = str_parse_uintmax(longer, &val, &endp); - test_assert_idx(ret == 0, i); - test_assert_idx(val == u64tests[i].val, i); - test_assert_idx(*endp == 'x', i); } T_END; } test_end(); @@ -146,9 +139,6 @@ void test_strnum(void) { - test_begin("test_strnum - size check"); - test_assert(sizeof(uintmax_t) == sizeof(uint64_t)); - test_end(); /* If the above isn't true, then we do expect some failures possibly */ test_str_to_u64(); test_str_to_u32(); From dovecot at dovecot.org Tue Sep 16 10:38:48 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 16 Sep 2014 10:38:48 +0000 Subject: dovecot-2.2: lib: test-strnum - add size-oblivious str_to/parse_... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c58aaa79d647 changeset: 17813:c58aaa79d647 user: Phil Carmody date: Tue Sep 16 13:38:30 2014 +0300 description: lib: test-strnum - add size-oblivious str_to/parse_uintmax tests Test a value of every bit-length. Also test the 10/9*MAX corner case. And due to crappy helper functions, test lots of leading zeroes too! Signed-off-by: Phil Carmody diffstat: src/lib/test-strnum.c | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 73 insertions(+), 0 deletions(-) diffs (90 lines): diff -r 138fd3ccc9da -r c58aaa79d647 src/lib/test-strnum.c --- a/src/lib/test-strnum.c Tue Sep 16 13:38:30 2014 +0300 +++ b/src/lib/test-strnum.c Tue Sep 16 13:38:30 2014 +0300 @@ -7,6 +7,78 @@ #define INVALID(n) { #n, -1, 0 } #define VALID(n) { #n, 0, n } +/* always pads with leading zeros to a size of 9 digits */ +static int crappy_uintmax_to_str(char *into, uintmax_t val) +{ +#define BIGBASE 1000000000u +#define STRINGIFY(s) #s +#define STRINGIFY2(s) STRINGIFY(s) + int len = 0; + if(val >= BIGBASE) { + len = crappy_uintmax_to_str(into, val/BIGBASE); + } + i_snprintf(into + len, 10, "%09lu", val % BIGBASE); + return len + strlen(STRINGIFY2(BIGBASE))-2; +#undef STRINGIFY2 +#undef STRINGIFY +#undef BIGBASE +} + +static void test_str_to_uintmax(void) +{ + unsigned int i=0; + int randrange = rand()%15+1; /* when 1, will max out on 1s */ + uintmax_t value = 0; + int len, ret; + char buff[50]; /* totally assumes < 159 bits */ + + test_begin("str_to_uintmax in range"); + while (i < sizeof(uintmax_t)*CHAR_BIT) { + uintmax_t value_back; + const char *endp; + + value = (value << 1) + 1; + if (value >= 64) + value -= rand()%randrange; /* don't always test the same numbers */ + len = crappy_uintmax_to_str(buff, value); + ret = str_to_uintmax(buff, &value_back); + test_assert_idx(ret == 0, i); + test_assert_idx(value == value_back, i); + + /* test with trailing noise */ + buff[len] = 'x'; /* don't even null-terminate, let's be evil */ + value_back = 0x1234567890123456; + ret = str_to_uintmax(buff, &value_back); + test_assert_idx(ret < 0, i); + test_assert_idx(value_back == 0x1234567890123456, i); + ret = str_parse_uintmax(buff, &value_back, &endp); + test_assert_idx(ret == 0, i); + test_assert_idx(value_back == value, i); + test_assert_idx(endp == &buff[len], i); + i++; + } + test_end(); + + /* not knowing exactly how large a uintmax_t is, we have to construct + the troublesome near-10/9*MAX strings manually by appending digits + to a MAX/9 string which we can easily create. Do a wider range + of 30 rather than the obvious 10, just in case - all are too large.*/ + test_begin("str_to_uintmax overflow corner case"); + value = UINTMAX_MAX/9-1; + len = crappy_uintmax_to_str(buff, value); + buff[len] = '0'; + buff[len+1] = '\0'; + for(i = 0; i <= 30; ++i) { + int j = len + 1; + while (buff[--j] == '9') + buff[j] = '0'; + buff[j]++; + ret = str_to_uintmax(buff, &value); + test_assert_idx(ret < 0 && value == UINTMAX_MAX/9-1, i); + } + test_end(); +} + static void test_str_to_u64(void) { unsigned int i; @@ -140,6 +212,7 @@ void test_strnum(void) { /* If the above isn't true, then we do expect some failures possibly */ + test_str_to_uintmax(); test_str_to_u64(); test_str_to_u32(); test_str_to_llong(); From dovecot at dovecot.org Tue Sep 16 11:04:09 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 16 Sep 2014 11:04:09 +0000 Subject: dovecot-2.2: lib: test-strnum - fix format size mismatch Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/56b565986dd3 changeset: 17814:56b565986dd3 user: Phil Carmody date: Tue Sep 16 14:03:52 2014 +0300 description: lib: test-strnum - fix format size mismatch Signed-off-by: Phil Carmody diffstat: src/lib/test-strnum.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r c58aaa79d647 -r 56b565986dd3 src/lib/test-strnum.c --- a/src/lib/test-strnum.c Tue Sep 16 13:38:30 2014 +0300 +++ b/src/lib/test-strnum.c Tue Sep 16 14:03:52 2014 +0300 @@ -17,7 +17,7 @@ if(val >= BIGBASE) { len = crappy_uintmax_to_str(into, val/BIGBASE); } - i_snprintf(into + len, 10, "%09lu", val % BIGBASE); + i_snprintf(into + len, 10, "%09lu", (unsigned long)(val % BIGBASE)); return len + strlen(STRINGIFY2(BIGBASE))-2; #undef STRINGIFY2 #undef STRINGIFY From dovecot at dovecot.org Tue Sep 16 11:32:30 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 16 Sep 2014 11:32:30 +0000 Subject: dovecot-2.2: fts: dovecot-expunges.log wasn't closed at deinit Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e82ad7f1c58f changeset: 17815:e82ad7f1c58f user: Timo Sirainen date: Tue Sep 16 14:32:20 2014 +0300 description: fts: dovecot-expunges.log wasn't closed at deinit diffstat: src/plugins/fts/fts-expunge-log.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 56b565986dd3 -r e82ad7f1c58f src/plugins/fts/fts-expunge-log.c --- a/src/plugins/fts/fts-expunge-log.c Tue Sep 16 14:03:52 2014 +0300 +++ b/src/plugins/fts/fts-expunge-log.c Tue Sep 16 14:32:20 2014 +0300 @@ -78,6 +78,8 @@ struct fts_expunge_log *log = *_log; *_log = NULL; + if (log->fd != -1) + i_close_fd(&log->fd); i_free(log->path); i_free(log); } From dovecot at dovecot.org Tue Sep 16 12:24:02 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 16 Sep 2014 12:24:02 +0000 Subject: dovecot-2.2: doveadm fts rescan: For virtual namespaces just mar... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8c2cb7d01a78 changeset: 17816:8c2cb7d01a78 user: Timo Sirainen date: Tue Sep 16 15:23:51 2014 +0300 description: doveadm fts rescan: For virtual namespaces just mark the last indexed UID to 0. diffstat: src/plugins/fts-solr/fts-backend-solr.c | 24 +----------------------- src/plugins/fts/fts-api-private.h | 1 + src/plugins/fts/fts-api.c | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 23 deletions(-) diffs (99 lines): diff -r e82ad7f1c58f -r 8c2cb7d01a78 src/plugins/fts-solr/fts-backend-solr.c --- a/src/plugins/fts-solr/fts-backend-solr.c Tue Sep 16 14:32:20 2014 +0300 +++ b/src/plugins/fts-solr/fts-backend-solr.c Tue Sep 16 15:23:51 2014 +0300 @@ -581,31 +581,9 @@ static int fts_backend_solr_rescan(struct fts_backend *backend) { - struct mailbox_list_iterate_context *iter; - const struct mailbox_info *info; - struct mailbox *box; - int ret = 0; - /* FIXME: proper rescan needed. for now we'll just reset the last-uids */ - iter = mailbox_list_iter_init(backend->ns->list, "*", - MAILBOX_LIST_ITER_SKIP_ALIASES | - MAILBOX_LIST_ITER_NO_AUTO_BOXES); - while ((info = mailbox_list_iter_next(iter)) != NULL) { - if ((info->flags & - (MAILBOX_NONEXISTENT | MAILBOX_NOSELECT)) != 0) - continue; - - box = mailbox_alloc(info->ns->list, info->vname, 0); - if (mailbox_open(box) == 0) { - if (fts_index_set_last_uid(box, 0) < 0) - ret = -1; - } - mailbox_free(&box); - } - if (mailbox_list_iter_deinit(&iter) < 0) - ret = -1; - return ret; + return fts_backend_reset_last_uids(backend); } static int fts_backend_solr_optimize(struct fts_backend *backend ATTR_UNUSED) diff -r e82ad7f1c58f -r 8c2cb7d01a78 src/plugins/fts/fts-api-private.h --- a/src/plugins/fts/fts-api-private.h Tue Sep 16 14:32:20 2014 +0300 +++ b/src/plugins/fts/fts-api-private.h Tue Sep 16 15:23:51 2014 +0300 @@ -110,6 +110,7 @@ const struct fts_index_header *hdr); int ATTR_NOWARN_UNUSED_RESULT fts_index_set_last_uid(struct mailbox *box, uint32_t last_uid); +int fts_backend_reset_last_uids(struct fts_backend *backend); int fts_index_have_compatible_settings(struct mailbox_list *list, uint32_t checksum); diff -r e82ad7f1c58f -r 8c2cb7d01a78 src/plugins/fts/fts-api.c --- a/src/plugins/fts/fts-api.c Tue Sep 16 14:32:20 2014 +0300 +++ b/src/plugins/fts/fts-api.c Tue Sep 16 15:23:51 2014 +0300 @@ -6,6 +6,7 @@ #include "mail-index.h" #include "mail-namespace.h" #include "mail-storage-private.h" +#include "mailbox-list-iter.h" #include "mail-search.h" #include "../virtual/virtual-storage.h" #include "fts-api-private.h" @@ -198,8 +199,40 @@ return backend->v.refresh(backend); } +int fts_backend_reset_last_uids(struct fts_backend *backend) +{ + struct mailbox_list_iterate_context *iter; + const struct mailbox_info *info; + struct mailbox *box; + int ret = 0; + + iter = mailbox_list_iter_init(backend->ns->list, "*", + MAILBOX_LIST_ITER_SKIP_ALIASES | + MAILBOX_LIST_ITER_NO_AUTO_BOXES); + while ((info = mailbox_list_iter_next(iter)) != NULL) { + if ((info->flags & + (MAILBOX_NONEXISTENT | MAILBOX_NOSELECT)) != 0) + continue; + + box = mailbox_alloc(info->ns->list, info->vname, 0); + if (mailbox_open(box) == 0) { + if (fts_index_set_last_uid(box, 0) < 0) + ret = -1; + } + mailbox_free(&box); + } + if (mailbox_list_iter_deinit(&iter) < 0) + ret = -1; + return ret; +} + int fts_backend_rescan(struct fts_backend *backend) { + if (strcmp(backend->ns->storage->name, VIRTUAL_STORAGE_NAME) == 0) { + /* just reset the last-uids for a virtual storage. */ + return fts_backend_reset_last_uids(backend); + } + return backend->v.rescan == NULL ? 0 : backend->v.rescan(backend); } From dovecot at dovecot.org Tue Sep 16 12:43:54 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 16 Sep 2014 12:43:54 +0000 Subject: dovecot-2.2: lib-storage: SEARCH_MAILBOX* value is now also comp... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ad16c74f4f36 changeset: 17817:ad16c74f4f36 user: Timo Sirainen date: Tue Sep 16 15:43:42 2014 +0300 description: lib-storage: SEARCH_MAILBOX* value is now also compared to to the (virtual) mailbox name. This fixes for example "doveadm fetch uid mailbox virtual/all" diffstat: src/lib-storage/index/index-search.c | 11 +++++++++++ 1 files changed, 11 insertions(+), 0 deletions(-) diffs (32 lines): diff -r 8c2cb7d01a78 -r ad16c74f4f36 src/lib-storage/index/index-search.c --- a/src/lib-storage/index/index-search.c Tue Sep 16 15:23:51 2014 +0300 +++ b/src/lib-storage/index/index-search.c Tue Sep 16 15:43:42 2014 +0300 @@ -239,10 +239,19 @@ static int search_arg_match_mailbox(struct index_search_context *ctx, struct mail_search_arg *arg) { + struct mailbox *box = ctx->cur_mail->box; const char *str; switch (arg->type) { case SEARCH_MAILBOX: + /* first try to match the mailbox name itself. this is + important when using "mailbox virtual/foo" parameter foin + doveadm's search query, otherwise we can never fetch + anything with doveadm from virtual mailboxes because the + mailbox parameter is compared to the mail's backend + mailbox. */ + if (strcmp(box->vname, arg->value.str) == 0) + return 1; if (mail_get_special(ctx->cur_mail, MAIL_FETCH_MAILBOX_NAME, &str) < 0) return -1; @@ -251,6 +260,8 @@ return strcasecmp(arg->value.str, "INBOX") == 0; return strcmp(str, arg->value.str) == 0; case SEARCH_MAILBOX_GLOB: + if (imap_match(arg->value.mailbox_glob, box->vname) == IMAP_MATCH_YES) + return 1; if (mail_get_special(ctx->cur_mail, MAIL_FETCH_MAILBOX_NAME, &str) < 0) return -1; From dovecot at dovecot.org Tue Sep 16 17:29:23 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 16 Sep 2014 17:29:23 +0000 Subject: dovecot-2.2: lib: hash-format - fix leak on _init() failure path Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4374e04f6aae changeset: 17818:4374e04f6aae user: Phil Carmody date: Tue Sep 16 20:29:04 2014 +0300 description: lib: hash-format - fix leak on _init() failure path If any of the analysis/parse helpers return failure, then the whole pool will leak. Signed-off-by: Phil Carmody diffstat: src/lib/hash-format.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r ad16c74f4f36 -r 4374e04f6aae src/lib/hash-format.c --- a/src/lib/hash-format.c Tue Sep 16 15:43:42 2014 +0300 +++ b/src/lib/hash-format.c Tue Sep 16 20:29:04 2014 +0300 @@ -134,6 +134,7 @@ } T_END; if (ret < 0) { *error_r = t_strdup(*error_r); + pool_unref(&pool); return -1; } *format_r = format; From dovecot at dovecot.org Tue Sep 16 17:29:24 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 16 Sep 2014 17:29:24 +0000 Subject: dovecot-2.2: lib: test-var-expand - disambiguate tests Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8a969c11e1af changeset: 17819:8a969c11e1af user: Phil Carmody date: Tue Sep 16 20:29:04 2014 +0300 description: lib: test-var-expand - disambiguate tests Two test cases had the same name. Signed-off-by: Phil Carmody diffstat: src/lib/test-var-expand.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (21 lines): diff -r 4374e04f6aae -r 8a969c11e1af src/lib/test-var-expand.c --- a/src/lib/test-var-expand.c Tue Sep 16 20:29:04 2014 +0300 +++ b/src/lib/test-var-expand.c Tue Sep 16 20:29:04 2014 +0300 @@ -34,7 +34,7 @@ string_t *str = t_str_new(128); unsigned int i; - test_begin("var_expand"); + test_begin("var_expand - ranges"); for (i = 0; i < N_ELEMENTS(tests); i++) { str_truncate(str, 0); var_expand(str, tests[i].in, table); @@ -66,7 +66,7 @@ tests[1].out = my_pid; env_put("FOO=baR"); - test_begin("var_expand"); + test_begin("var_expand - builtin"); for (i = 0; i < N_ELEMENTS(tests); i++) { str_truncate(str, 0); var_expand(str, tests[i].in, table); From dovecot at dovecot.org Tue Sep 16 17:59:58 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 16 Sep 2014 17:59:58 +0000 Subject: dovecot-2.2: quota: Quota recalculation didn't include INBOX in ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2baf1377f57a changeset: 17820:2baf1377f57a user: Timo Sirainen date: Tue Sep 16 20:59:48 2014 +0300 description: quota: Quota recalculation didn't include INBOX in some configurations. If one inbox=yes and another inbox=no namespace shared the same mail location, and the inbox=no namespace was first, only it was used to list all mailboxes. diffstat: src/plugins/quota/quota-private.h | 1 + src/plugins/quota/quota.c | 57 +++++++++++++++++++++++++++++--------- 2 files changed, 44 insertions(+), 14 deletions(-) diffs (93 lines): diff -r 8a969c11e1af -r 2baf1377f57a src/plugins/quota/quota-private.h --- a/src/plugins/quota/quota-private.h Tue Sep 16 20:29:04 2014 +0300 +++ b/src/plugins/quota/quota-private.h Tue Sep 16 20:59:48 2014 +0300 @@ -15,6 +15,7 @@ ARRAY(struct quota_root *) roots; ARRAY(struct mail_namespace *) namespaces; + struct mail_namespace *unwanted_ns; }; struct quota_settings { diff -r 8a969c11e1af -r 2baf1377f57a src/plugins/quota/quota.c --- a/src/plugins/quota/quota.c Tue Sep 16 20:29:04 2014 +0300 +++ b/src/plugins/quota/quota.c Tue Sep 16 20:59:48 2014 +0300 @@ -418,28 +418,55 @@ return enabled ? 1 : 0; } +static bool +quota_is_duplicate_namespace(struct quota *quota, struct mail_namespace *ns) +{ + struct mail_namespace *const *namespaces; + unsigned int i, count; + const char *path, *path2; + + if (!mailbox_list_get_root_path(ns->list, + MAILBOX_LIST_PATH_TYPE_MAILBOX, &path)) + return TRUE; + + namespaces = array_get("a->namespaces, &count); + for (i = 0; i < count; i++) { + if (mailbox_list_get_root_path(namespaces[i]->list, + MAILBOX_LIST_PATH_TYPE_MAILBOX, &path2) && + strcmp(path, path2) == 0) { + /* duplicate path */ + if ((ns->flags & NAMESPACE_FLAG_INBOX_USER) == 0) + return TRUE; + + /* this is inbox=yes namespace, but the earlier one + that had the same location was inbox=no. we need to + include the INBOX also in quota calculations, so we + can't just ignore this namespace. but since we've + already called backend's namespace_added(), we can't + just remove it either. so just mark the old one as + unwanted namespace. + + an alternative would be to do a bit larger change so + namespaces wouldn't be added until + mail_namespaces_created() hook is called */ + i_assert(quota->unwanted_ns == NULL); + quota->unwanted_ns = namespaces[i]; + return FALSE; + } + } + return FALSE; +} + void quota_add_user_namespace(struct quota *quota, struct mail_namespace *ns) { struct quota_root *const *roots; - struct mail_namespace *const *namespaces; struct quota_backend **backends; - const char *path, *path2; unsigned int i, j, count; /* first check if there already exists a namespace with the exact same path. we don't want to count them twice. */ - if (mailbox_list_get_root_path(ns->list, MAILBOX_LIST_PATH_TYPE_MAILBOX, - &path)) { - namespaces = array_get("a->namespaces, &count); - for (i = 0; i < count; i++) { - if (mailbox_list_get_root_path(namespaces[i]->list, - MAILBOX_LIST_PATH_TYPE_MAILBOX, &path2) && - strcmp(path, path2) == 0) { - /* duplicate */ - return; - } - } - } + if (quota_is_duplicate_namespace(quota, ns)) + return; array_append("a->namespaces, &ns, 1); @@ -507,6 +534,8 @@ if (mailbox_list_get_storage(&list, "", &storage) == 0 && (storage->class_flags & MAIL_STORAGE_CLASS_FLAG_NOQUOTA) != 0) return FALSE; + if (root->quota->unwanted_ns == ns) + return FALSE; if (root->ns_prefix != NULL) { if (root->ns != ns) From dovecot at dovecot.org Tue Sep 16 20:23:08 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 16 Sep 2014 20:23:08 +0000 Subject: dovecot-2.2: virtual: Using modseq-based syncing while mailbox w... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/21757e66053f changeset: 17821:21757e66053f user: Timo Sirainen date: Tue Sep 16 23:22:59 2014 +0300 description: virtual: Using modseq-based syncing while mailbox was open didn't handle expunges correctly. The expunges were handled only after EXPUNGE command was given, which should have been done only for mails that no longer matched the search query but still existed. diffstat: src/plugins/virtual/virtual-sync.c | 15 ++++++++++++++- 1 files changed, 14 insertions(+), 1 deletions(-) diffs (40 lines): diff -r 2baf1377f57a -r 21757e66053f src/plugins/virtual/virtual-sync.c --- a/src/plugins/virtual/virtual-sync.c Tue Sep 16 20:59:48 2014 +0300 +++ b/src/plugins/virtual/virtual-sync.c Tue Sep 16 23:22:59 2014 +0300 @@ -720,7 +720,8 @@ MAILBOX_SEARCH_RESULT_FLAG_QUEUE_SYNC; struct mail_index_view *view = bbox->box->view; struct mail_search_result *result; - ARRAY_TYPE(seq_range) removed_uids, added_uids, flag_update_uids; + ARRAY_TYPE(seq_range) expunged_uids = ARRAY_INIT, removed_uids; + ARRAY_TYPE(seq_range) added_uids, flag_update_uids; uint64_t modseq, old_highest_modseq; uint32_t seq, uid, old_msg_count; @@ -733,7 +734,15 @@ result = mailbox_search_result_alloc(bbox->box, bbox->search_args, result_flags); mailbox_search_result_initial_done(result); + i_assert(array_count(&result->removed_uids) == 0); virtual_sync_backend_handle_old_vmsgs(ctx, bbox, result); + if (array_count(&result->removed_uids) > 0) { + /* these are all expunged messages. treat them separately from + "no longer matching messages" (=removed_uids) */ + t_array_init(&expunged_uids, array_count(&result->removed_uids)); + array_append_array(&expunged_uids, &result->removed_uids); + array_clear(&result->removed_uids); + } /* get list of changed old messages (messages already once seen by virtual index), based on modseq changes. (we'll assume all modseq @@ -766,6 +775,10 @@ t_array_init(&removed_uids, 128); t_array_init(&added_uids, 128); mailbox_search_result_sync(result, &removed_uids, &added_uids); + if (array_is_created(&expunged_uids)) { + seq_range_array_remove_seq_range(&removed_uids, &expunged_uids); + virtual_sync_mailbox_box_remove(ctx, bbox, &expunged_uids); + } if (ctx->expunge_removed) virtual_sync_mailbox_box_remove(ctx, bbox, &removed_uids); else { From dovecot at dovecot.org Thu Sep 18 00:19:44 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 18 Sep 2014 00:19:44 +0000 Subject: dovecot-2.2: fs-posix: When deleting a directory, automatically ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/677cdc451031 changeset: 17822:677cdc451031 user: Timo Sirainen date: Thu Sep 18 02:19:32 2014 +0200 description: fs-posix: When deleting a directory, automatically fallback to rmdir()ing it diffstat: src/lib-fs/fs-posix.c | 12 ++++++++++-- 1 files changed, 10 insertions(+), 2 deletions(-) diffs (22 lines): diff -r 21757e66053f -r 677cdc451031 src/lib-fs/fs-posix.c --- a/src/lib-fs/fs-posix.c Tue Sep 16 23:22:59 2014 +0300 +++ b/src/lib-fs/fs-posix.c Thu Sep 18 02:19:32 2014 +0200 @@ -644,8 +644,16 @@ struct posix_fs *fs = (struct posix_fs *)_file->fs; if (unlink(_file->path) < 0) { - fs_set_error(_file->fs, "unlink(%s) failed: %m", _file->path); - return -1; + if (!UNLINK_EISDIR(errno)) { + fs_set_error(_file->fs, "unlink(%s) failed: %m", _file->path); + return -1; + } + /* attempting to delete a directory. convert it to rmdir() + automatically. */ + if (rmdir(_file->path) < 0) { + fs_set_error(_file->fs, "rmdir(%s) failed: %m", _file->path); + return -1; + } } (void)fs_posix_rmdir_parents(fs, _file->path); return 0; From dovecot at dovecot.org Thu Sep 18 15:40:04 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 18 Sep 2014 15:40:04 +0000 Subject: dovecot-2.2: doveadm dict iter: Use recursion flag only if -R pa... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9b65f3f583a3 changeset: 17823:9b65f3f583a3 user: Timo Sirainen date: Thu Sep 18 17:39:49 2014 +0200 description: doveadm dict iter: Use recursion flag only if -R parameter is specified. diffstat: src/doveadm/doveadm-dict.c | 30 ++++++++++++++++++++++++------ 1 files changed, 24 insertions(+), 6 deletions(-) diffs (73 lines): diff -r 677cdc451031 -r 9b65f3f583a3 src/doveadm/doveadm-dict.c --- a/src/doveadm/doveadm-dict.c Thu Sep 18 02:19:32 2014 +0200 +++ b/src/doveadm/doveadm-dict.c Thu Sep 18 17:39:49 2014 +0200 @@ -11,15 +11,22 @@ static void dict_cmd_help(doveadm_command_t *cmd); static struct dict * -cmd_dict_init(int *argc, char **argv[], int own_arg_count, int key_arg_idx, - doveadm_command_t *cmd) +cmd_dict_init_full(int *argc, char **argv[], int own_arg_count, int key_arg_idx, + doveadm_command_t *cmd, bool *recurse) { + const char *getopt_args = recurse == NULL ? "u:" : "Ru:"; struct dict *dict; const char *error, *username = ""; int c; - while ((c = getopt(*argc, *argv, "u:")) > 0) { + if (recurse != NULL) + *recurse = FALSE; + + while ((c = getopt(*argc, *argv, getopt_args)) > 0) { switch (c) { + case 'R': + *recurse = TRUE; + break; case 'u': username = optarg; break; @@ -56,6 +63,15 @@ return dict; } +static struct dict * +cmd_dict_init(int *argc, char **argv[], + int own_arg_count, int key_arg_idx, + doveadm_command_t *cmd) +{ + return cmd_dict_init_full(argc, argv, own_arg_count, + key_arg_idx, cmd, NULL); +} + static void cmd_dict_get(int argc, char *argv[]) { struct dict *dict; @@ -139,14 +155,16 @@ struct dict *dict; struct dict_iterate_context *iter; const char *key, *value; + bool recurse; - dict = cmd_dict_init(&argc, &argv, 1, 0, cmd_dict_iter); + dict = cmd_dict_init_full(&argc, &argv, 1, 0, cmd_dict_iter, &recurse); doveadm_print_init(DOVEADM_PRINT_TYPE_TAB); doveadm_print_header_simple("key"); doveadm_print_header_simple("value"); - iter = dict_iterate_init(dict, argv[0], DICT_ITERATE_FLAG_RECURSE); + iter = dict_iterate_init(dict, argv[0], + recurse ? DICT_ITERATE_FLAG_RECURSE : 0); while (dict_iterate(iter, &key, &value)) { doveadm_print(key); doveadm_print(value); @@ -163,7 +181,7 @@ { cmd_dict_set, "dict set", "[-u ] " }, { cmd_dict_unset, "dict unset", "[-u ] " }, { cmd_dict_inc, "dict inc", "[-u ] " }, - { cmd_dict_iter, "dict iter", "[-u ] " } + { cmd_dict_iter, "dict iter", "[-u ] [-R] " } }; static void dict_cmd_help(doveadm_command_t *cmd) From dovecot at dovecot.org Thu Sep 18 15:46:30 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 18 Sep 2014 15:46:30 +0000 Subject: dovecot-2.2: lib-dict: dict-fs implements now minimal iteration ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/979e5a007248 changeset: 17824:979e5a007248 user: Timo Sirainen date: Thu Sep 18 17:46:18 2014 +0200 description: lib-dict: dict-fs implements now minimal iteration support. diffstat: src/lib-dict/dict-fs.c | 94 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 files changed, 91 insertions(+), 3 deletions(-) diffs (118 lines): diff -r 9b65f3f583a3 -r 979e5a007248 src/lib-dict/dict-fs.c --- a/src/lib-dict/dict-fs.c Thu Sep 18 17:39:49 2014 +0200 +++ b/src/lib-dict/dict-fs.c Thu Sep 18 17:46:18 2014 +0200 @@ -14,6 +14,16 @@ char *username; }; +struct fs_dict_iterate_context { + struct dict_iterate_context ctx; + const char **paths; + unsigned int path_idx; + enum dict_iterate_flags flags; + pool_t value_pool; + struct fs_iter *fs_iter; + bool failed; +}; + static int fs_dict_init(struct dict *driver, const char *uri, const struct dict_settings *set, @@ -106,6 +116,84 @@ return ret; } +static struct dict_iterate_context * +fs_dict_iterate_init(struct dict *_dict, const char *const *paths, + enum dict_iterate_flags flags) +{ + struct fs_dict *dict = (struct fs_dict *)_dict; + struct fs_dict_iterate_context *iter; + + /* these flags are not supported for now */ + i_assert((flags & DICT_ITERATE_FLAG_RECURSE) == 0); + i_assert((flags & (DICT_ITERATE_FLAG_SORT_BY_KEY | + DICT_ITERATE_FLAG_SORT_BY_VALUE)) == 0); + + iter = i_new(struct fs_dict_iterate_context, 1); + iter->ctx.dict = _dict; + iter->paths = p_strarray_dup(default_pool, paths); + iter->flags = flags; + iter->value_pool = pool_alloconly_create("iterate value pool", 128); + iter->fs_iter = fs_iter_init(dict->fs, + fs_dict_get_full_key(dict, paths[0]), 0); + return &iter->ctx; +} + +static bool fs_dict_iterate(struct dict_iterate_context *ctx, + const char **key_r, const char **value_r) +{ + struct fs_dict_iterate_context *iter = + (struct fs_dict_iterate_context *)ctx; + struct fs_dict *dict = (struct fs_dict *)ctx->dict; + const char *path; + int ret; + + *key_r = fs_iter_next(iter->fs_iter); + if (*key_r == NULL) { + if (fs_iter_deinit(&iter->fs_iter) < 0) { + iter->failed = TRUE; + return FALSE; + } + if (iter->paths[++iter->path_idx] == NULL) + return FALSE; + path = fs_dict_get_full_key(dict, iter->paths[iter->path_idx]); + iter->fs_iter = fs_iter_init(dict->fs, path, 0); + return fs_dict_iterate(ctx, key_r, value_r); + } + if ((iter->flags & DICT_ITERATE_FLAG_NO_VALUE) != 0) { + *value_r = NULL; + return TRUE; + } + p_clear(iter->value_pool); + path = t_strconcat(iter->paths[iter->path_idx], *key_r, NULL); + if ((ret = fs_dict_lookup(ctx->dict, iter->value_pool, path, value_r)) < 0) { + /* I/O error */ + iter->failed = TRUE; + return FALSE; + } else if (ret == 0) { + /* file was just deleted, just skip to next one */ + return fs_dict_iterate(ctx, key_r, value_r); + } + return TRUE; +} + +static int fs_dict_iterate_deinit(struct dict_iterate_context *ctx) +{ + struct fs_dict_iterate_context *iter = + (struct fs_dict_iterate_context *)ctx; + int ret; + + if (iter->fs_iter != NULL) { + if (fs_iter_deinit(&iter->fs_iter) < 0) + iter->failed = TRUE; + } + ret = iter->failed ? -1 : 0; + + pool_unref(&iter->value_pool); + i_free(iter->paths); + i_free(iter); + return ret; +} + static struct dict_transaction_context * fs_dict_transaction_init(struct dict *_dict) { @@ -186,9 +274,9 @@ fs_dict_deinit, NULL, fs_dict_lookup, - NULL, - NULL, - NULL, + fs_dict_iterate_init, + fs_dict_iterate, + fs_dict_iterate_deinit, fs_dict_transaction_init, fs_dict_transaction_commit, dict_transaction_memory_rollback, From dovecot at dovecot.org Mon Sep 22 12:58:19 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 22 Sep 2014 12:58:19 +0000 Subject: dovecot-2.2: lib: bsearch - make BINARY_NUMBER_SEARCH more widel... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/12015689471c changeset: 17825:12015689471c user: Phil Carmody date: Mon Sep 22 15:56:31 2014 +0300 description: lib: bsearch - make BINARY_NUMBER_SEARCH more widely usable This template is more widely usable if we do not hard-code into it the method of accessing the value being compared. For the default case we already use, this accessor is just a simple array dereferencing macro. As rewriting with the array access happens in the preprocessor, the code generated is completely unchanged. Expected future use: : #define MY_GETTER(array, index) ((array)[(index)].index_field) : #define MY_BINARY_SEARCH(data, count, value, idx_r) \ : BINARY_NUMERIC_SEARCH(MY_GETTER, data, count, value, idx_r); Signed-off-by: Phil Carmody diffstat: src/lib/bsearch-insert-pos.h | 15 ++++++++++----- 1 files changed, 10 insertions(+), 5 deletions(-) diffs (39 lines): diff -r 979e5a007248 -r 12015689471c src/lib/bsearch-insert-pos.h --- a/src/lib/bsearch-insert-pos.h Thu Sep 18 17:46:18 2014 +0200 +++ b/src/lib/bsearch-insert-pos.h Mon Sep 22 15:56:31 2014 +0300 @@ -1,18 +1,19 @@ #ifndef BSEARCH_INSERT_POS_H #define BSEARCH_INSERT_POS_H -/* Binary search template */ -#define BINARY_NUMBER_SEARCH(data, count, value, idx_r) \ +/* Binary search template - getdata must be the name of a pure function + or a function-like macro that takes the two obvious parameters. */ +#define BINARY_NUMERIC_SEARCH(getdata, data, count, value, idx_r) \ unsigned int idx, left_idx, right_idx; \ \ i_assert((count) < INT_MAX); \ - idx = 0; left_idx = 0; right_idx = (count); \ + left_idx = 0; right_idx = (count); \ while (left_idx < right_idx) { \ idx = (left_idx + right_idx) / 2; \ \ - if ((data)[idx] < (value)) \ + if (getdata(data, idx) < (value)) \ left_idx = idx+1; \ - else if ((data)[idx] > (value)) \ + else if (getdata(data, idx) > (value))\ right_idx = idx; \ else { \ *(idx_r) = idx; \ @@ -21,6 +22,10 @@ } \ return FALSE +#define BINARY_SEARCH_ARRAY_GET(array, index) ((array)[(index)]) +#define BINARY_NUMBER_SEARCH(data, count, value, idx_r) \ + BINARY_NUMERIC_SEARCH(BINARY_SEARCH_ARRAY_GET, data, count, value, idx_r); + /* If key is found, returns TRUE and sets idx_r to the position where the key was found. If key isn't found, returns FALSE and sets idx_r to the position where the key should be inserted. */ From dovecot at dovecot.org Mon Sep 22 13:08:41 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 22 Sep 2014 13:08:41 +0000 Subject: dovecot-2.2: lib-master: Increased instances file's dotlock time... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a25880e35b09 changeset: 17826:a25880e35b09 user: Timo Sirainen date: Mon Sep 22 16:08:25 2014 +0300 description: lib-master: Increased instances file's dotlock timeout to 2 seconds. The 1 second would have been enough, except the current dotlocking code uses second-level precision in checking timeouts, so it could end up trying the lock only once. A better fix would be to fix the dotlocking code to use millisecond-level precision (internally), but since this is the only user so far where it would have mattered, it's not really worth the effort (yet). diffstat: src/lib-master/master-instance.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 12015689471c -r a25880e35b09 src/lib-master/master-instance.c --- a/src/lib-master/master-instance.c Mon Sep 22 15:56:31 2014 +0300 +++ b/src/lib-master/master-instance.c Mon Sep 22 16:08:25 2014 +0300 @@ -29,7 +29,7 @@ }; static const struct dotlock_settings dotlock_set = { - .timeout = 1, + .timeout = 2, .stale_timeout = 60 }; From dovecot at dovecot.org Tue Sep 23 13:59:15 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 23 Sep 2014 13:59:15 +0000 Subject: dovecot-2.2: imap: Don't send MODSEQ in untagged FETCH reply if ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4fc0fbed97f9 changeset: 17827:4fc0fbed97f9 user: Timo Sirainen date: Tue Sep 23 16:58:59 2014 +0300 description: imap: Don't send MODSEQ in untagged FETCH reply if IMAP client didn't enable CONDSTORE/QRESYNC. We should have been checking client->enabled_features, not mailbox_get_enabled_features(). The former contains the features enabled by the IMAP client, while the latter contains also some automatically added features. diffstat: src/imap/imap-sync.c | 3 +-- 1 files changed, 1 insertions(+), 2 deletions(-) diffs (13 lines): diff -r a25880e35b09 -r 4fc0fbed97f9 src/imap/imap-sync.c --- a/src/imap/imap-sync.c Mon Sep 22 16:08:25 2014 +0300 +++ b/src/imap/imap-sync.c Tue Sep 23 16:58:59 2014 +0300 @@ -415,8 +415,7 @@ str_printfa(str, "* %u FETCH (", ctx->seq); if (ctx->imap_flags & IMAP_SYNC_FLAG_SEND_UID) str_printfa(str, "UID %u ", ctx->mail->uid); - if ((mailbox_get_enabled_features(ctx->box) & - MAILBOX_FEATURE_CONDSTORE) != 0 && + if ((ctx->client->enabled_features & MAILBOX_FEATURE_CONDSTORE) != 0 && !ctx->client->nonpermanent_modseqs) { imap_sync_add_modseq(ctx, str); str_append_c(str, ' '); From dovecot at dovecot.org Wed Sep 24 19:54:43 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 24 Sep 2014 19:54:43 +0000 Subject: dovecot-2.2: man: Added doveadm-acl.1. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/07cb8ad8b063 changeset: 17828:07cb8ad8b063 user: Pascal Volk date: Wed Sep 24 19:50:37 2014 +0000 description: man: Added doveadm-acl.1. diffstat: .hgignore | 2 +- doc/man/Makefile.am | 2 + doc/man/doveadm-acl.1.in | 271 +++++++++++++++++++++++++++++++++++++++++++++++ doc/man/doveadm.1.in | 9 +- 4 files changed, 281 insertions(+), 3 deletions(-) diffs (truncated from 328 to 300 lines): diff -r 4fc0fbed97f9 -r 07cb8ad8b063 .hgignore --- a/.hgignore Tue Sep 23 16:58:59 2014 +0300 +++ b/.hgignore Wed Sep 24 19:50:37 2014 +0000 @@ -105,5 +105,5 @@ syntax: regexp src/.*/test-[^\.]*$ -doc/man/doveadm-(altmove|auth|batch|deduplicate|director|dump|exec|expunge|fetch|flags|import|instance|index|force-resync|help|kick|log|mailbox|mount|move|penalty|purge|pw|quota|search|user|who)\.1$ +doc/man/doveadm-(acl|altmove|auth|batch|deduplicate|director|dump|exec|expunge|fetch|flags|import|instance|index|force-resync|help|kick|log|mailbox|mount|move|penalty|purge|pw|quota|search|user|who)\.1$ doc/man/(doveadm|doveconf|dovecot-lda|dovecot|dsync)\.1$ diff -r 4fc0fbed97f9 -r 07cb8ad8b063 doc/man/Makefile.am --- a/doc/man/Makefile.am Tue Sep 23 16:58:59 2014 +0300 +++ b/doc/man/Makefile.am Wed Sep 24 19:50:37 2014 +0000 @@ -14,6 +14,7 @@ nodist_man1_MANS = \ doveadm.1 \ + doveadm-acl.1 \ doveadm-altmove.1 \ doveadm-auth.1 \ doveadm-batch.1 \ @@ -56,6 +57,7 @@ EXTRA_DIST = \ doveadm.1.in \ + doveadm-acl.1.in \ doveadm-altmove.1.in \ doveadm-auth.1.in \ doveadm-batch.1.in \ diff -r 4fc0fbed97f9 -r 07cb8ad8b063 doc/man/doveadm-acl.1.in --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/doc/man/doveadm-acl.1.in Wed Sep 24 19:50:37 2014 +0000 @@ -0,0 +1,271 @@ +.\" Copyright (c) 2014 Dovecot authors, see the included COPYING file +.TH DOVEADM\-ACL 1 "2014-09-24" "Dovecot v2.2" "Dovecot" +.SH NAME +doveadm\-acl \- Manage Access Control List (ACL) +.\"------------------------------------------------------------------------ +.SH SYNOPSIS +.BR doveadm " [" \-Dv ] +[\fB\-f\fP \fIformatter\fP] +.BI acl \ command +.RI [ OPTIONS ]\ [ ARGUMENTS ] +.\"------------------------------------------------------------------------ +.SH DESCRIPTION +The +.B doveadm acl +.I COMMANDS +can be used to execute various Access Control List related actions. +.\"------------------------------------------------------------------------ + at INCLUDE:global-options-formatter@ +.\" --- command specific options --- "/. +.PP +This command uses by default the output formatter +.B flow +(without the +.IR key = +prefix). +.PP +Command specific +.IR options : +.\"------------------------------------- + at INCLUDE:option-A@ +.\"------------------------------------- + at INCLUDE:option-S-socket@ +.\"------------------------------------- + at INCLUDE:option-u-user@ +.\"------------------------------------------------------------------------ +.SH ARGUMENTS +.TP +.I id +The id (identifier) is one of: +.RS +.RS +.TP 4 +* +.BR group\-override =\c +.I group_name +.\"----------------- +.TP +* +.BR user =\c +.I user_name +.\"----------------- +.TP +* +.B owner +.\"----------------- +.TP +* +.BR group =\c +.I group_name +.\"----------------- +.TP +* +.B authenticated +.\"----------------- +.TP +* +.BR anyone " (or " anonymous ", which is an alias for anyone)" +.\"----------------- +.RE +.PP +The ACLs are processed in the precedence given above, so for example if you +have given read\-access to a group, you can still remove that from specific +users inside the group. +.br +Group\-override identifier allows you to override users\(aq ACLs. +Probably the most useful reason to do this is to temporarily disable +access for some users. +For example: +.PP +.nf +user=timo rw +group\-override=tempdisabled +.fi +.PP +Now if timo is a member of the tempdisabled group, he has no access to the +mailbox. +This wouldn\(aqt be possible with a normal group identifier, because the +.B user=timo +would override it. +.RE +.\"------------------------------------- +.TP +.I mailbox +The name of the mailbox, for which the ACL manipulation should be done. +It\(aqs also possible to use the wildcard characters +.RB \(dq * "\(dq and/or \(dq" ? \(dq +in the mailbox name. +.\"------------------------------------- +.TP +.I right +Dovecot ACL right name. This isn\(aqt the same as the IMAP ACL letters, +which aren\(aqt currently supported. +Here is a mapping of the IMAP ACL letters to Dovecot ACL names: +.RS +.RS +.TP 4 +.B l \(-> lookup +.I Mailbox +is visible in mailbox list. +.I Mailbox +can be subscribed to. +.\"----------------- +.TP +.B r \(-> read +.I Mailbox +can be opened for reading. +.\"----------------- +.TP +.B w \(-> write +Message flags and keywords can be changed, except +.BR \(rsSeen " and " \(rsDeleted . +.\"----------------- +.TP +.B s \(-> write\-seen +.B \(rsSeen +flag can be changed. +.\"----------------- +.TP +.B t \(-> write\-deleted +.B \(rsDeleted +flag can be changed. +.\"----------------- +.TP +.B i \(-> insert +Messages can be written or copied to the +.IR mailbox . +.\"----------------- +.TP +.B p \(-> post +Messages can be posted to the +.I mailbox +by +.BR dovecot\-lda , +e.g. from Sieve scripts. +.\"----------------- +.TP +.B e \(-> expunge +Messages can be expunged. +.\"----------------- +.TP +.B k \(-> create +Mailboxes can be created/renamed directly under this +.I mailbox +(but not necessarily under its children, see +.I ACL Inheritance +in the wiki). +.br +Note: Renaming also requires the delete right. +.\"----------------- +.TP +.B x \(-> delete +.I Mailbox +can be deleted. +.\"----------------- +.TP +.B a \(-> admin +Administration rights to the +.I mailbox +(currently: ability to change ACLs for +.IR mailbox ). +.RE +.RE +.\"------------------------------------------------------------------------ +.SH COMMANDS +.SS acl add +.B doveadm acl add +[\fB\-u\fP \fIuser\fP|\fB\-A\fP] +[\fB\-S\fP \fIsocket_path\fP] +.I mailbox id right +.RI [ right " ...]" +.PP +Add ACL rights to the +.IR mailbox / id . +If the +.I id +already exists, the existing rights are preserved. +.\"------------------------------------- +.SS acl debug +.B doveadm acl debug +[\fB\-u\fP \fIuser\fP|\fB\-A\fP] +[\fB\-S\fP \fIsocket_path\fP] +.I mailbox +.PP +This command can be used to debug why a shared mailbox isn\(aqt +accessible to the user. +It will list exactly what the problem is. +.\"------------------------------------- +.SS acl delete +.B doveadm acl delete +[\fB\-u\fP \fIuser\fP|\fB\-A\fP] +[\fB\-S\fP \fIsocket_path\fP] +.I mailbox id +.PP +Remove the whole ACL entry for the +.IR mailbox / id . +.\"------------------------------------- +.SS acl get +.B doveadm acl get +[\fB\-u\fP \fIuser\fP|\fB\-A\fP] +[\fB\-S\fP \fIsocket_path\fP] +.RB [ \-m ] +.I mailbox +.PP +Show all the ACLs for the +.IR mailbox . +.\"------------------------------------- +.SS acl recalc +.B doveadm acl recalc +[\fB\-u\fP \fIuser\fP|\fB\-A\fP] +[\fB\-S\fP \fIsocket_path\fP] +.PP +Make sure the +.IR user \(aqs +shared mailboxes exist correctly in the +.IR acl_shared_dict . +.\"------------------------------------- +.SS acl remove +.B doveadm acl remove +[\fB\-u\fP \fIuser\fP|\fB\-A\fP] +[\fB\-S\fP \fIsocket_path\fP] +.I mailbox id right +.RI [ right " ...]" +.PP +Remove the specified ACL rights from the +.IR mailbox / id . +If all rights are removed, the entry still exists without any rights. +.\"------------------------------------- +.SS acl rights +.B doveadm acl rights +[\fB\-u\fP \fIuser\fP|\fB\-A\fP] +[\fB\-S\fP \fIsocket_path\fP] +.I mailbox +.PP +Show the +.IR user \(aqs +current ACL rights for the +.IR mailbox . +.\"------------------------------------- +.SS acl set +.B doveadm acl set +[\fB\-u\fP \fIuser\fP|\fB\-A\fP] +[\fB\-S\fP \fIsocket_path\fP] +.I mailbox id right +.RI [ right " ...]" +.PP +Set ACL rights to the +.IR mailbox / id . +If the +.I id +already exists, the existing rights are replaced. +.\"------------------------------------------------------------------------ + at INCLUDE:reporting-bugs@ +.\"------------------------------------------------------------------------ +.SH SEE ALSO +.BR doveadm (1), +.BR dovecot\-lda (1) +.\"------------------------------------- From dovecot at dovecot.org Wed Sep 24 20:03:35 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 24 Sep 2014 20:03:35 +0000 Subject: dovecot-2.2: man: Added doveadm-fts.1. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e1cc249ad417 changeset: 17829:e1cc249ad417 user: Pascal Volk date: Wed Sep 24 20:01:46 2014 +0000 description: man: Added doveadm-fts.1. diffstat: .hgignore | 2 +- doc/man/Makefile.am | 2 + doc/man/doveadm-fts.1.in | 76 ++++++++++++++++++++++++++++++++++++++++++++++++ doc/man/doveadm.1.in | 5 +++ 4 files changed, 84 insertions(+), 1 deletions(-) diffs (125 lines): diff -r 07cb8ad8b063 -r e1cc249ad417 .hgignore --- a/.hgignore Wed Sep 24 19:50:37 2014 +0000 +++ b/.hgignore Wed Sep 24 20:01:46 2014 +0000 @@ -105,5 +105,5 @@ syntax: regexp src/.*/test-[^\.]*$ -doc/man/doveadm-(acl|altmove|auth|batch|deduplicate|director|dump|exec|expunge|fetch|flags|import|instance|index|force-resync|help|kick|log|mailbox|mount|move|penalty|purge|pw|quota|search|user|who)\.1$ +doc/man/doveadm-(acl|altmove|auth|batch|deduplicate|director|dump|exec|expunge|fetch|flags|fts|import|instance|index|force-resync|help|kick|log|mailbox|mount|move|penalty|purge|pw|quota|search|user|who)\.1$ doc/man/(doveadm|doveconf|dovecot-lda|dovecot|dsync)\.1$ diff -r 07cb8ad8b063 -r e1cc249ad417 doc/man/Makefile.am --- a/doc/man/Makefile.am Wed Sep 24 19:50:37 2014 +0000 +++ b/doc/man/Makefile.am Wed Sep 24 20:01:46 2014 +0000 @@ -25,6 +25,7 @@ doveadm-expunge.1 \ doveadm-fetch.1 \ doveadm-flags.1 \ + doveadm-fts.1 \ doveadm-import.1 \ doveadm-instance.1 \ doveadm-index.1 \ @@ -68,6 +69,7 @@ doveadm-expunge.1.in \ doveadm-fetch.1.in \ doveadm-flags.1.in \ + doveadm-fts.1.in \ doveadm-import.1.in \ doveadm-instance.1.in \ doveadm-index.1.in \ diff -r 07cb8ad8b063 -r e1cc249ad417 doc/man/doveadm-fts.1.in --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/doc/man/doveadm-fts.1.in Wed Sep 24 20:01:46 2014 +0000 @@ -0,0 +1,76 @@ +.\" Copyright (c) 2014 Dovecot authors, see the included COPYING file +.TH DOVEADM\-FTS 1 "2014-09-24" "Dovecot v2.2" "Dovecot" +.SH NAME +doveadm\-fts \- Manipulate the Full Text Search (FTS) index +.\"------------------------------------------------------------------------ +.SH SYNOPSIS +.BR doveadm " [" \-Dv ] +.\" [\fB\-f\fP \fIformatter\fP] +.BI fts \ command +.RI [ OPTIONS ]\ [ ARGUMENTS ] +.\"------------------------------------------------------------------------ +.SH DESCRIPTION +The +doveadm fts +.I COMMANDS +can be used to manipulate the Full Text Search (FTS) index. +.\"------------------------------------------------------------------------ +.\" @INCLUDE:global-options-formatter@ + at INCLUDE:global-options@ +.\" --- command specific options --- "/. +.PP +This command uses by default the output formatter +.B flow +(without the +.IR key = +prefix). +.PP +Command specific +.IR options : +.\"------------------------------------- + at INCLUDE:option-A@ +.\"------------------------------------- + at INCLUDE:option-S-socket@ +.\"------------------------------------- + at INCLUDE:option-u-user@ +.\"------------------------------------------------------------------------ +.SH ARGUMENTS +.TP +.I namespace +The name of a namespace, e.g. the name of the shared namespace. +When no namespace was given, the user\(aqs private namespace will be used. +.\"------------------------------------------------------------------------ +.SH COMMANDS +.SS fts optimize +.B doveadm fts optimize +[\fB\-u\fP \fIuser\fP|\fB\-A\fP] +[\fB\-S\fP \fIsocket_path\fP] +.RI [ namespace ] +.PP +Optimize the full text search index. +This is also done automatically by the full text search engines, but this +enforces it to happen. +.\"------------------------------------- +.SS fts rescan +.B doveadm fts rescan +[\fB\-u\fP \fIuser\fP|\fB\-A\fP] +[\fB\-S\fP \fIsocket_path\fP] +.RI [ namespace ] +.PP +Scan what mails exist in the full text search index and compare those to +what actually exist in mailboxes. +This removes mails from the index that have already been expunged and +makes sure that the next +.B doveadm index +will index all the missing mails (if any). +.\"------------------------------------------------------------------------ + at INCLUDE:reporting-bugs@ +.\"------------------------------------------------------------------------ +.SH SEE ALSO +.BR doveadm (1), +.BR doveadm\-index (1) +.\"------------------------------------- +.PP +Additional resources: +.IP "Full text search indexing" +http://wiki2.dovecot.org/Plugins/FTS \ No newline at end of file diff -r 07cb8ad8b063 -r e1cc249ad417 doc/man/doveadm.1.in --- a/doc/man/doveadm.1.in Wed Sep 24 19:50:37 2014 +0000 +++ b/doc/man/doveadm.1.in Wed Sep 24 20:01:46 2014 +0000 @@ -142,6 +142,11 @@ Add, remove or replace messages\(aq flags. .\"------------------------------------- .TP +.B doveadm fts +.BR doveadm\-fts (1), +Manipulate the Full Text Search (FTS) index. +.\"------------------------------------- +.TP .B doveadm force\-resync .BR doveadm\-force\-resync (1), Repair broken mailboxes, in case Dovecot doesn\(aqt automatically do that. From dovecot at dovecot.org Wed Sep 24 21:44:06 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 24 Sep 2014 21:44:06 +0000 Subject: dovecot-2.2: imapc: If connection isn't in selected state when d... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d467f4c20abb changeset: 17830:d467f4c20abb user: Timo Sirainen date: Thu Sep 25 00:40:48 2014 +0300 description: imapc: If connection isn't in selected state when deleting mailbox, don't try to UNSELECT it. diffstat: src/lib-imap-client/imapc-client.h | 2 ++ src/lib-imap-client/imapc-connection.c | 14 ++++++++++++++ src/lib-storage/index/imapc/imapc-list.c | 16 ++++++++++------ 3 files changed, 26 insertions(+), 6 deletions(-) diffs (74 lines): diff -r e1cc249ad417 -r d467f4c20abb src/lib-imap-client/imapc-client.h --- a/src/lib-imap-client/imapc-client.h Wed Sep 24 20:01:46 2014 +0000 +++ b/src/lib-imap-client/imapc-client.h Thu Sep 25 00:40:48 2014 +0300 @@ -146,11 +146,13 @@ imapc_command_callback_t *callback, void *context); void imapc_command_set_flags(struct imapc_command *cmd, enum imapc_command_flags flags); +bool imapc_command_connection_is_selected(struct imapc_command *cmd); void imapc_command_send(struct imapc_command *cmd, const char *cmd_str); void imapc_command_sendf(struct imapc_command *cmd, const char *cmd_fmt, ...) ATTR_FORMAT(2, 3); void imapc_command_sendvf(struct imapc_command *cmd, const char *cmd_fmt, va_list args) ATTR_FORMAT(2, 0); +void imapc_command_abort(struct imapc_command **cmd); void imapc_client_register_untagged(struct imapc_client *client, imapc_untagged_callback_t *callback, diff -r e1cc249ad417 -r d467f4c20abb src/lib-imap-client/imapc-connection.c --- a/src/lib-imap-client/imapc-connection.c Wed Sep 24 20:01:46 2014 +0000 +++ b/src/lib-imap-client/imapc-connection.c Thu Sep 25 00:40:48 2014 +0300 @@ -1481,6 +1481,14 @@ pool_unref(&cmd->pool); } +void imapc_command_abort(struct imapc_command **_cmd) +{ + struct imapc_command *cmd = *_cmd; + + *_cmd = NULL; + imapc_command_free(cmd); +} + static void imapc_command_timeout(struct imapc_connection *conn) { struct imapc_command *const *cmds; @@ -1780,6 +1788,12 @@ box->pending_box_command_count++; } +bool imapc_command_connection_is_selected(struct imapc_command *cmd) +{ + return cmd->conn->selected_box != NULL || + cmd->conn->selecting_box != NULL; +} + void imapc_command_send(struct imapc_command *cmd, const char *cmd_str) { unsigned int len = strlen(cmd_str); diff -r e1cc249ad417 -r d467f4c20abb src/lib-storage/index/imapc/imapc-list.c --- a/src/lib-storage/index/imapc/imapc-list.c Wed Sep 24 20:01:46 2014 +0000 +++ b/src/lib-storage/index/imapc/imapc-list.c Thu Sep 25 00:40:48 2014 +0300 @@ -800,12 +800,16 @@ capa = imapc_client_get_capabilities(list->client->client); cmd = imapc_list_simple_context_init(&ctx, list); - imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_SELECT); - if ((capa & IMAPC_CAPABILITY_UNSELECT) != 0) - imapc_command_sendf(cmd, "UNSELECT"); - else - imapc_command_sendf(cmd, "SELECT \"~~~\""); - imapc_simple_run(&ctx); + if (!imapc_command_connection_is_selected(cmd)) + imapc_command_abort(&cmd); + else { + imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_SELECT); + if ((capa & IMAPC_CAPABILITY_UNSELECT) != 0) + imapc_command_sendf(cmd, "UNSELECT"); + else + imapc_command_sendf(cmd, "SELECT \"~~~\""); + imapc_simple_run(&ctx); + } cmd = imapc_list_simple_context_init(&ctx, list); imapc_command_sendf(cmd, "DELETE %s", name); From dovecot at dovecot.org Wed Sep 24 21:44:06 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 24 Sep 2014 21:44:06 +0000 Subject: dovecot-2.2: imapc: Fixed potential crash when trying to send NO... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/47ffe55be1ed changeset: 17831:47ffe55be1ed user: Timo Sirainen date: Thu Sep 25 00:41:15 2014 +0300 description: imapc: Fixed potential crash when trying to send NOOP before mailbox was fully opened. diffstat: src/lib-storage/index/imapc/imapc-storage.c | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diffs (15 lines): diff -r d467f4c20abb -r 47ffe55be1ed src/lib-storage/index/imapc/imapc-storage.c --- a/src/lib-storage/index/imapc/imapc-storage.c Thu Sep 25 00:40:48 2014 +0300 +++ b/src/lib-storage/index/imapc/imapc-storage.c Thu Sep 25 00:41:15 2014 +0300 @@ -150,6 +150,11 @@ struct imapc_command *cmd; struct imapc_simple_context sctx; + if (mbox->client_box == NULL) { + /* mailbox opening hasn't finished yet */ + return; + } + imapc_simple_context_init(&sctx, mbox->storage->client); cmd = imapc_client_mailbox_cmd(mbox->client_box, imapc_simple_callback, &sctx); From dovecot at dovecot.org Wed Sep 24 21:44:06 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 24 Sep 2014 21:44:06 +0000 Subject: dovecot-2.2: lib-storage: Headers were sometimes added to doveco... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6d4e7ccb24ea changeset: 17832:6d4e7ccb24ea user: Timo Sirainen date: Thu Sep 25 00:42:56 2014 +0300 description: lib-storage: Headers were sometimes added to dovecot.index.cache with wrong line number value. This happened if the header parsing was done multiple times for the same mail. The end result being that when fetching multiple headers as a stream, the headers could have been returned in the wrong order. This messed up at least dsync's header hash comparisons. diffstat: src/lib-storage/index/index-mail-headers.c | 6 ++++++ src/lib-storage/index/index-mail.h | 1 + 2 files changed, 7 insertions(+), 0 deletions(-) diffs (41 lines): diff -r 47ffe55be1ed -r 6d4e7ccb24ea src/lib-storage/index/index-mail-headers.c --- a/src/lib-storage/index/index-mail-headers.c Thu Sep 25 00:41:15 2014 +0300 +++ b/src/lib-storage/index/index-mail-headers.c Thu Sep 25 00:42:56 2014 +0300 @@ -130,6 +130,7 @@ } mail->data.dont_cache_field_idx = UINT_MAX; + mail->data.header_parser_initialized = FALSE; } static unsigned int @@ -247,6 +248,9 @@ array_idx_set(&mail->header_match, field_idx, &mail->header_match_value); } + mail->data.header_parser_initialized = TRUE; + mail->data.parse_line_num = 0; + memset(&mail->data.parse_line, 0, sizeof(mail->data.parse_line)); } static void index_mail_parse_finish_imap_envelope(struct index_mail *mail) @@ -276,6 +280,8 @@ unsigned int field_idx, count; uint8_t *match; + i_assert(data->header_parser_initialized); + data->parse_line_num++; if (data->save_bodystructure_header) { diff -r 47ffe55be1ed -r 6d4e7ccb24ea src/lib-storage/index/index-mail.h --- a/src/lib-storage/index/index-mail.h Thu Sep 25 00:41:15 2014 +0300 +++ b/src/lib-storage/index/index-mail.h Thu Sep 25 00:42:56 2014 +0300 @@ -126,6 +126,7 @@ unsigned int initialized_wrapper_stream:1; unsigned int destroy_callback_set:1; unsigned int prefetch_sent:1; + unsigned int header_parser_initialized:1; }; struct index_mail { From dovecot at dovecot.org Wed Sep 24 22:13:30 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 24 Sep 2014 22:13:30 +0000 Subject: dovecot-2.2: lib-storage: Added mailbox_get_index_path() and cac... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/936435377584 changeset: 17833:936435377584 user: Timo Sirainen date: Thu Sep 25 01:13:12 2014 +0300 description: lib-storage: Added mailbox_get_index_path() and caching of the index path. This solves some potential assert-crashes on race conditions when opening a mailbox while it's being deleted. diffstat: src/lib-storage/index/index-rebuild.c | 4 +--- src/lib-storage/mail-storage-private.h | 5 +++++ src/lib-storage/mail-storage.c | 17 +++++++++++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) diffs (77 lines): diff -r 6d4e7ccb24ea -r 936435377584 src/lib-storage/index/index-rebuild.c --- a/src/lib-storage/index/index-rebuild.c Thu Sep 25 00:42:56 2014 +0300 +++ b/src/lib-storage/index/index-rebuild.c Thu Sep 25 01:13:12 2014 +0300 @@ -167,9 +167,7 @@ mail_cache_reset(box->cache); /* if backup index file exists, try to use it */ - if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, - &index_dir) <= 0) - i_unreached(); + index_dir = mailbox_get_index_path(box); backup_path = t_strconcat(box->index_prefix, ".backup", NULL); ctx->backup_index = mail_index_alloc(index_dir, backup_path); diff -r 6d4e7ccb24ea -r 936435377584 src/lib-storage/mail-storage-private.h --- a/src/lib-storage/mail-storage-private.h Thu Sep 25 00:42:56 2014 +0300 +++ b/src/lib-storage/mail-storage-private.h Thu Sep 25 01:13:12 2014 +0300 @@ -291,6 +291,9 @@ /* Filled lazily when mailbox is opened, use mailbox_get_path() to access it */ const char *_path; + /* Filled lazily when mailbox is opened, use mailbox_get_index_path() + to access it */ + const char *_index_path; /* default vfuncs for new struct mails. */ const struct mail_vfuncs *mail_vfuncs; @@ -646,6 +649,8 @@ The mailbox must already be opened and the caller must know that the storage has mailbox files (i.e. NULL/empty path is never returned). */ const char *mailbox_get_path(struct mailbox *box) ATTR_PURE; +/* Similar to mailbox_get_path() but for MAILBOX_LIST_PATH_TYPE_INDEX. */ +const char *mailbox_get_index_path(struct mailbox *box) ATTR_PURE; /* Wrapper to mailbox_list_get_path() */ int mailbox_get_path_to(struct mailbox *box, enum mailbox_list_path_type type, const char **path_r); diff -r 6d4e7ccb24ea -r 936435377584 src/lib-storage/mail-storage.c --- a/src/lib-storage/mail-storage.c Thu Sep 25 00:42:56 2014 +0300 +++ b/src/lib-storage/mail-storage.c Thu Sep 25 01:13:12 2014 +0300 @@ -2245,6 +2245,14 @@ *path_r = box->_path; return 1; } + if (type == MAILBOX_LIST_PATH_TYPE_INDEX && box->_index_path != NULL) { + if (box->_index_path[0] == '\0') { + *path_r = NULL; + return 0; + } + *path_r = box->_index_path; + return 1; + } ret = mailbox_list_get_path(box->list, box->name, type, path_r); if (ret < 0) { mail_storage_copy_list_error(box->storage, box->list); @@ -2252,6 +2260,8 @@ } if (type == MAILBOX_LIST_PATH_TYPE_MAILBOX && box->_path == NULL) box->_path = ret == 0 ? "" : p_strdup(box->pool, *path_r); + if (type == MAILBOX_LIST_PATH_TYPE_INDEX && box->_index_path == NULL) + box->_index_path = ret == 0 ? "" : p_strdup(box->pool, *path_r); return ret; } @@ -2262,6 +2272,13 @@ return box->_path; } +const char *mailbox_get_index_path(struct mailbox *box) +{ + i_assert(box->_index_path != NULL); + i_assert(box->_index_path[0] != '\0'); + return box->_index_path; +} + static void mailbox_get_permissions_if_not_set(struct mailbox *box) { if (box->_perm.file_create_mode != 0) From dovecot at dovecot.org Wed Sep 24 23:55:52 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 24 Sep 2014 23:55:52 +0000 Subject: dovecot-2.2: fs-posix: fs_read_stream() now returns an istream t... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/65963be31ceb changeset: 17834:65963be31ceb user: Timo Sirainen date: Thu Sep 25 02:55:34 2014 +0300 description: fs-posix: fs_read_stream() now returns an istream that can live even after the fs_file is closed. diffstat: src/lib-fs/fs-posix.c | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-) diffs (18 lines): diff -r 936435377584 -r 65963be31ceb src/lib-fs/fs-posix.c --- a/src/lib-fs/fs-posix.c Thu Sep 25 01:13:12 2014 +0300 +++ b/src/lib-fs/fs-posix.c Thu Sep 25 02:55:34 2014 +0300 @@ -350,10 +350,11 @@ if (file->fd == -1 && fs_posix_open(file) < 0) input = i_stream_create_error_str(errno, "%s", fs_last_error(_file->fs)); - else - input = i_stream_create_fd(file->fd, max_buffer_size, FALSE); + else { + /* the stream could live even after the fs_file */ + input = i_stream_create_fd_autoclose(&file->fd, max_buffer_size); + } i_stream_set_name(input, _file->path); - i_stream_add_destroy_callback(input, fs_posix_file_close, _file); return input; } From dovecot at dovecot.org Thu Sep 25 10:29:03 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 25 Sep 2014 10:29:03 +0000 Subject: dovecot-2.2: lib: Added file_wait_lock_error() and file_try_lock... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/87f10e2fac95 changeset: 17835:87f10e2fac95 user: Timo Sirainen date: Thu Sep 25 13:26:20 2014 +0300 description: lib: Added file_wait_lock_error() and file_try_lock_error() diffstat: src/lib/file-lock.c | 84 +++++++++++++++++++++++++++++++++++++++------------- src/lib/file-lock.h | 11 ++++++ 2 files changed, 74 insertions(+), 21 deletions(-) diffs (193 lines): diff -r 65963be31ceb -r 87f10e2fac95 src/lib/file-lock.c --- a/src/lib/file-lock.c Thu Sep 25 02:55:34 2014 +0300 +++ b/src/lib/file-lock.c Thu Sep 25 13:26:20 2014 +0300 @@ -48,10 +48,19 @@ return file_wait_lock(fd, path, lock_type, lock_method, 0, lock_r); } +int file_try_lock_error(int fd, const char *path, int lock_type, + enum file_lock_method lock_method, + struct file_lock **lock_r, const char **error_r) +{ + return file_wait_lock_error(fd, path, lock_type, lock_method, 0, + lock_r, error_r); +} + static int file_lock_do(int fd, const char *path, int lock_type, enum file_lock_method lock_method, - unsigned int timeout_secs) + unsigned int timeout_secs, const char **error_r) { + const char *lock_type_str; int ret; i_assert(fd != -1); @@ -59,13 +68,17 @@ if (timeout_secs != 0) alarm(timeout_secs); + lock_type_str = lock_type == F_UNLCK ? "unlock" : + (lock_type == F_RDLCK ? "read-lock" : "write-lock"); + switch (lock_method) { case FILE_LOCK_METHOD_FCNTL: { #ifndef HAVE_FCNTL - i_fatal("fcntl() locks not supported"); + *error_r = t_strdup_printf( + "Can't lock file %s: fcntl() locks not supported", path); + return -1; #else struct flock fl; - const char *errstr; fl.l_type = lock_type; fl.l_whence = SEEK_SET; @@ -81,6 +94,9 @@ if (timeout_secs == 0 && (errno == EACCES || errno == EAGAIN)) { /* locked by another process */ + *error_r = t_strdup_printf( + "fcntl(%s, %s, F_SETLK) locking failed: %m " + "(File is already locked)", path, lock_type_str); return 0; } @@ -89,20 +105,22 @@ even if not, we probably want to be killed so stop blocking. */ errno = EAGAIN; + *error_r = t_strdup_printf( + "fcntl(%s, %s, F_SETLKW) locking failed: " + "Timed out after %u seconds", + path, lock_type_str, timeout_secs); return 0; } - errstr = errno != EACCES ? strerror(errno) : - "File is locked by another process (EACCES)"; - i_error("fcntl(%s) locking failed for file %s: %s", - lock_type == F_UNLCK ? "unlock" : - lock_type == F_RDLCK ? "read-lock" : "write-lock", - path, errstr); + *error_r = t_strdup_printf("fcntl(%s, %s, %s) locking failed: %m", + path, lock_type_str, timeout_secs == 0 ? "F_SETLK" : "F_SETLKW"); return -1; #endif } case FILE_LOCK_METHOD_FLOCK: { #ifndef HAVE_FLOCK - i_fatal("flock() locks not supported"); + *error_r = t_strdup_printf( + "Can't lock file %s: flock() not supported", path); + return -1; #else int operation = timeout_secs != 0 ? 0 : LOCK_NB; @@ -124,16 +142,22 @@ if (ret == 0) break; - if (errno == EWOULDBLOCK || errno == EINTR) { - /* a) locked by another process, - b) timeouted */ - errno = EAGAIN; + if (timeout_secs == 0 && errno == EWOULDBLOCK) { + /* locked by another process */ + *error_r = t_strdup_printf( + "flock(%s, %s) failed: %m " + "(File is already locked)", path, lock_type_str); return 0; } - i_error("flock(%s) locking failed for file %s: %m", - lock_type == F_UNLCK ? "unlock" : - lock_type == F_RDLCK ? "read-lock" : "write-lock", - path); + if (errno == EINTR) { + errno = EAGAIN; + *error_r = t_strdup_printf("flock(%s, %s) failed: " + "Timed out after %u seconds", + path, lock_type_str, timeout_secs); + return 0; + } + *error_r = t_strdup_printf("flock(%s, %s) failed: %m", + path, lock_type_str); return -1; #endif } @@ -150,10 +174,25 @@ unsigned int timeout_secs, struct file_lock **lock_r) { + const char *error; + int ret; + + ret = file_wait_lock_error(fd, path, lock_type, lock_method, + timeout_secs, lock_r, &error); + if (ret < 0) + i_error("%s", error); + return ret; +} + +int file_wait_lock_error(int fd, const char *path, int lock_type, + enum file_lock_method lock_method, + unsigned int timeout_secs, + struct file_lock **lock_r, const char **error_r) +{ struct file_lock *lock; int ret; - ret = file_lock_do(fd, path, lock_type, lock_method, timeout_secs); + ret = file_lock_do(fd, path, lock_type, lock_method, timeout_secs, error_r); if (ret <= 0) return ret; @@ -168,18 +207,21 @@ int file_lock_try_update(struct file_lock *lock, int lock_type) { + const char *error; + return file_lock_do(lock->fd, lock->path, lock_type, - lock->lock_method, 0); + lock->lock_method, 0, &error); } void file_unlock(struct file_lock **_lock) { struct file_lock *lock = *_lock; + const char *error; *_lock = NULL; if (file_lock_do(lock->fd, lock->path, F_UNLCK, - lock->lock_method, 0) == 0) { + lock->lock_method, 0, &error) == 0) { /* this shouldn't happen */ i_error("file_unlock(%s) failed: %m", lock->path); } diff -r 65963be31ceb -r 87f10e2fac95 src/lib/file-lock.h --- a/src/lib/file-lock.h Thu Sep 25 02:55:34 2014 +0300 +++ b/src/lib/file-lock.h Thu Sep 25 13:26:20 2014 +0300 @@ -25,12 +25,23 @@ int file_try_lock(int fd, const char *path, int lock_type, enum file_lock_method lock_method, struct file_lock **lock_r); +/* Like file_try_lock(), but return the error message as a string instead + of logging it. Also when returning 0 an error message is returned. */ +int file_try_lock_error(int fd, const char *path, int lock_type, + enum file_lock_method lock_method, + struct file_lock **lock_r, const char **error_r); /* Like lock_try_lock(), but return 0 only after having tried to lock for timeout_secs. */ int file_wait_lock(int fd, const char *path, int lock_type, enum file_lock_method lock_method, unsigned int timeout_secs, struct file_lock **lock_r); +/* Like file_wait_lock(), but return the error message as a string instead + of logging it. Also when returning 0 an error message is returned. */ +int file_wait_lock_error(int fd, const char *path, int lock_type, + enum file_lock_method lock_method, + unsigned int timeout_secs, + struct file_lock **lock_r, const char **error_r); /* Change the lock type. */ int file_lock_try_update(struct file_lock *lock, int lock_type); From dovecot at dovecot.org Thu Sep 25 10:29:03 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 25 Sep 2014 10:29:03 +0000 Subject: dovecot-2.2: Avoid logging warnings about increasing memory pool... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/90891ca867f3 changeset: 17836:90891ca867f3 user: Timo Sirainen date: Thu Sep 25 13:28:39 2014 +0300 description: Avoid logging warnings about increasing memory pool/data stack with DEBUG on. These could have a minor effect on memory usage, but shouldn't be much. diffstat: src/config/config-filter.c | 8 ++++---- src/config/config-parser.c | 22 ++++++++++++++++++---- src/config/config-request.c | 12 +++++++----- src/config/doveconf.c | 2 +- src/doveadm/dsync/dsync-brain-mailbox.c | 2 +- src/doveadm/dsync/dsync-ibc-pipe.c | 2 +- src/lib-master/master-instance.c | 2 +- src/lib-master/master-service-settings.c | 2 +- src/lib-settings/settings-parser.c | 4 ++-- src/lib-storage/mail-storage-service.c | 2 +- src/lib-storage/mail-user.c | 2 +- src/master/service.c | 2 +- 12 files changed, 39 insertions(+), 23 deletions(-) diffs (240 lines): diff -r 87f10e2fac95 -r 90891ca867f3 src/config/config-filter.c --- a/src/config/config-filter.c Thu Sep 25 13:26:20 2014 +0300 +++ b/src/config/config-filter.c Thu Sep 25 13:28:39 2014 +0300 @@ -185,7 +185,7 @@ } static struct config_filter_parser *const * -config_filter_find_all(struct config_filter_context *ctx, +config_filter_find_all(struct config_filter_context *ctx, pool_t pool, const char *const *modules, const struct config_filter *filter, struct master_service_settings_output *output_r) @@ -196,8 +196,8 @@ memset(output_r, 0, sizeof(*output_r)); - t_array_init(&matches, 8); - t_array_init(&service_names, 8); + p_array_init(&matches, pool, 8); + p_array_init(&service_names, pool, 8); for (i = 0; ctx->parsers[i] != NULL; i++) { const struct config_filter *mask = &ctx->parsers[i]->filter; @@ -322,7 +322,7 @@ with an error. Merging SET_STRLIST types requires settings_parser_apply_changes() to work a bit unintuitively by letting the destination settings override the source settings. */ - src = config_filter_find_all(ctx, modules, filter, output_r); + src = config_filter_find_all(ctx, pool, modules, filter, output_r); /* all of them should have the same number of parsers. duplicate our initial parsers from the first match */ diff -r 87f10e2fac95 -r 90891ca867f3 src/config/config-parser.c --- a/src/config/config-parser.c Thu Sep 25 13:26:20 2014 +0300 +++ b/src/config/config-parser.c Thu Sep 25 13:28:39 2014 +0300 @@ -336,6 +336,10 @@ const struct config_module_parser *p, const char **error_r) { + const char *error; + char *error_dup = NULL; + bool ok; + for (; p->root != NULL; p++) { /* skip checking settings we don't care about */ if (!config_module_want_parser(ctx->root_parsers, @@ -343,8 +347,17 @@ continue; settings_parse_var_skip(p->parser); - if (!settings_parser_check(p->parser, ctx->pool, error_r)) + T_BEGIN { + ok = settings_parser_check(p->parser, ctx->pool, &error); + if (!ok) + error_dup = i_strdup(error); + } T_END; + if (!ok) { + i_assert(error_dup != NULL); + *error_r = t_strdup(error_dup); + i_free(error_dup); return -1; + } } return 0; } @@ -392,7 +405,7 @@ return -1; } - tmp_pool = pool_alloconly_create("config parsers check", 1024*64); + tmp_pool = pool_alloconly_create(MEMPOOL_GROWING"config parsers check", 1024*64); parsers = array_get(&ctx->all_parsers, &count); i_assert(count > 0 && parsers[count-1] == NULL); count--; @@ -909,7 +922,7 @@ } memset(&ctx, 0, sizeof(ctx)); - ctx.pool = pool_alloconly_create("config file parser", 1024*256); + ctx.pool = pool_alloconly_create(MEMPOOL_GROWING"config file parser", 1024*256); ctx.path = path; ctx.hide_errors = fd == -1; @@ -940,8 +953,9 @@ i_stream_create_from_data("", 0); i_stream_set_return_partial_line(ctx.cur_input->input, TRUE); old_settings_init(&ctx); - if (hook_config_parser_begin != NULL) + if (hook_config_parser_begin != NULL) T_BEGIN { hook_config_parser_begin(&ctx); + } T_END; prevfile: while ((line = i_stream_read_next_line(ctx.cur_input->input)) != NULL) { diff -r 87f10e2fac95 -r 90891ca867f3 src/config/config-request.c --- a/src/config/config-request.c Thu Sep 25 13:26:20 2014 +0300 +++ b/src/config/config-request.c Thu Sep 25 13:28:39 2014 +0300 @@ -357,8 +357,8 @@ ctx->callback = callback; ctx->context = context; ctx->scope = scope; - ctx->value = t_str_new(256); - ctx->prefix = t_str_new(64); + ctx->value = str_new(pool, 256); + ctx->prefix = str_new(pool, 64); hash_table_create(&ctx->keys, ctx->pool, 0, str_hash, strcmp); return ctx; } @@ -419,9 +419,11 @@ ctx->modules, parser->root)) continue; - settings_export(ctx, parser->root, FALSE, - settings_parser_get(parser->parser), - settings_parser_get_changes(parser->parser)); + T_BEGIN { + settings_export(ctx, parser->root, FALSE, + settings_parser_get(parser->parser), + settings_parser_get_changes(parser->parser)); + } T_END; if ((ctx->flags & CONFIG_DUMP_FLAG_CHECK_SETTINGS) != 0) { settings_parse_var_skip(parser->parser); diff -r 87f10e2fac95 -r 90891ca867f3 src/config/doveconf.c --- a/src/config/doveconf.c Thu Sep 25 13:26:20 2014 +0300 +++ b/src/config/doveconf.c Thu Sep 25 13:28:39 2014 +0300 @@ -129,7 +129,7 @@ enum config_dump_flags flags; pool_t pool; - pool = pool_alloconly_create("config human strings", 1024*32); + pool = pool_alloconly_create(MEMPOOL_GROWING"config human strings", 1024*32); ctx = p_new(pool, struct config_dump_human_context, 1); ctx->pool = pool; ctx->list_prefix = str_new(ctx->pool, 128); diff -r 87f10e2fac95 -r 90891ca867f3 src/doveadm/dsync/dsync-brain-mailbox.c --- a/src/doveadm/dsync/dsync-brain-mailbox.c Thu Sep 25 13:26:20 2014 +0300 +++ b/src/doveadm/dsync/dsync-brain-mailbox.c Thu Sep 25 13:28:39 2014 +0300 @@ -144,7 +144,7 @@ p_clear(brain->dsync_box_pool); else { brain->dsync_box_pool = - pool_alloconly_create("dsync brain box pool", 1024); + pool_alloconly_create(MEMPOOL_GROWING"dsync brain box pool", 2048); } dsync_mailbox_cache_field_dup(&brain->local_dsync_box.cache_fields, &local_dsync_box->cache_fields, diff -r 87f10e2fac95 -r 90891ca867f3 src/doveadm/dsync/dsync-ibc-pipe.c --- a/src/doveadm/dsync/dsync-ibc-pipe.c Thu Sep 25 13:26:20 2014 +0300 +++ b/src/doveadm/dsync/dsync-ibc-pipe.c Thu Sep 25 13:28:39 2014 +0300 @@ -62,7 +62,7 @@ pools = array_get_modifiable(&pipe->pools, &count); if (count == 0) - return pool_alloconly_create("pipe item pool", 1024); + return pool_alloconly_create(MEMPOOL_GROWING"pipe item pool", 1024); ret = pools[count-1]; array_delete(&pipe->pools, count-1, 1); diff -r 87f10e2fac95 -r 90891ca867f3 src/lib-master/master-instance.c --- a/src/lib-master/master-instance.c Thu Sep 25 13:26:20 2014 +0300 +++ b/src/lib-master/master-instance.c Thu Sep 25 13:28:39 2014 +0300 @@ -38,7 +38,7 @@ struct master_instance_list *list; pool_t pool; - pool = pool_alloconly_create("master instances", 256); + pool = pool_alloconly_create(MEMPOOL_GROWING"master instances", 256); list = p_new(pool, struct master_instance_list, 1); list->pool = pool; list->path = p_strdup(pool, path); diff -r 87f10e2fac95 -r 90891ca867f3 src/lib-master/master-service-settings.c --- a/src/lib-master/master-service-settings.c Thu Sep 25 13:26:20 2014 +0300 +++ b/src/lib-master/master-service-settings.c Thu Sep 25 13:28:39 2014 +0300 @@ -413,7 +413,7 @@ p_clear(service->set_pool); } else { service->set_pool = - pool_alloconly_create("master service settings", 8192); + pool_alloconly_create("master service settings", 16384); } p_array_init(&all_roots, service->set_pool, 8); diff -r 87f10e2fac95 -r 90891ca867f3 src/lib-settings/settings-parser.c --- a/src/lib-settings/settings-parser.c Thu Sep 25 13:26:20 2014 +0300 +++ b/src/lib-settings/settings-parser.c Thu Sep 25 13:28:39 2014 +0300 @@ -118,7 +118,7 @@ memset(&info, 0, sizeof(info)); info = *def->list_info; - for (i = 0; i < count; i++) { + for (i = 0; i < count; i++) T_BEGIN { new_set = p_malloc(ctx->set_pool, info.struct_size); array_append(arr, &new_set, 1); @@ -148,7 +148,7 @@ info.defaults = children[i]; setting_parser_copy_defaults(ctx, &info, new_link); - } + } T_END; } static void diff -r 87f10e2fac95 -r 90891ca867f3 src/lib-storage/mail-storage-service.c --- a/src/lib-storage/mail-storage-service.c Thu Sep 25 13:26:20 2014 +0300 +++ b/src/lib-storage/mail-storage-service.c Thu Sep 25 13:28:39 2014 +0300 @@ -1022,7 +1022,7 @@ pool_t user_pool, temp_pool; int ret = 1; - user_pool = pool_alloconly_create("mail storage service user", 1024*6); + user_pool = pool_alloconly_create(MEMPOOL_GROWING"mail storage service user", 1024*6); flags = mail_storage_service_input_get_flags(ctx, input); if ((flags & MAIL_STORAGE_SERVICE_FLAG_TEMP_PRIV_DROP) != 0 && diff -r 87f10e2fac95 -r 90891ca867f3 src/lib-storage/mail-user.c --- a/src/lib-storage/mail-user.c Thu Sep 25 13:26:20 2014 +0300 +++ b/src/lib-storage/mail-user.c Thu Sep 25 13:28:39 2014 +0300 @@ -49,7 +49,7 @@ i_assert(username != NULL); i_assert(*username != '\0'); - pool = pool_alloconly_create("mail user", 16*1024); + pool = pool_alloconly_create(MEMPOOL_GROWING"mail user", 16*1024); user = p_new(pool, struct mail_user, 1); user->pool = pool; user->refcount = 1; diff -r 87f10e2fac95 -r 90891ca867f3 src/master/service.c --- a/src/master/service.c Thu Sep 25 13:26:20 2014 +0300 +++ b/src/master/service.c Thu Sep 25 13:28:39 2014 +0300 @@ -497,7 +497,7 @@ { pool_t pool; - pool = pool_alloconly_create("services pool", 4096); + pool = pool_alloconly_create("services pool", 32768); if (services_create_real(set, pool, services_r, error_r) < 0) { pool_unref(&pool); return -1; From dovecot at dovecot.org Thu Sep 25 10:38:29 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 25 Sep 2014 10:38:29 +0000 Subject: dovecot-2.2: auth: Mark memory pools as growing and use the same... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/569d41d21ec3 changeset: 17837:569d41d21ec3 user: Timo Sirainen date: Thu Sep 25 13:38:09 2014 +0300 description: auth: Mark memory pools as growing and use the same sizes for all mechanisms. Mainly to have DEBUG log fewer warnings. diffstat: src/auth/auth-request.c | 2 +- src/auth/db-dict.c | 2 +- src/auth/db-ldap.c | 2 +- src/auth/mech-anonymous.c | 2 +- src/auth/mech-apop.c | 2 +- src/auth/mech-cram-md5.c | 2 +- src/auth/mech-digest-md5.c | 2 +- src/auth/mech-dovecot-token.c | 2 +- src/auth/mech-external.c | 2 +- src/auth/mech-gssapi.c | 2 +- src/auth/mech-login.c | 2 +- src/auth/mech-ntlm.c | 2 +- src/auth/mech-otp.c | 2 +- src/auth/mech-plain.c | 2 +- src/auth/mech-rpa.c | 2 +- src/auth/mech-scram-sha1.c | 2 +- src/auth/mech-skey.c | 2 +- src/auth/mech-winbind.c | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diffs (216 lines): diff -r 90891ca867f3 -r 569d41d21ec3 src/auth/auth-request.c --- a/src/auth/auth-request.c Thu Sep 25 13:28:39 2014 +0300 +++ b/src/auth/auth-request.c Thu Sep 25 13:38:09 2014 +0300 @@ -73,7 +73,7 @@ struct auth_request *request; pool_t pool; - pool = pool_alloconly_create("auth_request", 1024); + pool = pool_alloconly_create(MEMPOOL_GROWING"auth_request", 1024); request = p_new(pool, struct auth_request, 1); request->pool = pool; diff -r 90891ca867f3 -r 569d41d21ec3 src/auth/db-dict.c --- a/src/auth/db-dict.c Thu Sep 25 13:28:39 2014 +0300 +++ b/src/auth/db-dict.c Thu Sep 25 13:38:09 2014 +0300 @@ -443,7 +443,7 @@ pool_t pool; int ret; - pool = pool_alloconly_create("auth dict lookup", 1024); + pool = pool_alloconly_create(MEMPOOL_GROWING"auth dict lookup", 1024); iter = p_new(pool, struct db_dict_value_iter, 1); iter->pool = pool; iter->conn = conn; diff -r 90891ca867f3 -r 569d41d21ec3 src/auth/db-ldap.c --- a/src/auth/db-ldap.c Thu Sep 25 13:28:39 2014 +0300 +++ b/src/auth/db-ldap.c Thu Sep 25 13:38:09 2014 +0300 @@ -1505,7 +1505,7 @@ const char *suffix; pool_t pool; - pool = pool_alloconly_create("ldap result iter", 1024); + pool = pool_alloconly_create(MEMPOOL_GROWING"ldap result iter", 1024); ctx = p_new(pool, struct db_ldap_result_iterate_context, 1); ctx->pool = pool; ctx->auth_request = ldap_request->request.auth_request; diff -r 90891ca867f3 -r 569d41d21ec3 src/auth/mech-anonymous.c --- a/src/auth/mech-anonymous.c Thu Sep 25 13:28:39 2014 +0300 +++ b/src/auth/mech-anonymous.c Thu Sep 25 13:38:09 2014 +0300 @@ -29,7 +29,7 @@ struct auth_request *request; pool_t pool; - pool = pool_alloconly_create("anonymous_auth_request", 512); + pool = pool_alloconly_create(MEMPOOL_GROWING"anonymous_auth_request", 2048); request = p_new(pool, struct auth_request, 1); request->pool = pool; return request; diff -r 90891ca867f3 -r 569d41d21ec3 src/auth/mech-apop.c --- a/src/auth/mech-apop.c Thu Sep 25 13:28:39 2014 +0300 +++ b/src/auth/mech-apop.c Thu Sep 25 13:38:09 2014 +0300 @@ -145,7 +145,7 @@ struct apop_auth_request *request; pool_t pool; - pool = pool_alloconly_create("apop_auth_request", 1024); + pool = pool_alloconly_create(MEMPOOL_GROWING"apop_auth_request", 2048); request = p_new(pool, struct apop_auth_request, 1); request->pool = pool; diff -r 90891ca867f3 -r 569d41d21ec3 src/auth/mech-cram-md5.c --- a/src/auth/mech-cram-md5.c Thu Sep 25 13:28:39 2014 +0300 +++ b/src/auth/mech-cram-md5.c Thu Sep 25 13:38:09 2014 +0300 @@ -173,7 +173,7 @@ struct cram_auth_request *request; pool_t pool; - pool = pool_alloconly_create("cram_md5_auth_request", 2048); + pool = pool_alloconly_create(MEMPOOL_GROWING"cram_md5_auth_request", 2048); request = p_new(pool, struct cram_auth_request, 1); request->pool = pool; diff -r 90891ca867f3 -r 569d41d21ec3 src/auth/mech-digest-md5.c --- a/src/auth/mech-digest-md5.c Thu Sep 25 13:28:39 2014 +0300 +++ b/src/auth/mech-digest-md5.c Thu Sep 25 13:38:09 2014 +0300 @@ -592,7 +592,7 @@ struct digest_auth_request *request; pool_t pool; - pool = pool_alloconly_create("digest_md5_auth_request", 2048); + pool = pool_alloconly_create(MEMPOOL_GROWING"digest_md5_auth_request", 2048); request = p_new(pool, struct digest_auth_request, 1); request->pool = pool; request->qop = QOP_AUTH; diff -r 90891ca867f3 -r 569d41d21ec3 src/auth/mech-dovecot-token.c --- a/src/auth/mech-dovecot-token.c Thu Sep 25 13:28:39 2014 +0300 +++ b/src/auth/mech-dovecot-token.c Thu Sep 25 13:38:09 2014 +0300 @@ -69,7 +69,7 @@ struct auth_request *request; pool_t pool; - pool = pool_alloconly_create("dovecot_token_auth_request", 512); + pool = pool_alloconly_create(MEMPOOL_GROWING"dovecot_token_auth_request", 512); request = p_new(pool, struct auth_request, 1); request->pool = pool; return request; diff -r 90891ca867f3 -r 569d41d21ec3 src/auth/mech-external.c --- a/src/auth/mech-external.c Thu Sep 25 13:28:39 2014 +0300 +++ b/src/auth/mech-external.c Thu Sep 25 13:38:09 2014 +0300 @@ -45,7 +45,7 @@ struct auth_request *request; pool_t pool; - pool = pool_alloconly_create("external_auth_request", 2048); + pool = pool_alloconly_create(MEMPOOL_GROWING"external_auth_request", 2048); request = p_new(pool, struct auth_request, 1); request->pool = pool; return request; diff -r 90891ca867f3 -r 569d41d21ec3 src/auth/mech-gssapi.c --- a/src/auth/mech-gssapi.c Thu Sep 25 13:28:39 2014 +0300 +++ b/src/auth/mech-gssapi.c Thu Sep 25 13:38:09 2014 +0300 @@ -122,7 +122,7 @@ struct gssapi_auth_request *request; pool_t pool; - pool = pool_alloconly_create("gssapi_auth_request", 1024); + pool = pool_alloconly_create(MEMPOOL_GROWING"gssapi_auth_request", 2048); request = p_new(pool, struct gssapi_auth_request, 1); request->pool = pool; diff -r 90891ca867f3 -r 569d41d21ec3 src/auth/mech-login.c --- a/src/auth/mech-login.c Thu Sep 25 13:28:39 2014 +0300 +++ b/src/auth/mech-login.c Thu Sep 25 13:38:09 2014 +0300 @@ -57,7 +57,7 @@ struct auth_request *request; pool_t pool; - pool = pool_alloconly_create("login_auth_request", 1024); + pool = pool_alloconly_create(MEMPOOL_GROWING"login_auth_request", 2048); request = p_new(pool, struct auth_request, 1); request->pool = pool; return request; diff -r 90891ca867f3 -r 569d41d21ec3 src/auth/mech-ntlm.c --- a/src/auth/mech-ntlm.c Thu Sep 25 13:28:39 2014 +0300 +++ b/src/auth/mech-ntlm.c Thu Sep 25 13:38:09 2014 +0300 @@ -238,7 +238,7 @@ struct ntlm_auth_request *request; pool_t pool; - pool = pool_alloconly_create("ntlm_auth_request", 1024); + pool = pool_alloconly_create(MEMPOOL_GROWING"ntlm_auth_request", 2048); request = p_new(pool, struct ntlm_auth_request, 1); request->pool = pool; diff -r 90891ca867f3 -r 569d41d21ec3 src/auth/mech-otp.c --- a/src/auth/mech-otp.c Thu Sep 25 13:28:39 2014 +0300 +++ b/src/auth/mech-otp.c Thu Sep 25 13:38:09 2014 +0300 @@ -237,7 +237,7 @@ otp_lock_init(); - pool = pool_alloconly_create("otp_auth_request", 256); + pool = pool_alloconly_create(MEMPOOL_GROWING"otp_auth_request", 2048); request = p_new(pool, struct otp_auth_request, 1); request->pool = pool; request->lock = FALSE; diff -r 90891ca867f3 -r 569d41d21ec3 src/auth/mech-plain.c --- a/src/auth/mech-plain.c Thu Sep 25 13:28:39 2014 +0300 +++ b/src/auth/mech-plain.c Thu Sep 25 13:38:09 2014 +0300 @@ -68,7 +68,7 @@ struct auth_request *request; pool_t pool; - pool = pool_alloconly_create("plain_auth_request", 2048); + pool = pool_alloconly_create(MEMPOOL_GROWING"plain_auth_request", 2048); request = p_new(pool, struct auth_request, 1); request->pool = pool; return request; diff -r 90891ca867f3 -r 569d41d21ec3 src/auth/mech-rpa.c --- a/src/auth/mech-rpa.c Thu Sep 25 13:28:39 2014 +0300 +++ b/src/auth/mech-rpa.c Thu Sep 25 13:38:09 2014 +0300 @@ -567,7 +567,7 @@ struct rpa_auth_request *request; pool_t pool; - pool = pool_alloconly_create("rpa_auth_request", 1024); + pool = pool_alloconly_create(MEMPOOL_GROWING"rpa_auth_request", 2048); request = p_new(pool, struct rpa_auth_request, 1); request->pool = pool; request->phase = 0; diff -r 90891ca867f3 -r 569d41d21ec3 src/auth/mech-scram-sha1.c --- a/src/auth/mech-scram-sha1.c Thu Sep 25 13:28:39 2014 +0300 +++ b/src/auth/mech-scram-sha1.c Thu Sep 25 13:38:09 2014 +0300 @@ -390,7 +390,7 @@ struct scram_auth_request *request; pool_t pool; - pool = pool_alloconly_create("scram_sha1_auth_request", 2048); + pool = pool_alloconly_create(MEMPOOL_GROWING"scram_sha1_auth_request", 2048); request = p_new(pool, struct scram_auth_request, 1); request->pool = pool; diff -r 90891ca867f3 -r 569d41d21ec3 src/auth/mech-skey.c --- a/src/auth/mech-skey.c Thu Sep 25 13:28:39 2014 +0300 +++ b/src/auth/mech-skey.c Thu Sep 25 13:38:09 2014 +0300 @@ -176,7 +176,7 @@ otp_lock_init(); - pool = pool_alloconly_create("skey_auth_request", 256); + pool = pool_alloconly_create(MEMPOOL_GROWING"skey_auth_request", 2048); request = p_new(pool, struct otp_auth_request, 1); request->pool = pool; request->lock = FALSE; diff -r 90891ca867f3 -r 569d41d21ec3 src/auth/mech-winbind.c --- a/src/auth/mech-winbind.c Thu Sep 25 13:28:39 2014 +0300 +++ b/src/auth/mech-winbind.c Thu Sep 25 13:38:09 2014 +0300 @@ -313,7 +313,7 @@ struct winbind_auth_request *request; pool_t pool; - pool = pool_alloconly_create("winbind_auth_request", 1024); + pool = pool_alloconly_create(MEMPOOL_GROWING"winbind_auth_request", 2048); request = p_new(pool, struct winbind_auth_request, 1); request->auth_request.pool = pool; From dovecot at dovecot.org Thu Sep 25 10:41:14 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 25 Sep 2014 10:41:14 +0000 Subject: dovecot-2.2: Mark more memory pools as growing Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/33c434858f41 changeset: 17838:33c434858f41 user: Timo Sirainen date: Thu Sep 25 13:40:54 2014 +0300 description: Mark more memory pools as growing diffstat: src/lib-http/http-server-request.c | 2 +- src/lib-index/mail-cache-lookup.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diffs (24 lines): diff -r 569d41d21ec3 -r 33c434858f41 src/lib-http/http-server-request.c --- a/src/lib-http/http-server-request.c Thu Sep 25 13:38:09 2014 +0300 +++ b/src/lib-http/http-server-request.c Thu Sep 25 13:40:54 2014 +0300 @@ -10,7 +10,7 @@ pool_t pool; struct http_server_request *req; - pool = pool_alloconly_create("http_message", 4096); + pool = pool_alloconly_create(MEMPOOL_GROWING"http_server_request", 4096); req = p_new(pool, struct http_server_request, 1); req->pool = pool; req->refcount = 1; diff -r 569d41d21ec3 -r 33c434858f41 src/lib-index/mail-cache-lookup.c --- a/src/lib-index/mail-cache-lookup.c Thu Sep 25 13:38:09 2014 +0300 +++ b/src/lib-index/mail-cache-lookup.c Thu Sep 25 13:40:54 2014 +0300 @@ -542,7 +542,7 @@ /* lookup the fields */ memset(&ctx, 0, sizeof(ctx)); ctx.view = view; - ctx.pool = *pool_r = pool_alloconly_create("mail cache headers", 1024); + ctx.pool = *pool_r = pool_alloconly_create(MEMPOOL_GROWING"mail cache headers", 1024); t_array_init(&ctx.lines, 32); mail_cache_lookup_iter_init(view, seq, &iter); From dovecot at dovecot.org Thu Sep 25 12:16:03 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 25 Sep 2014 12:16:03 +0000 Subject: dovecot-2.2: lib: Code cleanup - check io_loop_context_add/remov... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e588792b4f20 changeset: 17839:e588792b4f20 user: Timo Sirainen date: Thu Sep 25 15:15:20 2014 +0300 description: lib: Code cleanup - check io_loop_context_add/remove_callbacks() callback parameters. diffstat: src/lib-storage/mail-storage-service.c | 8 ++------ src/lib/ioloop.c | 2 ++ src/lib/ioloop.h | 10 ++++++++++ src/plugins/stats/stats-plugin.c | 6 ++---- 4 files changed, 16 insertions(+), 10 deletions(-) diffs (91 lines): diff -r 33c434858f41 -r e588792b4f20 src/lib-storage/mail-storage-service.c --- a/src/lib-storage/mail-storage-service.c Thu Sep 25 13:40:54 2014 +0300 +++ b/src/lib-storage/mail-storage-service.c Thu Sep 25 15:15:20 2014 +0300 @@ -695,17 +695,13 @@ return 0; } -static void mail_storage_service_io_activate(void *context) +static void mail_storage_service_io_activate(struct mail_storage_service_user *user) { - struct mail_storage_service_user *user = context; - i_set_failure_prefix("%s", user->log_prefix); } -static void mail_storage_service_io_deactivate(void *context) +static void mail_storage_service_io_deactivate(struct mail_storage_service_user *user) { - struct mail_storage_service_user *user = context; - i_set_failure_prefix("%s", user->service_ctx->default_log_prefix); } diff -r 33c434858f41 -r e588792b4f20 src/lib/ioloop.c --- a/src/lib/ioloop.c Thu Sep 25 13:40:54 2014 +0300 +++ b/src/lib/ioloop.c Thu Sep 25 15:15:20 2014 +0300 @@ -674,6 +674,7 @@ i_free(ctx); } +#undef io_loop_context_add_callbacks void io_loop_context_add_callbacks(struct ioloop_context *ctx, io_callback_t *activate, io_callback_t *deactivate, void *context) @@ -688,6 +689,7 @@ array_append(&ctx->callbacks, &cb, 1); } +#undef io_loop_context_remove_callbacks void io_loop_context_remove_callbacks(struct ioloop_context *ctx, io_callback_t *activate, io_callback_t *deactivate, void *context) diff -r 33c434858f41 -r e588792b4f20 src/lib/ioloop.h --- a/src/lib/ioloop.h Thu Sep 25 13:40:54 2014 +0300 +++ b/src/lib/ioloop.h Thu Sep 25 15:15:20 2014 +0300 @@ -142,10 +142,20 @@ void io_loop_context_add_callbacks(struct ioloop_context *ctx, io_callback_t *activate, io_callback_t *deactivate, void *context); +#define io_loop_context_add_callbacks(ctx, activate, deactivate, context) \ + io_loop_context_add_callbacks(ctx, 1 ? (io_callback_t *)activate : \ + CALLBACK_TYPECHECK(activate, void (*)(typeof(context))) + \ + CALLBACK_TYPECHECK(deactivate, void (*)(typeof(context))), \ + (io_callback_t *)deactivate, context) /* Remove callbacks with the given callbacks and context. */ void io_loop_context_remove_callbacks(struct ioloop_context *ctx, io_callback_t *activate, io_callback_t *deactivate, void *context); +#define io_loop_context_remove_callbacks(ctx, activate, deactivate, context) \ + io_loop_context_remove_callbacks(ctx, 1 ? (io_callback_t *)activate : \ + CALLBACK_TYPECHECK(activate, void (*)(typeof(context))) + \ + CALLBACK_TYPECHECK(deactivate, void (*)(typeof(context))), \ + (io_callback_t *)deactivate, context) /* Returns the current context set to ioloop. */ struct ioloop_context *io_loop_get_current_context(struct ioloop *ioloop); diff -r 33c434858f41 -r e588792b4f20 src/plugins/stats/stats-plugin.c --- a/src/plugins/stats/stats-plugin.c Thu Sep 25 13:40:54 2014 +0300 +++ b/src/plugins/stats/stats-plugin.c Thu Sep 25 15:15:20 2014 +0300 @@ -201,9 +201,8 @@ user_trans_stats_get(suser, &stats_r->trans_stats); } -static void stats_io_activate(void *context) +static void stats_io_activate(struct mail_user *user) { - struct mail_user *user = context; struct stats_user *suser = STATS_USER_CONTEXT(user); if (stats_user_count == 1) { @@ -519,9 +518,8 @@ session_stats_refresh(user); } -static void stats_io_deactivate(void *context) +static void stats_io_deactivate(struct mail_user *user) { - struct mail_user *user = context; struct stats_user *suser = STATS_USER_CONTEXT(user); unsigned int last_update_secs; From dovecot at dovecot.org Thu Sep 25 12:50:24 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 25 Sep 2014 12:50:24 +0000 Subject: dovecot-2.2: lib-storage: Allow calling storage service's ioloop... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a3352c6c483e changeset: 17840:a3352c6c483e user: Timo Sirainen date: Thu Sep 25 15:37:01 2014 +0300 description: lib-storage: Allow calling storage service's ioloop activate/deactivate functions manually. diffstat: src/lib-storage/mail-storage-service.c | 17 +++++++++++------ src/lib-storage/mail-storage-service.h | 9 +++++++++ 2 files changed, 20 insertions(+), 6 deletions(-) diffs (68 lines): diff -r e588792b4f20 -r a3352c6c483e src/lib-storage/mail-storage-service.c --- a/src/lib-storage/mail-storage-service.c Thu Sep 25 15:15:20 2014 +0300 +++ b/src/lib-storage/mail-storage-service.c Thu Sep 25 15:37:01 2014 +0300 @@ -695,16 +695,21 @@ return 0; } -static void mail_storage_service_io_activate(struct mail_storage_service_user *user) +void mail_storage_service_io_activate_user(struct mail_storage_service_user *user) { i_set_failure_prefix("%s", user->log_prefix); } -static void mail_storage_service_io_deactivate(struct mail_storage_service_user *user) +void mail_storage_service_io_deactivate_user(struct mail_storage_service_user *user) { i_set_failure_prefix("%s", user->service_ctx->default_log_prefix); } +void mail_storage_service_io_deactivate(struct mail_storage_service_ctx *ctx) +{ + i_set_failure_prefix("%s", ctx->default_log_prefix); +} + static void mail_storage_service_init_log(struct mail_storage_service_ctx *ctx, struct mail_storage_service_user *user, @@ -726,8 +731,8 @@ i_set_failure_send_prefix(user->log_prefix); user->ioloop_ctx = io_loop_context_new(current_ioloop); io_loop_context_add_callbacks(user->ioloop_ctx, - mail_storage_service_io_activate, - mail_storage_service_io_deactivate, + mail_storage_service_io_activate_user, + mail_storage_service_io_deactivate_user, user); } @@ -1275,8 +1280,8 @@ if (user->ioloop_ctx != NULL) { io_loop_context_remove_callbacks(user->ioloop_ctx, - mail_storage_service_io_activate, - mail_storage_service_io_deactivate, user); + mail_storage_service_io_activate_user, + mail_storage_service_io_deactivate_user, user); io_loop_context_unref(&user->ioloop_ctx); } settings_parser_deinit(&user->set_parser); diff -r e588792b4f20 -r a3352c6c483e src/lib-storage/mail-storage-service.h --- a/src/lib-storage/mail-storage-service.h Thu Sep 25 15:15:20 2014 +0300 +++ b/src/lib-storage/mail-storage-service.h Thu Sep 25 15:37:01 2014 +0300 @@ -109,6 +109,15 @@ const char **username_r); void mail_storage_service_deinit(struct mail_storage_service_ctx **ctx); +/* Activate user context. Normally this is called automatically by the ioloop, + but e.g. during loops at deinit where all users are being destroyed, it's + useful to call this to set the correct user-specific log prefix. */ +void mail_storage_service_io_activate_user(struct mail_storage_service_user *user); +/* Deactivate user context. This only switches back to non-user-specific + log prefix. */ +void mail_storage_service_io_deactivate_user(struct mail_storage_service_user *user); +void mail_storage_service_io_deactivate(struct mail_storage_service_ctx *ctx); + /* Return the settings pointed to by set_root parameter in _init(). The settings contain all the changes done by userdb lookups. */ void **mail_storage_service_user_get_set(struct mail_storage_service_user *user); From dovecot at dovecot.org Thu Sep 25 12:50:29 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 25 Sep 2014 12:50:29 +0000 Subject: dovecot-2.2: imap, pop3: Show user's proper log prefix when dein... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/576da94fd669 changeset: 17841:576da94fd669 user: Timo Sirainen date: Thu Sep 25 15:37:48 2014 +0300 description: imap, pop3: Show user's proper log prefix when deinitializing users at exit. diffstat: src/imap/imap-client.c | 4 +++- src/imap/imap-client.h | 3 ++- src/imap/main.c | 4 +++- src/pop3/main.c | 2 +- src/pop3/pop3-client.c | 4 +++- src/pop3/pop3-client.h | 3 ++- 6 files changed, 14 insertions(+), 6 deletions(-) diffs (119 lines): diff -r a3352c6c483e -r 576da94fd669 src/imap/imap-client.c --- a/src/imap/imap-client.c Thu Sep 25 15:37:01 2014 +0300 +++ b/src/imap/imap-client.c Thu Sep 25 15:37:48 2014 +0300 @@ -1111,12 +1111,14 @@ array_clear(&client->search_updates); } -void clients_destroy_all(void) +void clients_destroy_all(struct mail_storage_service_ctx *storage_service) { while (imap_clients != NULL) { client_send_line(imap_clients, "* BYE Server shutting down."); + mail_storage_service_io_activate_user(imap_clients->service_user); client_destroy(imap_clients, "Server shutting down."); } + mail_storage_service_io_deactivate(storage_service); } struct imap_client_vfuncs imap_client_vfuncs = { diff -r a3352c6c483e -r 576da94fd669 src/imap/imap-client.h --- a/src/imap/imap-client.h Thu Sep 25 15:37:01 2014 +0300 +++ b/src/imap/imap-client.h Thu Sep 25 15:37:48 2014 +0300 @@ -12,6 +12,7 @@ struct client; struct mail_storage; +struct mail_storage_service_ctx; struct imap_parser; struct imap_arg; struct imap_urlauth_context; @@ -244,6 +245,6 @@ bool client_handle_input(struct client *client); int client_output(struct client *client); -void clients_destroy_all(void); +void clients_destroy_all(struct mail_storage_service_ctx *storage_service); #endif diff -r a3352c6c483e -r 576da94fd669 src/imap/main.c --- a/src/imap/main.c Thu Sep 25 15:37:01 2014 +0300 +++ b/src/imap/main.c Thu Sep 25 15:37:48 2014 +0300 @@ -114,6 +114,7 @@ for (client = imap_clients; client != NULL; client = next) { next = client->next; + mail_storage_service_io_activate_user(client->service_user); last_io = I_MAX(client->last_input, client->last_output); if (last_io <= stop_timestamp) client_kill_idle(client); @@ -124,6 +125,7 @@ client_kill_idle, client); } } + mail_storage_service_io_deactivate(storage_service); } struct client_input { @@ -408,7 +410,7 @@ if (io_loop_is_running(current_ioloop)) master_service_run(master_service, client_connected); - clients_destroy_all(); + clients_destroy_all(storage_service); if (master_login != NULL) master_login_deinit(&master_login); diff -r a3352c6c483e -r 576da94fd669 src/pop3/main.c --- a/src/pop3/main.c Thu Sep 25 15:37:01 2014 +0300 +++ b/src/pop3/main.c Thu Sep 25 15:37:48 2014 +0300 @@ -275,7 +275,7 @@ if (io_loop_is_running(current_ioloop)) master_service_run(master_service, client_connected); - clients_destroy_all(); + clients_destroy_all(storage_service); if (master_login != NULL) master_login_deinit(&master_login); diff -r a3352c6c483e -r 576da94fd669 src/pop3/pop3-client.c --- a/src/pop3/pop3-client.c Thu Sep 25 15:37:01 2014 +0300 +++ b/src/pop3/pop3-client.c Thu Sep 25 15:37:48 2014 +0300 @@ -837,15 +837,17 @@ } } -void clients_destroy_all(void) +void clients_destroy_all(struct mail_storage_service_ctx *storage_service) { while (pop3_clients != NULL) { if (pop3_clients->cmd == NULL) { client_send_line(pop3_clients, "-ERR [SYS/TEMP] Server shutting down."); } + mail_storage_service_io_activate_user(pop3_clients->service_user); client_destroy(pop3_clients, "Server shutting down."); } + mail_storage_service_io_deactivate(storage_service); } struct pop3_client_vfuncs pop3_client_vfuncs = { diff -r a3352c6c483e -r 576da94fd669 src/pop3/pop3-client.h --- a/src/pop3/pop3-client.h Thu Sep 25 15:37:01 2014 +0300 +++ b/src/pop3/pop3-client.h Thu Sep 25 15:37:48 2014 +0300 @@ -5,6 +5,7 @@ struct client; struct mail_storage; +struct mail_storage_service_ctx; typedef void command_func_t(struct client *client); @@ -133,6 +134,6 @@ bool client_handle_input(struct client *client); bool client_update_mails(struct client *client); -void clients_destroy_all(void); +void clients_destroy_all(struct mail_storage_service_ctx *storage_service); #endif From dovecot at dovecot.org Thu Sep 25 12:50:29 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 25 Sep 2014 12:50:29 +0000 Subject: dovecot-2.2: lib: If file_wait_lock*() fails, try to include the... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6ed1eee94d31 changeset: 17842:6ed1eee94d31 user: Timo Sirainen date: Thu Sep 25 15:50:03 2014 +0300 description: lib: If file_wait_lock*() fails, try to include the current pid holding the lock in error message. This is currently supported via fcntl(F_GETLK) and also via Linux /proc/locks. diffstat: src/lib/file-lock.c | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 files changed, 94 insertions(+), 4 deletions(-) diffs (131 lines): diff -r 576da94fd669 -r 6ed1eee94d31 src/lib/file-lock.c --- a/src/lib/file-lock.c Thu Sep 25 15:37:48 2014 +0300 +++ b/src/lib/file-lock.c Thu Sep 25 15:50:03 2014 +0300 @@ -1,8 +1,10 @@ /* Copyright (c) 2002-2014 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "istream.h" #include "file-lock.h" +#include #ifdef HAVE_FLOCK # include #endif @@ -56,6 +58,92 @@ lock_r, error_r); } +static const char * +file_lock_find_fcntl(int lock_fd, int lock_type) +{ + struct flock fl; + + memset(&fl, 0, sizeof(fl)); + fl.l_type = lock_type; + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 0; + + if (fcntl(lock_fd, F_GETLK, &fl) < 0 || + fl.l_type == F_UNLCK || fl.l_pid == -1 || fl.l_pid == 0) + return ""; + return t_strdup_printf(" (%s lock held by pid %ld)", + fl.l_type == F_RDLCK ? "READ" : "WRITE", (long)fl.l_pid); +} + +static const char * +file_lock_find_proc_locks(int lock_fd ATTR_UNUSED) +{ + /* do anything except Linux support this? don't bother trying it for + OSes we don't know about. */ +#ifdef __linux__ + static bool have_proc_locks = TRUE; + struct stat st; + char node_buf[MAX_INT_STRLEN*3 + 2 + 1]; + struct istream *input; + const char *line, *lock_type = ""; + pid_t pid = 0; + int fd; + + if (!have_proc_locks) + return FALSE; + + if (fstat(lock_fd, &st) < 0) + return ""; + i_snprintf(node_buf, sizeof(node_buf), "%02x:%02x:%llu", + major(st.st_dev), minor(st.st_dev), + (unsigned long long)st.st_ino); + fd = open("/proc/locks", O_RDONLY); + if (fd == -1) { + have_proc_locks = FALSE; + return ""; + } + input = i_stream_create_fd_autoclose(&fd, 512); + while (pid == 0 && (line = i_stream_read_next_line(input)) != NULL) T_BEGIN { + const char *const *args = t_strsplit_spaces(line, " "); + + /* number: FLOCK/POSIX ADVISORY READ/WRITE pid + major:minor:inode region-start region-end */ + if (str_array_length(args) < 8) + continue; + if (strcmp(args[5], node_buf) == 0) { + lock_type = strcmp(args[3], "READ") == 0 ? + "READ" : "WRITE"; + if (str_to_pid(args[4], &pid) < 0) + pid = 0; + } + } T_END; + i_stream_destroy(&input); + if (pid == 0) { + /* not found */ + return ""; + } + if (pid == getpid()) + return " (BUG: lock is held by our own process)"; + return t_strdup_printf(" (%s lock held by pid %ld)", lock_type, (long)pid); +#else + return ""; +#endif +} + +static const char * +file_lock_find(int lock_fd, enum file_lock_method lock_method, int lock_type) +{ + const char *ret; + + if (lock_method == FILE_LOCK_METHOD_FCNTL) { + ret = file_lock_find_fcntl(lock_fd, lock_type); + if (ret[0] != '\0') + return ret; + } + return file_lock_find_proc_locks(lock_fd); +} + static int file_lock_do(int fd, const char *path, int lock_type, enum file_lock_method lock_method, unsigned int timeout_secs, const char **error_r) @@ -107,8 +195,9 @@ errno = EAGAIN; *error_r = t_strdup_printf( "fcntl(%s, %s, F_SETLKW) locking failed: " - "Timed out after %u seconds", - path, lock_type_str, timeout_secs); + "Timed out after %u seconds%s", + path, lock_type_str, timeout_secs, + file_lock_find(fd, lock_method, lock_type)); return 0; } *error_r = t_strdup_printf("fcntl(%s, %s, %s) locking failed: %m", @@ -152,8 +241,9 @@ if (errno == EINTR) { errno = EAGAIN; *error_r = t_strdup_printf("flock(%s, %s) failed: " - "Timed out after %u seconds", - path, lock_type_str, timeout_secs); + "Timed out after %u seconds%s", + path, lock_type_str, timeout_secs, + file_lock_find(fd, lock_method, lock_type)); return 0; } *error_r = t_strdup_printf("flock(%s, %s) failed: %m", From dovecot at dovecot.org Thu Sep 25 16:43:50 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 25 Sep 2014 16:43:50 +0000 Subject: dovecot-2.2: imap: die-callback didn't set log prefixes correctl... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/abc434a32140 changeset: 17843:abc434a32140 user: Timo Sirainen date: Thu Sep 25 19:43:28 2014 +0300 description: imap: die-callback didn't set log prefixes correctly for client handlers Calling mail_storage_service_io_activate_user() unfortunately doesn't set the ioloop context itself, so timeouts weren't inheriting it. It seemed a bit difficult to change this, so for now I just moved the call to the timeout handler. diffstat: src/imap/main.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (29 lines): diff -r 6ed1eee94d31 -r abc434a32140 src/imap/main.c --- a/src/imap/main.c Thu Sep 25 15:50:03 2014 +0300 +++ b/src/imap/main.c Thu Sep 25 19:43:28 2014 +0300 @@ -101,7 +101,9 @@ return; client_send_line(client, "* BYE Server shutting down."); + mail_storage_service_io_activate_user(client->service_user); client_destroy(client, "Server shutting down."); + mail_storage_service_io_deactivate(storage_service); } static void imap_die(void) @@ -114,7 +116,6 @@ for (client = imap_clients; client != NULL; client = next) { next = client->next; - mail_storage_service_io_activate_user(client->service_user); last_io = I_MAX(client->last_input, client->last_output); if (last_io <= stop_timestamp) client_kill_idle(client); @@ -125,7 +126,6 @@ client_kill_idle, client); } } - mail_storage_service_io_deactivate(storage_service); } struct client_input { From dovecot at dovecot.org Thu Sep 25 21:32:24 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 25 Sep 2014 21:32:24 +0000 Subject: dovecot-2.2: lib-storage: Don't assume that we must read the who... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/35c3194900b7 changeset: 17844:35c3194900b7 user: Timo Sirainen date: Fri Sep 26 00:32:03 2014 +0300 description: lib-storage: Don't assume that we must read the whole message to get its virtual size. This is true only with mbox format. All the other formats have various metadata available where it's stored. diffstat: src/lib-storage/index/index-mail.c | 15 --------------- 1 files changed, 0 insertions(+), 15 deletions(-) diffs (25 lines): diff -r abc434a32140 -r 35c3194900b7 src/lib-storage/index/index-mail.c --- a/src/lib-storage/index/index-mail.c Thu Sep 25 19:43:28 2014 +0300 +++ b/src/lib-storage/index/index-mail.c Fri Sep 26 00:32:03 2014 +0300 @@ -1538,21 +1538,6 @@ (void)mail_get_hdr_stream(_mail, NULL, &input); } } - - if ((data->wanted_fields & MAIL_FETCH_VIRTUAL_SIZE) != 0 && - data->virtual_size == (uoff_t)-1 && - _mail->lookup_abort == MAIL_LOOKUP_ABORT_NEVER && - ((data->access_part & (READ_HDR | PARSE_HDR)) == 0 || - (data->access_part & (READ_BODY | PARSE_BODY)) == 0)) { - /* we want virtual size, and we'd prefer not to read the entire - message for it. see if it's possible. */ - uoff_t vsize; - - _mail->lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE; - if (mail_get_virtual_size(_mail, &vsize) < 0) - mail->data.access_part |= READ_HDR | READ_BODY; - _mail->lookup_abort = MAIL_LOOKUP_ABORT_NEVER; - } } void index_mail_set_seq(struct mail *_mail, uint32_t seq, bool saving) From dovecot at dovecot.org Mon Sep 29 11:16:32 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Sep 2014 11:16:32 +0000 Subject: dovecot-2.2: dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to a... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d2e1b3f6d13b changeset: 17845:d2e1b3f6d13b user: Timo Sirainen date: Mon Sep 29 14:16:06 2014 +0300 description: dsync: Added DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH to avoid opening mails unnecessarily. diffstat: src/doveadm/dsync/dsync-brain-mailbox.c | 2 + src/doveadm/dsync/dsync-brain-private.h | 1 + src/doveadm/dsync/dsync-brain.c | 2 + src/doveadm/dsync/dsync-brain.h | 6 ++- src/doveadm/dsync/dsync-ibc-stream.c | 1 + src/doveadm/dsync/dsync-mail.c | 26 ++++++++++---- src/doveadm/dsync/dsync-mail.h | 25 ++++++++----- src/doveadm/dsync/dsync-mailbox-export.c | 21 +++++++---- src/doveadm/dsync/dsync-mailbox-export.h | 3 +- src/doveadm/dsync/dsync-mailbox-import.c | 56 ++++++++++++++++++++++++------- 10 files changed, 103 insertions(+), 40 deletions(-) diffs (truncated from 359 to 300 lines): diff -r 35c3194900b7 -r d2e1b3f6d13b src/doveadm/dsync/dsync-brain-mailbox.c --- a/src/doveadm/dsync/dsync-brain-mailbox.c Fri Sep 26 00:32:03 2014 +0300 +++ b/src/doveadm/dsync/dsync-brain-mailbox.c Mon Sep 29 14:16:06 2014 +0300 @@ -304,6 +304,8 @@ (brain->local_dsync_box.have_save_guids || (brain->backup_send && brain->local_dsync_box.have_guids))) exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS; + if (brain->no_mail_prefetch) + exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL; brain->box_exporter = brain->backup_recv ? NULL : dsync_mailbox_export_init(brain->box, brain->log_scan, diff -r 35c3194900b7 -r d2e1b3f6d13b src/doveadm/dsync/dsync-brain-private.h --- a/src/doveadm/dsync/dsync-brain-private.h Fri Sep 26 00:32:03 2014 +0300 +++ b/src/doveadm/dsync/dsync-brain-private.h Mon Sep 29 14:16:06 2014 +0300 @@ -100,6 +100,7 @@ unsigned int sync_visible_namespaces:1; unsigned int no_mail_sync:1; unsigned int no_backup_overwrite:1; + unsigned int no_mail_prefetch:1; unsigned int changes_during_sync:1; unsigned int require_full_resync:1; unsigned int verbose_proctitle:1; diff -r 35c3194900b7 -r d2e1b3f6d13b src/doveadm/dsync/dsync-brain.c --- a/src/doveadm/dsync/dsync-brain.c Fri Sep 26 00:32:03 2014 +0300 +++ b/src/doveadm/dsync/dsync-brain.c Mon Sep 29 14:16:06 2014 +0300 @@ -137,6 +137,8 @@ brain->no_mail_sync = (flags & DSYNC_BRAIN_FLAG_NO_MAIL_SYNC) != 0; brain->no_backup_overwrite = (flags & DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE) != 0; + brain->no_mail_prefetch = + (flags & DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH) != 0; } struct dsync_brain * diff -r 35c3194900b7 -r d2e1b3f6d13b src/doveadm/dsync/dsync-brain.h --- a/src/doveadm/dsync/dsync-brain.h Fri Sep 26 00:32:03 2014 +0300 +++ b/src/doveadm/dsync/dsync-brain.h Mon Sep 29 14:16:06 2014 +0300 @@ -22,7 +22,11 @@ DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE = 0x40, /* Run storage purge on the remote after syncing. Useful with e.g. a nightly doveadm backup. */ - DSYNC_BRAIN_FLAG_PURGE_REMOTE = 0x80 + DSYNC_BRAIN_FLAG_PURGE_REMOTE = 0x80, + /* Don't prefetch mail bodies until they're actually needed. This works + only with pipe ibc. It's useful if most of the mails can be copied + directly within filesystem without having to read them. */ + DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH = 0x100 }; enum dsync_brain_sync_type { diff -r 35c3194900b7 -r d2e1b3f6d13b src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Fri Sep 26 00:32:03 2014 +0300 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Mon Sep 29 14:16:06 2014 +0300 @@ -1681,6 +1681,7 @@ struct dsync_serializer_encoder *encoder; string_t *str = t_str_new(128); + i_assert(!mail->minimal_fields); i_assert(ibc->value_output == NULL); str_append_c(str, items[ITEM_MAIL].chr); diff -r 35c3194900b7 -r d2e1b3f6d13b src/doveadm/dsync/dsync-mail.c --- a/src/doveadm/dsync/dsync-mail.c Fri Sep 26 00:32:03 2014 +0300 +++ b/src/doveadm/dsync/dsync-mail.c Mon Sep 29 14:16:06 2014 +0300 @@ -60,10 +60,10 @@ return ret; } -int dsync_mail_fill(struct mail *mail, struct dsync_mail *dmail_r, - const char **error_field_r) +int dsync_mail_fill(struct mail *mail, bool minimal_fill, + struct dsync_mail *dmail_r, const char **error_field_r) { - const char *guid, *str; + const char *guid; memset(dmail_r, 0, sizeof(*dmail_r)); @@ -76,6 +76,22 @@ dmail_r->input_mail = mail; dmail_r->input_mail_uid = mail->uid; + + if (mail_get_save_date(mail, &dmail_r->saved_date) < 0) { + *error_field_r = "saved-date"; + return -1; + } + if (!minimal_fill) + return dsync_mail_fill_nonminimal(mail, dmail_r, error_field_r); + dmail_r->minimal_fields = TRUE; + return 0; +} + +int dsync_mail_fill_nonminimal(struct mail *mail, struct dsync_mail *dmail_r, + const char **error_field_r) +{ + const char *str; + if (mail_get_stream(mail, NULL, NULL, &dmail_r->input) < 0) { *error_field_r = "body"; return -1; @@ -97,10 +113,6 @@ *error_field_r = "received-date"; return -1; } - if (mail_get_save_date(mail, &dmail_r->saved_date) < 0) { - *error_field_r = "saved-date"; - return -1; - } return 0; } diff -r 35c3194900b7 -r d2e1b3f6d13b src/doveadm/dsync/dsync-mail.h --- a/src/doveadm/dsync/dsync-mail.h Fri Sep 26 00:32:03 2014 +0300 +++ b/src/doveadm/dsync/dsync-mail.h Mon Sep 29 14:16:06 2014 +0300 @@ -10,21 +10,24 @@ /* either GUID="" or uid=0 */ const char *guid; uint32_t uid; - - const char *pop3_uidl; - unsigned int pop3_order; - time_t received_date; time_t saved_date; - /* Input stream containing the message text, or NULL if all instances - of the message were already expunged from this mailbox. */ - struct istream *input; - /* If non-NULL, we're syncing within the dsync process using ibc-pipe. This mail can be used to mailbox_copy() the mail. */ struct mail *input_mail; /* Verify that this equals to input_mail->uid */ uint32_t input_mail_uid; + + /* TRUE if the following fields aren't set, because minimal_fill=TRUE + parameter was used. */ + bool minimal_fields; + + const char *pop3_uidl; + unsigned int pop3_order; + time_t received_date; + /* Input stream containing the message text, or NULL if all instances + of the message were already expunged from this mailbox. */ + struct istream *input; }; struct dsync_mail_request { @@ -80,8 +83,10 @@ dsync_mail_get_hash_headers(struct mailbox *box); int dsync_mail_get_hdr_hash(struct mail *mail, const char **hdr_hash_r); -int dsync_mail_fill(struct mail *mail, struct dsync_mail *dmail_r, - const char **error_field_r); +int dsync_mail_fill(struct mail *mail, bool minimal_fill, + struct dsync_mail *dmail_r, const char **error_field_r); +int dsync_mail_fill_nonminimal(struct mail *mail, struct dsync_mail *dmail_r, + const char **error_field_r); void dsync_mail_change_dup(pool_t pool, const struct dsync_mail_change *src, struct dsync_mail_change *dest_r); diff -r 35c3194900b7 -r d2e1b3f6d13b src/doveadm/dsync/dsync-mailbox-export.c --- a/src/doveadm/dsync/dsync-mailbox-export.c Fri Sep 26 00:32:03 2014 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-export.c Mon Sep 29 14:16:06 2014 +0300 @@ -57,6 +57,7 @@ unsigned int body_search_initialized:1; unsigned int auto_export_mails:1; unsigned int mails_have_guids:1; + unsigned int minimal_dmail_fill:1; unsigned int return_all_mails:1; }; @@ -474,6 +475,8 @@ (flags & DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS) != 0; exporter->mails_have_guids = (flags & DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS) != 0; + exporter->minimal_dmail_fill = + (flags & DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL) != 0; p_array_init(&exporter->requested_uids, pool, 16); p_array_init(&exporter->search_uids, pool, 16); hash_table_create(&exporter->export_guids, pool, 0, str_hash, strcmp); @@ -657,6 +660,7 @@ const struct seq_range *uids; char *guid; const char *const_guid; + enum mail_fetch_field wanted_fields; struct dsync_mail_guid_instances *instances; const struct seq_range *range; unsigned int i, count; @@ -714,16 +718,16 @@ array_append_array(&exporter->search_uids, &exporter->requested_uids); array_clear(&exporter->requested_uids); + wanted_fields = MAIL_FETCH_GUID | MAIL_FETCH_SAVE_DATE; + if (!exporter->minimal_dmail_fill) { + wanted_fields |= MAIL_FETCH_RECEIVED_DATE | + MAIL_FETCH_UIDL_BACKEND | MAIL_FETCH_POP3_ORDER | + MAIL_FETCH_STREAM_HEADER | MAIL_FETCH_STREAM_BODY; + } exporter->search_count += seq_range_count(&sarg->value.seqset); exporter->search_ctx = mailbox_search_init(exporter->trans, search_args, NULL, - MAIL_FETCH_GUID | - MAIL_FETCH_UIDL_BACKEND | - MAIL_FETCH_POP3_ORDER | - MAIL_FETCH_RECEIVED_DATE | - MAIL_FETCH_SAVE_DATE | - MAIL_FETCH_STREAM_HEADER | - MAIL_FETCH_STREAM_BODY, NULL); + wanted_fields, NULL); mail_search_args_unref(&search_args); return array_count(&sarg->value.seqset) > 0 ? 1 : 0; } @@ -748,7 +752,8 @@ struct dsync_mail_guid_instances *instances; const char *error_field; - if (dsync_mail_fill(mail, &exporter->dsync_mail, &error_field) < 0) + if (dsync_mail_fill(mail, exporter->minimal_dmail_fill, + &exporter->dsync_mail, &error_field) < 0) return dsync_mail_error(exporter, mail, error_field); instances = *exporter->dsync_mail.guid == '\0' ? NULL : diff -r 35c3194900b7 -r d2e1b3f6d13b src/doveadm/dsync/dsync-mailbox-export.h --- a/src/doveadm/dsync/dsync-mailbox-export.h Fri Sep 26 00:32:03 2014 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-export.h Mon Sep 29 14:16:06 2014 +0300 @@ -3,7 +3,8 @@ enum dsync_mailbox_exporter_flags { DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS = 0x01, - DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS = 0x02 + DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS = 0x02, + DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL = 0x04 }; struct dsync_mailbox_exporter * diff -r 35c3194900b7 -r d2e1b3f6d13b src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Fri Sep 26 00:32:03 2014 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Mon Sep 29 14:16:06 2014 +0300 @@ -1544,7 +1544,7 @@ if (!mail_set_uid(importer->mail, uid)) return 0; - if (dsync_mail_fill(importer->mail, dmail_r, &error_field) < 0) { + if (dsync_mail_fill(importer->mail, TRUE, dmail_r, &error_field) < 0) { errstr = mailbox_get_last_error(importer->mail->box, &error); if (error == MAIL_ERROR_EXPUNGED) return 0; @@ -1905,6 +1905,17 @@ return 0; } +static void +dsync_mailbox_save_set_nonminimal(struct mail_save_context *save_ctx, + const struct dsync_mail *mail) +{ + if (mail->pop3_uidl != NULL && *mail->pop3_uidl != '\0') + mailbox_save_set_pop3_uidl(save_ctx, mail->pop3_uidl); + if (mail->pop3_order > 0) + mailbox_save_set_pop3_order(save_ctx, mail->pop3_order); + mailbox_save_set_received_date(save_ctx, mail->received_date, 0); +} + static struct mail_save_context * dsync_mailbox_save_init(struct dsync_mailbox_importer *importer, const struct dsync_mail *mail, @@ -1919,11 +1930,9 @@ if (mail->saved_date != 0) mailbox_save_set_save_date(save_ctx, mail->saved_date); dsync_mailbox_save_set_metadata(importer, save_ctx, newmail->change); - if (mail->pop3_uidl != NULL && *mail->pop3_uidl != '\0') - mailbox_save_set_pop3_uidl(save_ctx, mail->pop3_uidl); - if (mail->pop3_order > 0) - mailbox_save_set_pop3_order(save_ctx, mail->pop3_order); - mailbox_save_set_received_date(save_ctx, mail->received_date, 0); + + if (!mail->minimal_fields) + dsync_mailbox_save_set_nonminimal(save_ctx, mail); return save_ctx; } @@ -1934,6 +1943,7 @@ struct importer_new_mail **all_newmails_forcopy) { struct mail_save_context *save_ctx; + struct istream *input; ssize_t ret; bool save_failed = FALSE; @@ -1960,22 +1970,42 @@ return; } /* fallback to saving from remote stream */ + if (mail->minimal_fields) { + struct dsync_mail mail2; + const char *error_field; - if (mail->input == NULL) { + i_assert(mail->input_mail != NULL); From dovecot at dovecot.org Mon Sep 29 11:58:07 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Sep 2014 11:58:07 +0000 Subject: dovecot-2.2: lib-http: Fixed assert-crash when http_client_conne... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/475ff24ac365 changeset: 17846:475ff24ac365 user: Timo Sirainen date: Mon Sep 29 14:57:42 2014 +0300 description: lib-http: Fixed assert-crash when http_client_connection_disconnect() is called multiple times. diffstat: src/lib-http/http-client-connection.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r d2e1b3f6d13b -r 475ff24ac365 src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Mon Sep 29 14:16:06 2014 +0300 +++ b/src/lib-http/http-client-connection.c Mon Sep 29 14:57:42 2014 +0300 @@ -1127,6 +1127,7 @@ /* the stream is still accessed by lib-http caller. */ i_stream_remove_destroy_callback(conn->incoming_payload, http_client_payload_destroyed); + conn->incoming_payload = NULL; } connection_disconnect(&conn->conn); From dovecot at dovecot.org Mon Sep 29 13:45:44 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Sep 2014 13:45:44 +0000 Subject: dovecot-2.2: lib-lda: smtp_client_deinit() may have returned NUL... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d7c05be052a3 changeset: 17847:d7c05be052a3 user: Timo Sirainen date: Mon Sep 29 15:56:02 2014 +0300 description: lib-lda: smtp_client_deinit() may have returned NULL error strings. diffstat: src/lib-lda/smtp-client.c | 24 +++++++++++++++--------- 1 files changed, 15 insertions(+), 9 deletions(-) diffs (74 lines): diff -r 475ff24ac365 -r d7c05be052a3 src/lib-lda/smtp-client.c --- a/src/lib-lda/smtp-client.c Mon Sep 29 14:57:42 2014 +0300 +++ b/src/lib-lda/smtp-client.c Mon Sep 29 15:56:02 2014 +0300 @@ -246,7 +246,8 @@ } } -static int smtp_client_send_flush(struct smtp_client *smtp_client) +static int +smtp_client_send_flush(struct smtp_client *smtp_client, const char **error_r) { struct lmtp_client_settings client_set; struct lmtp_client *client; @@ -261,18 +262,21 @@ host = t_strdup_until(host, p); if (str_to_uint(p + 1, &port) < 0 || port == 0 || port > 65535) { - i_error("Invalid port in submission_host: %s", p+1); + *error_r = t_strdup_printf( + "Invalid port in submission_host: %s", p+1); return -1; } } if (o_stream_nfinish(smtp_client->output) < 0) { - i_error("write(%s) failed: %m", smtp_client->temp_path); + *error_r = t_strdup_printf("write(%s) failed: %m", + smtp_client->temp_path); return -1; } if (o_stream_seek(smtp_client->output, 0) < 0) { - i_error("lseek(%s) failed: %m", smtp_client->temp_path); + *error_r = t_strdup_printf("lseek(%s) failed: %m", + smtp_client->temp_path); return -1; } @@ -289,6 +293,8 @@ host, port) < 0) { lmtp_client_deinit(&client); io_loop_destroy(&ioloop); + *error_r = t_strdup_printf("Couldn't connect to %s:%u", + host, port); return -1; } @@ -307,9 +313,11 @@ if (smtp_client->success) return 1; - else if (smtp_client->tempfail) + else if (smtp_client->tempfail) { + i_assert(smtp_client->error != NULL); + *error_r = t_strdup(smtp_client->error); return -1; - else + } else return 0; } @@ -326,11 +334,9 @@ } /* the mail has been written to a file. now actually send it. */ - ret = smtp_client_send_flush(client); + ret = smtp_client_send_flush(client, error_r); o_stream_destroy(&client->output); - - *error_r = t_strdup(client->error); pool_unref(&client->pool); return ret; } From dovecot at dovecot.org Mon Sep 29 19:44:27 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Sep 2014 19:44:27 +0000 Subject: dovecot-2.2: lib: If i_stream_default_seek_nonseekable() can't s... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ee9cebb11e0e changeset: 17848:ee9cebb11e0e user: Timo Sirainen date: Mon Sep 29 22:43:44 2014 +0300 description: lib: If i_stream_default_seek_nonseekable() can't seek, set a more readable error message. diffstat: src/lib/istream.c | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diffs (15 lines): diff -r d7c05be052a3 -r ee9cebb11e0e src/lib/istream.c --- a/src/lib/istream.c Mon Sep 29 15:56:02 2014 +0300 +++ b/src/lib/istream.c Mon Sep 29 22:43:44 2014 +0300 @@ -745,6 +745,11 @@ available = stream->pos - stream->skip; if (available == 0) { + io_stream_set_error(&stream->iostream, + "Can't seek to offset %"PRIuUOFF_T + ", because we have data only up to offset %" + PRIuUOFF_T" (eof=%d)", v_offset, + stream->istream.v_offset, stream->istream.eof); stream->istream.stream_errno = ESPIPE; return; } From dovecot at dovecot.org Mon Sep 29 19:44:28 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 29 Sep 2014 19:44:28 +0000 Subject: dovecot-2.2: lib: Improved istream-file's failing read()/pread()... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/18fc72a4a318 changeset: 17849:18fc72a4a318 user: Timo Sirainen date: Mon Sep 29 22:44:03 2014 +0300 description: lib: Improved istream-file's failing read()/pread() error message. diffstat: src/lib/istream-file.c | 15 +++++++++++++-- 1 files changed, 13 insertions(+), 2 deletions(-) diffs (42 lines): diff -r ee9cebb11e0e -r 18fc72a4a318 src/lib/istream-file.c --- a/src/lib/istream-file.c Mon Sep 29 22:43:44 2014 +0300 +++ b/src/lib/istream-file.c Mon Sep 29 22:44:03 2014 +0300 @@ -54,6 +54,7 @@ static ssize_t i_stream_file_read(struct istream_private *stream) { struct file_istream *fstream = (struct file_istream *) stream; + uoff_t offset; size_t size; ssize_t ret; @@ -65,11 +66,11 @@ return -1; } + offset = stream->istream.v_offset + (stream->pos - stream->skip); do { if (fstream->file) { ret = pread(stream->fd, stream->w_buffer + stream->pos, - size, stream->istream.v_offset + - (stream->pos - stream->skip)); + size, offset); } else if (fstream->seen_eof) { /* don't try to read() again. EOF from keyboard (^D) requires this to work right. */ @@ -97,6 +98,16 @@ /* if we get EBADF for a valid fd, it means something's really wrong and we'd better just crash. */ i_assert(errno != EBADF); + if (fstream->file) { + io_stream_set_error(&stream->iostream, + "pread(size=%"PRIuSIZE_T + " offset=%"PRIuUOFF_T") failed: %m", + size, offset); + } else { + io_stream_set_error(&stream->iostream, + "read(size=%"PRIuSIZE_T") failed: %m", + size); + } stream->istream.stream_errno = errno; return -1; } From dovecot at dovecot.org Tue Sep 30 20:50:25 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 30 Sep 2014 20:50:25 +0000 Subject: dovecot-2.2: lib-sasl: Added reference counting to lib-sasl init... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4c18abbf8b94 changeset: 17850:4c18abbf8b94 user: Stephan Bosch date: Tue Sep 30 23:49:35 2014 +0300 description: lib-sasl: Added reference counting to lib-sasl initialization. diffstat: src/lib-sasl/dsasl-client.c | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diffs (29 lines): diff -r 18fc72a4a318 -r 4c18abbf8b94 src/lib-sasl/dsasl-client.c --- a/src/lib-sasl/dsasl-client.c Mon Sep 29 22:44:03 2014 +0300 +++ b/src/lib-sasl/dsasl-client.c Tue Sep 30 23:49:35 2014 +0300 @@ -5,6 +5,7 @@ #include "safe-memset.h" #include "dsasl-client-private.h" +static int init_refcount = 0; static ARRAY(const struct dsasl_client_mech *) dsasl_mechanisms = ARRAY_INIT; static const struct dsasl_client_mech * @@ -93,6 +94,9 @@ void dsasl_clients_init(void) { + if (init_refcount++ > 0) + return; + i_array_init(&dsasl_mechanisms, 8); dsasl_client_mech_register(&dsasl_client_mech_plain); dsasl_client_mech_register(&dsasl_client_mech_login); @@ -100,5 +104,7 @@ void dsasl_clients_deinit(void) { + if (--init_refcount > 0) + return; array_free(&dsasl_mechanisms); } From dovecot at dovecot.org Tue Sep 30 20:50:25 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 30 Sep 2014 20:50:25 +0000 Subject: dovecot-2.2: lib-http: client: Fixed crash occurring sometimes w... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3773fec28b4d changeset: 17851:3773fec28b4d user: Stephan Bosch date: Tue Sep 30 23:49:45 2014 +0300 description: lib-http: client: Fixed crash occurring sometimes when delay timeout expired. Forgot to drop destroyed requests from delay queue. diffstat: src/lib-http/http-client-queue.c | 15 ++++++++++++++- src/lib-http/http-client-request.c | 3 +++ 2 files changed, 17 insertions(+), 1 deletions(-) diffs (46 lines): diff -r 4c18abbf8b94 -r 3773fec28b4d src/lib-http/http-client-queue.c --- a/src/lib-http/http-client-queue.c Tue Sep 30 23:49:35 2014 +0300 +++ b/src/lib-http/http-client-queue.c Tue Sep 30 23:49:45 2014 +0300 @@ -144,15 +144,28 @@ http_client_queue_drop_request(struct http_client_queue *queue, struct http_client_request *req) { - ARRAY_TYPE(http_client_request) *req_arr = &queue->request_queue; + ARRAY_TYPE(http_client_request) *req_arr; struct http_client_request **req_idx; + /* remove from main queue */ + req_arr = &queue->request_queue; array_foreach_modifiable(req_arr, req_idx) { if (*req_idx == req) { array_delete(req_arr, array_foreach_idx(req_arr, req_idx), 1); break; } } + + /* remove from delay queue */ + if (req->release_time.tv_sec > 0) { + req_arr = &queue->delayed_request_queue; + array_foreach_modifiable(req_arr, req_idx) { + if (*req_idx == req) { + array_delete(req_arr, array_foreach_idx(req_arr, req_idx), 1); + break; + } + } + } } static bool diff -r 4c18abbf8b94 -r 3773fec28b4d src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Tue Sep 30 23:49:35 2014 +0300 +++ b/src/lib-http/http-client-request.c Tue Sep 30 23:49:45 2014 +0300 @@ -163,6 +163,9 @@ http_client_request_debug(req, "Destroy (requests left=%d)", client->pending_requests); + if (req->queue != NULL) + http_client_queue_drop_request(req->queue, req); + if (client->pending_requests == 0 && client->ioloop != NULL) io_loop_stop(client->ioloop); From dovecot at dovecot.org Tue Sep 30 20:50:25 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 30 Sep 2014 20:50:25 +0000 Subject: dovecot-2.2: lib-http: client: The http_client_queue_fail() func... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b7678ce561c1 changeset: 17852:b7678ce561c1 user: Stephan Bosch date: Tue Sep 30 23:49:52 2014 +0300 description: lib-http: client: The http_client_queue_fail() function aborted requests in an unsafe manner. Copied queues before freeing the requests, because the destroyed requests modify the queue. diffstat: src/lib-http/http-client-queue.c | 21 ++++++++++++++------- 1 files changed, 14 insertions(+), 7 deletions(-) diffs (37 lines): diff -r 3773fec28b4d -r b7678ce561c1 src/lib-http/http-client-queue.c --- a/src/lib-http/http-client-queue.c Tue Sep 30 23:49:45 2014 +0300 +++ b/src/lib-http/http-client-queue.c Tue Sep 30 23:49:52 2014 +0300 @@ -125,19 +125,26 @@ void http_client_queue_fail(struct http_client_queue *queue, unsigned int status, const char *error) { - struct http_client_request **req; + ARRAY_TYPE(http_client_request) *req_arr, treqs; + struct http_client_request **req_idx; /* abort all pending requests */ - array_foreach_modifiable(&queue->request_queue, req) { - http_client_request_error(*req, status, error); + req_arr = &queue->request_queue; + t_array_init(&treqs, array_count(req_arr)); + array_copy(&treqs.arr, 0, &req_arr->arr, 0, array_count(req_arr)); + array_foreach_modifiable(&treqs, req_idx) { + http_client_request_error(*req_idx, status, error); } - array_clear(&queue->request_queue); + array_clear(req_arr); /* abort all delayed requests */ - array_foreach_modifiable(&queue->delayed_request_queue, req) { - http_client_request_error(*req, status, error); + req_arr = &queue->delayed_request_queue; + array_clear(&treqs); + array_copy(&treqs.arr, 0, &req_arr->arr, 0, array_count(req_arr)); + array_foreach_modifiable(&treqs, req_idx) { + http_client_request_error(*req_idx, status, error); } - array_clear(&queue->delayed_request_queue); + array_clear(req_arr); } void From dovecot at dovecot.org Tue Sep 30 21:04:01 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 30 Sep 2014 21:04:01 +0000 Subject: dovecot-2.2: lib: Added support for setting timeouts at absolute... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d80395660f27 changeset: 17853:d80395660f27 user: Stephan Bosch date: Wed Oct 01 00:03:17 2014 +0300 description: lib: Added support for setting timeouts at absolute time, rather than relative to current time. diffstat: src/lib/ioloop-private.h | 2 ++ src/lib/ioloop.c | 37 ++++++++++++++++++++++++++++++++----- src/lib/ioloop.h | 7 +++++++ 3 files changed, 41 insertions(+), 5 deletions(-) diffs (103 lines): diff -r b7678ce561c1 -r d80395660f27 src/lib/ioloop-private.h --- a/src/lib/ioloop-private.h Tue Sep 30 23:49:52 2014 +0300 +++ b/src/lib/ioloop-private.h Wed Oct 01 00:03:17 2014 +0300 @@ -69,6 +69,8 @@ struct ioloop *ioloop; struct ioloop_context *ctx; + + unsigned int one_shot:1; }; struct ioloop_context_callback { diff -r b7678ce561c1 -r d80395660f27 src/lib/ioloop.c --- a/src/lib/ioloop.c Tue Sep 30 23:49:52 2014 +0300 +++ b/src/lib/ioloop.c Wed Oct 01 00:03:17 2014 +0300 @@ -194,15 +194,14 @@ } } -#undef timeout_add -struct timeout *timeout_add(unsigned int msecs, unsigned int source_linenum, +static struct timeout * +timeout_add_common(unsigned int source_linenum, timeout_callback_t *callback, void *context) { struct timeout *timeout; timeout = i_new(struct timeout, 1); timeout->source_linenum = source_linenum; - timeout->msecs = msecs; timeout->ioloop = current_ioloop; timeout->callback = callback; @@ -213,6 +212,18 @@ io_loop_context_ref(timeout->ctx); } + return timeout; +} + +#undef timeout_add +struct timeout *timeout_add(unsigned int msecs, unsigned int source_linenum, + timeout_callback_t *callback, void *context) +{ + struct timeout *timeout; + + timeout = timeout_add_common(source_linenum, callback, context); + timeout->msecs = msecs; + timeout_update_next(timeout, timeout->ioloop->running ? NULL : &ioloop_timeval); priorityq_add(timeout->ioloop->timeouts, &timeout->item); @@ -227,6 +238,20 @@ return timeout_add(msecs, source_linenum, callback, context); } +#undef timeout_add_absolute +struct timeout *timeout_add_absolute(const struct timeval *time, + unsigned int source_linenum, + timeout_callback_t *callback, void *context) +{ + struct timeout *timeout; + + timeout = timeout_add_common(source_linenum, callback, context); + timeout->one_shot = TRUE; + + priorityq_add(timeout->ioloop->timeouts, &timeout->item); + return timeout; +} + static void timeout_free(struct timeout *timeout) { if (timeout->ctx != NULL) @@ -407,8 +432,10 @@ if (timeout_get_wait_time(timeout, &tv, &tv_call) > 0) break; - /* update timeout's next_run and reposition it in the queue */ - timeout_reset_timeval(timeout, &tv_call); + if (!timeout->one_shot) { + /* update timeout's next_run and reposition it in the queue */ + timeout_reset_timeval(timeout, &tv_call); + } if (timeout->ctx != NULL) io_loop_context_activate(timeout->ctx); diff -r b7678ce561c1 -r d80395660f27 src/lib/ioloop.h --- a/src/lib/ioloop.h Tue Sep 30 23:49:52 2014 +0300 +++ b/src/lib/ioloop.h Wed Oct 01 00:03:17 2014 +0300 @@ -97,6 +97,13 @@ timeout_add_short(msecs, __LINE__ + \ CALLBACK_TYPECHECK(callback, void (*)(typeof(context))), \ (io_callback_t *)callback, context) +struct timeout *timeout_add_absolute(const struct timeval *time, + unsigned int source_linenum, + timeout_callback_t *callback, void *context) ATTR_NULL(4); +#define timeout_add_absolute(time, callback, context) \ + timeout_add_absolute(time, __LINE__ + \ + CALLBACK_TYPECHECK(callback, void (*)(typeof(context))), \ + (io_callback_t *)callback, context) /* Remove timeout handler, and set timeout pointer to NULL. */ void timeout_remove(struct timeout **timeout); /* Reset timeout so it's next run after now+msecs. */ From dovecot at dovecot.org Tue Sep 30 21:09:21 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 30 Sep 2014 21:09:21 +0000 Subject: dovecot-2.2: lib: Added support for adding milliseconds to struc... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/bad8e61e2e88 changeset: 17854:bad8e61e2e88 user: Stephan Bosch date: Wed Oct 01 00:06:19 2014 +0300 description: lib: Added support for adding milliseconds to struct timeval time values. diffstat: src/lib/time-util.h | 22 ++++++++++++++++++++++ 1 files changed, 22 insertions(+), 0 deletions(-) diffs (32 lines): diff -r d80395660f27 -r bad8e61e2e88 src/lib/time-util.h --- a/src/lib/time-util.h Wed Oct 01 00:03:17 2014 +0300 +++ b/src/lib/time-util.h Wed Oct 01 00:06:19 2014 +0300 @@ -11,6 +11,28 @@ long long timeval_diff_usecs(const struct timeval *tv1, const struct timeval *tv2); +static inline void +timeval_add_msecs(struct timeval *tv, unsigned int msecs) +{ + tv->tv_sec += msecs / 1000; + tv->tv_usec += (msecs % 1000) * 1000; + if (tv->tv_usec >= 1000000) { + tv->tv_sec++; + tv->tv_usec -= 1000000; + } +} + +static inline void +timeval_sub_msecs(struct timeval *tv, unsigned int msecs) +{ + tv->tv_sec -= msecs / 1000; + tv->tv_usec -= (msecs % 1000) * 1000; + if (tv->tv_usec < 0) { + tv->tv_sec--; + tv->tv_usec += 1000000; + } +} + /* Wrapper to strftime() */ const char *t_strflocaltime(const char *fmt, time_t t) ATTR_STRFTIME(1); From dovecot at dovecot.org Tue Sep 30 21:14:45 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 30 Sep 2014 21:14:45 +0000 Subject: dovecot-2.2: lib: Created timeval_cmp_margin(). Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e45dda2d7cd6 changeset: 17855:e45dda2d7cd6 user: Stephan Bosch date: Wed Oct 01 00:13:36 2014 +0300 description: lib: Created timeval_cmp_margin(). It's identical to timeval_cmp(), except that it ignores tv_usec differences smaller than a specified margin. diffstat: src/lib/time-util.c | 15 +++++++++++++++ src/lib/time-util.h | 3 +++ 2 files changed, 18 insertions(+), 0 deletions(-) diffs (38 lines): diff -r bad8e61e2e88 -r e45dda2d7cd6 src/lib/time-util.c --- a/src/lib/time-util.c Wed Oct 01 00:06:19 2014 +0300 +++ b/src/lib/time-util.c Wed Oct 01 00:13:36 2014 +0300 @@ -20,6 +20,21 @@ return 0; } +int timeval_cmp_margin(const struct timeval *tv1, const struct timeval *tv2, + unsigned int usec_margin) +{ + if (tv1->tv_sec < tv2->tv_sec) + return -1; + if (tv1->tv_sec > tv2->tv_sec) + return 1; + + if (tv1->tv_usec - tv2->tv_usec < (int)usec_margin) + return -1; + if (tv1->tv_usec - tv2->tv_usec > (int)usec_margin) + return -1; + return 0; +} + int timeval_diff_msecs(const struct timeval *tv1, const struct timeval *tv2) { return timeval_diff_usecs(tv1, tv2) / 1000; diff -r bad8e61e2e88 -r e45dda2d7cd6 src/lib/time-util.h --- a/src/lib/time-util.h Wed Oct 01 00:06:19 2014 +0300 +++ b/src/lib/time-util.h Wed Oct 01 00:13:36 2014 +0300 @@ -5,6 +5,9 @@ /* Returns -1 if tv1tv2, 0 if they're equal. */ int timeval_cmp(const struct timeval *tv1, const struct timeval *tv2); +/* Same as timeval_cmp, but tv->usecs must differ by at least usec_margin */ +int timeval_cmp_margin(const struct timeval *tv1, const struct timeval *tv2, + unsigned int usec_margin); /* Returns tv1-tv2 in milliseconds. */ int timeval_diff_msecs(const struct timeval *tv1, const struct timeval *tv2); /* Returns tv1-tv2 in microseconds. */ From dovecot at dovecot.org Tue Sep 30 21:17:47 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 30 Sep 2014 21:17:47 +0000 Subject: dovecot-2.2: lib: Fixed earlier timeout_add_absolute() commit. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/df53b5ccc2ba changeset: 17856:df53b5ccc2ba user: Timo Sirainen date: Wed Oct 01 00:17:09 2014 +0300 description: lib: Fixed earlier timeout_add_absolute() commit. I removed too much from Stephan's patch. diffstat: src/lib/ioloop.c | 8 +++++--- 1 files changed, 5 insertions(+), 3 deletions(-) diffs (23 lines): diff -r e45dda2d7cd6 -r df53b5ccc2ba src/lib/ioloop.c --- a/src/lib/ioloop.c Wed Oct 01 00:13:36 2014 +0300 +++ b/src/lib/ioloop.c Wed Oct 01 00:17:09 2014 +0300 @@ -239,14 +239,16 @@ } #undef timeout_add_absolute -struct timeout *timeout_add_absolute(const struct timeval *time, - unsigned int source_linenum, - timeout_callback_t *callback, void *context) +struct timeout * +timeout_add_absolute(const struct timeval *time, + unsigned int source_linenum, + timeout_callback_t *callback, void *context) { struct timeout *timeout; timeout = timeout_add_common(source_linenum, callback, context); timeout->one_shot = TRUE; + timeout->next_run = *time; priorityq_add(timeout->ioloop->timeouts, &timeout->item); return timeout;