From dovecot at dovecot.org Tue Aug 4 08:31:46 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 04 Aug 2015 08:31:46 +0000 Subject: dovecot-2.2: lib-storage: escape_char and broken_char settings w... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/233802591cfb changeset: 18917:233802591cfb user: Timo Sirainen date: Tue Aug 04 11:31:17 2015 +0300 description: lib-storage: escape_char and broken_char settings weren't copied on mailbox_list_create() diffstat: src/lib-storage/mailbox-list.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 32d72cb26f9e -r 233802591cfb src/lib-storage/mailbox-list.c --- a/src/lib-storage/mailbox-list.c Mon Jul 20 10:33:39 2015 +0300 +++ b/src/lib-storage/mailbox-list.c Tue Aug 04 11:31:17 2015 +0300 @@ -192,6 +192,8 @@ list->set.mailbox_dir_name = p_strconcat(list->pool, set->mailbox_dir_name, "/", NULL); } + list->set.escape_char = set->escape_char; + list->set.broken_char = set->broken_char; list->set.utf8 = set->utf8; if (list->v.init != NULL) { From dovecot at dovecot.org Thu Aug 6 09:18:20 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 06 Aug 2015 09:18:20 +0000 Subject: dovecot-2.2: stats: If process was used for multiple sessions, s... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/46c34f3a73a0 changeset: 18918:46c34f3a73a0 user: Timo Sirainen date: Thu Aug 06 12:17:45 2015 +0300 description: stats: If process was used for multiple sessions, some of the fields weren't set correctly. For example the second user's disk_output was increased at startup by the same amount as what the previous user's last disk_output value was. diffstat: src/plugins/stats/stats-plugin.c | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diffs (15 lines): diff -r 233802591cfb -r 46c34f3a73a0 src/plugins/stats/stats-plugin.c --- a/src/plugins/stats/stats-plugin.c Tue Aug 04 11:31:17 2015 +0300 +++ b/src/plugins/stats/stats-plugin.c Thu Aug 06 12:17:45 2015 +0300 @@ -440,6 +440,11 @@ suser->to_stats_timeout = timeout_add(suser->refresh_secs*1000, session_stats_refresh_timeout, user); + /* fill the initial values. this is necessary for the process-global + values (e.g. getrusage()) if the process is reused for multiple + users. otherwise the next user will start with the previous one's + last values. */ + mail_user_stats_fill(user, suser->pre_io_stats); } static struct mail_storage_hooks stats_mail_storage_hooks = { From dovecot at dovecot.org Thu Aug 6 19:36:57 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 06 Aug 2015 19:36:57 +0000 Subject: dovecot-2.2: lib-storage: Make sure tryagain_r is never randomly... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/98158ef015f7 changeset: 18919:98158ef015f7 user: Timo Sirainen date: Thu Aug 06 22:36:20 2015 +0300 description: lib-storage: Make sure tryagain_r is never randomly set in mailbox_search_next_nonblock() This infinite looping in some fts plugin failure situations. diffstat: src/lib-storage/mail-storage.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 46c34f3a73a0 -r 98158ef015f7 src/lib-storage/mail-storage.c --- a/src/lib-storage/mail-storage.c Thu Aug 06 12:17:45 2015 +0300 +++ b/src/lib-storage/mail-storage.c Thu Aug 06 22:36:20 2015 +0300 @@ -1761,6 +1761,7 @@ struct mailbox *box = ctx->transaction->box; *mail_r = NULL; + *tryagain_r = FALSE; if (!box->v.search_next_nonblock(ctx, mail_r, tryagain_r)) return FALSE; From pigeonhole at rename-it.nl Thu Aug 6 20:23:09 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 06 Aug 2015 22:23:09 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: Properly implemented checking... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/49ac5b0000fd changeset: 2084:49ac5b0000fd user: Stephan Bosch date: Thu Aug 06 22:23:03 2015 +0200 description: lib-sieve: Properly implemented checking of ABI version for plugins. diffstat: configure.ac | 2 ++ pigeonhole-config.h.in | 3 +++ src/lib-sieve/sieve-plugins.c | 2 +- src/lib-sieve/storage/ldap/sieve-ldap-storage.c | 2 ++ src/plugins/sieve-extprograms/sieve-extprograms-plugin.c | 2 ++ 5 files changed, 10 insertions(+), 1 deletions(-) diffs (59 lines): diff -r f8374f7c6e92 -r 49ac5b0000fd configure.ac --- a/configure.ac Sat Jul 25 00:53:51 2015 +0200 +++ b/configure.ac Thu Aug 06 22:23:03 2015 +0200 @@ -1,4 +1,6 @@ AC_INIT([Pigeonhole], [0.4.8], [dovecot at dovecot.org], [dovecot-2.2-pigeonhole]) +AC_DEFINE_UNQUOTED([PIGEONHOLE_ABI_VERSION], "0.4.ABIv1($PACKAGE_VERSION)", [Pigeonhole ABI version]) + AC_CONFIG_AUX_DIR([.]) AC_CONFIG_SRCDIR([src]) AC_CONFIG_MACRO_DIR([m4]) diff -r f8374f7c6e92 -r 49ac5b0000fd pigeonhole-config.h.in --- a/pigeonhole-config.h.in Sat Jul 25 00:53:51 2015 +0200 +++ b/pigeonhole-config.h.in Thu Aug 06 22:23:03 2015 +0200 @@ -5,6 +5,9 @@ /* Define to the version of Pigeonhole for Dovecot. */ #undef PIGEONHOLE_VERSION +/* Pigeonhole ABI version */ +#undef PIGEONHOLE_ABI_VERSION + /* Define to build unfinished features/extensions. */ #undef HAVE_SIEVE_UNFINISHED diff -r f8374f7c6e92 -r 49ac5b0000fd src/lib-sieve/sieve-plugins.c --- a/src/lib-sieve/sieve-plugins.c Sat Jul 25 00:53:51 2015 +0200 +++ b/src/lib-sieve/sieve-plugins.c Thu Aug 06 22:23:03 2015 +0200 @@ -77,7 +77,7 @@ path = MODULEDIR"/sieve"; memset(&mod_set, 0, sizeof(mod_set)); - mod_set.abi_version = PIGEONHOLE_VERSION; + mod_set.abi_version = PIGEONHOLE_ABI_VERSION; mod_set.require_init_funcs = TRUE; mod_set.debug = FALSE; diff -r f8374f7c6e92 -r 49ac5b0000fd src/lib-sieve/storage/ldap/sieve-ldap-storage.c --- a/src/lib-sieve/storage/ldap/sieve-ldap-storage.c Sat Jul 25 00:53:51 2015 +0200 +++ b/src/lib-sieve/storage/ldap/sieve-ldap-storage.c Thu Aug 06 22:23:03 2015 +0200 @@ -184,6 +184,8 @@ #ifndef SIEVE_BUILTIN_LDAP /* Building a plugin */ +const char *sieve_storage_ldap_plugin_version = PIGEONHOLE_ABI_VERSION; + void sieve_storage_ldap_plugin_load (struct sieve_instance *svinst, void **context); void sieve_storage_ldap_plugin_unload diff -r f8374f7c6e92 -r 49ac5b0000fd src/plugins/sieve-extprograms/sieve-extprograms-plugin.c --- a/src/plugins/sieve-extprograms/sieve-extprograms-plugin.c Sat Jul 25 00:53:51 2015 +0200 +++ b/src/plugins/sieve-extprograms/sieve-extprograms-plugin.c Thu Aug 06 22:23:03 2015 +0200 @@ -18,6 +18,8 @@ const struct sieve_extension *ext_execute; }; +const char *sieve_extprograms_plugin_version = PIGEONHOLE_ABI_VERSION; + void sieve_extprograms_plugin_load (struct sieve_instance *svinst, void **context) { From dovecot at dovecot.org Fri Aug 7 08:32:03 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 07 Aug 2015 08:32:03 +0000 Subject: dovecot-2.2: fts: If Tika returns 500, retry it a couple of time... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/35d3777cc4d2 changeset: 18920:35d3777cc4d2 user: Timo Sirainen date: Fri Aug 07 11:31:29 2015 +0300 description: fts: If Tika returns 500, retry it a couple of times and then fallback to ignoring the problem. diffstat: src/plugins/fts/fts-parser-tika.c | 15 +++++++++++++++ 1 files changed, 15 insertions(+), 0 deletions(-) diffs (25 lines): diff -r 98158ef015f7 -r 35d3777cc4d2 src/plugins/fts/fts-parser-tika.c --- a/src/plugins/fts/fts-parser-tika.c Thu Aug 06 22:36:20 2015 +0300 +++ b/src/plugins/fts/fts-parser-tika.c Fri Aug 07 11:31:29 2015 +0300 @@ -103,6 +103,21 @@ } parser->payload = i_stream_create_from_data("", 0); break; + case 500: + /* Server Error - the problem could be anything (in Tika or + HTTP server or proxy) and might be retriable, but Tika has + trouble processing some documents and throws up this error + every time for those documents. So we try retrying this a + couple of times, but if that doesn't work we'll just ignore + it. */ + if (http_client_request_try_retry(parser->http_req)) + return; + i_info("fts_tika: PUT %s failed: %u %s - ignoring", + mail_user_plugin_getenv(parser->user, "fts_tika"), + response->status, response->reason); + parser->payload = i_stream_create_from_data("", 0); + break; + default: i_error("fts_tika: PUT %s failed: %u %s", mail_user_plugin_getenv(parser->user, "fts_tika"), From dovecot at dovecot.org Fri Aug 7 09:59:10 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 07 Aug 2015 09:59:10 +0000 Subject: dovecot-2.2: lib: istream-timeout could have triggered timeout t... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c143d037fd54 changeset: 18921:c143d037fd54 user: Timo Sirainen date: Fri Aug 07 12:58:30 2015 +0300 description: lib: istream-timeout could have triggered timeout too early after long-running code. diffstat: src/lib/istream-timeout.c | 30 ++++++++++++++++++++---------- 1 files changed, 20 insertions(+), 10 deletions(-) diffs (57 lines): diff -r 35d3777cc4d2 -r c143d037fd54 src/lib/istream-timeout.c --- a/src/lib/istream-timeout.c Fri Aug 07 11:31:29 2015 +0300 +++ b/src/lib/istream-timeout.c Fri Aug 07 12:58:30 2015 +0300 @@ -41,6 +41,13 @@ unsigned int msecs; int diff; + if (tstream->update_timestamp) { + /* we came here after a long-running code. timeouts are handled + before IOs, so wait for i_stream_read() to be called again + before assuming that we've timed out. */ + return; + } + timeout_remove(&tstream->to); diff = timeval_diff_msecs(&ioloop_timeval, &tstream->last_read_timestamp); @@ -64,6 +71,17 @@ i_stream_set_input_pending(tstream->istream.parent, TRUE); } +static void i_stream_timeout_set_pending(struct timeout_istream *tstream) +{ + /* make sure we get called again on the next ioloop run. this updates + the timeout to the timestamp where we actually would have wanted to + start waiting for more data (so if there is long-running code + outside the ioloop it's not counted) */ + tstream->update_timestamp = TRUE; + tstream->last_read_timestamp = ioloop_timeval; + i_stream_set_input_pending(&tstream->istream.istream, TRUE); +} + static ssize_t i_stream_timeout_read(struct istream_private *stream) { @@ -90,19 +108,11 @@ tstream->to = tstream->timeout_msecs == 0 ? NULL : timeout_add(tstream->timeout_msecs, i_stream_timeout, tstream); - tstream->update_timestamp = TRUE; - tstream->last_read_timestamp = ioloop_timeval; + i_stream_timeout_set_pending(tstream); } else if (ret > 0 && tstream->to != NULL) { /* we read something, reset the timeout */ timeout_reset(tstream->to); - /* make sure we get called again on the next ioloop run. - this updates the timeout to the timestamp where we actually - would have wanted to start waiting for more data (so if - there is long-running code outside the ioloop it's not - counted) */ - tstream->update_timestamp = TRUE; - tstream->last_read_timestamp = ioloop_timeval; - i_stream_set_input_pending(&stream->istream, TRUE); + i_stream_timeout_set_pending(tstream); } else if (tstream->update_timestamp) { tstream->update_timestamp = FALSE; tstream->last_read_timestamp = ioloop_timeval; From dovecot at dovecot.org Fri Aug 7 10:32:43 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 07 Aug 2015 10:32:43 +0000 Subject: dovecot-2.2: fts-solr: "highest UID for mailbox" lookup was actu... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/91005181b997 changeset: 18922:91005181b997 user: Timo Sirainen date: Fri Aug 07 13:32:02 2015 +0300 description: fts-solr: "highest UID for mailbox" lookup was actually returning "highest UID for user" If the default operator was OR instead of AND. This affected indexing mails in newly created mailboxes. diffstat: src/plugins/fts-solr/fts-backend-solr-old.c | 4 ++-- src/plugins/fts-solr/fts-backend-solr.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diffs (28 lines): diff -r c143d037fd54 -r 91005181b997 src/plugins/fts-solr/fts-backend-solr-old.c --- a/src/plugins/fts-solr/fts-backend-solr-old.c Fri Aug 07 12:58:30 2015 +0300 +++ b/src/plugins/fts-solr/fts-backend-solr-old.c Fri Aug 07 13:32:02 2015 +0300 @@ -306,10 +306,10 @@ box_name = fts_box_get_root(box, &ns); mailbox_get_open_status(box, STATUS_UIDVALIDITY, &status); - str_printfa(str, "uidv:%u+box:", status.uidvalidity); + str_printfa(str, "uidv:%u+AND+box:", status.uidvalidity); solr_quote_http(str, box_name); solr_add_ns_query_http(str, backend, ns); - str_append(str, "+user:"); + str_append(str, "+AND+user:"); solr_quote_http(str, ns->user->username); pool = pool_alloconly_create("solr last uid lookup", 1024); diff -r c143d037fd54 -r 91005181b997 src/plugins/fts-solr/fts-backend-solr.c --- a/src/plugins/fts-solr/fts-backend-solr.c Fri Aug 07 12:58:30 2015 +0300 +++ b/src/plugins/fts-solr/fts-backend-solr.c Fri Aug 07 13:32:02 2015 +0300 @@ -202,7 +202,7 @@ if (fts_mailbox_get_guid(box, &box_guid) < 0) return -1; - str_printfa(str, "box:%s+user:", box_guid); + str_printfa(str, "box:%s+AND+user:", box_guid); if (_backend->ns->owner != NULL) solr_quote_http(str, _backend->ns->owner->username); else From dovecot at dovecot.org Fri Aug 7 12:31:13 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 07 Aug 2015 12:31:13 +0000 Subject: dovecot-2.2: dsync: Renamed -F parameter to -O to avoid a conflict Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/573118da2eaa changeset: 18923:573118da2eaa user: Timo Sirainen date: Fri Aug 07 15:13:34 2015 +0300 description: dsync: Renamed -F parameter to -O to avoid a conflict doveadm mail commands already had a generic -F parameter. diffstat: src/doveadm/doveadm-dsync.c | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) diffs (31 lines): diff -r 91005181b997 -r 573118da2eaa src/doveadm/doveadm-dsync.c --- a/src/doveadm/doveadm-dsync.c Fri Aug 07 13:32:02 2015 +0300 +++ b/src/doveadm/doveadm-dsync.c Fri Aug 07 15:13:34 2015 +0300 @@ -38,7 +38,7 @@ #include #include -#define DSYNC_COMMON_GETOPT_ARGS "+1a:dEfF:g:l:m:n:NPr:Rs:t:Ux:" +#define DSYNC_COMMON_GETOPT_ARGS "+1a:dEfg:l:m:n:NO:Pr:Rs:t:Ux:" #define DSYNC_REMOTE_CMD_EXIT_WAIT_SECS 30 /* The broken_char is mainly set to get a proper error message when trying to convert a mailbox with a name that can't be used properly translated between @@ -941,15 +941,15 @@ case 'f': ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_FULL; break; - case 'F': { + case 'O': { const char *str = optarg; if (strchr(str, ' ') != NULL) - i_fatal("-F parameter doesn't support multiple flags currently"); + i_fatal("-O parameter doesn't support multiple flags currently"); if (str[0] == '-') str++; if (str[0] == '\\' && imap_parse_system_flag(str) == 0) - i_fatal("Invalid system flag given for -F parameter: '%s'", str); + i_fatal("Invalid system flag given for -O parameter: '%s'", str); ctx->sync_flags = optarg; break; } From dovecot at dovecot.org Fri Aug 7 12:31:16 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 07 Aug 2015 12:31:16 +0000 Subject: dovecot-2.2: stats: Added session= filter to doveadm dump se... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f611d66e56f6 changeset: 18924:f611d66e56f6 user: Timo Sirainen date: Fri Aug 07 15:30:26 2015 +0300 description: stats: Added session= filter to doveadm dump session|command diffstat: src/stats/client-export.c | 9 +++++++-- 1 files changed, 7 insertions(+), 2 deletions(-) diffs (40 lines): diff -r 573118da2eaa -r f611d66e56f6 src/stats/client-export.c --- a/src/stats/client-export.c Fri Aug 07 15:13:34 2015 +0300 +++ b/src/stats/client-export.c Fri Aug 07 15:30:26 2015 +0300 @@ -28,7 +28,7 @@ }; struct mail_export_filter { - const char *user, *domain; + const char *user, *domain, *session; struct ip_addr ip; unsigned int ip_bits; time_t since; @@ -65,7 +65,7 @@ unsigned long l; /* filters: - user= | domain= + user= | domain= | session= ip=[/] since= connected @@ -76,6 +76,8 @@ filter_r->user = p_strdup(pool, *args + 5); else if (strncmp(*args, "domain=", 7) == 0) filter_r->domain = p_strdup(pool, *args + 7); + else if (strncmp(*args, "session=", 8) == 0) + filter_r->session = p_strdup(pool, *args + 8); else if (strncmp(*args, "ip=", 3) == 0) { if (net_parse_range(*args + 3, &filter_r->ip, &filter_r->ip_bits) < 0) { @@ -134,6 +136,9 @@ return FALSE; if (filter->since > session->last_update.tv_sec) return FALSE; + if (filter->session != NULL && + strcmp(session->id, filter->session) != 0) + return FALSE; if (filter->user != NULL && !wildcard_match(session->user->name, filter->user)) return FALSE; From dovecot at dovecot.org Fri Aug 7 12:33:16 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 07 Aug 2015 12:33:16 +0000 Subject: dovecot-2.2: doveadm: Make sure we can't accidentally add duplic... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/00f306fc5bde changeset: 18926:00f306fc5bde user: Timo Sirainen date: Fri Aug 07 15:32:28 2015 +0300 description: doveadm: Make sure we can't accidentally add duplicate getopt args. diffstat: src/doveadm/doveadm-mail.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r ff17864ba6e0 -r 00f306fc5bde src/doveadm/doveadm-mail.c --- a/src/doveadm/doveadm-mail.c Fri Aug 07 15:32:14 2015 +0300 +++ b/src/doveadm/doveadm-mail.c Fri Aug 07 15:32:28 2015 +0300 @@ -549,6 +549,8 @@ /* keep context's getopt_args first in case it contains '+' */ if (ctx->getopt_args != NULL) getopt_args = t_strconcat(ctx->getopt_args, getopt_args, NULL); + i_assert(master_getopt_str_is_valid(getopt_args)); + ctx->cur_username = getenv("USER"); wildcard_user = NULL; while ((c = getopt(argc, argv, getopt_args)) > 0) { From dovecot at dovecot.org Fri Aug 7 12:33:15 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 07 Aug 2015 12:33:15 +0000 Subject: dovecot-2.2: lib-master: Make sure we can't accidentally add dup... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ff17864ba6e0 changeset: 18925:ff17864ba6e0 user: Timo Sirainen date: Fri Aug 07 15:32:14 2015 +0300 description: lib-master: Make sure we can't accidentally add duplicate getopt args. diffstat: src/lib-master/master-service.c | 19 +++++++++++++++++++ src/lib-master/master-service.h | 3 +++ 2 files changed, 22 insertions(+), 0 deletions(-) diffs (49 lines): diff -r f611d66e56f6 -r ff17864ba6e0 src/lib-master/master-service.c --- a/src/lib-master/master-service.c Fri Aug 07 15:30:26 2015 +0300 +++ b/src/lib-master/master-service.c Fri Aug 07 15:32:14 2015 +0300 @@ -251,6 +251,8 @@ { int c; + i_assert(master_getopt_str_is_valid(service->getopt_str)); + while ((c = getopt(service->argc, service->argv, service->getopt_str)) > 0) { if (!master_service_parse_option(service, c, optarg)) @@ -259,6 +261,23 @@ return c; } +bool master_getopt_str_is_valid(const char *str) +{ + unsigned int i, j; + + /* make sure there are no duplicates. there are few enough characters + that this should be fast enough. */ + for (i = 0; str[i] != '\0'; i++) { + if (str[i] == ':' || str[i] == '+' || str[i] == '-') + continue; + for (j = i+1; str[j] != '\0'; j++) { + if (str[i] == str[j]) + return FALSE; + } + } + return TRUE; +} + void master_service_init_log(struct master_service *service, const char *prefix) { diff -r f611d66e56f6 -r ff17864ba6e0 src/lib-master/master-service.h --- a/src/lib-master/master-service.h Fri Aug 07 15:30:26 2015 +0300 +++ b/src/lib-master/master-service.h Fri Aug 07 15:32:14 2015 +0300 @@ -63,6 +63,9 @@ /* Call getopt() and handle internal parameters. Return values are the same as getopt()'s. */ int master_getopt(struct master_service *service); +/* Returns TRUE if str is a valid getopt_str. Currently this only checks for + duplicate args so they aren't accidentally added. */ +bool master_getopt_str_is_valid(const char *str); /* Parser command line option. Returns TRUE if processed. */ bool master_service_parse_option(struct master_service *service, int opt, const char *arg); From dovecot at dovecot.org Sun Aug 9 18:55:29 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 09 Aug 2015 18:55:29 +0000 Subject: dovecot-2.2: man: doveadm-import -s description updated Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e7cb20b4785a changeset: 18927:e7cb20b4785a user: Timo Sirainen date: Sun Aug 09 21:54:33 2015 +0300 description: man: doveadm-import -s description updated The previously mentioned subscriptions-file is specific to Maildir format. diffstat: doc/man/doveadm-import.1.in | 6 ++---- 1 files changed, 2 insertions(+), 4 deletions(-) diffs (21 lines): diff -r 00f306fc5bde -r e7cb20b4785a doc/man/doveadm-import.1.in --- a/doc/man/doveadm-import.1.in Fri Aug 07 15:32:28 2015 +0300 +++ b/doc/man/doveadm-import.1.in Sun Aug 09 21:54:33 2015 +0300 @@ -1,5 +1,5 @@ .\" Copyright (c) 2010-2015 Dovecot authors, see the included COPYING file -.TH DOVEADM\-IMPORT 1 "2015-05-09" "Dovecot v2.2" "Dovecot" +.TH DOVEADM\-IMPORT 1 "2015-08-09" "Dovecot v2.2" "Dovecot" .SH NAME doveadm\-import \- Import messages matching given search query .\"------------------------------------------------------------------------ @@ -72,9 +72,7 @@ .B \-s option is present, .I dest_parent -and all new mailboxes under it will be listed in the -.I subscriptions -file. +and all new mailboxes under it will be subscribed to. .\"------------------------------------- @INCLUDE:option-u-user@ .\"------------------------------------------------------------------------ From dovecot at dovecot.org Mon Aug 10 12:36:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 10 Aug 2015 12:36:25 +0000 Subject: dovecot-2.2: lda: Removed unnecessary/duplicate 'k' from getopt_... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1890ab5a841f changeset: 18928:1890ab5a841f user: Timo Sirainen date: Mon Aug 10 15:35:43 2015 +0300 description: lda: Removed unnecessary/duplicate 'k' from getopt_args This caused assert-crash at startup after ff17864ba6e0. diffstat: src/lda/main.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r e7cb20b4785a -r 1890ab5a841f src/lda/main.c --- a/src/lda/main.c Sun Aug 09 21:54:33 2015 +0300 +++ b/src/lda/main.c Mon Aug 10 15:35:43 2015 +0300 @@ -310,7 +310,7 @@ master_service = master_service_init("lda", MASTER_SERVICE_FLAG_STANDALONE | MASTER_SERVICE_FLAG_DONT_LOG_TO_STDERR, - &argc, &argv, "a:d:ef:km:p:r:"); + &argc, &argv, "a:d:ef:m:p:r:"); memset(&ctx, 0, sizeof(ctx)); ctx.session = mail_deliver_session_init(); From dovecot at dovecot.org Mon Aug 10 12:40:30 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 10 Aug 2015 12:40:30 +0000 Subject: dovecot-2.2: rawlog: Removed unnecessary/duplicate 'o' from geto... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6bebbf0e1ec8 changeset: 18929:6bebbf0e1ec8 user: Timo Sirainen date: Mon Aug 10 15:39:50 2015 +0300 description: rawlog: Removed unnecessary/duplicate 'o' from getopt_args This caused assert-crash at startup after ff17864ba6e0. diffstat: src/util/rawlog.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 1890ab5a841f -r 6bebbf0e1ec8 src/util/rawlog.c --- a/src/util/rawlog.c Mon Aug 10 15:35:43 2015 +0300 +++ b/src/util/rawlog.c Mon Aug 10 15:39:50 2015 +0300 @@ -383,7 +383,7 @@ int c; master_service = master_service_init("rawlog", 0, - &argc, &argv, "+f:obIt"); + &argc, &argv, "+f:bIt"); while ((c = master_getopt(master_service)) > 0) { switch (c) { case 'f': From dovecot at dovecot.org Tue Aug 11 11:27:33 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 11 Aug 2015 11:27:33 +0000 Subject: dovecot-2.2: lib-fts: Install headers on make install. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/79b8efbbad96 changeset: 18930:79b8efbbad96 user: Timo Sirainen date: Tue Aug 11 14:27:01 2015 +0300 description: lib-fts: Install headers on make install. diffstat: src/lib-fts/Makefile.am | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (22 lines): diff -r 6bebbf0e1ec8 -r 79b8efbbad96 src/lib-fts/Makefile.am --- a/src/lib-fts/Makefile.am Mon Aug 10 15:39:50 2015 +0300 +++ b/src/lib-fts/Makefile.am Tue Aug 11 14:27:01 2015 +0300 @@ -73,7 +73,7 @@ fts-tokenizer-generic.c \ $(ICU_SOURCES) -noinst_HEADERS = \ +headers = \ fts-common.h \ fts-filter.h \ fts-filter-private.h \ @@ -83,6 +83,9 @@ fts-tokenizer-private.h \ fts-tokenizer-generic-private.h +pkginc_libdir=$(pkgincludedir) +pkginc_lib_HEADERS = $(headers) + test_programs = \ $(ICU_TESTS) \ $(TEST_FTS_LANGUAGE) \ From dovecot at dovecot.org Wed Aug 12 09:52:50 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 12 Aug 2015 09:52:50 +0000 Subject: dovecot-2.2: pop3: Added pop3_delete_type setting with values "d... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4fcf4d745a27 changeset: 18931:4fcf4d745a27 user: Timo Sirainen date: Wed Aug 12 12:52:06 2015 +0300 description: pop3: Added pop3_delete_type setting with values "default", "expunge" or "flag". This is related to pop3_deleted_flag setting. The main behefit here is that you can now hide messages from POP3 by setting pop3_deleted_flag, but without changing the actual deletion behavior by setting pop3_delete_type=expunge. diffstat: src/pop3/pop3-commands.c | 11 ++++++++--- src/pop3/pop3-settings.c | 22 +++++++++++++++++++++- src/pop3/pop3-settings.h | 6 ++++++ 3 files changed, 35 insertions(+), 4 deletions(-) diffs (93 lines): diff -r 79b8efbbad96 -r 4fcf4d745a27 src/pop3/pop3-commands.c --- a/src/pop3/pop3-commands.c Tue Aug 11 14:27:01 2015 +0300 +++ b/src/pop3/pop3-commands.c Wed Aug 12 12:52:06 2015 +0300 @@ -219,10 +219,15 @@ static void client_expunge(struct client *client, struct mail *mail) { - if (client->deleted_kw != NULL) + switch (client->set->parsed_delete_type) { + case POP3_DELETE_TYPE_EXPUNGE: + mail_expunge(mail); + break; + case POP3_DELETE_TYPE_FLAG: + i_assert(client->deleted_kw != NULL); mail_update_keywords(mail, MODIFY_ADD, client->deleted_kw); - else - mail_expunge(mail); + break; + } client->expunged_count++; } diff -r 79b8efbbad96 -r 4fcf4d745a27 src/pop3/pop3-settings.c --- a/src/pop3/pop3-settings.c Tue Aug 11 14:27:01 2015 +0300 +++ b/src/pop3/pop3-settings.c Wed Aug 12 12:52:06 2015 +0300 @@ -72,6 +72,7 @@ DEF(SET_STR, pop3_logout_format), DEF(SET_ENUM, pop3_uidl_duplicates), DEF(SET_STR, pop3_deleted_flag), + DEF(SET_ENUM, pop3_delete_type), SETTING_DEFINE_LIST_END }; @@ -88,7 +89,8 @@ .pop3_client_workarounds = "", .pop3_logout_format = "top=%t/%p, retr=%r/%b, del=%d/%m, size=%s", .pop3_uidl_duplicates = "allow:rename", - .pop3_deleted_flag = "" + .pop3_deleted_flag = "", + .pop3_delete_type = "default:expunge:flag" }; static const struct setting_parser_info *pop3_setting_dependencies[] = { @@ -156,6 +158,24 @@ if (pop3_settings_parse_workarounds(set, error_r) < 0) return FALSE; + if (strcmp(set->pop3_delete_type, "default") == 0) { + if (set->pop3_deleted_flag[0] == '\0') + set->parsed_delete_type = POP3_DELETE_TYPE_EXPUNGE; + else + set->parsed_delete_type = POP3_DELETE_TYPE_FLAG; + } else if (strcmp(set->pop3_delete_type, "expunge") == 0) { + set->parsed_delete_type = POP3_DELETE_TYPE_EXPUNGE; + } else if (strcmp(set->pop3_delete_type, "flag") == 0) { + if (set->pop3_deleted_flag[0] == '\0') { + *error_r = "pop3_delete_type=flag, but pop3_deleted_flag not set"; + return FALSE; + } + set->parsed_delete_type = POP3_DELETE_TYPE_FLAG; + } else { + *error_r = t_strdup_printf("pop3_delete_type: Unknown value '%s'", + set->pop3_delete_type); + return FALSE; + } return TRUE; } /* */ diff -r 79b8efbbad96 -r 4fcf4d745a27 src/pop3/pop3-settings.h --- a/src/pop3/pop3-settings.h Tue Aug 11 14:27:01 2015 +0300 +++ b/src/pop3/pop3-settings.h Wed Aug 12 12:52:06 2015 +0300 @@ -8,6 +8,10 @@ WORKAROUND_OUTLOOK_NO_NULS = 0x01, WORKAROUND_OE_NS_EOH = 0x02 }; +enum pop3_delete_type { + POP3_DELETE_TYPE_EXPUNGE = 0, + POP3_DELETE_TYPE_FLAG, +}; /* */ struct pop3_settings { @@ -24,8 +28,10 @@ const char *pop3_logout_format; const char *pop3_uidl_duplicates; const char *pop3_deleted_flag; + const char *pop3_delete_type; enum pop3_client_workarounds parsed_workarounds; + enum pop3_delete_type parsed_delete_type; }; extern const struct setting_parser_info pop3_setting_parser_info; From dovecot at dovecot.org Sat Aug 15 10:15:21 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 15 Aug 2015 10:15:21 +0000 Subject: dovecot-2.2: dsync: If mailbox is already deleted locally, don't... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/40e220761a41 changeset: 18932:40e220761a41 user: Timo Sirainen date: Sat Aug 15 12:05:34 2015 +0200 description: dsync: If mailbox is already deleted locally, don't attempt to delete it again. This only caused an unnecessary desync warning / exit code. diffstat: src/doveadm/dsync/dsync-mailbox-tree-sync.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 4fcf4d745a27 -r 40e220761a41 src/doveadm/dsync/dsync-mailbox-tree-sync.c --- a/src/doveadm/dsync/dsync-mailbox-tree-sync.c Wed Aug 12 12:52:06 2015 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-tree-sync.c Sat Aug 15 12:05:34 2015 +0200 @@ -164,7 +164,8 @@ guid_128_to_string(node->mailbox_guid), reason); } - if (tree == ctx->local_tree) { + if (tree == ctx->local_tree && + node->existence != DSYNC_MAILBOX_NODE_DELETED) { /* delete this mailbox locally */ i_assert(ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL); change = array_append_space(&ctx->changes); From dovecot at dovecot.org Sat Aug 15 10:15:30 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 15 Aug 2015 10:15:30 +0000 Subject: dovecot-2.2: dsync: Fixed syncing a recursive delete of mailbox ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1ed8418031b6 changeset: 18933:1ed8418031b6 user: Timo Sirainen date: Sat Aug 15 12:14:35 2015 +0200 description: dsync: Fixed syncing a recursive delete of mailbox with children. For example if 1/2/3 exists in both sides and then 1/2/3, 1/2 and 1 are deleted from one side, the next dsync would delete 1/2/3, but leave 1/2 and 1 undeleted. diffstat: src/doveadm/dsync/dsync-mailbox-tree-sync.c | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) diffs (27 lines): diff -r 40e220761a41 -r 1ed8418031b6 src/doveadm/dsync/dsync-mailbox-tree-sync.c --- a/src/doveadm/dsync/dsync-mailbox-tree-sync.c Sat Aug 15 12:05:34 2015 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-tree-sync.c Sat Aug 15 12:14:35 2015 +0200 @@ -1289,19 +1289,19 @@ sync_subscription(ctx, local_node, remote_node); if (local_node->existence == DSYNC_MAILBOX_NODE_DELETED && - local_node->first_child == NULL && + !node_has_existent_children(local_node, TRUE) && remote_node->existence == DSYNC_MAILBOX_NODE_EXISTS && ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE) { /* delete from remote */ - i_assert(remote_node->first_child == NULL); + i_assert(!node_has_existent_children(remote_node, TRUE)); remote_node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT; } if (remote_node->existence == DSYNC_MAILBOX_NODE_DELETED && - remote_node->first_child == NULL && + !node_has_existent_children(remote_node, TRUE) && local_node->existence == DSYNC_MAILBOX_NODE_EXISTS && ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL) { /* delete from local */ - i_assert(local_node->first_child == NULL); + i_assert(!node_has_existent_children(local_node, TRUE)); local_node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT; sync_add_dir_change(ctx, local_node, DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_DIR); From dovecot at dovecot.org Sun Aug 16 11:14:55 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 16 Aug 2015 11:14:55 +0000 Subject: dovecot-2.2: lib-fs: fs_write() didn't update stats.write_count ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/295fd771e02d changeset: 18934:295fd771e02d user: Timo Sirainen date: Sun Aug 16 13:07:21 2015 +0200 description: lib-fs: fs_write() didn't update stats.write_count correctly. If backend implemented asynchronous write(), the write_count was updated multiple times. If backend didn't implement write(), then it was counted twice. diffstat: src/lib-fs/fs-api.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (17 lines): diff -r 1ed8418031b6 -r 295fd771e02d src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Sat Aug 15 12:14:35 2015 +0200 +++ b/src/lib-fs/fs-api.c Sun Aug 16 13:07:21 2015 +0200 @@ -530,11 +530,12 @@ { int ret; - file->fs->stats.write_count++; if (file->fs->v.write != NULL) { T_BEGIN { ret = file->fs->v.write(file, data, size); } T_END; + if (!(ret < 0 && errno == EAGAIN)) + file->fs->stats.write_count++; return ret; } From dovecot at dovecot.org Sun Aug 16 11:15:00 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 16 Aug 2015 11:15:00 +0000 Subject: dovecot-2.2: lib-fs: Small code cleanup for handling write_strea... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2f1378beeef6 changeset: 18935:2f1378beeef6 user: Timo Sirainen date: Sun Aug 16 13:10:48 2015 +0200 description: lib-fs: Small code cleanup for handling write_stream_finish() diffstat: src/lib-fs/fs-api.c | 39 ++++++++++++++++++++------------------- 1 files changed, 20 insertions(+), 19 deletions(-) diffs (69 lines): diff -r 295fd771e02d -r 2f1378beeef6 src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Sun Aug 16 13:07:21 2015 +0200 +++ b/src/lib-fs/fs-api.c Sun Aug 16 13:10:48 2015 +0200 @@ -555,9 +555,25 @@ return file->output; } +static int fs_write_stream_finish_int(struct fs_file *file, bool success) +{ + int ret; + + T_BEGIN { + ret = file->fs->v.write_stream_finish(file, success); + } T_END; + if (ret == 0) { + file->metadata_changed = FALSE; + } else { + /* write didn't finish yet. this shouldn't happen if we + indicated a failure. */ + i_assert(success); + } + return ret; +} + int fs_write_stream_finish(struct fs_file *file, struct ostream **output) { - int ret; bool success = TRUE; i_assert(*output == file->output || *output == NULL); @@ -573,24 +589,12 @@ success = FALSE; } } - T_BEGIN { - ret = file->fs->v.write_stream_finish(file, success); - } T_END; - if (ret != 0) - file->metadata_changed = FALSE; - return ret; + return fs_write_stream_finish_int(file, success); } int fs_write_stream_finish_async(struct fs_file *file) { - int ret; - - T_BEGIN { - ret = file->fs->v.write_stream_finish(file, TRUE); - } T_END; - if (ret != 0) - file->metadata_changed = FALSE; - return ret; + return fs_write_stream_finish_int(file, TRUE); } void fs_write_stream_abort(struct fs_file *file, struct ostream **output) @@ -600,10 +604,7 @@ *output = NULL; if (file->output != NULL) o_stream_ignore_last_errors(file->output); - T_BEGIN { - (void)file->fs->v.write_stream_finish(file, FALSE); - } T_END; - file->metadata_changed = FALSE; + (void)fs_write_stream_finish_int(file, FALSE); } void fs_write_set_hash(struct fs_file *file, const struct hash_method *method, From dovecot at dovecot.org Sun Aug 16 11:15:00 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 16 Aug 2015 11:15:00 +0000 Subject: dovecot-2.2: lib-fs: fs_copy*() didn't update metadata_changed c... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e8e838cb9663 changeset: 18936:e8e838cb9663 user: Timo Sirainen date: Sun Aug 16 13:14:11 2015 +0200 description: lib-fs: fs_copy*() didn't update metadata_changed correctly. It was supposed to be (only) set when the copying actually finished. diffstat: src/lib-fs/fs-api.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (21 lines): diff -r 2f1378beeef6 -r e8e838cb9663 src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Sun Aug 16 13:10:48 2015 +0200 +++ b/src/lib-fs/fs-api.c Sun Aug 16 13:14:11 2015 +0200 @@ -761,7 +761,7 @@ T_BEGIN { ret = src->fs->v.copy(src, dest); } T_END; - if (ret < 0 || errno != EAGAIN) + if (!(ret < 0 && errno == EAGAIN)) dest->metadata_changed = FALSE; return ret; } @@ -773,7 +773,7 @@ T_BEGIN { ret = dest->fs->v.copy(NULL, dest); } T_END; - if (ret < 0 || errno != EAGAIN) + if (!(ret < 0 && errno == EAGAIN)) dest->metadata_changed = FALSE; return ret; } From dovecot at dovecot.org Sun Aug 16 11:17:47 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 16 Aug 2015 11:17:47 +0000 Subject: dovecot-2.2: lib-fs: Fix to earlier write_stream_finish() commit Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f04240346fa0 changeset: 18937:f04240346fa0 user: Timo Sirainen date: Sun Aug 16 13:17:06 2015 +0200 description: lib-fs: Fix to earlier write_stream_finish() commit 2f1378beeef6 accidentally reversed == and != check. diffstat: src/lib-fs/fs-api.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r e8e838cb9663 -r f04240346fa0 src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Sun Aug 16 13:14:11 2015 +0200 +++ b/src/lib-fs/fs-api.c Sun Aug 16 13:17:06 2015 +0200 @@ -562,7 +562,7 @@ T_BEGIN { ret = file->fs->v.write_stream_finish(file, success); } T_END; - if (ret == 0) { + if (ret != 0) { file->metadata_changed = FALSE; } else { /* write didn't finish yet. this shouldn't happen if we From dovecot at dovecot.org Sun Aug 16 11:23:03 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sun, 16 Aug 2015 11:23:03 +0000 Subject: dovecot-2.2: lib-fs: Moved enum fs_op from fs-randomfail.c to fs... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/38c3d7b44dd5 changeset: 18938:38c3d7b44dd5 user: Timo Sirainen date: Sun Aug 16 13:22:14 2015 +0200 description: lib-fs: Moved enum fs_op from fs-randomfail.c to fs-api-private.h These are useful for other purposes as well. diffstat: src/lib-fs/fs-api-private.h | 17 +++++++++++++++++ src/lib-fs/fs-randomfail.c | 16 ---------------- 2 files changed, 17 insertions(+), 16 deletions(-) diffs (53 lines): diff -r f04240346fa0 -r 38c3d7b44dd5 src/lib-fs/fs-api-private.h --- a/src/lib-fs/fs-api-private.h Sun Aug 16 13:17:06 2015 +0200 +++ b/src/lib-fs/fs-api-private.h Sun Aug 16 13:22:14 2015 +0200 @@ -4,6 +4,23 @@ #include "fs-api.h" #include "module-context.h" +enum fs_op { + FS_OP_WAIT, + FS_OP_METADATA, + FS_OP_PREFETCH, + FS_OP_READ, + FS_OP_WRITE, + FS_OP_LOCK, + FS_OP_EXISTS, + FS_OP_STAT, + FS_OP_COPY, + FS_OP_RENAME, + FS_OP_DELETE, + FS_OP_ITER, + + FS_OP_COUNT +}; + struct fs_api_module_register { unsigned int id; }; diff -r f04240346fa0 -r 38c3d7b44dd5 src/lib-fs/fs-randomfail.c --- a/src/lib-fs/fs-randomfail.c Sun Aug 16 13:17:06 2015 +0200 +++ b/src/lib-fs/fs-randomfail.c Sun Aug 16 13:22:14 2015 +0200 @@ -13,22 +13,6 @@ #define RANDOMFAIL_ERROR "Random failure injection" -enum fs_op { - FS_OP_WAIT, - FS_OP_METADATA, - FS_OP_PREFETCH, - FS_OP_READ, - FS_OP_WRITE, - FS_OP_LOCK, - FS_OP_EXISTS, - FS_OP_STAT, - FS_OP_COPY, - FS_OP_RENAME, - FS_OP_DELETE, - FS_OP_ITER, - - FS_OP_COUNT -}; static const char *fs_op_names[FS_OP_COUNT] = { "wait", "metadata", "prefetch", "read", "write", "lock", "exists", "stat", "copy", "rename", "delete", "iter" From pigeonhole at rename-it.nl Sun Aug 16 17:00:37 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 16 Aug 2015 19:00:37 +0200 Subject: dovecot-2.2-pigeonhole: sieve-filter: Fixed handling of failure-... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/469f9794408b changeset: 2085:469f9794408b user: Stephan Bosch date: Sun Aug 16 19:00:29 2015 +0200 description: sieve-filter: Fixed handling of failure-related implicit keep when there is an explicit default destination folder. The message was left in the source mailbox, even though it was stored in the destination folder. diffstat: src/sieve-tools/sieve-filter.c | 21 +++++++++++++++++++-- 1 files changed, 19 insertions(+), 2 deletions(-) diffs (66 lines): diff -r 49ac5b0000fd -r 469f9794408b src/sieve-tools/sieve-filter.c --- a/src/sieve-tools/sieve-filter.c Thu Aug 06 22:23:03 2015 +0200 +++ b/src/sieve-tools/sieve-filter.c Sun Aug 16 19:00:29 2015 +0200 @@ -58,6 +58,7 @@ unsigned int execute:1; unsigned int source_write:1; + unsigned int default_move:1; }; struct sieve_filter_context { @@ -227,6 +228,17 @@ return -1; case SIEVE_EXEC_FAILURE: case SIEVE_EXEC_TEMP_FAILURE: + if ( source_write && execute && sfctx->data->default_move && + !estatus.keep_original && estatus.message_saved ) { + /* The implicit keep action moved message to default mailbox, so + the source message still needs to be expunged */ + sieve_error(ehandler, NULL, + "sieve script execution failed for this message; " + "message moved to default mailbox"); + mail_expunge(mail); + return 0; + } + /* Fall through */ case SIEVE_EXEC_KEEP_FAILED: sieve_error(ehandler, NULL, "sieve script execution failed for this message; " @@ -360,7 +372,7 @@ struct sieve_binary *main_sbin; struct sieve_script_env scriptenv; struct sieve_error_handler *ehandler; - bool force_compile, execute, source_write, verbose; + bool force_compile, execute, source_write, verbose, default_move; struct mail_namespace *ns; struct mailbox *src_box = NULL, *move_box = NULL; enum mailbox_flags open_flags = MAILBOX_FLAG_IGNORE_ACLS; @@ -374,7 +386,8 @@ /* Parse arguments */ dst_mailbox = move_mailbox = NULL; - force_compile = execute = source_write = verbose = FALSE; + force_compile = execute = source_write = default_move = FALSE; + verbose = FALSE; while ((c = sieve_tool_getopt(sieve_tool)) > 0) { switch (c) { case 'm': @@ -475,6 +488,9 @@ if ( dst_mailbox == NULL ) { dst_mailbox = src_mailbox; + } else { + /* The (implicit) keep action will move the message */ + default_move = TRUE; } /* Finish tool initialization */ @@ -552,6 +568,7 @@ sfdata.ehandler = ehandler; sfdata.execute = execute; sfdata.source_write = source_write; + sfdata.default_move = default_move; /* Apply Sieve filter to all messages found */ (void) filter_mailbox(&sfdata, src_box); From dovecot at dovecot.org Mon Aug 17 09:57:12 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 17 Aug 2015 09:57:12 +0000 Subject: dovecot-2.2: lib-master: Code cleanup - split master_status_upda... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/10df63e74ed6 changeset: 18939:10df63e74ed6 user: Timo Sirainen date: Mon Aug 17 12:51:43 2015 +0300 description: lib-master: Code cleanup - split master_status_update() to two functions. No functional changes. diffstat: src/lib-master/master-service.c | 76 ++++++++++++++++++++++------------------ 1 files changed, 41 insertions(+), 35 deletions(-) diffs (95 lines): diff -r 38c3d7b44dd5 -r 10df63e74ed6 src/lib-master/master-service.c --- a/src/lib-master/master-service.c Sun Aug 16 13:22:14 2015 +0200 +++ b/src/lib-master/master-service.c Mon Aug 17 12:51:43 2015 +0300 @@ -991,9 +991,48 @@ return FALSE; } +static void +master_status_send(struct master_service *service, bool important_update) +{ + ssize_t ret; + + if (service->to_status != NULL) + timeout_remove(&service->to_status); + + ret = write(MASTER_STATUS_FD, &service->master_status, + sizeof(service->master_status)); + if (ret == sizeof(service->master_status)) { + /* success */ + if (service->io_status_write != NULL) { + /* delayed important update sent successfully */ + io_remove(&service->io_status_write); + } + service->last_sent_status_time = ioloop_time; + service->last_sent_status_avail_count = + service->master_status.available_count; + service->initial_status_sent = TRUE; + } else if (ret >= 0) { + /* shouldn't happen? */ + i_error("write(master_status_fd) returned %d", (int)ret); + service->master_status.pid = 0; + } else if (errno != EAGAIN) { + /* failure */ + if (errno != EPIPE) + i_error("write(master_status_fd) failed: %m"); + service->master_status.pid = 0; + } else if (important_update) { + /* reader is busy, but it's important to get this notification + through. send it when possible. */ + if (service->io_status_write == NULL) { + service->io_status_write = + io_add(MASTER_STATUS_FD, IO_WRITE, + master_status_update, service); + } + } +} + void master_status_update(struct master_service *service) { - ssize_t ret; bool important_update; if ((service->flags & MASTER_SERVICE_FLAG_UPDATE_PROCTITLE) != 0 && @@ -1030,40 +1069,7 @@ io_remove(&service->io_status_write); return; } - - if (service->to_status != NULL) - timeout_remove(&service->to_status); - - ret = write(MASTER_STATUS_FD, &service->master_status, - sizeof(service->master_status)); - if (ret == sizeof(service->master_status)) { - /* success */ - if (service->io_status_write != NULL) { - /* delayed important update sent successfully */ - io_remove(&service->io_status_write); - } - service->last_sent_status_time = ioloop_time; - service->last_sent_status_avail_count = - service->master_status.available_count; - service->initial_status_sent = TRUE; - } else if (ret >= 0) { - /* shouldn't happen? */ - i_error("write(master_status_fd) returned %d", (int)ret); - service->master_status.pid = 0; - } else if (errno != EAGAIN) { - /* failure */ - if (errno != EPIPE) - i_error("write(master_status_fd) failed: %m"); - service->master_status.pid = 0; - } else if (important_update) { - /* reader is busy, but it's important to get this notification - through. send it when possible. */ - if (service->io_status_write == NULL) { - service->io_status_write = - io_add(MASTER_STATUS_FD, IO_WRITE, - master_status_update, service); - } - } + master_status_send(service, important_update); } bool version_string_verify(const char *line, const char *service_name, From dovecot at dovecot.org Mon Aug 17 09:57:13 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 17 Aug 2015 09:57:13 +0000 Subject: dovecot-2.2: lib-master: If idle-die callback returns FALSE, not... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ec912a6039f5 changeset: 18940:ec912a6039f5 user: Timo Sirainen date: Mon Aug 17 12:53:52 2015 +0300 description: lib-master: If idle-die callback returns FALSE, notify master that we don't want to die. This avoids the master thinking that we're ignoring its idle-kill signal and logging an error. diffstat: src/lib-master/master-service.c | 8 +++++++- 1 files changed, 7 insertions(+), 1 deletions(-) diffs (26 lines): diff -r 10df63e74ed6 -r ec912a6039f5 src/lib-master/master-service.c --- a/src/lib-master/master-service.c Mon Aug 17 12:51:43 2015 +0300 +++ b/src/lib-master/master-service.c Mon Aug 17 12:53:52 2015 +0300 @@ -45,6 +45,8 @@ static void master_service_io_listeners_close(struct master_service *service); static void master_service_refresh_login_state(struct master_service *service); +static void +master_status_send(struct master_service *service, bool important_update); const char *master_service_getopt_string(void) { @@ -73,8 +75,12 @@ return; if (service->idle_die_callback != NULL && - !service->idle_die_callback()) + !service->idle_die_callback()) { + /* we don't want to die - send a notification to master + so it doesn't think we're ignoring it completely. */ + master_status_send(service, FALSE); return; + } } service->killed = TRUE; From dovecot at dovecot.org Mon Aug 17 09:57:13 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 17 Aug 2015 09:57:13 +0000 Subject: dovecot-2.2: indexer: Removed counting worker processes as servi... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/622ce8582a8e changeset: 18941:622ce8582a8e user: Timo Sirainen date: Mon Aug 17 12:56:25 2015 +0300 description: indexer: Removed counting worker processes as service clients to prevent idle-kill. lib-master handles this internally now. Also if the max client count was reached, we assert-crashed: indexer: Panic: file master-service.c: line 672 (master_service_client_connection_created): assertion failed: (service->master_status.available_count > 0) diffstat: src/indexer/worker-pool.c | 6 ------ 1 files changed, 0 insertions(+), 6 deletions(-) diffs (30 lines): diff -r ec912a6039f5 -r 622ce8582a8e src/indexer/worker-pool.c --- a/src/indexer/worker-pool.c Mon Aug 17 12:53:52 2015 +0300 +++ b/src/indexer/worker-pool.c Mon Aug 17 12:56:25 2015 +0300 @@ -49,7 +49,6 @@ struct worker_connection_list *list = pool->busy_list; DLLIST_REMOVE(&pool->busy_list, list); - master_service_client_connection_destroyed(master_service); worker_connection_list_free(pool, list); } @@ -144,10 +143,6 @@ list = pool->idle_list; DLLIST_REMOVE(&pool->idle_list, list); DLLIST_PREPEND(&pool->busy_list, list); - /* treat worker connection as another client. this is required (once, - at least) so that master doesn't think we are busy doing nothing and - ignoring an idle-kill. */ - master_service_client_connection_created(master_service); *conn_r = list->conn; return TRUE; @@ -185,7 +180,6 @@ i_assert(list != NULL); DLLIST_REMOVE(&pool->busy_list, list); - master_service_client_connection_destroyed(master_service); if (!worker_connection_is_connected(conn)) worker_connection_list_free(pool, list); From dovecot at dovecot.org Mon Aug 17 10:46:07 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 17 Aug 2015 10:46:07 +0000 Subject: dovecot-2.2: lib-fts: Update comment on tr29 rules. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0381695e303f changeset: 18942:0381695e303f user: Teemu Huovila date: Mon Aug 17 13:14:44 2015 +0300 description: lib-fts: Update comment on tr29 rules. diffstat: src/lib-fts/fts-tokenizer-generic.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (14 lines): diff -r 622ce8582a8e -r 0381695e303f src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Mon Aug 17 12:56:25 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Mon Aug 17 13:14:44 2015 +0300 @@ -594,6 +594,10 @@ #29, but tailored for FTS purposes. http://www.unicode.org/reports/tr29/ + Note: The text of tr29 is a living standard, so it keeps + changing. In newer specs some characters are combined, like AHLetter + (ALetter | Hebrew_Letter) and MidNumLetQ (MidNumLet | Single_Quote). + Adaptions: * No word boundary at Start-Of-Text or End-of-Text (Wb1 and WB2). * Break just once, not before and after. From dovecot at dovecot.org Mon Aug 17 10:46:07 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 17 Aug 2015 10:46:07 +0000 Subject: dovecot-2.2: lib-fts: Add note about possible additional apostro... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0994a6619380 changeset: 18943:0994a6619380 user: Teemu Huovila date: Mon Aug 17 13:15:11 2015 +0300 description: lib-fts: Add note about possible additional apostrophe. diffstat: src/lib-fts/fts-common.h | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 0381695e303f -r 0994a6619380 src/lib-fts/fts-common.h --- a/src/lib-fts/fts-common.h Mon Aug 17 13:14:44 2015 +0300 +++ b/src/lib-fts/fts-common.h Mon Aug 17 13:15:11 2015 +0300 @@ -1,6 +1,7 @@ #ifndef FTS_COMMON_H #define FTS_COMMON_H +/* Some might consider 0x02BB an apostrophe also. */ #define IS_NONASCII_APOSTROPHE(c) \ ((c) == 0x2019 || (c) == 0xFF07) #define IS_APOSTROPHE(c) \ From dovecot at dovecot.org Mon Aug 17 10:46:08 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 17 Aug 2015 10:46:08 +0000 Subject: dovecot-2.2: lib-fts: Add Unicode TR29 rule WB5a setting to toke... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f7f6ec738683 changeset: 18944:f7f6ec738683 user: Teemu Huovila date: Mon Aug 17 13:18:03 2015 +0300 description: lib-fts: Add Unicode TR29 rule WB5a setting to tokenizer. Splits prefixing contracted words from base word. E.g. "l'homme" -> "l" "homme". Together with a language specific stopword list unnecessary contractions can thus be filtered away. This is disabled by default and only works with the TR29 algorithm. Enable by "fts_tokenizer_generic = algorithm=tr29 wb5a=yes" diffstat: src/lib-fts/fts-common.h | 37 ++++++++++++++++- src/lib-fts/fts-tokenizer-generic-private.h | 5 ++ src/lib-fts/fts-tokenizer-generic.c | 64 +++++++++++++++++++++++++-- src/lib-fts/test-fts-tokenizer.c | 65 ++++++++++++++++++++++++++++- 4 files changed, 163 insertions(+), 8 deletions(-) diffs (truncated from 334 to 300 lines): diff -r 0994a6619380 -r f7f6ec738683 src/lib-fts/fts-common.h --- a/src/lib-fts/fts-common.h Mon Aug 17 13:15:11 2015 +0300 +++ b/src/lib-fts/fts-common.h Mon Aug 17 13:18:03 2015 +0300 @@ -6,5 +6,40 @@ ((c) == 0x2019 || (c) == 0xFF07) #define IS_APOSTROPHE(c) \ ((c) == 0x0027 || IS_NONASCII_APOSTROPHE(c)) - +#define IS_WB5A_APOSTROPHE(c) \ + ((c) == 0x0027 || (c) == 0x2019) +/* The h letters are included because it is an exception in French. + A, E, H, I, O, U, Y, a, e, h, i, o, u, y */ +#define IS_ASCII_VOWEL(c) \ + ((c) == 0x0041 || (c) == 0x0045 || (c) == 0x0048 || (c) == 0x0049 || \ + (c) == 0x004F || (c) == 0x0055 || (c) == 0x0059 || (c) == 0x0061 || \ + (c) == 0x0065 || (c) == 0x0068 || (c) == 0x0069 || (c) == 0x006F || \ + (c) == 0x0075 || (c) == 0x0079) +#define IS_NONASCII_VOWEL(c) \ + /*latin capital letter a with grave, acute and circumflex*/ \ + ((c) == 0x00C0 || (c) == 0x00C1 || (c) == 0x00C2 || \ + /* latin capital letter e with grave, acute and circumflex */ \ + (c) == 0x00C8 || (c) == 0x00C9 || (c) == 0x00CA || \ + /* latin capital letter i with grave, acute and circumflex */ \ + (c) == 0x00CC || (c) == 0x00CD || (c) == 0x00CE || \ + /* latin capital letter o with grave, acute and circumflex */ \ + (c) == 0x00D2 || (c) == 0x00D3 || (c) == 0x00D4 || \ + /* latin capital letter u with grave, acute and circumflex */ \ + (c) == 0x00D9 || (c) == 0x00DA || (c) == 0x00DB || \ + /* latin capital letter y with acute */ \ + (c) == 0x00DD || \ + /* latin small letter a with grave, acute and circumflex */ \ + (c) == 0x00E0 || (c) == 0x00E1 || (c) == 0x00E2 || \ + /* latin small letter e with grave, acute and circumflex */ \ + (c) == 0x00E8 || (c) == 0x00E9 || (c) == 0x00EA || \ + /* latin small letter i with grave, acute and circumflex */ \ + (c) == 0x00EC || (c) == 0x00ED || (c) == 0x00EE || \ + /* latin small letter o with grave, acute and circumflex */ \ + (c) == 0x00F2 || (c) == 0x00F3 || (c) == 0x00F4 || \ + /* latin small letter u with grave, acute and circumflex */ \ + (c) == 0x00F9 || (c) == 0x00FA || (c) == 0x00FB || \ + /* latin small letter y with acute */ \ + (c) == 0x00FD ) +#define IS_VOWEL(c) \ + (IS_ASCII_VOWEL(c) || IS_NONASCII_VOWEL(c)) #endif diff -r 0994a6619380 -r f7f6ec738683 src/lib-fts/fts-tokenizer-generic-private.h --- a/src/lib-fts/fts-tokenizer-generic-private.h Mon Aug 17 13:15:11 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic-private.h Mon Aug 17 13:18:03 2015 +0300 @@ -40,6 +40,11 @@ struct generic_fts_tokenizer { struct fts_tokenizer tokenizer; unsigned int max_length; + bool wb5a; /* TR29 rule for prefix separation + in e.g. French or Italian. */ + bool seen_wb5a; + unichar_t prev_letter_c; + unichar_t letter_c; enum boundary_algorithm algorithm; enum letter_type prev_letter; enum letter_type prev_prev_letter; diff -r 0994a6619380 -r f7f6ec738683 src/lib-fts/fts-tokenizer-generic.c --- a/src/lib-fts/fts-tokenizer-generic.c Mon Aug 17 13:15:11 2015 +0300 +++ b/src/lib-fts/fts-tokenizer-generic.c Mon Aug 17 13:18:03 2015 +0300 @@ -2,6 +2,7 @@ #include "lib.h" #include "buffer.h" +#include "str.h" #include "unichar.h" #include "bsearch-insert-pos.h" #include "fts-common.h" @@ -11,6 +12,7 @@ #include "word-break-data.c" #define FTS_DEFAULT_TOKEN_MAX_LENGTH 30 +#define FTS_WB5A_PREFIX_MAX_LENGTH 3 /* Including apostrophe */ static unsigned char fts_ascii_word_breaks[128] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0-15 */ @@ -32,6 +34,7 @@ struct generic_fts_tokenizer *tok; unsigned int max_length = FTS_DEFAULT_TOKEN_MAX_LENGTH; enum boundary_algorithm algo = BOUNDARY_ALGORITHM_SIMPLE; + bool wb5a = FALSE; unsigned int i; for (i = 0; settings[i] != NULL; i += 2) { @@ -57,12 +60,22 @@ } else if (strcmp(key, "search") == 0) { /* tokenizing a search string - makes no difference to us */ + } else if (strcasecmp(key, "wb5a") == 0) { + if (strcasecmp(value, "no") == 0) + wb5a = FALSE; + else + wb5a = TRUE; } else { *error_r = t_strdup_printf("Unknown setting: %s", key); return -1; } } + if (wb5a && algo != BOUNDARY_ALGORITHM_TR29) { + *error_r = "Can not use WB5a for algorithms other than TR29."; + return -1; + } + tok = i_new(struct generic_fts_tokenizer, 1); if (algo == BOUNDARY_ALGORITHM_TR29) tok->tokenizer.v = &generic_tokenizer_vfuncs_tr29; @@ -70,6 +83,7 @@ tok->tokenizer.v = &generic_tokenizer_vfuncs_simple; tok->max_length = max_length; tok->algorithm = algo; + tok->wb5a = wb5a; tok->token = buffer_create_dynamic(default_pool, 64); *tokenizer_r = &tok->tokenizer; @@ -369,6 +383,14 @@ static bool letter_aletter(struct generic_fts_tokenizer *tok) { + + /* WB5a */ + if (tok->wb5a && tok->token->used <= FTS_WB5A_PREFIX_MAX_LENGTH) + if (IS_WB5A_APOSTROPHE(tok->prev_letter_c) && IS_VOWEL(tok->letter_c)) { + tok->seen_wb5a = TRUE; + return TRUE; + } + /* WB5 */ if (tok->prev_letter == LETTER_TYPE_ALETTER) return FALSE; @@ -489,8 +511,8 @@ return TRUE; /* Any / Any */ } + static bool letter_other(struct generic_fts_tokenizer *tok ATTR_UNUSED) - { return TRUE; /* Any / Any */ } @@ -498,11 +520,17 @@ static void add_prev_letter(struct generic_fts_tokenizer *tok, enum letter_type lt) { - if(tok->prev_letter != LETTER_TYPE_NONE) { + if(tok->prev_letter != LETTER_TYPE_NONE) tok->prev_prev_letter = tok->prev_letter; - tok->prev_letter = lt; - } else - tok->prev_letter = lt; + tok->prev_letter = lt; +} + +static void +add_letter_c(struct generic_fts_tokenizer *tok, unichar_t c) +{ + if(tok->letter_c != 0) + tok->prev_letter_c = tok->letter_c; + tok->letter_c = c; } /* @@ -544,6 +572,7 @@ return FALSE; } + static void fts_tokenizer_generic_tr29_current_token(struct generic_fts_tokenizer *tok, const char **token_r) @@ -569,12 +598,23 @@ tok->prev_prev_letter = LETTER_TYPE_NONE; tok->prev_letter = LETTER_TYPE_NONE; - *token_r = t_strndup(data, len); buffer_set_used_size(tok->token, 0); tok->untruncated_length = 0; } +static void wb5a_reinsert(struct generic_fts_tokenizer *tok) +{ + string_t *utf8_str = t_str_new(6); + + uni_ucs4_to_utf8_c(tok->letter_c, utf8_str); + buffer_insert(tok->token, 0, str_data(utf8_str), str_len(utf8_str)); + tok->prev_letter = letter_type(tok->letter_c); + tok->letter_c = 0; + tok->prev_letter_c = 0; + tok->seen_wb5a = FALSE; +} + struct letter_fn { bool (*fn)(struct generic_fts_tokenizer *tok); }; @@ -599,6 +639,8 @@ (ALetter | Hebrew_Letter) and MidNumLetQ (MidNumLet | Single_Quote). Adaptions: + * Added optional WB5a as a configurable option. The cut of prefix is + max FTS_WB5A_PREFIX chars. * No word boundary at Start-Of-Text or End-of-Text (Wb1 and WB2). * Break just once, not before and after. * Break at MidNumLet, except apostrophes (diverging from WB6/WB7). @@ -644,12 +686,22 @@ i += char_size; lt = letter_type(c); + /* The WB5a break is detected only when the "after + break" char is inspected. That char needs to be + reinserted as the "previous char". */ + if (tok->seen_wb5a) + wb5a_reinsert(tok); + if (tok->prev_letter == LETTER_TYPE_NONE && is_nontoken(lt)) { /* Skip non-token chars at the beginning of token */ i_assert(tok->token->used == 0); start_pos = i; continue; } + + if (tok->wb5a && tok->token->used <= FTS_WB5A_PREFIX_MAX_LENGTH) + add_letter_c(tok, c); + if (uni_found_word_boundary(tok, lt)) { i_assert(char_start_i >= start_pos && size >= start_pos); tok_append_truncated(tok, data + start_pos, diff -r 0994a6619380 -r f7f6ec738683 src/lib-fts/test-fts-tokenizer.c --- a/src/lib-fts/test-fts-tokenizer.c Mon Aug 17 13:15:11 2015 +0300 +++ b/src/lib-fts/test-fts-tokenizer.c Mon Aug 17 13:18:03 2015 +0300 @@ -52,8 +52,10 @@ "galore\xE2\x81\x9F""and\xE2\x81\x9Amore.\n\n", /* TR29 MinNumLet U+FF0E at end: u+FF0E is EF BC 8E */ - "hello world\xEF\xBC\x8E" + "hello world\xEF\xBC\x8E", + /* TR29 WB5a */ + "l\xE2\x80\x99homme l\xE2\x80\x99humanit\xC3\xA9 d\xE2\x80\x99immixtions qu\xE2\x80\x99il aujourd'hui que'euq" }; static void test_fts_tokenizer_find(void) @@ -172,6 +174,8 @@ "hello", "world", NULL, + "l'homme", "l'humanit\xC3\xA9", "d'immixtions", "qu'il", "aujourd'hui", "que'euq", NULL, + NULL }; struct fts_tokenizer *tok; @@ -229,6 +233,7 @@ "hello", "world", NULL, + "l'homme", "l'humanit\xC3\xA9", "d'immixtions", "qu'il", "aujourd'hui", "que'euq", NULL, NULL }; struct fts_tokenizer *tok; @@ -241,6 +246,63 @@ test_end(); } +const char *const tr29_settings_wb5a[] = {"algorithm", "tr29", "wb5a", "yes", NULL}; + +/* TODO: U+206F is in "Format" and therefore currently not word break. + This definitely needs to be remapped. */ +static void test_fts_tokenizer_generic_tr29_wb5a(void) +{ + static const char *const expected_output[] = { + "hello", "world", "And", + "there", "was", "text", "galor\xC3\xA9", + "abc", "example", "com", "Bar", "Baz", + "bar", "example", "org", "foo", "domain", + "1234567890123456789012345678?", + "12345678901234567890123456789", + "123456789012345678901234567890", + "and", "longlonglongabcdefghijklmnopqr", + "more", "Hello", "world", "3", "14", "3,14", "last", NULL, + + "1", NULL, + + "quoted", "text", "word", "hlo", "words", "you're", "bad", + "word", "pre", "post", NULL, + + "1234567890123456789012345678?", + "123456789012345678901234567x'", + "1234567890123456789012345678x'", + "1234567890123456789012345678x", + "1234567890123456789012345678x", + "12345678901234567890123456789x", + "12345678901234567890123456789x", + "123456789012345678901234567890", + "123456789012345678901234567890", + + "quoted", "text", "word", "hlo", "words", "you're789012345678901234567890", "bad", + "word", "pre", "post", NULL, From dovecot at dovecot.org Mon Aug 17 11:26:43 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 17 Aug 2015 11:26:43 +0000 Subject: dovecot-2.2: lib-fts: Minor cleanup - initialize err with U_ZERO... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6e89241d1995 changeset: 18945:6e89241d1995 user: Timo Sirainen date: Mon Aug 17 14:25:59 2015 +0300 description: lib-fts: Minor cleanup - initialize err with U_ZERO_ERROR instead of 0. They're the same though. diffstat: src/lib-fts/fts-icu.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r f7f6ec738683 -r 6e89241d1995 src/lib-fts/fts-icu.c --- a/src/lib-fts/fts-icu.c Mon Aug 17 13:18:03 2015 +0300 +++ b/src/lib-fts/fts-icu.c Mon Aug 17 14:25:59 2015 +0300 @@ -135,7 +135,7 @@ size_t avail_bytes, dest_pos = dest_utf8->used; char *dest_data; int dest_full_len; - UErrorCode err = 0; + UErrorCode err = U_ZERO_ERROR; avail_bytes = buffer_get_writable_size(dest_utf8) - dest_pos; dest_data = buffer_get_space_unsafe(dest_utf8, dest_pos, avail_bytes); From dovecot at dovecot.org Mon Aug 17 16:29:37 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 17 Aug 2015 16:29:37 +0000 Subject: dovecot-2.2: lazy-expunge: If MAIL_FETCH_REFCOUNT fails, log the... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c71c4dceee97 changeset: 18946:c71c4dceee97 user: Timo Sirainen date: Mon Aug 17 19:28:55 2015 +0300 description: lazy-expunge: If MAIL_FETCH_REFCOUNT fails, log the storage error. diffstat: src/plugins/lazy-expunge/lazy-expunge-plugin.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 6e89241d1995 -r c71c4dceee97 src/plugins/lazy-expunge/lazy-expunge-plugin.c --- a/src/plugins/lazy-expunge/lazy-expunge-plugin.c Mon Aug 17 14:25:59 2015 +0300 +++ b/src/plugins/lazy-expunge/lazy-expunge-plugin.c Mon Aug 17 19:28:55 2015 +0300 @@ -155,7 +155,8 @@ if (mail_get_special(_mail, MAIL_FETCH_REFCOUNT, &value) < 0) { mail_storage_set_critical(_mail->box->storage, - "lazy_expunge: Couldn't lookup message's refcount"); + "lazy_expunge: Couldn't lookup message's refcount: %s", + mailbox_get_last_error(_mail->box, NULL)); return -1; } if (*value == '\0') { From dovecot at dovecot.org Mon Aug 17 16:32:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 17 Aug 2015 16:32:25 +0000 Subject: dovecot-2.2: lazy-expunge: If MAIL_FETCH_REFCOUNT fails because ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/41fabc9aa4c7 changeset: 18947:41fabc9aa4c7 user: Timo Sirainen date: Mon Aug 17 19:31:42 2015 +0300 description: lazy-expunge: If MAIL_FETCH_REFCOUNT fails because mail is expunged, ignore the error. diffstat: src/plugins/lazy-expunge/lazy-expunge-plugin.c | 10 ++++++++-- 1 files changed, 8 insertions(+), 2 deletions(-) diffs (25 lines): diff -r c71c4dceee97 -r 41fabc9aa4c7 src/plugins/lazy-expunge/lazy-expunge-plugin.c --- a/src/plugins/lazy-expunge/lazy-expunge-plugin.c Mon Aug 17 19:28:55 2015 +0300 +++ b/src/plugins/lazy-expunge/lazy-expunge-plugin.c Mon Aug 17 19:31:42 2015 +0300 @@ -150,13 +150,19 @@ { struct lazy_expunge_transaction *lt = LAZY_EXPUNGE_CONTEXT(_mail->transaction); - const char *value; + const char *value, *errstr; unsigned long refcount; + enum mail_error error; if (mail_get_special(_mail, MAIL_FETCH_REFCOUNT, &value) < 0) { + errstr = mailbox_get_last_error(_mail->box, &error); + if (error == MAIL_ERROR_EXPUNGED) { + /* already expunged - just ignore it */ + return 0; + } mail_storage_set_critical(_mail->box->storage, "lazy_expunge: Couldn't lookup message's refcount: %s", - mailbox_get_last_error(_mail->box, NULL)); + errstr); return -1; } if (*value == '\0') { From pigeonhole at rename-it.nl Mon Aug 17 21:34:27 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 17 Aug 2015 23:34:27 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: Implemented the vnd.dovecot.e... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/166279fd3fd3 changeset: 2086:166279fd3fd3 user: Stephan Bosch date: Mon Aug 17 23:34:21 2015 +0200 description: lib-sieve: Implemented the vnd.dovecot.environment extension. This adds the vnd.dovecot.default-mailbox and vnd.dovecot.username environment items. It creates the 'env' variables namespace through which all environment items can be accessed directly. This extension is yet to be documented properly. More environment items are under consideration. diffstat: Makefile.am | 2 + configure.ac | 1 + src/lib-sieve/Makefile.am | 1 + src/lib-sieve/plugins/environment/ext-environment-common.c | 180 ++++++++---- src/lib-sieve/plugins/environment/ext-environment-common.h | 7 +- src/lib-sieve/plugins/environment/ext-environment.c | 16 +- src/lib-sieve/plugins/environment/sieve-ext-environment.h | 33 ++- src/lib-sieve/plugins/environment/tst-environment.c | 2 +- src/lib-sieve/plugins/vnd.dovecot/Makefile.am | 2 +- src/lib-sieve/sieve-extensions.c | 3 +- 10 files changed, 165 insertions(+), 82 deletions(-) diffs (truncated from 431 to 300 lines): diff -r 469f9794408b -r 166279fd3fd3 Makefile.am --- a/Makefile.am Sun Aug 16 19:00:29 2015 +0200 +++ b/Makefile.am Mon Aug 17 23:34:21 2015 +0200 @@ -162,6 +162,8 @@ tests/extensions/metadata/execute.svtest \ tests/extensions/metadata/errors.svtest \ tests/extensions/vnd.dovecot/debug/execute.svtest \ + tests/extensions/vnd.dovecot/environment/basic.svtest \ + tests/extensions/vnd.dovecot/environment/variables.svtest \ tests/deprecated/notify/basic.svtest \ tests/deprecated/notify/mailto.svtest \ tests/deprecated/notify/errors.svtest \ diff -r 469f9794408b -r 166279fd3fd3 configure.ac --- a/configure.ac Sun Aug 16 19:00:29 2015 +0200 +++ b/configure.ac Mon Aug 17 23:34:21 2015 +0200 @@ -211,6 +211,7 @@ src/lib-sieve/plugins/index/Makefile src/lib-sieve/plugins/vnd.dovecot/Makefile src/lib-sieve/plugins/vnd.dovecot/debug/Makefile +src/lib-sieve/plugins/vnd.dovecot/environment/Makefile src/lib-sieve-tool/Makefile src/lib-managesieve/Makefile src/plugins/Makefile diff -r 469f9794408b -r 166279fd3fd3 src/lib-sieve/Makefile.am --- a/src/lib-sieve/Makefile.am Sun Aug 16 19:00:29 2015 +0200 +++ b/src/lib-sieve/Makefile.am Mon Aug 17 23:34:21 2015 +0200 @@ -78,6 +78,7 @@ $(extdir)/index/libsieve_ext_index.la \ $(extdir)/metadata/libsieve_ext_metadata.la \ $(extdir)/vnd.dovecot/debug/libsieve_ext_debug.la \ + $(extdir)/vnd.dovecot/environment/libsieve_ext_vnd_environment.la \ $(unfinished_plugins) libdovecot_sieve_la_DEPENDENCIES = \ diff -r 469f9794408b -r 166279fd3fd3 src/lib-sieve/plugins/environment/ext-environment-common.c --- a/src/lib-sieve/plugins/environment/ext-environment-common.c Sun Aug 16 19:00:29 2015 +0200 +++ b/src/lib-sieve/plugins/environment/ext-environment-common.c Mon Aug 17 23:34:21 2015 +0200 @@ -6,14 +6,10 @@ #include "sieve-common.h" #include "sieve-extensions.h" +#include "sieve-interpreter.h" #include "ext-environment-common.h" -struct ext_environment_context { - HASH_TABLE(const char *, - const struct sieve_environment_item *) environment_items; -}; - /* * Core environment items */ @@ -30,71 +26,121 @@ static unsigned int core_env_items_count = N_ELEMENTS(core_env_items); /* + * Validator context + */ + +struct ext_environment_interpreter_context { + HASH_TABLE(const char *, + const struct sieve_environment_item *) environment_items; + + unsigned int active:1; +}; + +static void ext_environment_interpreter_extension_free + (const struct sieve_extension *ext, struct sieve_interpreter *interp, + void *context); + +struct sieve_interpreter_extension environment_interpreter_extension = { + .ext_def = &environment_extension, + .free = ext_environment_interpreter_extension_free, +}; + +static struct ext_environment_interpreter_context * +ext_environment_interpreter_context_create +(const struct sieve_extension *this_ext, struct sieve_interpreter *interp) +{ + pool_t pool = sieve_interpreter_pool(interp); + struct ext_environment_interpreter_context *ctx; + + ctx = p_new(pool, struct ext_environment_interpreter_context, 1); + + hash_table_create + (&ctx->environment_items, default_pool, 0, str_hash, strcmp); + + sieve_interpreter_extension_register + (interp, this_ext, &environment_interpreter_extension, (void *)ctx); + return ctx; +} + +static void ext_environment_interpreter_extension_free +(const struct sieve_extension *ext ATTR_UNUSED, + struct sieve_interpreter *interp ATTR_UNUSED, void *context) +{ + struct ext_environment_interpreter_context *ctx = + (struct ext_environment_interpreter_context *)context; + + hash_table_destroy(&ctx->environment_items); +} + +static struct ext_environment_interpreter_context * +ext_environment_interpreter_context_get +(const struct sieve_extension *this_ext, struct sieve_interpreter *interp) +{ + struct ext_environment_interpreter_context *ctx = + (struct ext_environment_interpreter_context *) + sieve_interpreter_extension_get_context(interp, this_ext); + + if ( ctx == NULL ) + ctx = ext_environment_interpreter_context_create(this_ext, interp); + + return ctx; +} + +void ext_environment_interpreter_init +(const struct sieve_extension *this_ext, struct sieve_interpreter *interp) +{ + struct ext_environment_interpreter_context *ctx; + unsigned int i; + + /* Create our context */ + ctx = ext_environment_interpreter_context_get(this_ext, interp); + + for ( i = 0; i < core_env_items_count; i++ ) { + const struct sieve_environment_item *item = core_env_items[i]; + hash_table_insert(ctx->environment_items, item->name, item); + } + + ctx->active = TRUE; +} + +bool sieve_ext_environment_is_active +(const struct sieve_extension *env_ext, struct sieve_interpreter *interp) +{ + struct ext_environment_interpreter_context *ctx = + ext_environment_interpreter_context_get(env_ext, interp); + + return ( ctx != NULL && ctx->active ); +} + +/* * Registration */ -static void ext_environment_item_register -(struct ext_environment_context *ectx, +void sieve_environment_item_register +(const struct sieve_extension *env_ext, struct sieve_interpreter *interp, const struct sieve_environment_item *item) { - hash_table_insert(ectx->environment_items, item->name, item); + struct ext_environment_interpreter_context *ctx; + + i_assert( sieve_extension_is(env_ext, environment_extension) ); + ctx = ext_environment_interpreter_context_get(env_ext, interp); + hash_table_insert(ctx->environment_items, item->name, item); } -void sieve_ext_environment_item_register -(const struct sieve_extension *ext, const struct sieve_environment_item *item) -{ - struct ext_environment_context *ectx = - (struct ext_environment_context *) ext->context; - - ext_environment_item_register(ectx, item); -} - -/* - * Initialization - */ - -bool ext_environment_init -(const struct sieve_extension *ext ATTR_UNUSED, void **context) -{ - struct ext_environment_context *ectx = - i_new(struct ext_environment_context, 1); - - unsigned int i; - - hash_table_create - (&ectx->environment_items, default_pool, 0, str_hash, strcmp); - - for ( i = 0; i < core_env_items_count; i++ ) { - ext_environment_item_register(ectx, core_env_items[i]); - } - - *context = (void *) ectx; - - return TRUE; -} - -void ext_environment_deinit(const struct sieve_extension *ext) -{ - struct ext_environment_context *ectx = - (struct ext_environment_context *) ext->context; - - hash_table_destroy(&ectx->environment_items); - i_free(ectx); -} - - /* * Retrieval */ const char *ext_environment_item_get_value -(const struct sieve_extension *ext, const char *name, - const struct sieve_script_env *senv) +(const struct sieve_extension *env_ext, + const struct sieve_runtime_env *renv, const char *name) { - struct ext_environment_context *ectx = - (struct ext_environment_context *) ext->context; + struct ext_environment_interpreter_context *ctx = + ext_environment_interpreter_context_get(env_ext, renv->interp); const struct sieve_environment_item *item = - hash_table_lookup(ectx->environment_items, name); + hash_table_lookup(ctx->environment_items, name); + + i_assert( sieve_extension_is(env_ext, environment_extension) ); if ( item == NULL ) return NULL; @@ -103,7 +149,7 @@ return item->value; if ( item->get_value != NULL ) - return item->get_value(ext->svinst, senv); + return item->get_value(renv); return NULL; } @@ -119,10 +165,9 @@ */ static const char *envit_domain_get_value -(struct sieve_instance *svinst, - const struct sieve_script_env *senv ATTR_UNUSED) +(const struct sieve_runtime_env *renv) { - return svinst->domainname; + return renv->svinst->domainname; } const struct sieve_environment_item domain_env_item = { @@ -137,10 +182,9 @@ */ static const char *envit_host_get_value -(struct sieve_instance *svinst, - const struct sieve_script_env *senv ATTR_UNUSED) +(const struct sieve_runtime_env *renv) { - return svinst->hostname; + return renv->svinst->hostname; } const struct sieve_environment_item host_env_item = { @@ -160,10 +204,9 @@ */ static const char *envit_location_get_value -(struct sieve_instance *svinst, - const struct sieve_script_env *senv ATTR_UNUSED) +(const struct sieve_runtime_env *renv) { - switch ( svinst->env_location ) { + switch ( renv->svinst->env_location ) { case SIEVE_ENV_LOCATION_MDA: return "MDA"; case SIEVE_ENV_LOCATION_MTA: @@ -190,10 +233,9 @@ */ static const char *envit_phase_get_value -(struct sieve_instance *svinst, - const struct sieve_script_env *senv ATTR_UNUSED) +(const struct sieve_runtime_env *renv) { - switch ( svinst->delivery_phase ) { + switch ( renv->svinst->delivery_phase ) { case SIEVE_DELIVERY_PHASE_PRE: return "pre"; case SIEVE_DELIVERY_PHASE_DURING: diff -r 469f9794408b -r 166279fd3fd3 src/lib-sieve/plugins/environment/ext-environment-common.h --- a/src/lib-sieve/plugins/environment/ext-environment-common.h Sun Aug 16 19:00:29 2015 +0200 +++ b/src/lib-sieve/plugins/environment/ext-environment-common.h Mon Aug 17 23:34:21 2015 +0200 @@ -47,11 +47,10 @@ void ext_environment_deinit(const struct sieve_extension *ext); /* - * Environment item retrieval + * Validator context */ -const char *ext_environment_item_get_value - (const struct sieve_extension *ext, const char *name, From pigeonhole at rename-it.nl Mon Aug 17 21:41:10 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 17 Aug 2015 23:41:10 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: vnd.dovecot.environment exten... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/c07900b0d045 changeset: 2087:c07900b0d045 user: Stephan Bosch date: Mon Aug 17 23:41:05 2015 +0200 description: lib-sieve: vnd.dovecot.environment extension: Forgot to add several files in previous commit. diffstat: src/lib-sieve/plugins/vnd.dovecot/environment/Makefile.am | 16 + src/lib-sieve/plugins/vnd.dovecot/environment/ext-vnd-environment-common.h | 40 + src/lib-sieve/plugins/vnd.dovecot/environment/ext-vnd-environment-items.c | 66 +++ src/lib-sieve/plugins/vnd.dovecot/environment/ext-vnd-environment-variables.c | 207 ++++++++++ src/lib-sieve/plugins/vnd.dovecot/environment/ext-vnd-environment.c | 102 ++++ tests/extensions/vnd.dovecot/environment/basic.svtest | 19 + tests/extensions/vnd.dovecot/environment/variables.svtest | 19 + 7 files changed, 469 insertions(+), 0 deletions(-) diffs (truncated from 497 to 300 lines): diff -r 166279fd3fd3 -r c07900b0d045 src/lib-sieve/plugins/vnd.dovecot/environment/Makefile.am --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-sieve/plugins/vnd.dovecot/environment/Makefile.am Mon Aug 17 23:41:05 2015 +0200 @@ -0,0 +1,16 @@ +noinst_LTLIBRARIES = libsieve_ext_vnd_environment.la + +AM_CPPFLAGS = \ + -I$(srcdir)/../../.. \ + -I$(srcdir)/../../environment \ + -I$(srcdir)/../../variables \ + $(LIBDOVECOT_INCLUDE) + +libsieve_ext_vnd_environment_la_SOURCES = \ + ext-vnd-environment.c \ + ext-vnd-environment-items.c \ + ext-vnd-environment-variables.c + +noinst_HEADERS = + ext-vnd-environment-common.h + diff -r 166279fd3fd3 -r c07900b0d045 src/lib-sieve/plugins/vnd.dovecot/environment/ext-vnd-environment-common.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-sieve/plugins/vnd.dovecot/environment/ext-vnd-environment-common.h Mon Aug 17 23:41:05 2015 +0200 @@ -0,0 +1,40 @@ +/* Copyright (c) 2002-2015 Pigeonhole authors, see the included COPYING file + */ + +#ifndef __EXT_VND_ENVIRONMENT_COMMON_H +#define __EXT_VND_ENVIRONMENT_COMMON_H + +#include "sieve-ext-environment.h" + +/* + * Extension + */ + +struct ext_vnd_environment_context { + const struct sieve_extension *env_ext; + const struct sieve_extension *var_ext; +}; + +const struct sieve_extension_def vnd_environment_extension; + +/* + * Operands + */ + +extern const struct sieve_operand_def environment_namespace_operand; + +/* + * Environment items + */ + +void ext_vnd_environment_items_register +(const struct sieve_extension *ext, const struct sieve_runtime_env *renv); + +/* + * Variables + */ + +void ext_environment_variables_init +(const struct sieve_extension *this_ext, struct sieve_validator *valdtr); + +#endif diff -r 166279fd3fd3 -r c07900b0d045 src/lib-sieve/plugins/vnd.dovecot/environment/ext-vnd-environment-items.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-sieve/plugins/vnd.dovecot/environment/ext-vnd-environment-items.c Mon Aug 17 23:41:05 2015 +0200 @@ -0,0 +1,66 @@ +/* Copyright (c) 2002-2015 Pigeonhole authors, see the included COPYING file + */ + +#include "lib.h" +#include "array.h" + +#include "sieve-extensions.h" +#include "sieve-commands.h" +#include "sieve-comparators.h" +#include "sieve-match-types.h" +#include "sieve-address-parts.h" + +#include "sieve-validator.h" +#include "sieve-generator.h" +#include "sieve-binary.h" +#include "sieve-interpreter.h" +#include "sieve-dump.h" + +#include "ext-vnd-environment-common.h" + +/* + * Environment items + */ + +/* default_mailbox */ + +static const char *envit_default_mailbox_get_value +(const struct sieve_runtime_env *renv) +{ + i_assert(renv->scriptenv->default_mailbox != NULL); + return renv->scriptenv->default_mailbox; +} + +const struct sieve_environment_item default_mailbox_env_item = { + .name = "vnd.dovecot.default-mailbox", + .get_value = envit_default_mailbox_get_value +}; + +/* username */ + +static const char *envit_username_get_value +(const struct sieve_runtime_env *renv) +{ + return renv->svinst->username; +} + +const struct sieve_environment_item username_env_item = { + .name = "vnd.dovecot.username", + .get_value = envit_username_get_value +}; + +/* + * Register + */ + +void ext_vnd_environment_items_register +(const struct sieve_extension *ext, const struct sieve_runtime_env *renv) +{ + struct ext_vnd_environment_context *ectx = + (struct ext_vnd_environment_context *) ext->context; + + sieve_environment_item_register + (ectx->env_ext, renv->interp, &default_mailbox_env_item); + sieve_environment_item_register + (ectx->env_ext, renv->interp, &username_env_item); +} diff -r 166279fd3fd3 -r c07900b0d045 src/lib-sieve/plugins/vnd.dovecot/environment/ext-vnd-environment-variables.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-sieve/plugins/vnd.dovecot/environment/ext-vnd-environment-variables.c Mon Aug 17 23:41:05 2015 +0200 @@ -0,0 +1,207 @@ +/* Copyright (c) 2015 Pigeonhole authors, see the included COPYING file + */ + +#include "lib.h" +#include "str.h" + +#include "sieve-common.h" +#include "sieve-ast.h" +#include "sieve-binary.h" +#include "sieve-code.h" +#include "sieve-commands.h" +#include "sieve-validator.h" +#include "sieve-generator.h" +#include "sieve-interpreter.h" +#include "sieve-dump.h" + +#include "sieve-ext-variables.h" + +#include "ext-vnd-environment-common.h" + +static bool vnspc_vnd_environment_validate + (struct sieve_validator *valdtr, + const struct sieve_variables_namespace *nspc, + struct sieve_ast_argument *arg, struct sieve_command *cmd, + ARRAY_TYPE(sieve_variable_name) *var_name, void **var_data, + bool assignment); +static bool vnspc_vnd_environment_generate + (const struct sieve_codegen_env *cgenv, + const struct sieve_variables_namespace *nspc, + struct sieve_ast_argument *arg, + struct sieve_command *cmd, void *var_data); +static bool vnspc_vnd_environment_dump_variable + (const struct sieve_dumptime_env *denv, + const struct sieve_variables_namespace *nspc, + const struct sieve_operand *oprnd, sieve_size_t *address); +static int vnspc_vnd_environment_read_variable + (const struct sieve_runtime_env *renv, + const struct sieve_variables_namespace *nspc, + const struct sieve_operand *oprnd, + sieve_size_t *address, string_t **str_r); + +static const struct sieve_variables_namespace_def +environment_namespace = { + SIEVE_OBJECT("env", &environment_namespace_operand, 0), + vnspc_vnd_environment_validate, + vnspc_vnd_environment_generate, + vnspc_vnd_environment_dump_variable, + vnspc_vnd_environment_read_variable +}; + +static bool vnspc_vnd_environment_validate +(struct sieve_validator *valdtr, + const struct sieve_variables_namespace *nspc ATTR_UNUSED, + struct sieve_ast_argument *arg, struct sieve_command *cmd ATTR_UNUSED, + ARRAY_TYPE(sieve_variable_name) *var_name, void **var_data, + bool assignment) +{ + struct sieve_ast *ast = arg->ast; + const struct sieve_variable_name *name_elements; + unsigned int i, count; + const char *variable; + string_t *name; + + /* Compose environment name from parsed variable name */ + name = t_str_new(64); + name_elements = array_get(var_name, &count); + i_assert(count > 1); + for (i = 1; i < count; i++) { + if ( name_elements[i].num_variable >= 0 ) { + sieve_argument_validate_error(valdtr, arg, + "vnd.dovecot.environment: invalid variable name within " + "env namespace `env.%d': " + "encountered numeric variable name", + name_elements[i].num_variable); + return FALSE; + } + if (str_len(name) > 0) + str_append_c(name, '.'); + str_append_str(name, name_elements[i].identifier); + } + + variable = str_c(name); + + if ( assignment ) { + sieve_argument_validate_error(valdtr, arg, + "vnd.dovecot.environment: cannot assign to environment " + "variable `env.%s'", variable); + return FALSE; + } + + *var_data = (void *) p_strdup(sieve_ast_pool(ast), variable); + return TRUE; +} + +static bool vnspc_vnd_environment_generate +(const struct sieve_codegen_env *cgenv, + const struct sieve_variables_namespace *nspc, + struct sieve_ast_argument *arg ATTR_UNUSED, + struct sieve_command *cmd ATTR_UNUSED, void *var_data) +{ + const struct sieve_extension *this_ext = SIEVE_OBJECT_EXTENSION(nspc); + const char *variable = (const char *) var_data; + struct ext_vnd_environment_context *ext_data; + + if ( this_ext == NULL ) + return FALSE; + + ext_data = (struct ext_vnd_environment_context *) this_ext->context; + + sieve_variables_opr_namespace_variable_emit + (cgenv->sblock, ext_data->var_ext, this_ext, &environment_namespace); + sieve_binary_emit_cstring(cgenv->sblock, variable); + + return TRUE; +} + +static bool vnspc_vnd_environment_dump_variable +(const struct sieve_dumptime_env *denv, + const struct sieve_variables_namespace *nspc ATTR_UNUSED, + const struct sieve_operand *oprnd, sieve_size_t *address) +{ + string_t *var_name; + + if ( !sieve_binary_read_string(denv->sblock, address, &var_name) ) + return FALSE; + + if ( oprnd->field_name != NULL ) + sieve_code_dumpf(denv, "%s: VAR ${env.%s}", + oprnd->field_name, str_c(var_name)); + else + sieve_code_dumpf(denv, "VAR ${env.%s}", + str_c(var_name)); + + return TRUE; +} + +static int vnspc_vnd_environment_read_variable +(const struct sieve_runtime_env *renv, + const struct sieve_variables_namespace *nspc, + const struct sieve_operand *oprnd, sieve_size_t *address, + string_t **str_r) +{ + const struct sieve_extension *this_ext = SIEVE_OBJECT_EXTENSION(nspc); + struct ext_vnd_environment_context *ectx = + (struct ext_vnd_environment_context *) this_ext->context; + string_t *var_name; + const char *ext_value; + + if ( !sieve_binary_read_string(renv->sblock, address, &var_name) ) { + sieve_runtime_trace_operand_error(renv, oprnd, + "environment variable operand corrupt: invalid name"); + return SIEVE_EXEC_BIN_CORRUPT; + } + + if ( str_r != NULL ) { + const char *vname = str_c(var_name); + + ext_value = ext_environment_item_get_value + (ectx->env_ext, renv, vname); + + if ( ext_value == NULL && strchr(vname, '_') != NULL) { + char *p, *aname; From pigeonhole at rename-it.nl Mon Aug 17 21:51:41 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 17 Aug 2015 23:51:41 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: variables extension: Added as... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/652aec3e8440 changeset: 2088:652aec3e8440 user: Stephan Bosch date: Mon Aug 17 23:51:35 2015 +0200 description: lib-sieve: variables extension: Added asserts to extension API to make sure variables extension instance gets passed as argument. diffstat: src/lib-sieve/plugins/variables/ext-variables-common.c | 13 ++++++++++--- src/lib-sieve/plugins/variables/ext-variables-dump.c | 6 ++++-- src/lib-sieve/plugins/variables/ext-variables-namespaces.c | 1 + src/lib-sieve/plugins/variables/ext-variables-operands.c | 3 +++ 4 files changed, 18 insertions(+), 5 deletions(-) diffs (80 lines): diff -r c07900b0d045 -r 652aec3e8440 src/lib-sieve/plugins/variables/ext-variables-common.c --- a/src/lib-sieve/plugins/variables/ext-variables-common.c Mon Aug 17 23:41:05 2015 +0200 +++ b/src/lib-sieve/plugins/variables/ext-variables-common.c Mon Aug 17 23:51:35 2015 +0200 @@ -608,8 +608,10 @@ struct ext_variables_validator_context *ext_variables_validator_context_get (const struct sieve_extension *this_ext, struct sieve_validator *valdtr) { - struct ext_variables_validator_context *ctx = - (struct ext_variables_validator_context *) + struct ext_variables_validator_context *ctx; + + i_assert( sieve_extension_is(this_ext, variables_extension) ); + ctx = (struct ext_variables_validator_context *) sieve_validator_extension_get_context(valdtr, this_ext); if ( ctx == NULL ) { @@ -768,8 +770,13 @@ ext_variables_interpreter_context_get (const struct sieve_extension *this_ext, struct sieve_interpreter *interp) { - return (struct ext_variables_interpreter_context *) + struct ext_variables_interpreter_context *ctx; + + i_assert( sieve_extension_is(this_ext, variables_extension) ); + ctx = (struct ext_variables_interpreter_context *) sieve_interpreter_extension_get_context(interp, this_ext); + + return ctx; } struct sieve_variable_storage *sieve_ext_variables_runtime_get_storage diff -r c07900b0d045 -r 652aec3e8440 src/lib-sieve/plugins/variables/ext-variables-dump.c --- a/src/lib-sieve/plugins/variables/ext-variables-dump.c Mon Aug 17 23:41:05 2015 +0200 +++ b/src/lib-sieve/plugins/variables/ext-variables-dump.c Mon Aug 17 23:51:35 2015 +0200 @@ -49,10 +49,12 @@ (const struct sieve_extension *this_ext, const struct sieve_dumptime_env *denv) { struct sieve_code_dumper *dumper = denv->cdumper; - struct ext_variables_dump_context *dctx = sieve_dump_extension_get_context - (dumper, this_ext); + struct ext_variables_dump_context *dctx; pool_t pool; + i_assert( sieve_extension_is(this_ext, variables_extension) ); + dctx = sieve_dump_extension_get_context(dumper, this_ext); + if ( dctx == NULL ) { /* Create dumper context */ pool = sieve_code_dumper_pool(dumper); diff -r c07900b0d045 -r 652aec3e8440 src/lib-sieve/plugins/variables/ext-variables-namespaces.c --- a/src/lib-sieve/plugins/variables/ext-variables-namespaces.c Mon Aug 17 23:41:05 2015 +0200 +++ b/src/lib-sieve/plugins/variables/ext-variables-namespaces.c Mon Aug 17 23:51:35 2015 +0200 @@ -182,6 +182,7 @@ const struct sieve_extension *ext, const struct sieve_variables_namespace_def *nspc_def) { + i_assert( sieve_extension_is(var_ext, variables_extension) ); sieve_operand_emit(sblock, var_ext, &namespace_variable_operand); sieve_opr_object_emit(sblock, ext, &nspc_def->obj_def); } diff -r c07900b0d045 -r 652aec3e8440 src/lib-sieve/plugins/variables/ext-variables-operands.c --- a/src/lib-sieve/plugins/variables/ext-variables-operands.c Mon Aug 17 23:41:05 2015 +0200 +++ b/src/lib-sieve/plugins/variables/ext-variables-operands.c Mon Aug 17 23:51:35 2015 +0200 @@ -52,6 +52,8 @@ (struct sieve_binary_block *sblock, const struct sieve_extension *var_ext, struct sieve_variable *var) { + i_assert( sieve_extension_is(var_ext, variables_extension) ); + if ( var->ext == NULL ) { /* Default variable storage */ (void) sieve_operand_emit(sblock, var_ext, &variable_operand); @@ -222,6 +224,7 @@ (struct sieve_binary_block *sblock, const struct sieve_extension *var_ext, unsigned int index) { + i_assert( sieve_extension_is(var_ext, variables_extension) ); (void) sieve_operand_emit(sblock, var_ext, &match_value_operand); (void) sieve_binary_emit_unsigned(sblock, index); } From pigeonhole at rename-it.nl Mon Aug 17 21:56:26 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 17 Aug 2015 23:56:26 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: Improved efficiency of readin... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/c5ccf2e6d3ad changeset: 2089:c5ccf2e6d3ad user: Stephan Bosch date: Mon Aug 17 23:56:21 2015 +0200 description: lib-sieve: Improved efficiency of reading a string-list operand that is actually a single string. diffstat: src/lib-sieve/sieve-code.c | 15 ++++++++++----- 1 files changed, 10 insertions(+), 5 deletions(-) diffs (26 lines): diff -r 652aec3e8440 -r c5ccf2e6d3ad src/lib-sieve/sieve-code.c --- a/src/lib-sieve/sieve-code.c Mon Aug 17 23:51:35 2015 +0200 +++ b/src/lib-sieve/sieve-code.c Mon Aug 17 23:56:21 2015 +0200 @@ -800,12 +800,17 @@ return SIEVE_EXEC_FAILURE; } - if ( (ret=intf->read(renv, oprnd, address, NULL)) <= 0 ) - return ret; + if ( strlist_r == NULL ) { + if ( (ret=intf->read(renv, oprnd, address, NULL)) <= 0 ) + return ret; + } else { + string_t *stritem; + if ( (ret=intf->read(renv, oprnd, address, &stritem)) <= 0 ) + return ret; - if ( strlist_r != NULL ) - *strlist_r = sieve_code_stringlist_create - (renv, oprnd->address, 1, *address); + *strlist_r = sieve_single_stringlist_create + (renv, stritem, FALSE); + } return SIEVE_EXEC_OK; } From dovecot at dovecot.org Tue Aug 18 18:02:02 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Aug 2015 18:02:02 +0000 Subject: dovecot-2.2: lib-master: Moved connection accepting code to its ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b9ccd6349654 changeset: 18948:b9ccd6349654 user: Timo Sirainen date: Tue Aug 18 20:03:28 2015 +0300 description: lib-master: Moved connection accepting code to its own functions. No functional changes. diffstat: src/lib-master/master-service-private.h | 5 +++ src/lib-master/master-service.c | 55 +++++++++++++++++++------------- 2 files changed, 38 insertions(+), 22 deletions(-) diffs (85 lines): diff -r 41fabc9aa4c7 -r b9ccd6349654 src/lib-master/master-service-private.h --- a/src/lib-master/master-service-private.h Mon Aug 17 19:31:42 2015 +0300 +++ b/src/lib-master/master-service-private.h Tue Aug 18 20:03:28 2015 +0300 @@ -83,4 +83,9 @@ void master_service_io_listeners_remove(struct master_service *service); void master_service_ssl_io_listeners_remove(struct master_service *service); +void master_service_client_connection_handled(struct master_service *service, + struct master_service_connection *conn); +void master_service_client_connection_callback(struct master_service *service, + struct master_service_connection *conn); + #endif diff -r 41fabc9aa4c7 -r b9ccd6349654 src/lib-master/master-service.c --- a/src/lib-master/master-service.c Mon Aug 17 19:31:42 2015 +0300 +++ b/src/lib-master/master-service.c Tue Aug 18 20:03:28 2015 +0300 @@ -680,6 +680,38 @@ master_status_update(service); } +void master_service_client_connection_handled(struct master_service *service, + struct master_service_connection *conn) +{ + if (!conn->accepted) { + if (close(conn->fd) < 0) + i_error("close(service connection) failed: %m"); + master_service_client_connection_destroyed(service); + } else if (conn->fifo) { + /* reading FIFOs stays open forever, don't count them + as real clients */ + master_service_client_connection_destroyed(service); + } + if (service->master_status.available_count == 0 && + service->service_count_left == 1) { + /* we're not going to accept any more connections after this. + go ahead and close the connection early. don't do this + before calling callback, because it may want to access + the listen_fd (e.g. to check socket permissions). */ + i_assert(service->listeners != NULL); + master_service_io_listeners_remove(service); + master_service_io_listeners_close(service); + } +} + +void master_service_client_connection_callback(struct master_service *service, + struct master_service_connection *conn) +{ + service->callback(conn); + + master_service_client_connection_handled(service, conn); +} + void master_service_client_connection_accept(struct master_service_connection *conn) { conn->accepted = TRUE; @@ -870,28 +902,7 @@ net_set_nonblock(conn.fd, TRUE); master_service_client_connection_created(service); - - service->callback(&conn); - - if (!conn.accepted) { - if (close(conn.fd) < 0) - i_error("close(service connection) failed: %m"); - master_service_client_connection_destroyed(service); - } else if (conn.fifo) { - /* reading FIFOs stays open forever, don't count them - as real clients */ - master_service_client_connection_destroyed(service); - } - if (service->master_status.available_count == 0 && - service->service_count_left == 1) { - /* we're not going to accept any more connections after this. - go ahead and close the connection early. don't do this - before calling callback, because it may want to access - the listen_fd (e.g. to check socket permissions). */ - i_assert(service->listeners != NULL); - master_service_io_listeners_remove(service); - master_service_io_listeners_close(service); - } + master_service_client_connection_callback(service, &conn); } static void io_listeners_init(struct master_service *service) From dovecot at dovecot.org Tue Aug 18 18:02:03 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Aug 2015 18:02:03 +0000 Subject: dovecot-2.2: lib-master: Added local and real IP addresses and p... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/71f4b77c519f changeset: 18949:71f4b77c519f user: Stephan Bosch date: Mon Jun 15 18:50:53 2015 +0200 description: lib-master: Added local and real IP addresses and ports to struct master_service_connection. diffstat: src/lib-master/master-service.c | 7 +++++++ src/lib-master/master-service.h | 7 +++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diffs (34 lines): diff -r b9ccd6349654 -r 71f4b77c519f src/lib-master/master-service.c --- a/src/lib-master/master-service.c Tue Aug 18 20:03:28 2015 +0300 +++ b/src/lib-master/master-service.c Mon Jun 15 18:50:53 2015 +0200 @@ -899,6 +899,13 @@ } conn.ssl = l->ssl; conn.name = l->name; + + (void)net_getsockname(conn.fd, &conn.local_ip, &conn.local_port); + conn.real_remote_ip = conn.remote_ip; + conn.real_remote_port = conn.remote_port; + conn.real_local_ip = conn.local_ip; + conn.real_local_port = conn.local_port; + net_set_nonblock(conn.fd, TRUE); master_service_client_connection_created(service); diff -r b9ccd6349654 -r 71f4b77c519f src/lib-master/master-service.h --- a/src/lib-master/master-service.h Tue Aug 18 20:03:28 2015 +0300 +++ b/src/lib-master/master-service.h Mon Jun 15 18:50:53 2015 +0200 @@ -40,8 +40,11 @@ int listen_fd; const char *name; - struct ip_addr remote_ip; - unsigned int remote_port; + struct ip_addr remote_ip, local_ip; + unsigned int remote_port, local_port; + + struct ip_addr real_remote_ip, real_local_ip; + unsigned int real_remote_port, real_local_port; unsigned int fifo:1; unsigned int ssl:1; From dovecot at dovecot.org Tue Aug 18 18:02:14 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Aug 2015 18:02:14 +0000 Subject: dovecot-2.2: lmtp, *-login: Use ip/port values from struct maste... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a0e8c6b88072 changeset: 18950:a0e8c6b88072 user: Stephan Bosch date: Mon Jun 15 18:50:53 2015 +0200 description: lmtp, *-login: Use ip/port values from struct master_service_connection instead of from the socket. This way, a proxy protocol like HAProxy can transparently override these addresses with what is seen by the proxy. diffstat: src/lmtp/client.c | 3 ++- src/login-common/client-common.c | 19 ++++++++++++++----- src/login-common/client-common.h | 6 ++++-- src/login-common/main.c | 22 +++++----------------- 4 files changed, 25 insertions(+), 25 deletions(-) diffs (134 lines): diff -r 71f4b77c519f -r a0e8c6b88072 src/lmtp/client.c --- a/src/lmtp/client.c Mon Jun 15 18:50:53 2015 +0200 +++ b/src/lmtp/client.c Mon Jun 15 18:50:53 2015 +0200 @@ -249,7 +249,8 @@ client->fd_out = fd_out; client->remote_ip = conn->remote_ip; client->remote_port = conn->remote_port; - (void)net_getsockname(conn->fd, &client->local_ip, &client->local_port); + client->local_ip = conn->local_ip; + client->local_port = conn->local_port; client->input = i_stream_create_fd(fd_in, CLIENT_MAX_INPUT_SIZE, FALSE); client->output = o_stream_create_fd(fd_out, (size_t)-1, FALSE); diff -r 71f4b77c519f -r a0e8c6b88072 src/login-common/client-common.c --- a/src/login-common/client-common.c Mon Jun 15 18:50:53 2015 +0200 +++ b/src/login-common/client-common.c Mon Jun 15 18:50:53 2015 +0200 @@ -103,10 +103,10 @@ struct client * client_create(int fd, bool ssl, pool_t pool, + const struct master_service_connection *conn, const struct login_settings *set, const struct master_service_ssl_settings *ssl_set, - void **other_sets, - const struct ip_addr *local_ip, const struct ip_addr *remote_ip) + void **other_sets) { struct client *client; @@ -125,13 +125,22 @@ client->pool = pool; client->set = set; client->ssl_set = ssl_set; - client->real_local_ip = client->local_ip = *local_ip; - client->real_remote_ip = client->ip = *remote_ip; + client->fd = fd; client->tls = ssl; + + client->local_ip = conn->local_ip; + client->local_port = conn->local_port; + client->ip = conn->remote_ip; + client->remote_port = conn->remote_port; + client->real_local_ip = conn->real_local_ip; + client->real_local_port = conn->real_local_port; + client->real_remote_ip = conn->real_remote_ip; + client->real_remote_port = conn->real_remote_port; + client->trusted = client_is_trusted(client); client->secured = ssl || client->trusted || - net_ip_compare(remote_ip, local_ip); + net_ip_compare(&conn->real_remote_ip, &conn->real_local_ip); client->proxy_ttl = LOGIN_PROXY_TTL; if (last_client == NULL) diff -r 71f4b77c519f -r a0e8c6b88072 src/login-common/client-common.h --- a/src/login-common/client-common.h Mon Jun 15 18:50:53 2015 +0200 +++ b/src/login-common/client-common.h Mon Jun 15 18:50:53 2015 +0200 @@ -34,6 +34,8 @@ #define AUTH_MASTER_WAITING_MSG \ "Waiting for authentication master process to respond.." +struct master_service_connection; + enum client_disconnect_reason { CLIENT_DISCONNECT_TIMEOUT, CLIENT_DISCONNECT_SYSTEM_SHUTDOWN, @@ -173,10 +175,10 @@ struct client * client_create(int fd, bool ssl, pool_t pool, + const struct master_service_connection *conn, const struct login_settings *set, const struct master_service_ssl_settings *ssl_set, - void **other_sets, - const struct ip_addr *local_ip, const struct ip_addr *remote_ip); + void **other_sets); void client_destroy(struct client *client, const char *reason); void client_destroy_success(struct client *client, const char *reason); void client_destroy_internal_failure(struct client *client); diff -r 71f4b77c519f -r a0e8c6b88072 src/login-common/main.c --- a/src/login-common/main.c Mon Jun 15 18:50:53 2015 +0200 +++ b/src/login-common/main.c Mon Jun 15 18:50:53 2015 +0200 @@ -112,27 +112,19 @@ { struct client *client; struct ssl_proxy *proxy; - struct ip_addr local_ip; const struct login_settings *set; const struct master_service_ssl_settings *ssl_set; - unsigned int local_port; pool_t pool; int fd_ssl; void **other_sets; - if (net_getsockname(conn->fd, &local_ip, &local_port) < 0) { - memset(&local_ip, 0, sizeof(local_ip)); - local_port = 0; - } - pool = pool_alloconly_create("login client", 8*1024); - set = login_settings_read(pool, &local_ip, + set = login_settings_read(pool, &conn->local_ip, &conn->remote_ip, NULL, &ssl_set, &other_sets); if (!ssl_connections && !conn->ssl) { - client = client_create(conn->fd, FALSE, pool, - set, ssl_set, other_sets, - &local_ip, &conn->remote_ip); + client = client_create(conn->fd, FALSE, pool, conn, + set, ssl_set, other_sets); } else { fd_ssl = ssl_proxy_alloc(conn->fd, &conn->remote_ip, pool, set, ssl_set, &proxy); @@ -143,17 +135,13 @@ return; } - client = client_create(fd_ssl, TRUE, pool, - set, ssl_set, other_sets, - &local_ip, &conn->remote_ip); + client = client_create(fd_ssl, TRUE, pool, conn, + set, ssl_set, other_sets); client->ssl_proxy = proxy; ssl_proxy_set_client(proxy, client); ssl_proxy_start(proxy); } - client->real_remote_port = client->remote_port = conn->remote_port; - client->real_local_port = client->local_port = local_port; - if (auth_client_to != NULL) timeout_remove(&auth_client_to); } From dovecot at dovecot.org Tue Aug 18 18:02:14 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Aug 2015 18:02:14 +0000 Subject: dovecot-2.2: master: Changed passing of listener settings from m... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/52368e60177c changeset: 18951:52368e60177c user: Timo Sirainen date: Tue Aug 18 20:23:45 2015 +0300 description: master: Changed passing of listener settings from master to process to be more flexible. This is needed to allow adding new listener settings, such as `haproxy'. Patch by Stephan Bosch - with some small changes. diffstat: src/lib-master/master-service-private.h | 12 +- src/lib-master/master-service.c | 138 +++++++++++++++---------------- src/master/service-process.c | 44 +++------ 3 files changed, 90 insertions(+), 104 deletions(-) diffs (truncated from 334 to 300 lines): diff -r a0e8c6b88072 -r 52368e60177c src/lib-master/master-service-private.h --- a/src/lib-master/master-service-private.h Mon Jun 15 18:50:53 2015 +0200 +++ b/src/lib-master/master-service-private.h Tue Aug 18 20:23:45 2015 +0300 @@ -6,10 +6,14 @@ struct master_service_listener { struct master_service *service; - int fd; + char *name; + + /* settings */ bool ssl; + + /* state */ + int fd; struct io *io; - const char *name; }; struct master_service { @@ -28,10 +32,8 @@ int config_fd; int syslog_facility; - unsigned int socket_count, ssl_socket_count; struct master_service_listener *listeners; - char **listener_names; - unsigned int listener_names_count; + unsigned int socket_count; struct io *io_status_write, *io_status_error; unsigned int service_count_left; diff -r a0e8c6b88072 -r 52368e60177c src/lib-master/master-service.c --- a/src/lib-master/master-service.c Mon Jun 15 18:50:53 2015 +0200 +++ b/src/lib-master/master-service.c Tue Aug 18 20:23:45 2015 +0300 @@ -106,13 +106,53 @@ } } +static void master_service_init_socket_listeners(struct master_service *service) +{ + unsigned int i; + const char *value; + bool have_ssl_sockets = FALSE; + + if (service->socket_count == 0) + return; + + service->listeners = + i_new(struct master_service_listener, service->socket_count); + + for (i = 0; i < service->socket_count; i++) { + struct master_service_listener *l = &service->listeners[i]; + + l->service = service; + l->fd = MASTER_LISTEN_FD_FIRST + i; + + value = getenv(t_strdup_printf("SOCKET%u_SETTINGS", i)); + if (value != NULL) { + const char *const *settings = + t_strsplit_tabescaped(value); + + if (*settings != NULL) { + l->name = i_strdup_empty(*settings); + settings++; + } + while (*settings != NULL) { + if (strcmp(*settings, "ssl") == 0) { + l->ssl = TRUE; + have_ssl_sockets = TRUE; + } + settings++; + } + } + } + service->want_ssl_settings = have_ssl_sockets || + (service->flags & MASTER_SERVICE_FLAG_USE_SSL_SETTINGS) != 0; +} + struct master_service * master_service_init(const char *name, enum master_service_flags flags, int *argc, char **argv[], const char *getopt_str) { struct master_service *service; + unsigned int count; const char *value; - unsigned int count; i_assert(name != NULL); @@ -177,21 +217,14 @@ } else { service->version_string = PACKAGE_VERSION; } + + /* listener configuration */ value = getenv("SOCKET_COUNT"); if (value != NULL) service->socket_count = atoi(value); - value = getenv("SSL_SOCKET_COUNT"); - if (value != NULL) - service->ssl_socket_count = atoi(value); - value = getenv("SOCKET_NAMES"); - if (value != NULL) { - service->listener_names = - p_strsplit_tabescaped(default_pool, value); - service->listener_names_count = - str_array_length((void *)service->listener_names); - } - service->want_ssl_settings = service->ssl_socket_count > 0 || - (flags & MASTER_SERVICE_FLAG_USE_SSL_SETTINGS) != 0; + T_BEGIN { + master_service_init_socket_listeners(service); + } T_END; /* set up some kind of logging until we know exactly how and where we want to log */ @@ -574,8 +607,9 @@ i_assert(listen_fd >= MASTER_LISTEN_FD_FIRST); i = listen_fd - MASTER_LISTEN_FD_FIRST; - return i < service->listener_names_count ? - service->listener_names[i] : ""; + i_assert(i < service->socket_count); + return service->listeners[i].name != NULL ? + service->listeners[i].name : ""; } void master_service_set_avail_overflow_callback(struct master_service *service, @@ -807,6 +841,7 @@ void master_service_deinit(struct master_service **_service) { struct master_service *service = *_service; + unsigned int i; *_service = NULL; @@ -836,8 +871,8 @@ lib_atexit_run(); io_loop_destroy(&service->ioloop); - if (service->listener_names != NULL) - p_strsplit_free(default_pool, service->listener_names); + for (i = 0; i < service->socket_count; i++) + i_free(service->listeners[i].name); i_free(service->listeners); i_free(service->getopt_str); i_free(service->name); @@ -912,29 +947,6 @@ master_service_client_connection_callback(service, &conn); } -static void io_listeners_init(struct master_service *service) -{ - unsigned int i; - - if (service->socket_count == 0) - return; - - service->listeners = - i_new(struct master_service_listener, service->socket_count); - - for (i = 0; i < service->socket_count; i++) { - struct master_service_listener *l = &service->listeners[i]; - - l->service = service; - l->fd = MASTER_LISTEN_FD_FIRST + i; - l->name = i < service->listener_names_count ? - service->listener_names[i] : ""; - - if (i >= service->socket_count - service->ssl_socket_count) - l->ssl = TRUE; - } -} - void master_service_io_listeners_add(struct master_service *service) { unsigned int i; @@ -942,9 +954,6 @@ if (service->stopping) return; - if (service->listeners == NULL) - io_listeners_init(service); - for (i = 0; i < service->socket_count; i++) { struct master_service_listener *l = &service->listeners[i]; @@ -959,11 +968,9 @@ { unsigned int i; - if (service->listeners != NULL) { - for (i = 0; i < service->socket_count; i++) { - if (service->listeners[i].io != NULL) - io_remove(&service->listeners[i].io); - } + for (i = 0; i < service->socket_count; i++) { + if (service->listeners[i].io != NULL) + io_remove(&service->listeners[i].io); } } @@ -971,12 +978,10 @@ { unsigned int i; - if (service->listeners != NULL) { - for (i = 0; i < service->socket_count; i++) { - if (service->listeners[i].io != NULL && - service->listeners[i].ssl) - io_remove(&service->listeners[i].io); - } + for (i = 0; i < service->socket_count; i++) { + if (service->listeners[i].io != NULL && + service->listeners[i].ssl) + io_remove(&service->listeners[i].io); } } @@ -984,24 +989,15 @@ { unsigned int i; - if (service->listeners != NULL) { - /* close via listeners. some fds might be pipes that are - currently handled as clients. we don't want to close them. */ - for (i = 0; i < service->socket_count; i++) { - if (service->listeners[i].fd != -1) { - if (close(service->listeners[i].fd) < 0) { - i_error("close(listener %d) failed: %m", - service->listeners[i].fd); - } - service->listeners[i].fd = -1; + /* close via listeners. some fds might be pipes that are + currently handled as clients. we don't want to close them. */ + for (i = 0; i < service->socket_count; i++) { + if (service->listeners[i].fd != -1) { + if (close(service->listeners[i].fd) < 0) { + i_error("close(listener %d) failed: %m", + service->listeners[i].fd); } - } - } else { - for (i = 0; i < service->socket_count; i++) { - int fd = MASTER_LISTEN_FD_FIRST + i; - - if (close(fd) < 0) - i_error("close(listener %d) failed: %m", fd); + service->listeners[i].fd = -1; } } } diff -r a0e8c6b88072 -r 52368e60177c src/master/service-process.c --- a/src/master/service-process.c Mon Jun 15 18:50:53 2015 +0200 +++ b/src/master/service-process.c Tue Aug 18 20:23:45 2015 +0300 @@ -58,9 +58,9 @@ { struct service_listener *const *listeners; ARRAY_TYPE(dup2) dups; - string_t *listener_names; + string_t *listener_settings; int fd = MASTER_LISTEN_FD_FIRST; - unsigned int i, count, socket_listener_count, ssl_socket_count; + unsigned int i, count, socket_listener_count; /* stdin/stdout is already redirected to /dev/null. Other master fds should have been opened with fd_close_on_exec() so we don't have to @@ -72,7 +72,6 @@ socket_listener_count = 0; listeners = array_get(&service->listeners, &count); t_array_init(&dups, count + 10); - listener_names = t_str_new(256); switch (service->type) { case SERVICE_TYPE_LOG: @@ -94,34 +93,25 @@ break; } - /* anvil/log fds have no names */ - for (i = MASTER_LISTEN_FD_FIRST; i < (unsigned int)fd; i++) - str_append_c(listener_names, '\t'); + /* add listeners */ + listener_settings = t_str_new(256); + for (i = 0; i < count; i++) { + if (listeners[i]->fd != -1) { + str_truncate(listener_settings, 0); + str_append_tabescaped(listener_settings, listeners[i]->name); - /* first add non-ssl listeners */ - for (i = 0; i < count; i++) { - if (listeners[i]->fd != -1 && - (listeners[i]->type != SERVICE_LISTENER_INET || - !listeners[i]->set.inetset.set->ssl)) { - str_append_tabescaped(listener_names, listeners[i]->name); - str_append_c(listener_names, '\t'); + if (listeners[i]->type == SERVICE_LISTENER_INET) { + if (listeners[i]->set.inetset.set->ssl) + str_append(listener_settings, "\tssl"); From dovecot at dovecot.org Tue Aug 18 18:02:19 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Aug 2015 18:02:19 +0000 Subject: dovecot-2.2: lib-master: Added support for HAProxy protocol. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4d7a83ddb644 changeset: 18952:4d7a83ddb644 user: Timo Sirainen date: Tue Aug 18 20:39:06 2015 +0300 description: lib-master: Added support for HAProxy protocol. Patch by Stephan Bosch - with some small changes. diffstat: src/lib-master/Makefile.am | 1 + src/lib-master/master-service-haproxy.c | 476 +++++++++++++++++++++++++++++++ src/lib-master/master-service-private.h | 9 + src/lib-master/master-service-settings.c | 8 +- src/lib-master/master-service-settings.h | 3 + src/lib-master/master-service.c | 9 +- src/lib-master/service-settings.h | 1 + src/master/master-settings.c | 4 +- src/master/service-process.c | 2 + 9 files changed, 510 insertions(+), 3 deletions(-) diffs (truncated from 643 to 300 lines): diff -r 52368e60177c -r 4d7a83ddb644 src/lib-master/Makefile.am --- a/src/lib-master/Makefile.am Tue Aug 18 20:23:45 2015 +0300 +++ b/src/lib-master/Makefile.am Tue Aug 18 20:39:06 2015 +0300 @@ -21,6 +21,7 @@ master-login.c \ master-login-auth.c \ master-service.c \ + master-service-haproxy.c \ master-service-settings.c \ master-service-settings-cache.c \ master-service-ssl.c \ diff -r 52368e60177c -r 4d7a83ddb644 src/lib-master/master-service-haproxy.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-master/master-service-haproxy.c Tue Aug 18 20:39:06 2015 +0300 @@ -0,0 +1,476 @@ +/* Copyright (c) 2013-2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "llist.h" +#include "ioloop.h" +#include "str-sanitize.h" +#include "master-service-private.h" +#include "master-service-settings.h" + +#define HAPROXY_V1_MAX_HEADER_SIZE (108) + +enum { + HAPROXY_CMD_LOCAL = 0x00, + HAPROXY_CMD_PROXY = 0x01 +}; + +enum { + HAPROXY_AF_UNSPEC = 0x00, + HAPROXY_AF_INET = 0x01, + HAPROXY_AF_INET6 = 0x02, + HAPROXY_AF_UNIX = 0x03 +}; + +enum { + HAPROXY_SOCK_UNSPEC = 0x00, + HAPROXY_SOCK_STREAM = 0x01, + HAPROXY_SOCK_DGRAM = 0x02 +}; + +static const char haproxy_v2sig[12] = + "\x0D\x0A\x0D\x0A\x00\x0D\x0A\x51\x55\x49\x54\x0A"; + +struct haproxy_header_v2 { + uint8_t sig[12]; + uint8_t ver_cmd; + uint8_t fam; + uint16_t len; +}; + +struct haproxy_data_v2 { + union { + struct { /* for TCP/UDP over IPv4, len = 12 */ + uint32_t src_addr; + uint32_t dst_addr; + uint16_t src_port; + uint16_t dst_port; + } ip4; + struct { /* for TCP/UDP over IPv6, len = 36 */ + uint8_t src_addr[16]; + uint8_t dst_addr[16]; + uint16_t src_port; + uint16_t dst_port; + } ip6; + struct { /* for AF_UNIX sockets, len = 216 */ + uint8_t src_addr[108]; + uint8_t dst_addr[108]; + } unx; + } addr; +}; + +struct master_service_haproxy_conn { + struct master_service_connection conn; + + struct master_service_haproxy_conn *prev, *next; + + struct master_service *service; + + struct io *io; + struct timeout *to; +}; + +static void +master_service_haproxy_conn_free(struct master_service_haproxy_conn *hpconn) +{ + struct master_service *service = hpconn->service; + + DLLIST_REMOVE(&service->haproxy_conns, hpconn); + + if (hpconn->io != NULL) + io_remove(&hpconn->io); + if (hpconn->to != NULL) + timeout_remove(&hpconn->to); + i_free(hpconn); +} + +static void +master_service_haproxy_conn_failure(struct master_service_haproxy_conn *hpconn) +{ + struct master_service *service = hpconn->service; + struct master_service_connection conn = hpconn->conn; + + master_service_haproxy_conn_free(hpconn); + master_service_client_connection_handled(service, &conn); +} + +static void +master_service_haproxy_conn_success(struct master_service_haproxy_conn *hpconn) +{ + struct master_service *service = hpconn->service; + struct master_service_connection conn = hpconn->conn; + + master_service_haproxy_conn_free(hpconn); + master_service_client_connection_callback(service, &conn); +} + +static void +master_service_haproxy_timeout(struct master_service_haproxy_conn *hpconn) +{ + i_error("haproxy: Client timed out (rip=%s)", + net_ip2addr(&hpconn->conn.remote_ip)); + master_service_haproxy_conn_failure(hpconn); +} + +static int +master_service_haproxy_read(struct master_service_haproxy_conn *hpconn) +{ + static union { + unsigned char v1_data[HAPROXY_V1_MAX_HEADER_SIZE]; + struct { + const struct haproxy_header_v2 hdr; + const struct haproxy_data_v2 data; + } v2; + } buf; + struct ip_addr *real_remote_ip = &hpconn->conn.remote_ip; + int fd = hpconn->conn.fd; + struct ip_addr local_ip, remote_ip; + unsigned int local_port, remote_port; + size_t size; + ssize_t ret; + + /* the protocol specification explicitly states that the protocol header + must be sent as one TCP frame, meaning that we will get it in full + with the first recv() call. + FIXME: still, it would be cleaner to allow reading it incrementally. + */ + do { + ret = recv(fd, &buf, sizeof(buf), MSG_PEEK); + } while (ret < 0 && errno == EINTR); + + if (ret < 0 && errno == EAGAIN) + return 0; + if (ret <= 0) { + i_info("haproxy: Client disconnected (rip=%s)", + net_ip2addr(real_remote_ip)); + return -1; + } + + /* don't update true connection data until we succeed */ + local_ip = hpconn->conn.local_ip; + remote_ip = hpconn->conn.remote_ip; + local_port = hpconn->conn.local_port; + remote_port = hpconn->conn.remote_port; + + /* protocol version 2 */ + if (ret >= (ssize_t)sizeof(buf.v2.hdr) && + memcmp(buf.v2.hdr.sig, haproxy_v2sig, + sizeof(buf.v2.hdr.sig)) == 0) { + const struct haproxy_header_v2 *hdr = &buf.v2.hdr; + const struct haproxy_data_v2 *data = &buf.v2.data; + size_t hdr_len; + + if ((hdr->ver_cmd & 0xf0) != 0x20) { + i_error("haproxy: Client disconnected: " + "Unsupported protocol version (version=%02x, rip=%s)", + (hdr->ver_cmd & 0xf0) >> 4, + net_ip2addr(real_remote_ip)); + return -1; + } + + hdr_len = ntohs(hdr->len); + size = sizeof(*hdr) + hdr_len; + if (ret < (ssize_t)size) { + i_error("haproxy(v2): Client disconnected: " + "Protocol payload length does not match header " + "(got=%"PRIuSIZE_T", expect=%"PRIuSIZE_T", rip=%s)", + (size_t)ret, size, net_ip2addr(real_remote_ip)); + return -1; + } + + switch (hdr->ver_cmd & 0x0f) { + case HAPROXY_CMD_LOCAL: + /* keep local connection address for LOCAL */ + /*i_debug("haproxy(v2): Local connection (rip=%s)", + net_ip2addr(real_remote_ip));*/ + break; + case HAPROXY_CMD_PROXY: + if ((hdr->fam & 0x0f) != HAPROXY_SOCK_STREAM) { + /* UDP makes no sense currently */ + i_error("haproxy(v2): Client disconnected: " + "Not using TCP (type=%02x, rip=%s)", + (hdr->fam & 0x0f), net_ip2addr(real_remote_ip)); + return -1; + } + switch ((hdr->fam & 0xf0) >> 4) { + case HAPROXY_AF_INET: + /* IPv4 */ + if (hdr_len < sizeof(data->addr.ip4)) { + i_error("haproxy(v2): Client disconnected: " + "IPv4 data is incomplete (rip=%s)", + net_ip2addr(real_remote_ip)); + return -1; + } + local_ip.family = AF_INET; + local_ip.u.ip4.s_addr = data->addr.ip4.dst_addr; + local_port = ntohs(data->addr.ip4.dst_port); + remote_ip.family = AF_INET; + remote_ip.u.ip4.s_addr = data->addr.ip4.src_addr; + remote_port = ntohs(data->addr.ip4.src_port); + break; + case HAPROXY_AF_INET6: + /* IPv6 */ + if (hdr_len < sizeof(data->addr.ip6)) { + i_error("haproxy(v2): Client disconnected: " + "IPv6 data is incomplete (rip=%s)", + net_ip2addr(real_remote_ip)); + return -1; + } + local_ip.family = AF_INET6; + memcpy(&local_ip.u.ip6.s6_addr, data->addr.ip6.dst_addr, 16); + local_port = ntohs(data->addr.ip6.dst_port); + remote_ip.family = AF_INET6; + memcpy(&remote_ip.u.ip6.s6_addr, data->addr.ip6.src_addr, 16); + remote_port = ntohs(data->addr.ip6.src_port); + break; + case HAPROXY_AF_UNSPEC: + case HAPROXY_AF_UNIX: + /* unsupported; ignored */ + i_error("haproxy(v2): Unsupported address family " + "(family=%02x, rip=%s)", (hdr->fam & 0xf0) >> 4, + net_ip2addr(real_remote_ip)); + break; + default: + /* unsupported; error */ + i_error("haproxy(v2): Client disconnected: " + "Unknown address family " + "(family=%02x, rip=%s)", (hdr->fam & 0xf0) >> 4, + net_ip2addr(real_remote_ip)); + return -1; + } + break; + default: + i_error("haproxy(v2): Client disconnected: " + "Invalid command (cmd=%02x, rip=%s)", + (hdr->ver_cmd & 0x0f), + net_ip2addr(real_remote_ip)); + return -1; /* not a supported command */ + } + + // FIXME: TLV vectors are ignored + // (useful to see whether proxied client is using SSL) + + /* protocol version 1 (soon obsolete) */ + } else if (ret >= 8 && memcmp(buf.v1_data, "PROXY", 5) == 0) { + unsigned char *data = buf.v1_data, *end; + const char *const *fields; + unsigned int family = 0; + + /* find end of header line */ + end = memchr(data, '\r', ret - 1); + if (end == NULL || end[1] != '\n') + return -1; + *end = '\0'; + size = end + 2 - data; + + /* magic */ + fields = t_strsplit((char *)data, " "); + i_assert(strcmp(*fields, "PROXY") == 0); + fields++; + + /* protocol */ + if (*fields == NULL) { + i_error("haproxy(v1): Client disconnected: " + "Field for proxied protocol is missing " + "(rip=%s)", net_ip2addr(real_remote_ip)); + return -1; + } + if (strcmp(*fields, "TCP4") == 0) { + family = AF_INET; + } else if (strcmp(*fields, "TCP6") == 0) { + family = AF_INET6; + } else if (strcmp(*fields, "UNKNOWN") == 0) { + family = 0; + } else { + i_error("haproxy(v1): Client disconnected: " + "Unknown proxied protocol " From dovecot at dovecot.org Tue Aug 18 18:02:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Aug 2015 18:02:25 +0000 Subject: dovecot-2.2: lib-http auth: Fixed segfault occurring in http_aut... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f73ed907fe64 changeset: 18953:f73ed907fe64 user: Stephan Bosch date: Tue Aug 18 20:39:24 2015 +0300 description: lib-http auth: Fixed segfault occurring in http_auth_challenge_copy() when there are no parameters. diffstat: src/lib-http/http-auth.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (13 lines): diff -r 4d7a83ddb644 -r f73ed907fe64 src/lib-http/http-auth.c --- a/src/lib-http/http-auth.c Tue Aug 18 20:39:06 2015 +0300 +++ b/src/lib-http/http-auth.c Tue Aug 18 20:39:24 2015 +0300 @@ -370,6 +370,9 @@ { const struct http_auth_param *sparam; + if (!array_is_created(src)) + return; + p_array_init(dst, pool, 4); array_foreach(src, sparam) { struct http_auth_param nparam; From dovecot at dovecot.org Tue Aug 18 18:02:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Aug 2015 18:02:25 +0000 Subject: dovecot-2.2: lib-http auth: Made second argument of http_auth_cr... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9d530e890669 changeset: 18954:9d530e890669 user: Stephan Bosch date: Tue Aug 18 20:39:24 2015 +0300 description: lib-http auth: Made second argument of http_auth_create_challenges() const as it should be. diffstat: src/lib-http/http-auth.c | 2 +- src/lib-http/http-auth.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diffs (24 lines): diff -r f73ed907fe64 -r 9d530e890669 src/lib-http/http-auth.c --- a/src/lib-http/http-auth.c Tue Aug 18 20:39:24 2015 +0300 +++ b/src/lib-http/http-auth.c Tue Aug 18 20:39:24 2015 +0300 @@ -313,7 +313,7 @@ } void http_auth_create_challenges(string_t *out, - ARRAY_TYPE(http_auth_challenge) *chlngs) + const ARRAY_TYPE(http_auth_challenge) *chlngs) { const struct http_auth_challenge *chlgs; unsigned int count, i; diff -r f73ed907fe64 -r 9d530e890669 src/lib-http/http-auth.h --- a/src/lib-http/http-auth.h Tue Aug 18 20:39:24 2015 +0300 +++ b/src/lib-http/http-auth.h Tue Aug 18 20:39:24 2015 +0300 @@ -43,7 +43,7 @@ void http_auth_create_challenge(string_t *out, const struct http_auth_challenge *chlng); void http_auth_create_challenges(string_t *out, - ARRAY_TYPE(http_auth_challenge) *chlngs); + const ARRAY_TYPE(http_auth_challenge) *chlngs); void http_auth_create_credentials(string_t *out, const struct http_auth_credentials *crdts); From dovecot at dovecot.org Tue Aug 18 18:02:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Aug 2015 18:02:25 +0000 Subject: dovecot-2.2: lib-http auth: Fixed assertion on the validity of t... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cc09fc9194d1 changeset: 18955:cc09fc9194d1 user: Stephan Bosch date: Tue Aug 18 20:39:24 2015 +0300 description: lib-http auth: Fixed assertion on the validity of the token68 encountered during conversion to string. diffstat: src/lib-http/http-auth.c | 24 ++++++++++++++---------- 1 files changed, 14 insertions(+), 10 deletions(-) diffs (50 lines): diff -r 9d530e890669 -r cc09fc9194d1 src/lib-http/http-auth.c --- a/src/lib-http/http-auth.c Tue Aug 18 20:39:24 2015 +0300 +++ b/src/lib-http/http-auth.c Tue Aug 18 20:39:24 2015 +0300 @@ -285,6 +285,18 @@ } } +static void http_auth_check_token68(const char *data) +{ + const char *p = data; + + /* Make sure we're not working with nonsense. */ + i_assert(http_char_is_token68(*p)); + for (p++; *p != '\0' && *p != '='; p++) + i_assert(http_char_is_token68(*p)); + for (; *p != '\0'; p++) + i_assert(*p == '='); +} + void http_auth_create_challenge(string_t *out, const struct http_auth_challenge *chlng) { @@ -296,12 +308,8 @@ str_append(out, chlng->scheme); if (chlng->data != NULL) { - const char *p; - /* SP token68 */ - for (p = chlng->data; *p != '\0'; p++) - i_assert(http_char_is_token68(*p)); - + http_auth_check_token68(chlng->data); str_append_c(out, ' '); str_append(out, chlng->data); @@ -343,12 +351,8 @@ str_append(out, crdts->scheme); if (crdts->data != NULL) { - const char *p; - /* SP token68 */ - for (p = crdts->data; *p != '\0'; p++) - i_assert(http_char_is_token68(*p)); - + http_auth_check_token68(crdts->data); str_append_c(out, ' '); str_append(out, crdts->data); From dovecot at dovecot.org Tue Aug 18 18:02:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Aug 2015 18:02:25 +0000 Subject: dovecot-2.2: lib-http auth: Implemented client-side handling of ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3e1ab97a392c changeset: 18956:3e1ab97a392c user: Stephan Bosch date: Tue Aug 18 20:39:24 2015 +0300 description: lib-http auth: Implemented client-side handling of basic authentication scheme. diffstat: src/lib-http/http-auth.c | 21 ++++++++++++++++++++- src/lib-http/http-auth.h | 2 ++ 2 files changed, 22 insertions(+), 1 deletions(-) diffs (54 lines): diff -r cc09fc9194d1 -r 3e1ab97a392c src/lib-http/http-auth.c --- a/src/lib-http/http-auth.c Tue Aug 18 20:39:24 2015 +0300 +++ b/src/lib-http/http-auth.c Tue Aug 18 20:39:24 2015 +0300 @@ -2,6 +2,7 @@ #include "lib.h" #include "str.h" +#include "base64.h" #include "array.h" #include "http-parser.h" @@ -420,7 +421,7 @@ const char *realm) { memset(chlng, 0, sizeof(*chlng)); - chlng->scheme = "basic"; + chlng->scheme = "Basic"; if (realm != NULL) { struct http_auth_param param; @@ -432,3 +433,21 @@ array_append(&chlng->params, ¶m, 1); } } + +void http_auth_basic_credentials_init(struct http_auth_credentials *crdts, + const char *username, const char *password) +{ + const char *auth; + string_t *data; + + i_assert(username != NULL && *username != '\0'); + i_assert(strchr(username, ':') == NULL); + + data = t_str_new(64); + auth = t_strconcat(username, ":", password, NULL); + base64_encode(auth, strlen(auth), data); + + memset(crdts, 0, sizeof(*crdts)); + crdts->scheme = "Basic"; + crdts->data = str_c(data); +} diff -r cc09fc9194d1 -r 3e1ab97a392c src/lib-http/http-auth.h --- a/src/lib-http/http-auth.h Tue Aug 18 20:39:24 2015 +0300 +++ b/src/lib-http/http-auth.h Tue Aug 18 20:39:24 2015 +0300 @@ -65,6 +65,8 @@ void http_auth_basic_challenge_init(struct http_auth_challenge *chlng, const char *realm); +void http_auth_basic_credentials_init(struct http_auth_credentials *crdts, + const char *username, const char *password); #endif From dovecot at dovecot.org Tue Aug 18 18:02:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Aug 2015 18:02:25 +0000 Subject: dovecot-2.2: lib-http: url: Make sure destination url struct is ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1d5d4d469b2b changeset: 18957:1d5d4d469b2b user: Stephan Bosch date: Tue Aug 18 20:39:24 2015 +0300 description: lib-http: url: Make sure destination url struct is cleared in http_url_copy(). diffstat: src/lib-http/http-url.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 3e1ab97a392c -r 1d5d4d469b2b src/lib-http/http-url.c --- a/src/lib-http/http-url.c Tue Aug 18 20:39:24 2015 +0300 +++ b/src/lib-http/http-url.c Tue Aug 18 20:39:24 2015 +0300 @@ -428,6 +428,7 @@ void http_url_copy_authority(pool_t pool, struct http_url *dest, const struct http_url *src) { + memset(dest, 0, sizeof(*dest)); dest->host_name = p_strdup(pool, src->host_name); if (src->have_host_ip) { dest->host_ip = src->host_ip; From dovecot at dovecot.org Tue Aug 18 18:02:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Aug 2015 18:02:25 +0000 Subject: dovecot-2.2: lib-http client: Minor whitespace cleanups in struc... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5653b61d4c73 changeset: 18958:5653b61d4c73 user: Stephan Bosch date: Tue Aug 18 20:39:24 2015 +0300 description: lib-http client: Minor whitespace cleanups in struct http_client_request. diffstat: src/lib-http/http-client-private.h | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) diffs (19 lines): diff -r 1d5d4d469b2b -r 5653b61d4c73 src/lib-http/http-client-private.h --- a/src/lib-http/http-client-private.h Tue Aug 18 20:39:24 2015 +0300 +++ b/src/lib-http/http-client-private.h Tue Aug 18 20:39:24 2015 +0300 @@ -95,11 +95,11 @@ enum http_request_state state; - unsigned int have_hdr_connection:1; - unsigned int have_hdr_date:1; + unsigned int have_hdr_body_spec:1; + unsigned int have_hdr_connection:1; + unsigned int have_hdr_date:1; unsigned int have_hdr_expect:1; - unsigned int have_hdr_host:1; - unsigned int have_hdr_body_spec:1; + unsigned int have_hdr_host:1; unsigned int have_hdr_user_agent:1; unsigned int payload_sync:1; From dovecot at dovecot.org Tue Aug 18 18:02:35 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Aug 2015 18:02:35 +0000 Subject: dovecot-2.2: lib-http client: Removed useless assignment of requ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/34f78bc1eed4 changeset: 18959:34f78bc1eed4 user: Stephan Bosch date: Tue Aug 18 20:39:24 2015 +0300 description: lib-http client: Removed useless assignment of request->authority in http_client_request_redirect(). It is assigned again in http_client_request_do_submit(). diffstat: src/lib-http/http-client-request.c | 4 ---- 1 files changed, 0 insertions(+), 4 deletions(-) diffs (14 lines): diff -r 5653b61d4c73 -r 34f78bc1eed4 src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Tue Aug 18 20:39:24 2015 +0300 +++ b/src/lib-http/http-client-request.c Tue Aug 18 20:39:24 2015 +0300 @@ -1022,10 +1022,6 @@ http_url_copy(req->pool, &req->origin_url, url); req->target = p_strdup(req->pool, target); - if (req->host_url == &req->origin_url) { - req->authority = - p_strdup(req->pool, http_url_create_authority(req->host_url)); - } req->host = NULL; req->conn = NULL; From dovecot at dovecot.org Tue Aug 18 18:02:35 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Aug 2015 18:02:35 +0000 Subject: dovecot-2.2: lib-http client: Implemented simple authentication ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/58694b53e730 changeset: 18960:58694b53e730 user: Stephan Bosch date: Tue Aug 18 20:39:24 2015 +0300 description: lib-http client: Implemented simple authentication using Basic scheme. diffstat: src/lib-http/http-client-private.h | 2 ++ src/lib-http/http-client-request.c | 28 ++++++++++++++++++++++++++++ src/lib-http/http-client.h | 3 +++ src/lib-http/test-http-client.c | 8 ++++++++ 4 files changed, 41 insertions(+), 0 deletions(-) diffs (116 lines): diff -r 34f78bc1eed4 -r 58694b53e730 src/lib-http/http-client-private.h --- a/src/lib-http/http-client-private.h Tue Aug 18 20:39:24 2015 +0300 +++ b/src/lib-http/http-client-private.h Tue Aug 18 20:39:24 2015 +0300 @@ -57,6 +57,7 @@ const char *method, *target; struct http_url origin_url; + const char *username, *password; const struct http_url *host_url; const char *authority; @@ -95,6 +96,7 @@ enum http_request_state state; + unsigned int have_hdr_authorization:1; unsigned int have_hdr_body_spec:1; unsigned int have_hdr_connection:1; unsigned int have_hdr_date:1; diff -r 34f78bc1eed4 -r 58694b53e730 src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Tue Aug 18 20:39:24 2015 +0300 +++ b/src/lib-http/http-client-request.c Tue Aug 18 20:39:24 2015 +0300 @@ -12,6 +12,7 @@ #include "dns-lookup.h" #include "http-url.h" #include "http-date.h" +#include "http-auth.h" #include "http-response-parser.h" #include "http-transfer.h" @@ -104,6 +105,11 @@ req = http_client_request_new(client, method, callback, context); http_url_copy_authority(req->pool, &req->origin_url, target_url); req->target = p_strdup(req->pool, http_url_create_target(target_url)); + if (target_url->user != NULL && *target_url->user != '\0' && + target_url->password != NULL) { + req->username = p_strdup(req->pool, target_url->user); + req->password = p_strdup(req->pool, target_url->password); + } return req; } @@ -226,6 +232,10 @@ /* mark presence of special headers */ switch (key[0]) { + case 'a': case 'A': + if (strcasecmp(key, "Authorization") == 0) + req->have_hdr_authorization = TRUE; + break; case 'c': case 'C': if (strcasecmp(key, "Connection") == 0) req->have_hdr_connection = TRUE; @@ -338,6 +348,13 @@ req->timeout_msecs = 0; } +void http_client_request_set_auth_simple(struct http_client_request *req, + const char *username, const char *password) +{ + req->username = p_strdup(req->pool, username); + req->password = p_strdup(req->pool, password); +} + void http_client_request_delay_until(struct http_client_request *req, time_t time) { @@ -744,6 +761,17 @@ str_append(rtext, http_date_create(req->date)); str_append(rtext, "\r\n"); } + if (!req->have_hdr_authorization && + req->username != NULL && req->password != NULL) { + struct http_auth_credentials auth_creds; + + http_auth_basic_credentials_init(&auth_creds, + req->username, req->password); + + str_append(rtext, "Authorization: "); + http_auth_create_credentials(rtext, &auth_creds); + str_append(rtext, "\r\n"); + } if (!req->have_hdr_user_agent && req->client->set.user_agent != NULL) { str_printfa(rtext, "User-Agent: %s\r\n", req->client->set.user_agent); diff -r 34f78bc1eed4 -r 58694b53e730 src/lib-http/http-client.h --- a/src/lib-http/http-client.h Tue Aug 18 20:39:24 2015 +0300 +++ b/src/lib-http/http-client.h Tue Aug 18 20:39:24 2015 +0300 @@ -208,6 +208,9 @@ void http_client_request_set_timeout(struct http_client_request *req, const struct timeval *time); +void http_client_request_set_auth_simple(struct http_client_request *req, + const char *username, const char *password); + void http_client_request_delay_until(struct http_client_request *req, time_t time); void http_client_request_delay(struct http_client_request *req, diff -r 34f78bc1eed4 -r 58694b53e730 src/lib-http/test-http-client.c --- a/src/lib-http/test-http-client.c Tue Aug 18 20:39:24 2015 +0300 +++ b/src/lib-http/test-http-client.c Tue Aug 18 20:39:24 2015 +0300 @@ -260,6 +260,14 @@ test_req = i_new(struct http_test_request, 1); http_req = http_client_request(http_client, + "GET", "jigsaw.w3.org", "/HTTP/Basic/", + got_request_response, test_req); + http_client_request_set_auth_simple + (http_req, "guest", "guest"); + http_client_request_submit(http_req); + + test_req = i_new(struct http_test_request, 1); + http_req = http_client_request(http_client, "PUT", "test.dovecot.org", "/http/put/put.php", got_request_response, test_req); post_payload = i_stream_create_file("Makefile.am", 10); From dovecot at dovecot.org Tue Aug 18 18:02:35 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Aug 2015 18:02:35 +0000 Subject: dovecot-2.2: lib-http client: Added inline function to check whe... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5c53d13a30ab changeset: 18961:5c53d13a30ab user: Stephan Bosch date: Tue Aug 18 20:39:24 2015 +0300 description: lib-http client: Added inline function to check whether request is directed at a proxy. diffstat: src/lib-http/http-client-private.h | 6 ++++++ src/lib-http/http-client-request.c | 3 ++- 2 files changed, 8 insertions(+), 1 deletions(-) diffs (29 lines): diff -r 58694b53e730 -r 5c53d13a30ab src/lib-http/http-client-private.h --- a/src/lib-http/http-client-private.h Tue Aug 18 20:39:24 2015 +0300 +++ b/src/lib-http/http-client-private.h Tue Aug 18 20:39:24 2015 +0300 @@ -412,6 +412,12 @@ } } +static inline bool +http_client_request_to_proxy(const struct http_client_request *req) +{ + return (req->host_url != &req->origin_url); +} + static inline const char * http_client_connection_label(struct http_client_connection *conn) { diff -r 58694b53e730 -r 5c53d13a30ab src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Tue Aug 18 20:39:24 2015 +0300 +++ b/src/lib-http/http-client-request.c Tue Aug 18 20:39:24 2015 +0300 @@ -797,7 +797,8 @@ o_stream_ref(output); } } - if (!req->have_hdr_connection && req->host_url == &req->origin_url) { + if (!req->have_hdr_connection && + !http_client_request_to_proxy(req)) { /* https://tools.ietf.org/html/rfc2068 Section 19.7.1: From dovecot at dovecot.org Tue Aug 18 18:02:36 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Aug 2015 18:02:36 +0000 Subject: dovecot-2.2: lib-http client: Implemented proxy authentication u... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cdf9e584c1c7 changeset: 18962:cdf9e584c1c7 user: Stephan Bosch date: Tue Aug 18 20:39:24 2015 +0300 description: lib-http client: Implemented proxy authentication using Basic scheme. diffstat: src/lib-http/http-client-request.c | 15 +++++++++++++++ src/lib-http/http-client.c | 11 +++++++++-- src/lib-http/http-client.h | 2 +- 3 files changed, 25 insertions(+), 3 deletions(-) diffs (72 lines): diff -r 5c53d13a30ab -r cdf9e584c1c7 src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Tue Aug 18 20:39:24 2015 +0300 +++ b/src/lib-http/http-client-request.c Tue Aug 18 20:39:24 2015 +0300 @@ -254,6 +254,9 @@ if (strcasecmp(key, "Host") == 0) req->have_hdr_host = TRUE; break; + case 'p': case 'P': + i_assert(strcasecmp(key, "Proxy-Authorization") != 0); + break; case 't': case 'T': if (strcasecmp(key, "Transfer-Encoding") == 0) req->have_hdr_body_spec = TRUE; @@ -734,6 +737,7 @@ static int http_client_request_send_real(struct http_client_request *req, const char **error_r) { + const struct http_client_settings *set = &req->client->set; struct http_client_connection *conn = req->conn; struct ostream *output = conn->conn.output; string_t *rtext = t_str_new(256); @@ -772,6 +776,17 @@ http_auth_create_credentials(rtext, &auth_creds); str_append(rtext, "\r\n"); } + if (http_client_request_to_proxy(req) && + set->proxy_username != NULL && set->proxy_password != NULL) { + struct http_auth_credentials auth_creds; + + http_auth_basic_credentials_init(&auth_creds, + set->proxy_username, set->proxy_password); + + str_append(rtext, "Proxy-Authorization: "); + http_auth_create_credentials(rtext, &auth_creds); + str_append(rtext, "\r\n"); + } if (!req->have_hdr_user_agent && req->client->set.user_agent != NULL) { str_printfa(rtext, "User-Agent: %s\r\n", req->client->set.user_agent); diff -r 5c53d13a30ab -r cdf9e584c1c7 src/lib-http/http-client.c --- a/src/lib-http/http-client.c Tue Aug 18 20:39:24 2015 +0300 +++ b/src/lib-http/http-client.c Tue Aug 18 20:39:24 2015 +0300 @@ -111,8 +111,15 @@ } else if (set->proxy_url != NULL) { client->set.proxy_url = http_url_clone(pool, set->proxy_url); } - client->set.proxy_username = p_strdup_empty(pool, set->proxy_username); - client->set.proxy_password = p_strdup_empty(pool, set->proxy_password); + if (set->proxy_username != NULL && *set->proxy_username != '\0') { + client->set.proxy_username = p_strdup_empty(pool, set->proxy_username); + client->set.proxy_password = p_strdup(pool, set->proxy_password); + } else if (set->proxy_url != NULL) { + client->set.proxy_username = + p_strdup_empty(pool, set->proxy_url->user); + client->set.proxy_password = + p_strdup(pool, set->proxy_url->password); + } client->set.max_idle_time_msecs = set->max_idle_time_msecs; client->set.max_parallel_connections = diff -r 5c53d13a30ab -r cdf9e584c1c7 src/lib-http/http-client.h --- a/src/lib-http/http-client.h Tue Aug 18 20:39:24 2015 +0300 +++ b/src/lib-http/http-client.h Tue Aug 18 20:39:24 2015 +0300 @@ -54,7 +54,7 @@ /* configuration for using a proxy */ const char *proxy_socket_path; /* FIXME: implement */ const struct http_url *proxy_url; - const char *proxy_username; /* FIXME: implement */ + const char *proxy_username; const char *proxy_password; const char *rawlog_dir; From dovecot at dovecot.org Tue Aug 18 18:02:36 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Aug 2015 18:02:36 +0000 Subject: dovecot-2.2: lib-http auth: Implemented cloning/copying of crede... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/07ceb84bf899 changeset: 18963:07ceb84bf899 user: Stephan Bosch date: Tue Aug 18 20:39:24 2015 +0300 description: lib-http auth: Implemented cloning/copying of credentials struct. diffstat: src/lib-http/http-auth.c | 23 +++++++++++++++++++++++ src/lib-http/http-auth.h | 7 +++++++ 2 files changed, 30 insertions(+), 0 deletions(-) diffs (50 lines): diff -r cdf9e584c1c7 -r 07ceb84bf899 src/lib-http/http-auth.c --- a/src/lib-http/http-auth.c Tue Aug 18 20:39:24 2015 +0300 +++ b/src/lib-http/http-auth.c Tue Aug 18 20:39:24 2015 +0300 @@ -413,6 +413,29 @@ return new; } +void http_auth_credentials_copy(pool_t pool, + struct http_auth_credentials *dst, + const struct http_auth_credentials *src) +{ + dst->scheme = p_strdup(pool, src->scheme); + if (src->data != NULL) + dst->data = p_strdup(pool, src->data); + else + http_auth_params_clone(pool, &dst->params, &src->params); +} + +struct http_auth_credentials * +http_auth_credentials_clone(pool_t pool, + const struct http_auth_credentials *src) +{ + struct http_auth_credentials *new; + + new = p_new(pool, struct http_auth_credentials, 1); + http_auth_credentials_copy(pool, new, src); + + return new; +} + /* * Simple schemes */ diff -r cdf9e584c1c7 -r 07ceb84bf899 src/lib-http/http-auth.h --- a/src/lib-http/http-auth.h Tue Aug 18 20:39:24 2015 +0300 +++ b/src/lib-http/http-auth.h Tue Aug 18 20:39:24 2015 +0300 @@ -59,6 +59,13 @@ http_auth_challenge_clone(pool_t pool, const struct http_auth_challenge *src); +void http_auth_credentials_copy(pool_t pool, + struct http_auth_credentials *dst, + const struct http_auth_credentials *src); +struct http_auth_credentials * +http_auth_credentials_clone(pool_t pool, + const struct http_auth_credentials *src); + /* * Simple schemes */ From dovecot at dovecot.org Tue Aug 18 18:02:36 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Aug 2015 18:02:36 +0000 Subject: dovecot-2.2: lib-http: url: Implemented functions to copy/clone ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c1ffec72a134 changeset: 18964:c1ffec72a134 user: Stephan Bosch date: Tue Aug 18 20:39:24 2015 +0300 description: lib-http: url: Implemented functions to copy/clone URLs including the userinfo part (normally skipped). diffstat: src/lib-http/http-url.c | 22 +++++++++++++++++++++- src/lib-http/http-url.h | 5 +++++ 2 files changed, 26 insertions(+), 1 deletions(-) diffs (55 lines): diff -r 07ceb84bf899 -r c1ffec72a134 src/lib-http/http-url.c --- a/src/lib-http/http-url.c Tue Aug 18 20:39:24 2015 +0300 +++ b/src/lib-http/http-url.c Tue Aug 18 20:39:24 2015 +0300 @@ -450,7 +450,15 @@ dest->enc_fragment = p_strdup(pool, src->enc_fragment); } -struct http_url *http_url_clone(pool_t pool,const struct http_url *src) +void http_url_copy_with_userinfo(pool_t pool, struct http_url *dest, + const struct http_url *src) +{ + http_url_copy(pool, dest, src); + dest->user = p_strdup(pool, src->user); + dest->password = p_strdup(pool, src->password); +} + +struct http_url *http_url_clone(pool_t pool, const struct http_url *src) { struct http_url *new_url; @@ -460,6 +468,18 @@ return new_url; } +struct http_url *http_url_clone_with_userinfo(pool_t pool, + const struct http_url *src) +{ + struct http_url *new_url; + + new_url = p_new(pool, struct http_url, 1); + http_url_copy_with_userinfo(pool, new_url, src); + + return new_url; +} + + /* * HTTP URL construction */ diff -r 07ceb84bf899 -r c1ffec72a134 src/lib-http/http-url.h --- a/src/lib-http/http-url.h Tue Aug 18 20:39:24 2015 +0300 +++ b/src/lib-http/http-url.h Tue Aug 18 20:39:24 2015 +0300 @@ -59,7 +59,12 @@ const struct http_url *src); void http_url_copy(pool_t pool, struct http_url *dest, const struct http_url *src); +void http_url_copy_with_userinfo(pool_t pool, struct http_url *dest, + const struct http_url *src); + struct http_url *http_url_clone(pool_t pool,const struct http_url *src); +struct http_url *http_url_clone_with_userinfo(pool_t pool, + const struct http_url *src); /* * HTTP URL construction From dovecot at dovecot.org Tue Aug 18 18:02:36 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Aug 2015 18:02:36 +0000 Subject: dovecot-2.2: lib-http: url: Implemented http_url_clone_authority... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/aa695f538fd0 changeset: 18965:aa695f538fd0 user: Stephan Bosch date: Tue Aug 18 20:39:24 2015 +0300 description: lib-http: url: Implemented http_url_clone_authority() to clone authority part of existing HTTP URL. diffstat: src/lib-http/http-url.c | 11 +++++++++++ src/lib-http/http-url.h | 3 +++ 2 files changed, 14 insertions(+), 0 deletions(-) diffs (34 lines): diff -r c1ffec72a134 -r aa695f538fd0 src/lib-http/http-url.c --- a/src/lib-http/http-url.c Tue Aug 18 20:39:24 2015 +0300 +++ b/src/lib-http/http-url.c Tue Aug 18 20:39:24 2015 +0300 @@ -441,6 +441,17 @@ dest->have_ssl = src->have_ssl; } +struct http_url *http_url_clone_authority(pool_t pool, + const struct http_url *src) +{ + struct http_url *new_url; + + new_url = p_new(pool, struct http_url, 1); + http_url_copy_authority(pool, new_url, src); + + return new_url; +} + void http_url_copy(pool_t pool, struct http_url *dest, const struct http_url *src) { diff -r c1ffec72a134 -r aa695f538fd0 src/lib-http/http-url.h --- a/src/lib-http/http-url.h Tue Aug 18 20:39:24 2015 +0300 +++ b/src/lib-http/http-url.h Tue Aug 18 20:39:24 2015 +0300 @@ -57,6 +57,9 @@ void http_url_copy_authority(pool_t pool, struct http_url *dest, const struct http_url *src); +struct http_url *http_url_clone_authority(pool_t pool, + const struct http_url *src); + void http_url_copy(pool_t pool, struct http_url *dest, const struct http_url *src); void http_url_copy_with_userinfo(pool_t pool, struct http_url *dest, From dovecot at dovecot.org Tue Aug 18 18:02:36 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Aug 2015 18:02:36 +0000 Subject: dovecot-2.2: lib-http: test-http-client double-freed memory Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4e6b44f371bd changeset: 18966:4e6b44f371bd user: Timo Sirainen date: Tue Aug 18 20:54:47 2015 +0300 description: lib-http: test-http-client double-freed memory diffstat: src/lib-http/test-http-client.c | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diffs (11 lines): diff -r aa695f538fd0 -r 4e6b44f371bd src/lib-http/test-http-client.c --- a/src/lib-http/test-http-client.c Tue Aug 18 20:39:24 2015 +0300 +++ b/src/lib-http/test-http-client.c Tue Aug 18 20:54:47 2015 +0300 @@ -181,7 +181,6 @@ http_client_request_set_ssl(http_req, TRUE); http_client_request_submit(http_req); http_client_request_abort(&http_req); - i_free(test_req); test_req = i_new(struct http_test_request, 1); http_req = http_client_request(http_client, From dovecot at dovecot.org Tue Aug 18 18:02:36 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Aug 2015 18:02:36 +0000 Subject: dovecot-2.2: lib-http: test-http-client may have failed to load ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5be374f2b1d8 changeset: 18967:5be374f2b1d8 user: Timo Sirainen date: Tue Aug 18 20:59:25 2015 +0300 description: lib-http: test-http-client may have failed to load SSL library plugin. safe_memset() wasn't necessarily compiled into test-http-client. This is a bit kludgy way of just adding it. Another possibility could have been to use -Wl,--whole-archive with GNU ld, but it's a bit tricky here. diffstat: src/lib-http/Makefile.am | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 4e6b44f371bd -r 5be374f2b1d8 src/lib-http/Makefile.am --- a/src/lib-http/Makefile.am Tue Aug 18 20:54:47 2015 +0300 +++ b/src/lib-http/Makefile.am Tue Aug 18 20:59:25 2015 +0300 @@ -139,6 +139,7 @@ ../lib-ssl-iostream/libssl_iostream.la \ ../lib-master/libmaster.la \ ../lib-settings/libsettings.la \ + ../lib/safe-memset.lo \ $(test_libs) test_http_client_DEPENDENCIES = \ libhttp.la \ From dovecot at dovecot.org Tue Aug 18 18:11:31 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Aug 2015 18:11:31 +0000 Subject: dovecot-2.2: lib-http: Previous test-http-client safe_memset() c... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/04be746494dc changeset: 18968:04be746494dc user: Timo Sirainen date: Tue Aug 18 21:10:41 2015 +0300 description: lib-http: Previous test-http-client safe_memset() change broke in some systems. Since there's no easy way to fix this in Makefile.am, just do it by actually using safe_memset().. diffstat: src/lib-http/Makefile.am | 1 - src/lib-http/test-http-client.c | 8 ++++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diffs (36 lines): diff -r 5be374f2b1d8 -r 04be746494dc src/lib-http/Makefile.am --- a/src/lib-http/Makefile.am Tue Aug 18 20:59:25 2015 +0300 +++ b/src/lib-http/Makefile.am Tue Aug 18 21:10:41 2015 +0300 @@ -139,7 +139,6 @@ ../lib-ssl-iostream/libssl_iostream.la \ ../lib-master/libmaster.la \ ../lib-settings/libsettings.la \ - ../lib/safe-memset.lo \ $(test_libs) test_http_client_DEPENDENCIES = \ libhttp.la \ diff -r 5be374f2b1d8 -r 04be746494dc src/lib-http/test-http-client.c --- a/src/lib-http/test-http-client.c Tue Aug 18 20:59:25 2015 +0300 +++ b/src/lib-http/test-http-client.c Tue Aug 18 21:10:41 2015 +0300 @@ -1,6 +1,7 @@ /* Copyright (c) 2013-2015 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "safe-memset.h" #include "ioloop.h" #include "istream.h" #include "write-full.h" @@ -342,8 +343,11 @@ ioloop = io_loop_create(); io_loop_set_running(ioloop); - - memset(&dns_set, 0, sizeof(dns_set)); + /* kludge: use safe_memset() here since otherwise it's not included in + the binary in all systems (but is in others! so linking + safe-memset.lo directly causes them to fail.) If safe_memset() isn't + included, libssl-iostream plugin loading fails. */ + safe_memset(&dns_set, 0, sizeof(dns_set)); dns_set.dns_client_socket_path = "/var/run/dovecot/dns-client"; dns_set.timeout_msecs = 30*1000; dns_set.idle_timeout_msecs = UINT_MAX; From dovecot at dovecot.org Tue Aug 18 20:02:52 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Aug 2015 20:02:52 +0000 Subject: dovecot-2.2: lib-http: client: Fixed handling of request timeout. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a08e79a3707f changeset: 18969:a08e79a3707f user: Stephan Bosch date: Tue Aug 18 23:02:01 2015 +0300 description: lib-http: client: Fixed handling of request timeout. It was inappropriately active when the client needed to take action. This showed particularly with large payloads sent using http_client_request_send_payload(). diffstat: src/lib-http/http-client-connection.c | 52 ++++++++++++++++++++++++---------- src/lib-http/http-client-private.h | 7 ++++ src/lib-http/http-client-request.c | 14 ++++++++- 3 files changed, 55 insertions(+), 18 deletions(-) diffs (172 lines): diff -r 04be746494dc -r a08e79a3707f src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Tue Aug 18 21:10:41 2015 +0300 +++ b/src/lib-http/http-client-connection.c Tue Aug 18 23:02:01 2015 +0300 @@ -266,6 +266,35 @@ msecs/1000, msecs%1000)); } +void http_client_connection_start_request_timeout( + struct http_client_connection *conn) +{ + unsigned int timeout_msecs = conn->client->set.request_timeout_msecs; + + if (timeout_msecs == 0) + ; + else if (conn->to_requests != NULL) + timeout_reset(conn->to_requests); + else { + conn->to_requests = timeout_add(timeout_msecs, + http_client_connection_request_timeout, conn); + } +} + +void http_client_connection_reset_request_timeout( + struct http_client_connection *conn) +{ + if (conn->to_requests != NULL) + timeout_reset(conn->to_requests); +} + +void http_client_connection_stop_request_timeout( + struct http_client_connection *conn) +{ + if (conn->to_requests != NULL) + timeout_remove(&conn->to_requests); +} + static void http_client_connection_continue_timeout(struct http_client_connection *conn) { @@ -316,14 +345,6 @@ if (conn->to_idle != NULL) timeout_remove(&conn->to_idle); - if (conn->client->set.request_timeout_msecs == 0) - ; - else if (conn->to_requests != NULL) - timeout_reset(conn->to_requests); - else { - conn->to_requests = timeout_add(conn->client->set.request_timeout_msecs, - http_client_connection_request_timeout, conn); - } req->conn = conn; conn->payload_continue = FALSE; if (conn->peer->no_payload_sync) @@ -489,8 +510,7 @@ io_remove(&conn->conn.io); /* we've received the request itself, and we can't reset the timeout during the payload reading. */ - if (conn->to_requests != NULL) - timeout_remove(&conn->to_requests); + http_client_connection_stop_request_timeout(conn); } conn->in_req_callback = TRUE; @@ -607,8 +627,9 @@ http_client_payload_finished(conn); finished++; } - if (conn->to_requests != NULL) - timeout_reset(conn->to_requests); + + /* we've seen activity from the server; reset request timeout */ + http_client_connection_reset_request_timeout(conn); /* get first waiting request */ reqs = array_get(&conn->request_wait_list, &count); @@ -786,8 +807,7 @@ payload_type = http_client_request_get_payload_type(req); } else { /* no more requests waiting for the connection */ - if (conn->to_requests != NULL) - timeout_remove(&conn->to_requests); + http_client_connection_stop_request_timeout(conn); req = NULL; payload_type = HTTP_RESPONSE_PAYLOAD_TYPE_ALLOWED; } @@ -839,8 +859,8 @@ const char *error; int ret; - if (conn->to_requests != NULL) - timeout_reset(conn->to_requests); + /* we've seen activity from the server; reset request timeout */ + http_client_connection_reset_request_timeout(conn); if ((ret = o_stream_flush(output)) <= 0) { if (ret < 0) { diff -r 04be746494dc -r a08e79a3707f src/lib-http/http-client-private.h --- a/src/lib-http/http-client-private.h Tue Aug 18 21:10:41 2015 +0300 +++ b/src/lib-http/http-client-private.h Tue Aug 18 23:02:01 2015 +0300 @@ -291,6 +291,12 @@ void http_client_connection_unref(struct http_client_connection **_conn); void http_client_connection_close(struct http_client_connection **_conn); int http_client_connection_output(struct http_client_connection *conn); +void http_client_connection_start_request_timeout( + struct http_client_connection *conn); +void http_client_connection_reset_request_timeout( + struct http_client_connection *conn); +void http_client_connection_stop_request_timeout( + struct http_client_connection *conn); unsigned int http_client_connection_count_pending(struct http_client_connection *conn); bool http_client_connection_is_ready(struct http_client_connection *conn); @@ -301,6 +307,7 @@ void http_client_connection_start_tunnel(struct http_client_connection **_conn, struct http_client_tunnel *tunnel); + unsigned int http_client_peer_addr_hash (const struct http_client_peer_addr *peer) ATTR_PURE; int http_client_peer_addr_cmp diff -r 04be746494dc -r a08e79a3707f src/lib-http/http-client-request.c --- a/src/lib-http/http-client-request.c Tue Aug 18 21:10:41 2015 +0300 +++ b/src/lib-http/http-client-request.c Tue Aug 18 23:02:01 2015 +0300 @@ -536,7 +536,10 @@ /* advance state only when request didn't get aborted in the mean time */ if (req->state != HTTP_REQUEST_STATE_ABORTED) { i_assert(req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT); + + /* we're now waiting for a response from the server */ req->state = HTTP_REQUEST_STATE_WAITING; + http_client_connection_start_request_timeout(req->conn); } /* release connection */ @@ -703,6 +706,7 @@ i_assert(ret >= 0); if (i_stream_is_eof(req->payload_input)) { + /* finished sending */ if (!req->payload_chunked && req->payload_input->v_offset - req->payload_offset != req->payload_size) { *error_r = t_strdup_printf("BUG: stream '%s' input size changed: " @@ -714,20 +718,26 @@ } if (req->payload_wait) { + /* this chunk of input is finished + (client needs to act; disable timeout) */ conn->output_locked = TRUE; + http_client_connection_stop_request_timeout(conn); if (req->client->ioloop != NULL) io_loop_stop(req->client->ioloop); } else { + /* finished sending payload */ http_client_request_finish_payload_out(req); } } else if (i_stream_get_data_size(req->payload_input) > 0) { - /* output is blocking */ + /* output is blocking (server needs to act; enable timeout) */ conn->output_locked = TRUE; + http_client_connection_start_request_timeout(conn); o_stream_set_flush_pending(output, TRUE); http_client_request_debug(req, "Partially sent payload"); } else { - /* input is blocking */ + /* input is blocking (client needs to act; disable timeout) */ conn->output_locked = TRUE; + http_client_connection_stop_request_timeout(conn); conn->io_req_payload = io_add_istream(req->payload_input, http_client_request_payload_input, req); } From dovecot at dovecot.org Tue Aug 18 20:08:05 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Aug 2015 20:08:05 +0000 Subject: dovecot-2.2: fts-solr: Fixed memory leak when indexing multiple ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9765b87823df changeset: 18970:9765b87823df user: Timo Sirainen date: Tue Aug 18 23:07:19 2015 +0300 description: fts-solr: Fixed memory leak when indexing multiple mailboxes in one update context. diffstat: src/plugins/fts-solr/fts-backend-solr.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r a08e79a3707f -r 9765b87823df src/plugins/fts-solr/fts-backend-solr.c --- a/src/plugins/fts-solr/fts-backend-solr.c Tue Aug 18 23:02:01 2015 +0300 +++ b/src/plugins/fts-solr/fts-backend-solr.c Tue Aug 18 23:07:19 2015 +0300 @@ -468,7 +468,8 @@ if (ctx->post == NULL) { i_assert(ctx->prev_uid == 0); - ctx->cmd = str_new(default_pool, SOLR_CMDBUF_SIZE); + if (ctx->cmd == NULL) + ctx->cmd = str_new(default_pool, SOLR_CMDBUF_SIZE); ctx->post = solr_connection_post_begin(backend->solr_conn); str_append(ctx->cmd, ""); } else { From dovecot at dovecot.org Tue Aug 18 20:18:32 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Aug 2015 20:18:32 +0000 Subject: dovecot-2.2: fts-solr: Fix to previous memory leak fix - second ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4707f1e7602c changeset: 18971:4707f1e7602c user: Timo Sirainen date: Tue Aug 18 23:17:47 2015 +0300 description: fts-solr: Fix to previous memory leak fix - second request was failing. Duplicated data was sent to Solr because string wasn't truncated. diffstat: src/plugins/fts-solr/fts-backend-solr.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 9765b87823df -r 4707f1e7602c src/plugins/fts-solr/fts-backend-solr.c --- a/src/plugins/fts-solr/fts-backend-solr.c Tue Aug 18 23:07:19 2015 +0300 +++ b/src/plugins/fts-solr/fts-backend-solr.c Tue Aug 18 23:17:47 2015 +0300 @@ -340,6 +340,7 @@ solr_connection_post_more(ctx->post, str_data(ctx->cmd), str_len(ctx->cmd)); + str_truncate(ctx->cmd, 0); return solr_connection_post_end(&ctx->post); } From dovecot at dovecot.org Tue Aug 18 20:21:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Aug 2015 20:21:25 +0000 Subject: dovecot-2.2: fts-solr: If HTTP request fails, log the status num... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ba4ac54e79a7 changeset: 18972:ba4ac54e79a7 user: Timo Sirainen date: Tue Aug 18 23:20:07 2015 +0300 description: fts-solr: If HTTP request fails, log the status number as well. diffstat: src/plugins/fts-solr/solr-connection.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (23 lines): diff -r 4707f1e7602c -r ba4ac54e79a7 src/plugins/fts-solr/solr-connection.c --- a/src/plugins/fts-solr/solr-connection.c Tue Aug 18 23:17:47 2015 +0300 +++ b/src/plugins/fts-solr/solr-connection.c Tue Aug 18 23:20:07 2015 +0300 @@ -383,7 +383,8 @@ struct solr_connection *conn) { if (response->status / 100 != 2) { - i_error("fts_solr: Lookup failed: %s", response->reason); + i_error("fts_solr: Lookup failed: %u %s", + response->status, response->reason); conn->request_status = -1; return; } @@ -452,7 +453,8 @@ struct solr_connection *conn) { if (response->status / 100 != 2) { - i_error("fts_solr: Indexing failed: %s", response->reason); + i_error("fts_solr: Indexing failed: %u %s", + response->status, response->reason); conn->request_status = -1; } } From dovecot at dovecot.org Tue Aug 18 20:21:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 18 Aug 2015 20:21:25 +0000 Subject: dovecot-2.2: fts-solr: Flush Solr indexing HTTP requests every 1... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/58f65e129b4d changeset: 18973:58f65e129b4d user: Timo Sirainen date: Tue Aug 18 23:20:35 2015 +0300 description: fts-solr: Flush Solr indexing HTTP requests every 1000 mails. This avoids the HTTP request from becoming too huge. diffstat: src/plugins/fts-solr/fts-backend-solr.c | 16 +++++++++++----- 1 files changed, 11 insertions(+), 5 deletions(-) diffs (68 lines): diff -r ba4ac54e79a7 -r 58f65e129b4d src/plugins/fts-solr/fts-backend-solr.c --- a/src/plugins/fts-solr/fts-backend-solr.c Tue Aug 18 23:20:07 2015 +0300 +++ b/src/plugins/fts-solr/fts-backend-solr.c Tue Aug 18 23:20:35 2015 +0300 @@ -27,6 +27,8 @@ #define SOLR_HEADER_LINE_MAX_TRUNC_SIZE 1024 #define SOLR_QUERY_MAX_MAILBOX_COUNT 10 +/* How often to flush indexing request to Solr before beginning a new one. */ +#define SOLR_MAIL_FLUSH_INTERVAL 1000 struct solr_fts_backend { struct fts_backend backend; @@ -51,6 +53,7 @@ ARRAY(struct solr_fts_field) fields; uint32_t last_indexed_uid; + unsigned int mails_since_flush; unsigned int tokenized_input:1; unsigned int last_indexed_uid_set:1; @@ -330,13 +333,14 @@ } static int -fts_backed_solr_build_commit(struct solr_fts_backend_update_context *ctx) +fts_backed_solr_build_flush(struct solr_fts_backend_update_context *ctx) { if (ctx->post == NULL) return 0; fts_backend_solr_doc_close(ctx); str_append(ctx->cmd, ""); + ctx->mails_since_flush = 0; solr_connection_post_more(ctx->post, str_data(ctx->cmd), str_len(ctx->cmd)); @@ -367,7 +371,7 @@ const char *str; int ret = _ctx->failed ? -1 : 0; - if (fts_backed_solr_build_commit(ctx) < 0) + if (fts_backed_solr_build_flush(ctx) < 0) ret = -1; if (ctx->documents_added || ctx->expunges) { @@ -405,7 +409,7 @@ if (ctx->prev_uid != 0) { /* flush solr between mailboxes, so we don't wrongly update last_uid before we know it has succeeded */ - if (fts_backed_solr_build_commit(ctx) < 0) + if (fts_backed_solr_build_flush(ctx) < 0) _ctx->failed = TRUE; else if (!_ctx->failed) fts_index_set_last_uid(ctx->cur_box, ctx->prev_uid); @@ -466,9 +470,11 @@ struct solr_fts_backend *backend = (struct solr_fts_backend *)ctx->ctx.backend; + if (ctx->mails_since_flush++ >= SOLR_MAIL_FLUSH_INTERVAL) { + if (fts_backed_solr_build_flush(ctx) < 0) + ctx->ctx.failed = TRUE; + } if (ctx->post == NULL) { - i_assert(ctx->prev_uid == 0); - if (ctx->cmd == NULL) ctx->cmd = str_new(default_pool, SOLR_CMDBUF_SIZE); ctx->post = solr_connection_post_begin(backend->solr_conn); From dovecot at dovecot.org Wed Aug 19 08:40:28 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 19 Aug 2015 08:40:28 +0000 Subject: dovecot-2.2: *-login: Removed dead assignment to make static ana... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0f442376beae changeset: 18974:0f442376beae user: Timo Sirainen date: Wed Aug 19 11:34:36 2015 +0300 description: *-login: Removed dead assignment to make static analyzer happy. diffstat: src/login-common/main.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (14 lines): diff -r 58f65e129b4d -r 0f442376beae src/login-common/main.c --- a/src/login-common/main.c Tue Aug 18 23:20:35 2015 +0300 +++ b/src/login-common/main.c Wed Aug 19 11:34:36 2015 +0300 @@ -123,8 +123,8 @@ &conn->remote_ip, NULL, &ssl_set, &other_sets); if (!ssl_connections && !conn->ssl) { - client = client_create(conn->fd, FALSE, pool, conn, - set, ssl_set, other_sets); + (void)client_create(conn->fd, FALSE, pool, conn, + set, ssl_set, other_sets); } else { fd_ssl = ssl_proxy_alloc(conn->fd, &conn->remote_ip, pool, set, ssl_set, &proxy); From dovecot at dovecot.org Wed Aug 19 09:57:43 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 19 Aug 2015 09:57:43 +0000 Subject: dovecot-2.2: lib-storage: Moved most of the \Recent flag handlin... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f78bd5be4e47 changeset: 18975:f78bd5be4e47 user: Timo Sirainen date: Wed Aug 19 12:56:55 2015 +0300 description: lib-storage: Moved most of the \Recent flag handling code to mailbox-recent-flags.c There are also some API changes, because functions were renamed and the recent_* fields were moved to struct mailbox. I'm not aware of any plugins using these though, except for index_mailbox_set_recent_seq() which for now is kept as a backwards compatibility macro. No changes were made to the actual code logic. diffstat: src/lib-storage/Makefile.am | 2 + src/lib-storage/index/cydir/cydir-sync.c | 4 +- src/lib-storage/index/dbox-multi/mdbox-sync.c | 6 +- src/lib-storage/index/dbox-single/sdbox-sync.c | 2 +- src/lib-storage/index/index-mail.c | 4 +- src/lib-storage/index/index-rebuild.c | 2 +- src/lib-storage/index/index-search.c | 2 +- src/lib-storage/index/index-status.c | 2 +- src/lib-storage/index/index-storage.c | 5 - src/lib-storage/index/index-storage.h | 15 +- src/lib-storage/index/index-sync.c | 120 +------------------- src/lib-storage/index/maildir/maildir-save.c | 2 +- src/lib-storage/index/maildir/maildir-sync-index.c | 4 +- src/lib-storage/index/maildir/maildir-util.c | 2 +- src/lib-storage/index/mbox/mbox-file.c | 3 +- src/lib-storage/index/mbox/mbox-sync.c | 6 +- src/lib-storage/index/pop3c/pop3c-sync.c | 2 +- src/lib-storage/index/raw/raw-sync.c | 2 +- src/lib-storage/mail-storage-private.h | 5 + src/lib-storage/mail-storage.c | 5 + src/lib-storage/mailbox-recent-flags.c | 93 ++++++++++++++++ src/lib-storage/mailbox-recent-flags.h | 17 ++ src/plugins/virtual/virtual-sync.c | 4 +- 23 files changed, 161 insertions(+), 148 deletions(-) diffs (truncated from 628 to 300 lines): diff -r 0f442376beae -r f78bd5be4e47 src/lib-storage/Makefile.am --- a/src/lib-storage/Makefile.am Wed Aug 19 11:34:36 2015 +0300 +++ b/src/lib-storage/Makefile.am Wed Aug 19 12:56:55 2015 +0300 @@ -51,6 +51,7 @@ mailbox-keywords.c \ mailbox-list.c \ mailbox-list-notify.c \ + mailbox-recent-flags.c \ mailbox-search-result.c \ mailbox-tree.c \ mailbox-uidvalidity.c @@ -79,6 +80,7 @@ mailbox-list-iter.h \ mailbox-list-private.h \ mailbox-list-notify.h \ + mailbox-recent-flags.h \ mailbox-search-result-private.h \ mailbox-tree.h \ mailbox-uidvalidity.h diff -r 0f442376beae -r f78bd5be4e47 src/lib-storage/index/cydir/cydir-sync.c --- a/src/lib-storage/index/cydir/cydir-sync.c Wed Aug 19 11:34:36 2015 +0300 +++ b/src/lib-storage/index/cydir/cydir-sync.c Wed Aug 19 12:56:55 2015 +0300 @@ -71,8 +71,8 @@ /* mark the newly seen messages as recent */ if (mail_index_lookup_seq_range(ctx->sync_view, hdr->first_recent_uid, hdr->next_uid, &seq1, &seq2)) { - index_mailbox_set_recent_seq(&ctx->mbox->box, ctx->sync_view, - seq1, seq2); + mailbox_recent_flags_set_seqs(&ctx->mbox->box, ctx->sync_view, + seq1, seq2); } while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) { diff -r 0f442376beae -r f78bd5be4e47 src/lib-storage/index/dbox-multi/mdbox-sync.c --- a/src/lib-storage/index/dbox-multi/mdbox-sync.c Wed Aug 19 11:34:36 2015 +0300 +++ b/src/lib-storage/index/dbox-multi/mdbox-sync.c Wed Aug 19 12:56:55 2015 +0300 @@ -154,8 +154,8 @@ /* mark the newly seen messages as recent */ if (mail_index_lookup_seq_range(ctx->sync_view, hdr->first_recent_uid, hdr->next_uid, &seq1, &seq2)) { - index_mailbox_set_recent_seq(&ctx->mbox->box, ctx->sync_view, - seq1, seq2); + mailbox_recent_flags_set_seqs(&ctx->mbox->box, ctx->sync_view, + seq1, seq2); } /* handle syncing records without map being locked. */ @@ -242,7 +242,7 @@ if (rebuild && (flags & MDBOX_SYNC_FLAG_NO_REBUILD) == 0) { if (mdbox_storage_rebuild_in_context(mbox->storage, atomic) < 0) return -1; - index_mailbox_reset_uidvalidity(&mbox->box); + mailbox_recent_flags_reset(&mbox->box); storage_rebuilt = TRUE; } diff -r 0f442376beae -r f78bd5be4e47 src/lib-storage/index/dbox-single/sdbox-sync.c --- a/src/lib-storage/index/dbox-single/sdbox-sync.c Wed Aug 19 11:34:36 2015 +0300 +++ b/src/lib-storage/index/dbox-single/sdbox-sync.c Wed Aug 19 12:56:55 2015 +0300 @@ -125,7 +125,7 @@ /* mark the newly seen messages as recent */ if (mail_index_lookup_seq_range(ctx->sync_view, hdr->first_recent_uid, hdr->next_uid, &seq1, &seq2)) - index_mailbox_set_recent_seq(box, ctx->sync_view, seq1, seq2); + mailbox_recent_flags_set_seqs(box, ctx->sync_view, seq1, seq2); while (mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) sdbox_sync_add(ctx, &sync_rec); diff -r 0f442376beae -r f78bd5be4e47 src/lib-storage/index/index-mail.c --- a/src/lib-storage/index/index-mail.c Wed Aug 19 11:34:36 2015 +0300 +++ b/src/lib-storage/index/index-mail.c Wed Aug 19 12:56:55 2015 +0300 @@ -195,7 +195,7 @@ flags = rec->flags & (MAIL_FLAGS_NONRECENT | MAIL_INDEX_MAIL_FLAG_BACKEND); - if (index_mailbox_is_recent(_mail->box, _mail->uid)) + if (mailbox_recent_flags_have_uid(_mail->box, _mail->uid)) flags |= MAIL_RECENT; if (index_mail_get_pvt(_mail)) { @@ -1980,7 +1980,7 @@ bool update_modseq = FALSE; if ((flags & MAIL_RECENT) == 0 && - index_mailbox_is_recent(_mail->box, _mail->uid)) + mailbox_recent_flags_have_uid(_mail->box, _mail->uid)) index_mail_drop_recent_flag(_mail); flags &= MAIL_FLAGS_NONRECENT | MAIL_INDEX_MAIL_FLAG_BACKEND; diff -r 0f442376beae -r f78bd5be4e47 src/lib-storage/index/index-rebuild.c --- a/src/lib-storage/index/index-rebuild.c Wed Aug 19 11:34:36 2015 +0300 +++ b/src/lib-storage/index/index-rebuild.c Wed Aug 19 12:56:55 2015 +0300 @@ -157,7 +157,7 @@ ctx->view = view; ctx->trans = trans; mail_index_reset(ctx->trans); - index_mailbox_reset_uidvalidity(box); + mailbox_recent_flags_reset(box); (void)mail_index_ext_lookup(box->index, "cache", &ctx->cache_ext_id); /* open cache and read the caching decisions. */ diff -r 0f442376beae -r f78bd5be4e47 src/lib-storage/index/index-search.c --- a/src/lib-storage/index/index-search.c Wed Aug 19 11:34:36 2015 +0300 +++ b/src/lib-storage/index/index-search.c Wed Aug 19 12:56:55 2015 +0300 @@ -178,7 +178,7 @@ may contain it. */ flags = rec->flags & ~MAIL_RECENT; if ((arg->value.flags & MAIL_RECENT) != 0 && - index_mailbox_is_recent(ctx->box, rec->uid)) + mailbox_recent_flags_have_uid(ctx->box, rec->uid)) flags |= MAIL_RECENT; if (ctx->box->view_pvt == NULL) { /* no private view (set by view syncing) -> diff -r 0f442376beae -r f78bd5be4e47 src/lib-storage/index/index-status.c --- a/src/lib-storage/index/index-status.c Wed Aug 19 11:34:36 2015 +0300 +++ b/src/lib-storage/index/index-status.c Wed Aug 19 12:56:55 2015 +0300 @@ -126,7 +126,7 @@ synced yet */ index_sync_update_recent_count(box); } - status_r->recent = index_mailbox_get_recent_count(box); + status_r->recent = mailbox_recent_flags_count(box); i_assert(status_r->recent <= status_r->messages); } if ((items & STATUS_UNSEEN) != 0) { diff -r 0f442376beae -r f78bd5be4e47 src/lib-storage/index/index-storage.c --- a/src/lib-storage/index/index-storage.c Wed Aug 19 11:34:36 2015 +0300 +++ b/src/lib-storage/index/index-storage.c Wed Aug 19 12:56:55 2015 +0300 @@ -369,11 +369,6 @@ ibox->keyword_names = NULL; i_free_and_null(ibox->cache_fields); - if (array_is_created(&ibox->recent_flags)) - array_free(&ibox->recent_flags); - ibox->recent_flags_prev_uid = 0; - ibox->recent_flags_count = 0; - ibox->sync_last_check = 0; } diff -r 0f442376beae -r f78bd5be4e47 src/lib-storage/index/index-storage.h --- a/src/lib-storage/index/index-storage.h Wed Aug 19 11:34:36 2015 +0300 +++ b/src/lib-storage/index/index-storage.h Wed Aug 19 12:56:55 2015 +0300 @@ -4,6 +4,7 @@ #include "file-dotlock.h" #include "mail-storage-private.h" #include "mail-index-private.h" +#include "mailbox-recent-flags.h" /* FIXME: remove in v2.3 */ #define MAILBOX_FULL_SYNC_INTERVAL 5 @@ -30,9 +31,7 @@ const ARRAY_TYPE(keywords) *keyword_names; struct mail_cache_field *cache_fields; - ARRAY_TYPE(seq_range) recent_flags; - uint32_t recent_flags_prev_uid, recent_flags_last_check_nextuid; - uint32_t recent_flags_count; + uint32_t recent_flags_last_check_nextuid; time_t sync_last_check; uint32_t list_index_sync_ext_id; @@ -76,13 +75,9 @@ bool index_storage_is_readonly(struct mailbox *box); bool index_storage_is_inconsistent(struct mailbox *box); -void index_mailbox_set_recent_uid(struct mailbox *box, uint32_t uid); -void index_mailbox_set_recent_seq(struct mailbox *box, - struct mail_index_view *view, - uint32_t seq1, uint32_t seq2); -bool index_mailbox_is_recent(struct mailbox *box, uint32_t uid); -unsigned int index_mailbox_get_recent_count(struct mailbox *box); -void index_mailbox_reset_uidvalidity(struct mailbox *box); +/* FIXME: for backwards compatibility - remove in v2.3 */ +#define index_mailbox_set_recent_seq(box, view, seq1, seq2) \ + mailbox_recent_flags_set_seqs(box, view, seq1, seq2) void index_mailbox_check_add(struct mailbox *box, const char *path); void index_mailbox_check_remove_all(struct mailbox *box); diff -r 0f442376beae -r f78bd5be4e47 src/lib-storage/index/index-sync.c --- a/src/lib-storage/index/index-sync.c Wed Aug 19 11:34:36 2015 +0300 +++ b/src/lib-storage/index/index-sync.c Wed Aug 19 12:56:55 2015 +0300 @@ -51,103 +51,6 @@ return TRUE; } -void index_mailbox_set_recent_uid(struct mailbox *box, uint32_t uid) -{ - struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); - - if (uid <= ibox->recent_flags_prev_uid) { - if (seq_range_exists(&ibox->recent_flags, uid)) - return; - - mail_storage_set_critical(box->storage, - "Recent flags state corrupted for mailbox %s", - box->vname); - array_clear(&ibox->recent_flags); - ibox->recent_flags_count = 0; - } - ibox->recent_flags_prev_uid = uid; - - seq_range_array_add_with_init(&ibox->recent_flags, 64, uid); - ibox->recent_flags_count++; -} - -void index_mailbox_set_recent_seq(struct mailbox *box, - struct mail_index_view *view, - uint32_t seq1, uint32_t seq2) -{ - uint32_t uid; - - for (; seq1 <= seq2; seq1++) { - mail_index_lookup_uid(view, seq1, &uid); - index_mailbox_set_recent_uid(box, uid); - } -} - -bool index_mailbox_is_recent(struct mailbox *box, uint32_t uid) -{ - struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); - - return array_is_created(&ibox->recent_flags) && - seq_range_exists(&ibox->recent_flags, uid); -} - -void index_mailbox_reset_uidvalidity(struct mailbox *box) -{ - struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); - - /* can't trust the currently cached recent flags anymore */ - if (array_is_created(&ibox->recent_flags)) - array_clear(&ibox->recent_flags); - ibox->recent_flags_count = 0; - ibox->recent_flags_prev_uid = 0; -} - -unsigned int index_mailbox_get_recent_count(struct mailbox *box) -{ - struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); - const struct mail_index_header *hdr; - const struct seq_range *range; - unsigned int i, count, recent_count; - - if (!array_is_created(&ibox->recent_flags)) - return 0; - - hdr = mail_index_get_header(box->view); - recent_count = ibox->recent_flags_count; - range = array_get(&ibox->recent_flags, &count); - for (i = count; i > 0; ) { - i--; - if (range[i].seq2 < hdr->next_uid) - break; - - if (range[i].seq1 >= hdr->next_uid) { - /* completely invisible to this view */ - recent_count -= range[i].seq2 - range[i].seq1 + 1; - } else { - /* partially invisible */ - recent_count -= range[i].seq2 - hdr->next_uid + 1; - break; - } - } - return recent_count; -} - -static void -index_mailbox_expunge_recent(struct mailbox *box, uint32_t seq1, uint32_t seq2) -{ - struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); - uint32_t uid; - - if (!array_is_created(&ibox->recent_flags)) - return; - - for (; seq1 <= seq2; seq1++) { - mail_index_lookup_uid(box->view, seq1, &uid); - if (seq_range_array_remove(&ibox->recent_flags, uid)) - ibox->recent_flags_count--; - } -} - static void index_view_sync_recs_get(struct index_mailbox_sync_context *ctx) { struct mail_index_view_sync_rec sync_rec; @@ -264,7 +167,7 @@ range = array_idx(ctx->expunges, ctx->expunge_pos); i_assert(range->seq2 <= ctx->messages_count); - index_mailbox_expunge_recent(ctx->ctx.box, range->seq1, range->seq2); + mailbox_recent_flags_expunge_seqs(ctx->ctx.box, range->seq1, range->seq2); ctx->messages_count -= range->seq2 - range->seq1 + 1; sync_rec_r->seq1 = range->seq1; @@ -310,13 +213,12 @@ static void index_mailbox_expunge_unseen_recent(struct index_mailbox_sync_context *ctx) { - struct index_mailbox_context *ibox = From dovecot at dovecot.org Wed Aug 19 10:42:44 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 19 Aug 2015 10:42:44 +0000 Subject: dovecot-2.2: lib-storage: Code cleanup - Don't keep a separate n... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3a8af59b379d changeset: 18976:3a8af59b379d user: Timo Sirainen date: Wed Aug 19 13:40:35 2015 +0300 description: lib-storage: Code cleanup - Don't keep a separate notify_ios list. diffstat: src/lib-storage/index/index-mailbox-check.c | 25 ++++--------------------- src/lib-storage/index/index-storage.h | 1 - 2 files changed, 4 insertions(+), 22 deletions(-) diffs (77 lines): diff -r f78bd5be4e47 -r 3a8af59b379d src/lib-storage/index/index-mailbox-check.c --- a/src/lib-storage/index/index-mailbox-check.c Wed Aug 19 12:56:55 2015 +0300 +++ b/src/lib-storage/index/index-mailbox-check.c Wed Aug 19 13:40:35 2015 +0300 @@ -16,11 +16,7 @@ char *path; time_t last_stamp; -}; - -struct index_notify_io { - struct index_notify_io *next; - struct io *io; + struct io *io_notify; }; static void notify_delay_callback(struct mailbox *box) @@ -71,21 +67,15 @@ struct index_notify_file *file; struct stat st; struct io *io = NULL; - struct index_notify_io *aio; i_assert(set->mailbox_idle_check_interval > 0); (void)io_add_notify(path, notify_callback, box, &io); - if (io != NULL) { - aio = i_new(struct index_notify_io, 1); - aio->io = io; - aio->next = ibox->notify_ios; - ibox->notify_ios = aio; - } file = i_new(struct index_notify_file, 1); file->path = i_strdup(path); file->last_stamp = stat(path, &st) < 0 ? 0 : st.st_mtime; + file->io_notify = io; file->next = ibox->notify_files; ibox->notify_files = file; @@ -104,24 +94,17 @@ { struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); struct index_notify_file *file; - struct index_notify_io *aio; while (ibox->notify_files != NULL) { file = ibox->notify_files; ibox->notify_files = file->next; + if (file->io_notify != NULL) + io_remove(&file->io_notify); i_free(file->path); i_free(file); } - while (ibox->notify_ios != NULL) { - aio = ibox->notify_ios; - ibox->notify_ios = aio->next; - - io_remove(&aio->io); - i_free(aio); - } - if (ibox->notify_delay_to != NULL) timeout_remove(&ibox->notify_delay_to); if (ibox->notify_to != NULL) diff -r f78bd5be4e47 -r 3a8af59b379d src/lib-storage/index/index-storage.h --- a/src/lib-storage/index/index-storage.h Wed Aug 19 12:56:55 2015 +0300 +++ b/src/lib-storage/index/index-storage.h Wed Aug 19 13:40:35 2015 +0300 @@ -23,7 +23,6 @@ struct timeout *notify_to, *notify_delay_to; struct index_notify_file *notify_files; - struct index_notify_io *notify_ios; time_t next_lock_notify; /* temporary */ enum mailbox_lock_notify_type last_notify_type; From dovecot at dovecot.org Wed Aug 19 10:42:45 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 19 Aug 2015 10:42:45 +0000 Subject: dovecot-2.2: lib: kqueue notification should trigger also on fil... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/412de6dea139 changeset: 18977:412de6dea139 user: Timo Sirainen date: Wed Aug 19 13:41:54 2015 +0300 description: lib: kqueue notification should trigger also on file renames. For example if dovecot.index.log is renamed to dovecot.index.log.2, we should notice that since there's now a new dovecot.index.log containing new changes. diffstat: src/lib/ioloop-notify-kqueue.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 3a8af59b379d -r 412de6dea139 src/lib/ioloop-notify-kqueue.c --- a/src/lib/ioloop-notify-kqueue.c Wed Aug 19 13:40:35 2015 +0300 +++ b/src/lib/ioloop-notify-kqueue.c Wed Aug 19 13:41:54 2015 +0300 @@ -138,7 +138,8 @@ event state transitions and not the current state. With this flag, the same event is only returned once. */ MY_EV_SET(&ev, fd, EVFILT_VNODE, EV_ADD | EV_CLEAR, - NOTE_DELETE | NOTE_WRITE | NOTE_EXTEND | NOTE_REVOKE, 0, io); + NOTE_DELETE | NOTE_RENAME | NOTE_WRITE | NOTE_EXTEND | + NOTE_REVOKE, 0, io); if (kevent(ctx->kq, &ev, 1, NULL, 0, NULL) < 0) { i_error("kevent(%d, %s) for notify failed: %m", fd, path); i_close_fd(&fd); From dovecot at dovecot.org Wed Aug 19 11:33:56 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 19 Aug 2015 11:33:56 +0000 Subject: dovecot-2.2: pop3: If transcation commit failed at QUIT, don't l... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ff9a9ffcd2af changeset: 18978:ff9a9ffcd2af user: Timo Sirainen date: Wed Aug 19 14:30:09 2015 +0300 description: pop3: If transcation commit failed at QUIT, don't log any messages as expunged. diffstat: src/pop3/pop3-client.c | 3 ++- src/pop3/pop3-client.h | 3 ++- src/pop3/pop3-commands.c | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diffs (53 lines): diff -r 412de6dea139 -r ff9a9ffcd2af src/pop3/pop3-client.c --- a/src/pop3/pop3-client.c Wed Aug 19 13:41:54 2015 +0300 +++ b/src/pop3/pop3-client.c Wed Aug 19 14:30:09 2015 +0300 @@ -535,7 +535,8 @@ tab[1].value = dec2str(client->top_count); tab[2].value = dec2str(client->retr_bytes); tab[3].value = dec2str(client->retr_count); - tab[4].value = dec2str(client->expunged_count); + tab[4].value = client->delete_success ? + dec2str(client->deleted_count) : 0; tab[5].value = dec2str(client->messages_count); tab[6].value = dec2str(client->total_size); tab[7].value = dec2str(client->input->v_offset); diff -r 412de6dea139 -r ff9a9ffcd2af src/pop3/pop3-client.h --- a/src/pop3/pop3-client.h Wed Aug 19 13:41:54 2015 +0300 +++ b/src/pop3/pop3-client.h Wed Aug 19 14:30:09 2015 +0300 @@ -62,7 +62,7 @@ unsigned int uid_validity; unsigned int messages_count; - unsigned int deleted_count, expunged_count, seen_change_count; + unsigned int deleted_count, seen_change_count; uoff_t total_size; uoff_t deleted_size; uint32_t last_seen_pop3_msn, lowest_retr_pop3_msn; @@ -100,6 +100,7 @@ unsigned int waiting_input:1; unsigned int anvil_sent:1; unsigned int message_uidls_save:1; + unsigned int delete_success:1; }; struct pop3_module_register { diff -r 412de6dea139 -r ff9a9ffcd2af src/pop3/pop3-commands.c --- a/src/pop3/pop3-commands.c Wed Aug 19 13:41:54 2015 +0300 +++ b/src/pop3/pop3-commands.c Wed Aug 19 14:30:09 2015 +0300 @@ -228,7 +228,6 @@ mail_update_keywords(mail, MODIFY_ADD, client->deleted_kw); break; } - client->expunged_count++; } bool client_update_mails(struct client *client) @@ -297,6 +296,8 @@ client_send_storage_error(client); client_disconnect(client, "Storage error during logout."); return 1; + } else { + client->delete_success = TRUE; } if (!client->deleted) From dovecot at dovecot.org Wed Aug 19 11:33:57 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 19 Aug 2015 11:33:57 +0000 Subject: dovecot-2.2: pop3: Added %{deleted_bytes} variable to pop3_logou... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/06351f47a15c changeset: 18979:06351f47a15c user: Timo Sirainen date: Wed Aug 19 14:33:09 2015 +0300 description: pop3: Added %{deleted_bytes} variable to pop3_logout_format diffstat: doc/example-config/conf.d/20-pop3.conf | 1 + src/pop3/pop3-client.c | 3 +++ 2 files changed, 4 insertions(+), 0 deletions(-) diffs (31 lines): diff -r ff9a9ffcd2af -r 06351f47a15c doc/example-config/conf.d/20-pop3.conf --- a/doc/example-config/conf.d/20-pop3.conf Wed Aug 19 14:30:09 2015 +0300 +++ b/doc/example-config/conf.d/20-pop3.conf Wed Aug 19 14:33:09 2015 +0300 @@ -73,6 +73,7 @@ # %r - number of RETR commands # %b - number of bytes sent to client as a result of RETR command # %d - number of deleted messages +# %{deleted_bytes} - number of bytes in deleted messages # %m - number of messages (before deletion) # %s - mailbox size in bytes (before deletion) # %u - old/new UIDL hash. may help finding out if UIDLs changed unexpectedly diff -r ff9a9ffcd2af -r 06351f47a15c src/pop3/pop3-client.c --- a/src/pop3/pop3-client.c Wed Aug 19 14:30:09 2015 +0300 +++ b/src/pop3/pop3-client.c Wed Aug 19 14:33:09 2015 +0300 @@ -523,6 +523,7 @@ { 'o', NULL, "output" }, { 'u', NULL, "uidl_change" }, { '\0', NULL, "session" }, + { 'd', NULL, "deleted_bytes" }, { '\0', NULL, NULL } }; struct var_expand_table *tab; @@ -547,6 +548,8 @@ else tab[9].value = ""; tab[10].value = client->session_id; + tab[11].value = client->delete_success ? + dec2str(client->deleted_size) : 0; str = t_str_new(128); var_expand(str, client->set->pop3_logout_format, tab); From dovecot at dovecot.org Wed Aug 19 11:48:41 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 19 Aug 2015 11:48:41 +0000 Subject: dovecot-2.2: Removed support for dnotify. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/39255150d521 changeset: 18980:39255150d521 user: Timo Sirainen date: Wed Aug 19 14:47:51 2015 +0300 description: Removed support for dnotify. Everybody should be using inotify by now. diffstat: configure.ac | 30 +---- doc/example-config/conf.d/10-mail.conf | 2 +- src/lib/Makefile.am | 1 - src/lib/ioloop-notify-dn.c | 220 --------------------------------- src/lib/ioloop-notify-fd.c | 2 +- src/master/main.c | 3 - 6 files changed, 3 insertions(+), 255 deletions(-) diffs (truncated from 319 to 300 lines): diff -r 06351f47a15c -r 39255150d521 configure.ac --- a/configure.ac Wed Aug 19 14:33:09 2015 +0300 +++ b/configure.ac Wed Aug 19 14:47:51 2015 +0300 @@ -64,7 +64,7 @@ ioloop=best) AC_ARG_WITH(notify, -AS_HELP_STRING([--with-notify=NOTIFY], [Specify the file system notification method to use (inotify, kqueue, dnotify, none; default is detected in the above order)]), +AS_HELP_STRING([--with-notify=NOTIFY], [Specify the file system notification method to use (inotify, kqueue, none; default is detected in the above order)]), notify=$withval, notify=) @@ -602,34 +602,6 @@ fi fi -if test "$notify" = "" || test "$notify" = "dnotify"; then - dnl * dnotify? - AC_CACHE_CHECK([whether we can use dnotify],i_cv_have_dnotify,[ - AC_TRY_COMPILE([ - #define _GNU_SOURCE - #include - #include - #include - ], [ - fcntl(0, F_SETSIG, SIGRTMIN); - fcntl(0, F_NOTIFY, DN_CREATE | DN_DELETE | DN_RENAME | DN_MULTISHOT); - ], [ - i_cv_have_dnotify=yes - ], [ - i_cv_have_dnotify=no - ]) - ]) - if test $i_cv_have_dnotify = yes; then - AC_DEFINE(IOLOOP_NOTIFY_DNOTIFY,, [Use Linux dnotify]) - have_notify=dnotify - notify=dnotify - else - if test "$notify" = "dnotify"; then - AC_MSG_ERROR([dnotify requested but not available]) - fi - fi -fi - if test "$have_notify" = "none"; then AC_DEFINE(IOLOOP_NOTIFY_NONE,, [No special notify support]) fi diff -r 06351f47a15c -r 39255150d521 doc/example-config/conf.d/10-mail.conf --- a/doc/example-config/conf.d/10-mail.conf Wed Aug 19 14:33:09 2015 +0300 +++ b/doc/example-config/conf.d/10-mail.conf Wed Aug 19 14:47:51 2015 +0300 @@ -215,7 +215,7 @@ # When IDLE command is running, mailbox is checked once in a while to see if # there are any new mails or other changes. This setting defines the minimum -# time to wait between those checks. Dovecot can also use dnotify, inotify and +# time to wait between those checks. Dovecot can also use inotify and # kqueue to find out immediately when changes occur. #mailbox_idle_check_interval = 30 secs diff -r 06351f47a15c -r 39255150d521 src/lib/Makefile.am --- a/src/lib/Makefile.am Wed Aug 19 14:33:09 2015 +0300 +++ b/src/lib/Makefile.am Wed Aug 19 14:47:51 2015 +0300 @@ -81,7 +81,6 @@ ioloop-iolist.c \ ioloop-notify-none.c \ ioloop-notify-fd.c \ - ioloop-notify-dn.c \ ioloop-notify-inotify.c \ ioloop-notify-kqueue.c \ ioloop-poll.c \ diff -r 06351f47a15c -r 39255150d521 src/lib/ioloop-notify-dn.c --- a/src/lib/ioloop-notify-dn.c Wed Aug 19 14:33:09 2015 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,220 +0,0 @@ -/* Copyright (c) 2003-2015 Dovecot authors, see the included COPYING file */ - -/* Logic is pretty much based on dnotify by Oskar Liljeblad. */ - -#define _GNU_SOURCE -#include "lib.h" - -#ifdef IOLOOP_NOTIFY_DNOTIFY - -#include "ioloop-private.h" -#include "ioloop-notify-fd.h" -#include "fd-set-nonblock.h" -#include "fd-close-on-exec.h" - -#include -#include -#include - -struct ioloop_notify_handler_context { - struct ioloop_notify_fd_context fd_ctx; - - struct io *event_io; - int event_pipe[2]; - - bool disabled; -}; - -static int sigrt_refcount = 0; - -static struct ioloop_notify_handler_context *io_loop_notify_handler_init(void); - -static void ioloop_dnotify_disable(struct ioloop_notify_handler_context *ctx) -{ - if (ctx->disabled) - return; - - if (--sigrt_refcount == 0) - signal(SIGRTMIN, SIG_IGN); - - if (close(ctx->event_pipe[0]) < 0) - i_error("close(dnotify pipe[0]) failed: %m"); - if (close(ctx->event_pipe[1]) < 0) - i_error("close(dnotify pipe[1]) failed: %m"); - ctx->disabled = TRUE; -} - -static void sigrt_handler(int signo ATTR_UNUSED, siginfo_t *si, - void *data ATTR_UNUSED) -{ - struct ioloop_notify_handler_context *ctx = - current_ioloop->notify_handler_context; - int saved_errno = errno; - int ret; - - if (ctx->disabled) - return; - - ret = write(ctx->event_pipe[1], &si->si_fd, sizeof(int)); - if (ret < 0 && errno != EINTR && errno != EAGAIN) { - i_error("write(dnotify pipe) failed: %m"); - ioloop_dnotify_disable(ctx); - } - - i_assert(ret <= 0 || ret == sizeof(int)); - - errno = saved_errno; -} - -static void dnotify_input(struct ioloop *ioloop) -{ - struct ioloop_notify_handler_context *ctx = - ioloop->notify_handler_context; - struct io_notify *io; - int fd_buf[256], i, ret; - - ret = read(ctx->event_pipe[0], fd_buf, sizeof(fd_buf)); - if (ret < 0) - i_fatal("read(dnotify pipe) failed: %m"); - if ((ret % sizeof(fd_buf[0])) != 0) - i_fatal("read(dnotify pipe) returned %d", ret); - ret /= sizeof(fd_buf[0]); - - if (gettimeofday(&ioloop_timeval, NULL) < 0) - i_fatal("gettimeofday(): %m"); - ioloop_time = ioloop_timeval.tv_sec; - - for (i = 0; i < ret; i++) { - io = io_notify_fd_find(&ctx->fd_ctx, fd_buf[i]); - if (io != NULL) - io_loop_call_io(&io->io); - } -} - -#undef io_add_notify -enum io_notify_result io_add_notify(const char *path, io_callback_t *callback, - void *context, struct io **io_r) -{ - struct ioloop_notify_handler_context *ctx = - current_ioloop->notify_handler_context; - int fd; - - *io_r = NULL; - - if (ctx == NULL) - ctx = io_loop_notify_handler_init(); - if (ctx->disabled) - return IO_NOTIFY_NOSUPPORT; - - fd = open(path, O_RDONLY); - if (fd == -1) { - /* ESTALE could happen with NFS. Don't bother giving an error - message then. */ - if (errno != ENOENT && errno != ESTALE) - i_error("open(%s) for dnotify failed: %m", path); - return IO_NOTIFY_NOTFOUND; - } - - if (fcntl(fd, F_SETSIG, SIGRTMIN) < 0) { - /* EINVAL means there's no realtime signals and no dnotify */ - if (errno != EINVAL) - i_error("fcntl(F_SETSIG) failed: %m"); - ioloop_dnotify_disable(ctx); - i_close_fd(&fd); - return IO_NOTIFY_NOSUPPORT; - } - if (fcntl(fd, F_NOTIFY, DN_CREATE | DN_DELETE | DN_RENAME | - DN_MULTISHOT) < 0) { - if (errno == ENOTDIR) { - /* we're trying to add dnotify to a non-directory fd. - fail silently. */ - } else { - /* dnotify not in kernel. disable it. */ - if (errno != EINVAL) - i_error("fcntl(F_NOTIFY) failed: %m"); - ioloop_dnotify_disable(ctx); - } - (void)fcntl(fd, F_SETSIG, 0); - i_close_fd(&fd); - return IO_NOTIFY_NOSUPPORT; - } - - if (ctx->event_io == NULL) { - ctx->event_io = io_add(ctx->event_pipe[0], IO_READ, - dnotify_input, current_ioloop); - } - - *io_r = io_notify_fd_add(&ctx->fd_ctx, fd, callback, context); - return IO_NOTIFY_ADDED; -} - -void io_loop_notify_remove(struct io *_io) -{ - struct ioloop_notify_handler_context *ctx = - _io->ioloop->notify_handler_context; - struct io_notify *io = (struct io_notify *)_io; - - if (fcntl(io->fd, F_NOTIFY, 0) < 0) - i_error("fcntl(F_NOTIFY, 0) failed: %m"); - if (fcntl(io->fd, F_SETSIG, 0) < 0) - i_error("fcntl(F_SETSIG, 0) failed: %m"); - if (close(io->fd)) - i_error("close(dnotify) failed: %m"); - - io_notify_fd_free(&ctx->fd_ctx, io); - - if (ctx->fd_ctx.notifies == NULL) - io_remove(&ctx->event_io); -} - -static struct ioloop_notify_handler_context *io_loop_notify_handler_init(void) -{ - struct ioloop_notify_handler_context *ctx; - struct sigaction act; - - ctx = current_ioloop->notify_handler_context = - i_new(struct ioloop_notify_handler_context, 1); - - if (pipe(ctx->event_pipe) < 0) { - ctx->disabled = TRUE; - i_error("dnotify: pipe() failed: %m"); - return ctx; - } - - fd_set_nonblock(ctx->event_pipe[0], TRUE); - fd_set_nonblock(ctx->event_pipe[1], TRUE); - - fd_close_on_exec(ctx->event_pipe[0], TRUE); - fd_close_on_exec(ctx->event_pipe[1], TRUE); - - if (sigrt_refcount++ == 0) { - /* SIGIO is sent if queue gets full. we'll just ignore it. */ - signal(SIGIO, SIG_IGN); - - act.sa_sigaction = sigrt_handler; - sigemptyset(&act.sa_mask); - act.sa_flags = SA_SIGINFO | SA_RESTART | SA_NODEFER; - - if (sigaction(SIGRTMIN, &act, NULL) < 0) { - if (errno == EINVAL) { - /* kernel is too old to understand even RT - signals, so there's no way dnotify works */ - ioloop_dnotify_disable(ctx); - } else { - i_fatal("sigaction(SIGRTMIN) failed: %m"); - } - } - } - return ctx; -} - -void io_loop_notify_handler_deinit(struct ioloop *ioloop) -{ - struct ioloop_notify_handler_context *ctx = - ioloop->notify_handler_context; - - ioloop_dnotify_disable(ctx); - i_free(ctx); -} - -#endif diff -r 06351f47a15c -r 39255150d521 src/lib/ioloop-notify-fd.c --- a/src/lib/ioloop-notify-fd.c Wed Aug 19 14:33:09 2015 +0300 +++ b/src/lib/ioloop-notify-fd.c Wed Aug 19 14:47:51 2015 +0300 @@ -4,7 +4,7 @@ #include "ioloop-private.h" #include "ioloop-notify-fd.h" From dovecot at dovecot.org Wed Aug 19 13:51:03 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 19 Aug 2015 13:51:03 +0000 Subject: dovecot-2.2: lib-storage: Moved index/index-mailbox-check.c code... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f84c81ed5d09 changeset: 18981:f84c81ed5d09 user: Timo Sirainen date: Wed Aug 19 15:12:09 2015 +0300 description: lib-storage: Moved index/index-mailbox-check.c code to mailbox-watch.c This changes the API, but adds backwards compatibility macros. diffstat: src/lib-storage/Makefile.am | 6 +- src/lib-storage/index/Makefile.am | 1 - src/lib-storage/index/cydir/cydir-storage.c | 4 +- src/lib-storage/index/dbox-common/dbox-storage.c | 4 +- src/lib-storage/index/index-mailbox-check.c | 112 ----------------------- src/lib-storage/index/index-storage.c | 2 +- src/lib-storage/index/index-storage.h | 9 +- src/lib-storage/index/index-sync.c | 4 +- src/lib-storage/index/maildir/maildir-storage.c | 6 +- src/lib-storage/index/mbox/mbox-storage.c | 4 +- src/lib-storage/mail-storage-private.h | 2 + src/lib-storage/mailbox-watch.c | 106 +++++++++++++++++++++ src/lib-storage/mailbox-watch.h | 7 + 13 files changed, 134 insertions(+), 133 deletions(-) diffs (truncated from 409 to 300 lines): diff -r 39255150d521 -r f84c81ed5d09 src/lib-storage/Makefile.am --- a/src/lib-storage/Makefile.am Wed Aug 19 14:47:51 2015 +0300 +++ b/src/lib-storage/Makefile.am Wed Aug 19 15:12:09 2015 +0300 @@ -54,7 +54,8 @@ mailbox-recent-flags.c \ mailbox-search-result.c \ mailbox-tree.c \ - mailbox-uidvalidity.c + mailbox-uidvalidity.c \ + mailbox-watch.c headers = \ fail-mail-storage.h \ @@ -83,7 +84,8 @@ mailbox-recent-flags.h \ mailbox-search-result-private.h \ mailbox-tree.h \ - mailbox-uidvalidity.h + mailbox-uidvalidity.h \ + mailbox-watch.h shlibs = \ @LINKED_STORAGE_LIBS@ \ diff -r 39255150d521 -r f84c81ed5d09 src/lib-storage/index/Makefile.am --- a/src/lib-storage/index/Makefile.am Wed Aug 19 14:47:51 2015 +0300 +++ b/src/lib-storage/index/Makefile.am Wed Aug 19 15:12:09 2015 +0300 @@ -19,7 +19,6 @@ index-mail.c \ index-mail-binary.c \ index-mail-headers.c \ - index-mailbox-check.c \ index-mailbox-size.c \ index-rebuild.c \ index-search.c \ diff -r 39255150d521 -r f84c81ed5d09 src/lib-storage/index/cydir/cydir-storage.c --- a/src/lib-storage/index/cydir/cydir-storage.c Wed Aug 19 14:47:51 2015 +0300 +++ b/src/lib-storage/index/cydir/cydir-storage.c Wed Aug 19 15:12:09 2015 +0300 @@ -103,9 +103,9 @@ static void cydir_notify_changes(struct mailbox *box) { if (box->notify_callback == NULL) - index_mailbox_check_remove_all(box); + mailbox_watch_remove_all(box); else - index_mailbox_check_add(box, mailbox_get_path(box)); + mailbox_watch_add(box, mailbox_get_path(box)); } struct mail_storage cydir_storage = { diff -r 39255150d521 -r f84c81ed5d09 src/lib-storage/index/dbox-common/dbox-storage.c --- a/src/lib-storage/index/dbox-common/dbox-storage.c Wed Aug 19 14:47:51 2015 +0300 +++ b/src/lib-storage/index/dbox-common/dbox-storage.c Wed Aug 19 15:12:09 2015 +0300 @@ -155,13 +155,13 @@ const char *dir, *path; if (box->notify_callback == NULL) - index_mailbox_check_remove_all(box); + mailbox_watch_remove_all(box); else { if (mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &dir) <= 0) return; path = t_strdup_printf("%s/"MAIL_INDEX_PREFIX".log", dir); - index_mailbox_check_add(box, path); + mailbox_watch_add(box, path); } } diff -r 39255150d521 -r f84c81ed5d09 src/lib-storage/index/index-mailbox-check.c --- a/src/lib-storage/index/index-mailbox-check.c Wed Aug 19 14:47:51 2015 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,112 +0,0 @@ -/* Copyright (c) 2002-2015 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "ioloop.h" -#include "index-storage.h" - -#include -#include -#include -#include - -#define NOTIFY_DELAY_MSECS 500 - -struct index_notify_file { - struct index_notify_file *next; - - char *path; - time_t last_stamp; - struct io *io_notify; -}; - -static void notify_delay_callback(struct mailbox *box) -{ - struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); - - if (ibox->notify_delay_to != NULL) - timeout_remove(&ibox->notify_delay_to); - box->notify_callback(box, box->notify_context); -} - -static void check_timeout(struct mailbox *box) -{ - struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); - struct index_notify_file *file; - struct stat st; - bool notify = FALSE; - - for (file = ibox->notify_files; file != NULL; file = file->next) { - if (stat(file->path, &st) == 0 && - file->last_stamp != st.st_mtime) { - file->last_stamp = st.st_mtime; - notify = TRUE; - } - } - - if (notify) - notify_delay_callback(box); -} - -static void notify_callback(struct mailbox *box) -{ - struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); - - timeout_reset(ibox->notify_to); - - if (ibox->notify_delay_to == NULL) { - ibox->notify_delay_to = - timeout_add_short(NOTIFY_DELAY_MSECS, - notify_delay_callback, box); - } -} - -void index_mailbox_check_add(struct mailbox *box, const char *path) -{ - struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); - const struct mail_storage_settings *set = box->storage->set; - struct index_notify_file *file; - struct stat st; - struct io *io = NULL; - - i_assert(set->mailbox_idle_check_interval > 0); - - (void)io_add_notify(path, notify_callback, box, &io); - - file = i_new(struct index_notify_file, 1); - file->path = i_strdup(path); - file->last_stamp = stat(path, &st) < 0 ? 0 : st.st_mtime; - file->io_notify = io; - - file->next = ibox->notify_files; - ibox->notify_files = file; - - /* we still add a timeout if we don't have one already, - * because we don't know what happens with [di]notify - * when the filesystem is remote (NFS, ...) */ - if (ibox->notify_to == NULL) { - ibox->notify_to = - timeout_add(set->mailbox_idle_check_interval * 1000, - check_timeout, box); - } -} - -void index_mailbox_check_remove_all(struct mailbox *box) -{ - struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); - struct index_notify_file *file; - - while (ibox->notify_files != NULL) { - file = ibox->notify_files; - ibox->notify_files = file->next; - - if (file->io_notify != NULL) - io_remove(&file->io_notify); - i_free(file->path); - i_free(file); - } - - if (ibox->notify_delay_to != NULL) - timeout_remove(&ibox->notify_delay_to); - if (ibox->notify_to != NULL) - timeout_remove(&ibox->notify_to); -} diff -r 39255150d521 -r f84c81ed5d09 src/lib-storage/index/index-storage.c --- a/src/lib-storage/index/index-storage.c Wed Aug 19 14:47:51 2015 +0300 +++ b/src/lib-storage/index/index-storage.c Wed Aug 19 15:12:09 2015 +0300 @@ -354,7 +354,7 @@ { struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(box); - index_mailbox_check_remove_all(box); + mailbox_watch_remove_all(box); if (box->input != NULL) i_stream_unref(&box->input); diff -r 39255150d521 -r f84c81ed5d09 src/lib-storage/index/index-storage.h --- a/src/lib-storage/index/index-storage.h Wed Aug 19 14:47:51 2015 +0300 +++ b/src/lib-storage/index/index-storage.h Wed Aug 19 15:12:09 2015 +0300 @@ -5,6 +5,7 @@ #include "mail-storage-private.h" #include "mail-index-private.h" #include "mailbox-recent-flags.h" /* FIXME: remove in v2.3 */ +#include "mailbox-watch.h" #define MAILBOX_FULL_SYNC_INTERVAL 5 @@ -21,9 +22,6 @@ union mailbox_module_context module_ctx; enum mail_index_open_flags index_flags; - struct timeout *notify_to, *notify_delay_to; - struct index_notify_file *notify_files; - time_t next_lock_notify; /* temporary */ enum mailbox_lock_notify_type last_notify_type; @@ -77,9 +75,8 @@ /* FIXME: for backwards compatibility - remove in v2.3 */ #define index_mailbox_set_recent_seq(box, view, seq1, seq2) \ mailbox_recent_flags_set_seqs(box, view, seq1, seq2) - -void index_mailbox_check_add(struct mailbox *box, const char *path); -void index_mailbox_check_remove_all(struct mailbox *box); +#define index_mailbox_check_add(box, path) mailbox_watch_add(box, path) +#define index_mailbox_check_remove_all(box) mailbox_watch_remove_all(box) enum mail_index_sync_flags index_storage_get_sync_flags(struct mailbox *box); bool index_mailbox_want_full_sync(struct mailbox *box, diff -r 39255150d521 -r f84c81ed5d09 src/lib-storage/index/index-sync.c --- a/src/lib-storage/index/index-sync.c Wed Aug 19 14:47:51 2015 +0300 +++ b/src/lib-storage/index/index-sync.c Wed Aug 19 15:12:09 2015 +0300 @@ -45,8 +45,8 @@ return FALSE; } - if (ibox->notify_to != NULL) - timeout_reset(ibox->notify_to); + if (box->to_notify != NULL) + timeout_reset(box->to_notify); ibox->sync_last_check = ioloop_time; return TRUE; } diff -r 39255150d521 -r f84c81ed5d09 src/lib-storage/index/maildir/maildir-storage.c --- a/src/lib-storage/index/maildir/maildir-storage.c Wed Aug 19 14:47:51 2015 +0300 +++ b/src/lib-storage/index/maildir/maildir-storage.c Wed Aug 19 15:12:09 2015 +0300 @@ -579,11 +579,11 @@ const char *box_path = mailbox_get_path(box); if (box->notify_callback == NULL) - index_mailbox_check_remove_all(&mbox->box); + mailbox_watch_remove_all(&mbox->box); else { - index_mailbox_check_add(&mbox->box, + mailbox_watch_add(&mbox->box, t_strconcat(box_path, "/new", NULL)); - index_mailbox_check_add(&mbox->box, + mailbox_watch_add(&mbox->box, t_strconcat(box_path, "/cur", NULL)); } } diff -r 39255150d521 -r f84c81ed5d09 src/lib-storage/index/mbox/mbox-storage.c --- a/src/lib-storage/index/mbox/mbox-storage.c Wed Aug 19 14:47:51 2015 +0300 +++ b/src/lib-storage/index/mbox/mbox-storage.c Wed Aug 19 15:12:09 2015 +0300 @@ -680,9 +680,9 @@ struct mbox_mailbox *mbox = (struct mbox_mailbox *)box; if (box->notify_callback == NULL) - index_mailbox_check_remove_all(box); + mailbox_watch_remove_all(box); else if (!mbox->no_mbox_file) - index_mailbox_check_add(box, mailbox_get_path(box)); + mailbox_watch_add(box, mailbox_get_path(box)); } static bool diff -r 39255150d521 -r f84c81ed5d09 src/lib-storage/mail-storage-private.h --- a/src/lib-storage/mail-storage-private.h Wed Aug 19 14:47:51 2015 +0300 +++ b/src/lib-storage/mail-storage-private.h Wed Aug 19 15:12:09 2015 +0300 @@ -329,6 +329,8 @@ /* Mailbox notification settings: */ mailbox_notify_callback_t *notify_callback; void *notify_context; + struct timeout *to_notify, *to_notify_delay; + struct mailbox_notify_file *notify_files; /* Increased by one for each new struct mailbox. */ unsigned int generation_sequence; diff -r 39255150d521 -r f84c81ed5d09 src/lib-storage/mailbox-watch.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/mailbox-watch.c Wed Aug 19 15:12:09 2015 +0300 @@ -0,0 +1,106 @@ +/* Copyright (c) 2002-2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "mail-storage-private.h" +#include "mailbox-watch.h" + +#include From dovecot at dovecot.org Wed Aug 19 13:51:04 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 19 Aug 2015 13:51:04 +0000 Subject: dovecot-2.2: lib: inotify doesn't need to watch for IN_CLOSE Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/84036f6687d5 changeset: 18982:84036f6687d5 user: Timo Sirainen date: Wed Aug 19 16:50:10 2015 +0300 description: lib: inotify doesn't need to watch for IN_CLOSE This doesn't indicate that the file was modified. We don't care if some reader accessed a watched file. diffstat: src/lib/ioloop-notify-inotify.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r f84c81ed5d09 -r 84036f6687d5 src/lib/ioloop-notify-inotify.c --- a/src/lib/ioloop-notify-inotify.c Wed Aug 19 15:12:09 2015 +0300 +++ b/src/lib/ioloop-notify-inotify.c Wed Aug 19 16:50:10 2015 +0300 @@ -101,7 +101,7 @@ wd = inotify_add_watch(ctx->inotify_fd, path, IN_CREATE | IN_DELETE | IN_DELETE_SELF | - IN_MOVE | IN_CLOSE | IN_MODIFY); + IN_MOVE | IN_MODIFY); if (wd < 0) { /* ESTALE could happen with NFS. Don't bother giving an error message then. */ From dovecot at dovecot.org Wed Aug 19 17:50:07 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 19 Aug 2015 17:50:07 +0000 Subject: dovecot-2.2: imap: If command has no imap_parser, don't crash wh... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/637e005a5559 changeset: 18983:637e005a5559 user: Timo Sirainen date: Wed Aug 19 20:49:12 2015 +0300 description: imap: If command has no imap_parser, don't crash when freeing the command. All the current commands have a parser though, so this doesn't actually fix anything. diffstat: src/imap/imap-client.c | 12 +++++++----- 1 files changed, 7 insertions(+), 5 deletions(-) diffs (22 lines): diff -r 84036f6687d5 -r 637e005a5559 src/imap/imap-client.c --- a/src/imap/imap-client.c Wed Aug 19 16:50:10 2015 +0300 +++ b/src/imap/imap-client.c Wed Aug 19 20:49:12 2015 +0300 @@ -699,11 +699,13 @@ if (client->mailbox_change_lock == cmd) client->mailbox_change_lock = NULL; - if (client->free_parser == NULL) { - imap_parser_reset(cmd->parser); - client->free_parser = cmd->parser; - } else if (cmd->parser != NULL) { - imap_parser_unref(&cmd->parser); + if (cmd->parser != NULL) { + if (client->free_parser == NULL) { + imap_parser_reset(cmd->parser); + client->free_parser = cmd->parser; + } else { + imap_parser_unref(&cmd->parser); + } } client->command_queue_size--; From dovecot at dovecot.org Wed Aug 19 18:25:41 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 19 Aug 2015 18:25:41 +0000 Subject: dovecot-2.2: lib: io_add_notify() wasn't setting struct io.sourc... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d26178d0557a changeset: 18984:d26178d0557a user: Timo Sirainen date: Wed Aug 19 21:24:50 2015 +0300 description: lib: io_add_notify() wasn't setting struct io.source_linenum diffstat: src/lib/ioloop-notify-inotify.c | 6 ++++-- src/lib/ioloop-notify-kqueue.c | 6 ++++-- src/lib/ioloop-notify-none.c | 1 + src/lib/ioloop.h | 7 ++++--- 4 files changed, 13 insertions(+), 7 deletions(-) diffs (75 lines): diff -r 637e005a5559 -r d26178d0557a src/lib/ioloop-notify-inotify.c --- a/src/lib/ioloop-notify-inotify.c Wed Aug 19 20:49:12 2015 +0300 +++ b/src/lib/ioloop-notify-inotify.c Wed Aug 19 21:24:50 2015 +0300 @@ -85,8 +85,9 @@ } #undef io_add_notify -enum io_notify_result io_add_notify(const char *path, io_callback_t *callback, - void *context, struct io **io_r) +enum io_notify_result +io_add_notify(const char *path, unsigned int source_linenum, + io_callback_t *callback, void *context, struct io **io_r) { struct ioloop_notify_handler_context *ctx = current_ioloop->notify_handler_context; @@ -125,6 +126,7 @@ } *io_r = io_notify_fd_add(&ctx->fd_ctx, wd, callback, context); + (*io_r)->source_linenum = source_linenum; return IO_NOTIFY_ADDED; } diff -r 637e005a5559 -r d26178d0557a src/lib/ioloop-notify-kqueue.c --- a/src/lib/ioloop-notify-kqueue.c Wed Aug 19 20:49:12 2015 +0300 +++ b/src/lib/ioloop-notify-kqueue.c Wed Aug 19 21:24:50 2015 +0300 @@ -104,8 +104,9 @@ } #undef io_add_notify -enum io_notify_result io_add_notify(const char *path, io_callback_t *callback, - void *context, struct io **io_r) +enum io_notify_result +io_add_notify(const char *path, unsigned int source_linenum, + io_callback_t *callback, void *context, struct io **io_r) { struct ioloop_notify_handler_context *ctx = current_ioloop->notify_handler_context; @@ -128,6 +129,7 @@ io = i_new(struct io_notify, 1); io->io.condition = IO_NOTIFY; + io->io.source_linenum = source_linenum; io->io.callback = callback; io->io.context = context; io->io.ioloop = current_ioloop; diff -r 637e005a5559 -r d26178d0557a src/lib/ioloop-notify-none.c --- a/src/lib/ioloop-notify-none.c Wed Aug 19 20:49:12 2015 +0300 +++ b/src/lib/ioloop-notify-none.c Wed Aug 19 21:24:50 2015 +0300 @@ -8,6 +8,7 @@ #undef io_add_notify enum io_notify_result io_add_notify(const char *path ATTR_UNUSED, + unsigned int source_linenum ATTR_UNUSED, io_callback_t *callback ATTR_UNUSED, void *context ATTR_UNUSED, struct io **io_r) { diff -r 637e005a5559 -r d26178d0557a src/lib/ioloop.h --- a/src/lib/ioloop.h Wed Aug 19 20:49:12 2015 +0300 +++ b/src/lib/ioloop.h Wed Aug 19 21:24:50 2015 +0300 @@ -55,10 +55,11 @@ CALLBACK_TYPECHECK(callback, void (*)(typeof(context))), \ (io_callback_t *)callback, context) enum io_notify_result -io_add_notify(const char *path, io_callback_t *callback, - void *context, struct io **io_r) ATTR_NULL(3); +io_add_notify(const char *path, unsigned int source_linenum, + io_callback_t *callback, void *context, + struct io **io_r) ATTR_NULL(3); #define io_add_notify(path, callback, context, io_r) \ - io_add_notify(path + \ + io_add_notify(path, __LINE__ + \ CALLBACK_TYPECHECK(callback, void (*)(typeof(context))), \ (io_callback_t *)callback, context, io_r) struct io *io_add_istream(struct istream *input, unsigned int source_linenum, From dovecot at dovecot.org Wed Aug 19 18:32:43 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 19 Aug 2015 18:32:43 +0000 Subject: dovecot-2.2: lib: Log notify IO leaks when ioloop is destroyed. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/818c1a77c025 changeset: 18985:818c1a77c025 user: Timo Sirainen date: Wed Aug 19 21:31:56 2015 +0300 description: lib: Log notify IO leaks when ioloop is destroyed. diffstat: src/lib/ioloop-notify-inotify.c | 10 ++++++++++ src/lib/ioloop-notify-kqueue.c | 26 ++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diffs (98 lines): diff -r d26178d0557a -r 818c1a77c025 src/lib/ioloop-notify-inotify.c --- a/src/lib/ioloop-notify-inotify.c Wed Aug 19 21:24:50 2015 +0300 +++ b/src/lib/ioloop-notify-inotify.c Wed Aug 19 21:31:56 2015 +0300 @@ -193,6 +193,16 @@ struct ioloop_notify_handler_context *ctx = ioloop->notify_handler_context; + while (ctx->fd_ctx.notifies != NULL) { + struct io_notify *io = ctx->fd_ctx.notifies; + struct io *_io = &io->io; + + i_warning("I/O notify leak: %p (line %u, fd %d)", + (void *)_io->callback, + _io->source_linenum, io->fd); + io_remove(&_io); + } + if (ctx->inotify_fd != -1) { if (close(ctx->inotify_fd) < 0) i_error("close(inotify) failed: %m"); diff -r d26178d0557a -r 818c1a77c025 src/lib/ioloop-notify-kqueue.c --- a/src/lib/ioloop-notify-kqueue.c Wed Aug 19 21:24:50 2015 +0300 +++ b/src/lib/ioloop-notify-kqueue.c Wed Aug 19 21:31:56 2015 +0300 @@ -10,6 +10,7 @@ #ifdef IOLOOP_NOTIFY_KQUEUE #include "ioloop-private.h" +#include "llist.h" #include "fd-close-on-exec.h" #include #include @@ -32,13 +33,23 @@ struct io io; int refcount; int fd; + struct io_notify *prev, *next; }; struct ioloop_notify_handler_context { int kq; struct io *event_io; + struct io_notify *notifies; }; +static void +io_loop_notify_free(struct ioloop_notify_handler_context *ctx, + struct io_notify *io) +{ + DLLIST_REMOVE(&ctx->notifies, io); + i_free(io); +} + static void event_callback(struct ioloop_notify_handler_context *ctx) { struct io_notify *io; @@ -74,7 +85,7 @@ io_loop_call_io(&io->io); if (--io->refcount == 0) - i_free(io); + io_loop_notify_free(ctx, io); } } @@ -96,6 +107,16 @@ struct ioloop_notify_handler_context *ctx = ioloop->notify_handler_context; + while (ctx->notifies != NULL) { + struct io_notify *io = ctx->notifies; + struct io *_io = &io->io; + + i_warning("I/O notify leak: %p (line %u, fd %d)", + (void *)_io->callback, + _io->source_linenum, io->fd); + io_remove(&_io); + } + if (ctx->event_io) io_remove(&ctx->event_io); if (close(ctx->kq) < 0) @@ -153,6 +174,7 @@ ctx->event_io = io_add(ctx->kq, IO_READ, event_callback, io->io.ioloop->notify_handler_context); } + DLLIST_PREPEND(&ctx->notifies, io); *io_r = &io->io; return IO_NOTIFY_ADDED; } @@ -172,7 +194,7 @@ io->fd = -1; if (--io->refcount == 0) - i_free(io); + io_loop_notify_free(ctx, io); } #endif From dovecot at dovecot.org Wed Aug 19 18:42:49 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 19 Aug 2015 18:42:49 +0000 Subject: dovecot-2.2: imap: Code cleanup Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/118594ed7d46 changeset: 18986:118594ed7d46 user: Timo Sirainen date: Wed Aug 19 21:41:55 2015 +0300 description: imap: Code cleanup Keep client_add_input() functionality separate from client_create_from_input(). Also this way temporary buffer doesn't need to be created to add the input in login_client_connected(). diffstat: src/imap/main.c | 68 ++++++++++++++++++++++++++++---------------------------- 1 files changed, 34 insertions(+), 34 deletions(-) diffs (139 lines): diff -r 818c1a77c025 -r 118594ed7d46 src/imap/main.c --- a/src/imap/main.c Wed Aug 19 21:31:56 2015 +0300 +++ b/src/imap/main.c Wed Aug 19 21:41:55 2015 +0300 @@ -159,13 +159,15 @@ } } -static void client_add_input(struct client *client, const buffer_t *buf) +static void +client_add_input(struct client *client, const unsigned char *client_input, + size_t client_input_size) { struct ostream *output; struct client_input input; - if (buf != NULL && buf->used > 0) { - client_parse_input(buf->data, buf->used, &input); + if (client_input_size > 0) { + client_parse_input(client_input, client_input_size, &input); if (input.input_size > 0 && !i_stream_add_data(client->input, input.input, input.input_size)) @@ -203,16 +205,14 @@ static int client_create_from_input(const struct mail_storage_service_input *input, - const struct master_login_client *login_client, - int fd_in, int fd_out, const buffer_t *input_buf, - const char **error_r) + int fd_in, int fd_out, + struct client **client_r, const char **error_r) { struct mail_storage_service_user *user; struct mail_user *mail_user; struct mail_namespace *ns; struct client *client; struct imap_settings *set; - enum mail_auth_request_flags flags; const char *errstr; enum mail_error mail_error; @@ -246,22 +246,15 @@ client = client_create(fd_in, fd_out, input->session_id, mail_user, user, set); - T_BEGIN { - client_add_input(client, input_buf); - } T_END; - - flags = login_client->auth_req.flags; - if ((flags & MAIL_AUTH_REQUEST_FLAG_TLS_COMPRESSION) != 0) - client->tls_compression = TRUE; + *client_r = client; return 0; } static void main_stdio_run(const char *username) { - struct master_login_client login_client; + struct client *client; struct mail_storage_service_input input; const char *value, *error, *input_base64; - buffer_t *input_buf; memset(&input, 0, sizeof(input)); input.module = input.service = "imap"; @@ -275,39 +268,39 @@ if ((value = getenv("LOCAL_IP")) != NULL) (void)net_addr2ip(value, &input.local_ip); + if (client_create_from_input(&input, STDIN_FILENO, STDOUT_FILENO, + &client, &error) < 0) + i_fatal("%s", error); input_base64 = getenv("CLIENT_INPUT"); - input_buf = input_base64 == NULL ? NULL : - t_base64_decode_str(input_base64); - - memset(&login_client, 0, sizeof(login_client)); - if (client_create_from_input(&input, &login_client, - STDIN_FILENO, STDOUT_FILENO, - input_buf, &error) < 0) - i_fatal("%s", error); + if (input_base64 == NULL) + client_add_input(client, NULL, 0); + else { + const buffer_t *input_buf = t_base64_decode_str(input_base64); + client_add_input(client, input_buf->data, input_buf->used); + } } static void -login_client_connected(const struct master_login_client *client, +login_client_connected(const struct master_login_client *login_client, const char *username, const char *const *extra_fields) { #define MSG_BYE_INTERNAL_ERROR "* BYE "MAIL_ERRSTR_CRITICAL_MSG"\r\n" struct mail_storage_service_input input; + struct client *client; + enum mail_auth_request_flags flags; const char *error; - buffer_t input_buf; memset(&input, 0, sizeof(input)); input.module = input.service = "imap"; - input.local_ip = client->auth_req.local_ip; - input.remote_ip = client->auth_req.remote_ip; + input.local_ip = login_client->auth_req.local_ip; + input.remote_ip = login_client->auth_req.remote_ip; input.username = username; input.userdb_fields = extra_fields; - input.session_id = client->session_id; + input.session_id = login_client->session_id; - buffer_create_from_const_data(&input_buf, client->data, - client->auth_req.data_size); - if (client_create_from_input(&input, client, client->fd, client->fd, - &input_buf, &error) < 0) { - int fd = client->fd; + if (client_create_from_input(&input, login_client->fd, login_client->fd, + &client, &error) < 0) { + int fd = login_client->fd; if (write(fd, MSG_BYE_INTERNAL_ERROR, strlen(MSG_BYE_INTERNAL_ERROR)) < 0) { @@ -317,7 +310,14 @@ i_error("%s", error); i_close_fd(&fd); master_service_client_connection_destroyed(master_service); + return; } + client_add_input(client, login_client->data, + login_client->auth_req.data_size); + + flags = login_client->auth_req.flags; + if ((flags & MAIL_AUTH_REQUEST_FLAG_TLS_COMPRESSION) != 0) + client->tls_compression = TRUE; } static void login_client_failed(const struct master_login_client *client, From dovecot at dovecot.org Thu Aug 20 13:05:39 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 20 Aug 2015 13:05:39 +0000 Subject: dovecot-2.2: imap: Fixed memory leak in SELECT QRESYNC error han... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9f815e781beb changeset: 18987:9f815e781beb user: Timo Sirainen date: Thu Aug 20 16:04:48 2015 +0300 description: imap: Fixed memory leak in SELECT QRESYNC error handling path. diffstat: src/imap/cmd-select.c | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diffs (24 lines): diff -r 118594ed7d46 -r 9f815e781beb src/imap/cmd-select.c --- a/src/imap/cmd-select.c Wed Aug 19 21:41:55 2015 +0300 +++ b/src/imap/cmd-select.c Thu Aug 20 16:04:48 2015 +0300 @@ -238,6 +238,7 @@ struct imap_fetch_context *fetch_ctx; struct mail_search_args *search_args; struct imap_fetch_qresync_args qresync_args; + int ret; search_args = mail_search_build_init(); search_args->args = p_new(search_args->pool, struct mail_search_arg, 1); @@ -273,10 +274,9 @@ ctx->cmd->context = ctx; return 0; } - if (imap_fetch_end(fetch_ctx) < 0) - return -1; + ret = imap_fetch_end(fetch_ctx); imap_fetch_free(&fetch_ctx); - return 1; + return ret < 0 ? -1 : 1; } static int From dovecot at dovecot.org Mon Aug 24 09:05:48 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 24 Aug 2015 09:05:48 +0000 Subject: dovecot-2.2: lib: connection_disconnect_reason() now returns the... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/984ce015e9d0 changeset: 18988:984ce015e9d0 user: Timo Sirainen date: Mon Aug 24 12:04:55 2015 +0300 description: lib: connection_disconnect_reason() now returns the full iostream error string. diffstat: src/lib/connection.c | 20 ++++++++++++++------ 1 files changed, 14 insertions(+), 6 deletions(-) diffs (33 lines): diff -r 9f815e781beb -r 984ce015e9d0 src/lib/connection.c --- a/src/lib/connection.c Thu Aug 20 16:04:48 2015 +0300 +++ b/src/lib/connection.c Mon Aug 24 12:04:55 2015 +0300 @@ -358,15 +358,23 @@ const char *connection_disconnect_reason(struct connection *conn) { - if (conn->input != NULL && conn->input->stream_errno != 0) + const char *errstr; + + if (conn->input != NULL && conn->input->stream_errno != 0) { errno = conn->input->stream_errno; - else if (conn->output != NULL && conn->output->stream_errno != 0) + errstr = i_stream_get_error(conn->input); + } else if (conn->output != NULL && conn->output->stream_errno != 0) { errno = conn->output->stream_errno; + errstr = o_stream_get_error(conn->output); + } else { + errno = 0; + errstr = ""; + } + + if (errno == 0 || errno == EPIPE) + return "Connection closed"; else - errno = 0; - - return errno == 0 || errno == EPIPE ? "Connection closed" : - t_strdup_printf("Connection closed: %m"); + return t_strdup_printf("Connection closed: %s", errstr); } void connection_switch_ioloop(struct connection *conn) From dovecot at dovecot.org Mon Aug 24 09:11:10 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 24 Aug 2015 09:11:10 +0000 Subject: dovecot-2.2: lib: Added io_stream_get_disconnect_reason() to ios... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ca2f5d3575b3 changeset: 18989:ca2f5d3575b3 user: Timo Sirainen date: Mon Aug 24 12:10:08 2015 +0300 description: lib: Added io_stream_get_disconnect_reason() to iostream.h diffstat: src/lib/Makefile.am | 1 + src/lib/iostream-private.h | 2 ++ src/lib/iostream.c | 24 ++++++++++++++++++++++++ src/lib/iostream.h | 8 ++++++++ 4 files changed, 35 insertions(+), 0 deletions(-) diffs (73 lines): diff -r 984ce015e9d0 -r ca2f5d3575b3 src/lib/Makefile.am --- a/src/lib/Makefile.am Mon Aug 24 12:04:55 2015 +0300 +++ b/src/lib/Makefile.am Mon Aug 24 12:10:08 2015 +0300 @@ -193,6 +193,7 @@ hostpid.h \ imem.h \ ipwd.h \ + iostream.h \ iostream-private.h \ iostream-rawlog.h \ iostream-rawlog-private.h \ diff -r 984ce015e9d0 -r ca2f5d3575b3 src/lib/iostream-private.h --- a/src/lib/iostream-private.h Mon Aug 24 12:04:55 2015 +0300 +++ b/src/lib/iostream-private.h Mon Aug 24 12:10:08 2015 +0300 @@ -1,6 +1,8 @@ #ifndef IOSTREAM_PRIVATE_H #define IOSTREAM_PRIVATE_H +#include "iostream.h" + /* This file is private to input stream and output stream implementations */ struct iostream_destroy_callback { diff -r 984ce015e9d0 -r ca2f5d3575b3 src/lib/iostream.c --- a/src/lib/iostream.c Mon Aug 24 12:04:55 2015 +0300 +++ b/src/lib/iostream.c Mon Aug 24 12:10:08 2015 +0300 @@ -2,6 +2,8 @@ #include "lib.h" #include "array.h" +#include "istream.h" +#include "ostream.h" #include "iostream-private.h" static void @@ -106,3 +108,25 @@ i_free(stream->error); stream->error = i_strdup_vprintf(fmt, args); } + +const char *io_stream_get_disconnect_reason(struct istream *input, + struct ostream *output) +{ + const char *errstr; + + if (input != NULL && input->stream_errno != 0) { + errno = input->stream_errno; + errstr = i_stream_get_error(input); + } else if (output != NULL && output->stream_errno != 0) { + errno = output->stream_errno; + errstr = o_stream_get_error(output); + } else { + errno = 0; + errstr = ""; + } + + if (errno == 0 || errno == EPIPE) + return "Connection closed"; + else + return t_strdup_printf("Connection closed: %s", errstr); +} diff -r 984ce015e9d0 -r ca2f5d3575b3 src/lib/iostream.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/iostream.h Mon Aug 24 12:10:08 2015 +0300 @@ -0,0 +1,8 @@ +#ifndef IOSTREAM_H +#define IOSTREAM_H + +/* Returns human-readable reason for why iostream was disconnected. */ +const char *io_stream_get_disconnect_reason(struct istream *input, + struct ostream *output); + +#endif From dovecot at dovecot.org Mon Aug 24 09:15:33 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 24 Aug 2015 09:15:33 +0000 Subject: dovecot-2.2: Use io_stream_get_disconnect_reason() instead of du... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cd2c95d82d4c changeset: 18990:cd2c95d82d4c user: Timo Sirainen date: Mon Aug 24 12:14:40 2015 +0300 description: Use io_stream_get_disconnect_reason() instead of duplicating its code all over the place. diffstat: src/imap/imap-client.c | 20 +++++--------------- src/lib/connection.c | 19 ++----------------- src/lmtp/client.c | 12 +++--------- src/login-common/client-common-auth.c | 10 ++-------- src/pop3/pop3-client.c | 19 +++++-------------- 5 files changed, 17 insertions(+), 63 deletions(-) diffs (187 lines): diff -r ca2f5d3575b3 -r cd2c95d82d4c src/imap/imap-client.c --- a/src/imap/imap-client.c Mon Aug 24 12:10:08 2015 +0300 +++ b/src/imap/imap-client.c Mon Aug 24 12:14:40 2015 +0300 @@ -6,6 +6,7 @@ #include "str.h" #include "hostpid.h" #include "net.h" +#include "iostream.h" #include "istream.h" #include "ostream.h" #include "time-util.h" @@ -248,19 +249,6 @@ return str_c(str); } -static const char *client_get_disconnect_reason(struct client *client) -{ - errno = client->input->stream_errno != 0 ? - client->input->stream_errno : - client->output->stream_errno; - if (errno == 0 || errno == EPIPE) - return "Connection closed"; - return t_strdup_printf("Connection closed: %s", - client->input->stream_errno != 0 ? - i_stream_get_error(client->input) : - o_stream_get_error(client->output)); -} - void client_destroy(struct client *client, const char *reason) { client->v.destroy(client, reason); @@ -275,8 +263,10 @@ if (!client->disconnected) { client->disconnected = TRUE; - if (reason == NULL) - reason = client_get_disconnect_reason(client); + if (reason == NULL) { + reason = io_stream_get_disconnect_reason(client->input, + client->output); + } i_info("%s %s", reason, client_stats(client)); } diff -r ca2f5d3575b3 -r cd2c95d82d4c src/lib/connection.c --- a/src/lib/connection.c Mon Aug 24 12:10:08 2015 +0300 +++ b/src/lib/connection.c Mon Aug 24 12:14:40 2015 +0300 @@ -4,6 +4,7 @@ #include "ioloop.h" #include "istream.h" #include "ostream.h" +#include "iostream.h" #include "net.h" #include "strescape.h" #include "llist.h" @@ -358,23 +359,7 @@ const char *connection_disconnect_reason(struct connection *conn) { - const char *errstr; - - if (conn->input != NULL && conn->input->stream_errno != 0) { - errno = conn->input->stream_errno; - errstr = i_stream_get_error(conn->input); - } else if (conn->output != NULL && conn->output->stream_errno != 0) { - errno = conn->output->stream_errno; - errstr = o_stream_get_error(conn->output); - } else { - errno = 0; - errstr = ""; - } - - if (errno == 0 || errno == EPIPE) - return "Connection closed"; - else - return t_strdup_printf("Connection closed: %s", errstr); + return io_stream_get_disconnect_reason(conn->input, conn->output); } void connection_switch_ioloop(struct connection *conn) diff -r ca2f5d3575b3 -r cd2c95d82d4c src/lmtp/client.c --- a/src/lmtp/client.c Mon Aug 24 12:10:08 2015 +0300 +++ b/src/lmtp/client.c Mon Aug 24 12:14:40 2015 +0300 @@ -5,6 +5,7 @@ #include "base64.h" #include "str.h" #include "llist.h" +#include "iostream.h" #include "istream.h" #include "ostream.h" #include "hostpid.h" @@ -323,16 +324,9 @@ err); } } - errno = client->input->stream_errno != 0 ? - client->input->stream_errno : - client->output->stream_errno; - if (errno == 0 || errno == EPIPE) - return "Connection closed"; - return t_strdup_printf("Connection closed: %s", - client->input->stream_errno != 0 ? - i_stream_get_error(client->input) : - o_stream_get_error(client->output)); + return io_stream_get_disconnect_reason(client->input, client->output); } + void client_disconnect(struct client *client, const char *prefix, const char *reason) { diff -r ca2f5d3575b3 -r cd2c95d82d4c src/login-common/client-common-auth.c --- a/src/login-common/client-common-auth.c Mon Aug 24 12:10:08 2015 +0300 +++ b/src/login-common/client-common-auth.c Mon Aug 24 12:14:40 2015 +0300 @@ -2,6 +2,7 @@ #include "hostpid.h" #include "login-common.h" +#include "iostream.h" #include "istream.h" #include "ostream.h" #include "str.h" @@ -216,13 +217,6 @@ client_auth_failed(client); } -static const char *get_disconnect_reason(struct istream *input) -{ - errno = input->stream_errno; - return errno == 0 || errno == EPIPE ? "Connection closed" : - t_strdup_printf("Connection closed: %m"); -} - static void proxy_input(struct client *client) { struct istream *input; @@ -262,7 +256,7 @@ "(state=%u, duration=%us)%s", login_proxy_get_host(client->login_proxy), login_proxy_get_port(client->login_proxy), - get_disconnect_reason(input), + io_stream_get_disconnect_reason(input, NULL), client->proxy_state, duration, line == NULL ? "" : t_strdup_printf( " - BUG: line not read: %s", line))); diff -r ca2f5d3575b3 -r cd2c95d82d4c src/pop3/pop3-client.c --- a/src/pop3/pop3-client.c Mon Aug 24 12:10:08 2015 +0300 +++ b/src/pop3/pop3-client.c Mon Aug 24 12:14:40 2015 +0300 @@ -4,6 +4,7 @@ #include "array.h" #include "ioloop.h" #include "net.h" +#include "iostream.h" #include "istream.h" #include "ostream.h" #include "crc32.h" @@ -556,18 +557,6 @@ return str_c(str); } -static const char *client_get_disconnect_reason(struct client *client) -{ - errno = client->input->stream_errno != 0 ? - client->input->stream_errno : - client->output->stream_errno; - if (errno == 0 || errno == EPIPE) - return "Connection closed"; - return t_strdup_printf("Connection closed: %s", - client->input->stream_errno != 0 ? - i_stream_get_error(client->input) : - o_stream_get_error(client->output)); -} void client_destroy(struct client *client, const char *reason) { client->v.destroy(client, reason); @@ -579,8 +568,10 @@ (void)client_update_mails(client); if (!client->disconnected) { - if (reason == NULL) - reason = client_get_disconnect_reason(client); + if (reason == NULL) { + reason = io_stream_get_disconnect_reason(client->input, + client->output); + } i_info("%s %s", reason, client_stats(client)); } From dovecot at dovecot.org Mon Aug 24 09:57:03 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 24 Aug 2015 09:57:03 +0000 Subject: dovecot-2.2: lib-storage: mail_user_var_expand_table() may not h... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/39710cec820b changeset: 18991:39710cec820b user: Timo Sirainen date: Mon Aug 24 12:56:07 2015 +0300 description: lib-storage: mail_user_var_expand_table() may not have returned %{auth_username} correctly diffstat: src/lib-storage/mail-user.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r cd2c95d82d4c -r 39710cec820b src/lib-storage/mail-user.c --- a/src/lib-storage/mail-user.c Mon Aug 24 12:14:40 2015 +0300 +++ b/src/lib-storage/mail-user.c Mon Aug 24 12:56:07 2015 +0300 @@ -248,7 +248,7 @@ tab[13].value = tab[2].value; } else { tab[11].value = user->auth_user; - tab[12].value = t_strcut(user->auth_user, '@'); + tab[12].value = p_strdup(user->pool, t_strcut(user->auth_user, '@')); tab[13].value = strchr(user->auth_user, '@'); } From dovecot at dovecot.org Mon Aug 24 11:05:19 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 24 Aug 2015 11:05:19 +0000 Subject: dovecot-2.2: lib-storage: mail_storage_service_user_free() needs... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cf620b80c380 changeset: 18992:cf620b80c380 user: Timo Sirainen date: Mon Aug 24 13:57:03 2015 +0300 description: lib-storage: mail_storage_service_user_free() needs to update log prefix if current user is freed. diffstat: src/lib-storage/mail-storage-service.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 39710cec820b -r cf620b80c380 src/lib-storage/mail-storage-service.c --- a/src/lib-storage/mail-storage-service.c Mon Aug 24 12:56:07 2015 +0300 +++ b/src/lib-storage/mail-storage-service.c Mon Aug 24 13:57:03 2015 +0300 @@ -1385,6 +1385,8 @@ io_loop_context_remove_callbacks(user->ioloop_ctx, mail_storage_service_io_activate_user, mail_storage_service_io_deactivate_user, user); + if (io_loop_get_current_context(current_ioloop) == user->ioloop_ctx) + mail_storage_service_io_deactivate_user(user); io_loop_context_unref(&user->ioloop_ctx); } settings_parser_deinit(&user->set_parser); From dovecot at dovecot.org Mon Aug 24 11:05:20 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 24 Aug 2015 11:05:20 +0000 Subject: dovecot-2.2: lib-storage: Added mailbox_recent_flags_set_uid_for... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4673b93b04c6 changeset: 18993:4673b93b04c6 user: Timo Sirainen date: Mon Aug 24 14:01:04 2015 +0300 description: lib-storage: Added mailbox_recent_flags_set_uid_forced() This allows adding recent flags to mails whose UID is lower than the existing recent mails. diffstat: src/lib-storage/mailbox-recent-flags.c | 5 +++++ src/lib-storage/mailbox-recent-flags.h | 1 + 2 files changed, 6 insertions(+), 0 deletions(-) diffs (26 lines): diff -r cf620b80c380 -r 4673b93b04c6 src/lib-storage/mailbox-recent-flags.c --- a/src/lib-storage/mailbox-recent-flags.c Mon Aug 24 13:57:03 2015 +0300 +++ b/src/lib-storage/mailbox-recent-flags.c Mon Aug 24 14:01:04 2015 +0300 @@ -16,6 +16,11 @@ array_clear(&box->recent_flags); box->recent_flags_count = 0; } + mailbox_recent_flags_set_uid_forced(box, uid); +} + +void mailbox_recent_flags_set_uid_forced(struct mailbox *box, uint32_t uid) +{ box->recent_flags_prev_uid = uid; seq_range_array_add_with_init(&box->recent_flags, 64, uid); diff -r cf620b80c380 -r 4673b93b04c6 src/lib-storage/mailbox-recent-flags.h --- a/src/lib-storage/mailbox-recent-flags.h Mon Aug 24 13:57:03 2015 +0300 +++ b/src/lib-storage/mailbox-recent-flags.h Mon Aug 24 14:01:04 2015 +0300 @@ -5,6 +5,7 @@ struct mail_index_view; void mailbox_recent_flags_set_uid(struct mailbox *box, uint32_t uid); +void mailbox_recent_flags_set_uid_forced(struct mailbox *box, uint32_t uid); void mailbox_recent_flags_set_seqs(struct mailbox *box, struct mail_index_view *view, uint32_t seq1, uint32_t seq2); From dovecot at dovecot.org Mon Aug 24 11:05:20 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 24 Aug 2015 11:05:20 +0000 Subject: dovecot-2.2: lib: Added io_loop_extract_notify_fd() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3083424cafcd changeset: 18994:3083424cafcd user: Timo Sirainen date: Mon Aug 24 14:02:52 2015 +0300 description: lib: Added io_loop_extract_notify_fd() diffstat: src/lib/ioloop-notify-inotify.c | 29 ++++++++++++++++++++++++++++- src/lib/ioloop-notify-kqueue.c | 27 +++++++++++++++++++++++++++ src/lib/ioloop-notify-none.c | 5 +++++ src/lib/ioloop.h | 8 ++++++++ 4 files changed, 68 insertions(+), 1 deletions(-) diffs (110 lines): diff -r 4673b93b04c6 -r 3083424cafcd src/lib/ioloop-notify-inotify.c --- a/src/lib/ioloop-notify-inotify.c Mon Aug 24 14:01:04 2015 +0300 +++ b/src/lib/ioloop-notify-inotify.c Mon Aug 24 14:02:52 2015 +0300 @@ -146,7 +146,7 @@ io_notify_fd_free(&ctx->fd_ctx, io); - if (ctx->fd_ctx.notifies == NULL) + if (ctx->fd_ctx.notifies == NULL && ctx->event_io != NULL) io_remove(&ctx->event_io); } @@ -211,4 +211,31 @@ i_free(ctx); } +int io_loop_extract_notify_fd(struct ioloop *ioloop) +{ + struct ioloop_notify_handler_context *ctx = + ioloop->notify_handler_context; + struct io_notify *io; + int fd, new_inotify_fd; + + if (ctx->inotify_fd == -1) + return -1; + + new_inotify_fd = inotify_init(); + if (new_inotify_fd == -1) { + if (errno != EMFILE) + i_error("inotify_init() failed: %m"); + else + ioloop_inotify_user_limit_exceeded(); + return -1; + } + for (io = ctx->fd_ctx.notifies; io != NULL; io = io->next) + io->fd = -1; + if (ctx->event_io != NULL) + io_remove(&ctx->event_io); + fd = ctx->inotify_fd; + ctx->inotify_fd = new_inotify_fd; + return fd; +} + #endif diff -r 4673b93b04c6 -r 3083424cafcd src/lib/ioloop-notify-kqueue.c --- a/src/lib/ioloop-notify-kqueue.c Mon Aug 24 14:01:04 2015 +0300 +++ b/src/lib/ioloop-notify-kqueue.c Mon Aug 24 14:02:52 2015 +0300 @@ -197,4 +197,31 @@ io_loop_notify_free(ctx, io); } +int io_loop_extract_notify_fd(struct ioloop *ioloop) +{ + struct ioloop_notify_handler_context *ctx = + ioloop->notify_handler_context; + struct io_notify *io; + int fd, new_kq; + + if (ctx->kq == -1) + return -1; + + new_kq = inotify_init(); + if (new_kq == -1) { + if (errno != EMFILE) + i_error("inotify_init() failed: %m"); + else + ioloop_inotify_user_limit_exceeded(); + return -1; + } + for (io = ctx->fd_ctx.notifies; io != NULL; io = io->next) + io->fd = -1; + if (ctx->event_io != NULL) + io_remove(&ctx->event_io); + fd = ctx->kq; + ctx->kq = new_kq; + return fd; +} + #endif diff -r 4673b93b04c6 -r 3083424cafcd src/lib/ioloop-notify-none.c --- a/src/lib/ioloop-notify-none.c Mon Aug 24 14:01:04 2015 +0300 +++ b/src/lib/ioloop-notify-none.c Mon Aug 24 14:02:52 2015 +0300 @@ -24,4 +24,9 @@ { } +int io_loop_extract_notify_fd(struct ioloop *ioloop ATTR_UNUSED) +{ + return -1; +} + #endif diff -r 4673b93b04c6 -r 3083424cafcd src/lib/ioloop.h --- a/src/lib/ioloop.h Mon Aug 24 14:01:04 2015 +0300 +++ b/src/lib/ioloop.h Mon Aug 24 14:02:52 2015 +0300 @@ -167,6 +167,14 @@ /* Returns the current context set to ioloop. */ struct ioloop_context *io_loop_get_current_context(struct ioloop *ioloop); +/* Returns fd, which contains all of the ioloop's current notifications. + When it becomes readable, there is a new notification. Calling this function + stops the existing notifications in the ioloop from working anymore. + This function's main idea is that the fd can be passed to another process, + which can use it to find out if an interesting notification happens. + Returns fd on success, -1 on error. */ +int io_loop_extract_notify_fd(struct ioloop *ioloop); + /* Move the given I/O into the current I/O loop if it's not already there. New I/O is returned, while the old one is freed. */ struct io *io_loop_move_io(struct io **io); From dovecot at dovecot.org Mon Aug 24 11:29:32 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 24 Aug 2015 11:29:32 +0000 Subject: dovecot-2.2: lib-storage: Added mailbox_watch_extract_notify_fd() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4da6efa3fd91 changeset: 18995:4da6efa3fd91 user: Timo Sirainen date: Mon Aug 24 14:14:59 2015 +0300 description: lib-storage: Added mailbox_watch_extract_notify_fd() diffstat: src/lib-storage/mailbox-watch.c | 49 +++++++++++++++++++++++++++++++++++++++++ src/lib-storage/mailbox-watch.h | 4 +++ 2 files changed, 53 insertions(+), 0 deletions(-) diffs (68 lines): diff -r 3083424cafcd -r 4da6efa3fd91 src/lib-storage/mailbox-watch.c --- a/src/lib-storage/mailbox-watch.c Mon Aug 24 14:02:52 2015 +0300 +++ b/src/lib-storage/mailbox-watch.c Mon Aug 24 14:14:59 2015 +0300 @@ -104,3 +104,52 @@ if (box->to_notify != NULL) timeout_remove(&box->to_notify); } + +static void notify_extract_callback(struct mailbox *box ATTR_UNUSED) +{ + i_unreached(); +} + +int mailbox_watch_extract_notify_fd(struct mailbox *box, const char **reason_r) +{ + struct ioloop *ioloop; + struct mailbox_notify_file *file; + struct io *io, *const *iop; + ARRAY(struct io *) temp_ios; + int ret; + bool failed = FALSE; + + /* add all the notify IOs to a new ioloop. */ + ioloop = io_loop_create(); + + t_array_init(&temp_ios, 8); + for (file = box->notify_files; file != NULL && !failed; file = file->next) { + switch (io_add_notify(file->path, notify_extract_callback, box, &io)) { + case IO_NOTIFY_ADDED: + array_append(&temp_ios, &io, 1); + break; + case IO_NOTIFY_NOTFOUND: + *reason_r = t_strdup_printf( + "%s not found - can't watch it", file->path); + failed = TRUE; + break; + case IO_NOTIFY_NOSUPPORT: + *reason_r = "Filesystem notifications not supported"; + failed = TRUE; + break; + } + } + if (failed) + ret = -1; + else { + ret = io_loop_extract_notify_fd(ioloop); + if (ret == -1) + *reason_r = "Couldn't extra notify fd"; + } + array_foreach(&temp_ios, iop) { + struct io *io = *iop; + io_remove(&io); + } + io_loop_destroy(&ioloop); + return ret; +} diff -r 3083424cafcd -r 4da6efa3fd91 src/lib-storage/mailbox-watch.h --- a/src/lib-storage/mailbox-watch.h Mon Aug 24 14:02:52 2015 +0300 +++ b/src/lib-storage/mailbox-watch.h Mon Aug 24 14:14:59 2015 +0300 @@ -4,4 +4,8 @@ void mailbox_watch_add(struct mailbox *box, const char *path); void mailbox_watch_remove_all(struct mailbox *box); +/* Create a new temporary ioloop, add all the watches back and call + io_loop_extract_notify_fd() on it. Returns fd on success, -1 on error. */ +int mailbox_watch_extract_notify_fd(struct mailbox *box, const char **reason_r); + #endif From dovecot at dovecot.org Mon Aug 24 11:29:32 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 24 Aug 2015 11:29:32 +0000 Subject: dovecot-2.2: Added imap-hibernate process for gathering IDLEing ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/64c73e6bd397 changeset: 18996:64c73e6bd397 user: Timo Sirainen date: Mon Aug 24 14:13:02 2015 +0300 description: Added imap-hibernate process for gathering IDLEing imap processes. imap_hibernate_timeout setting controls how quickly the connection is moved from imap process to imap-hibernate process. Some IMAP extensions like NOTIFY, SEARCH=CONTEXT and COMPRESS aren't supported yet. There's also a new X-STATE command, which can be used to export the current IMAP connection state to a string and later on imported to get back to the original state (a quick resync feature for IMAP clients). However, this command is disabled for now due to the current code being unoptimized for untrusted input. diffstat: .hgignore | 1 + configure.ac | 1 + src/Makefile.am | 1 + src/imap-hibernate/Makefile.am | 24 + src/imap-hibernate/imap-client.c | 530 ++++++++++++++++ src/imap-hibernate/imap-client.h | 35 + src/imap-hibernate/imap-hibernate-client.c | 262 +++++++ src/imap-hibernate/imap-hibernate-client.h | 9 + src/imap-hibernate/imap-hibernate-settings.c | 48 + src/imap-hibernate/imap-master-connection.c | 121 +++ src/imap-hibernate/imap-master-connection.h | 21 + src/imap-hibernate/main.c | 56 + src/imap/Makefile.am | 8 +- src/imap/cmd-idle.c | 49 +- src/imap/cmd-x-state.c | 68 ++ src/imap/imap-client-hibernate.c | 225 ++++++ src/imap/imap-client.c | 7 +- src/imap/imap-client.h | 19 + src/imap/imap-commands.c | 1 + src/imap/imap-commands.h | 1 + src/imap/imap-common.h | 7 + src/imap/imap-master-client.c | 321 +++++++++ src/imap/imap-master-client.h | 9 + src/imap/imap-settings.c | 8 +- src/imap/imap-settings.h | 1 + src/imap/imap-state.c | 889 +++++++++++++++++++++++++++ src/imap/imap-state.h | 30 + src/imap/main.c | 25 +- src/lib-storage/mail-user.c | 2 + src/plugins/imap-zlib/imap-zlib-plugin.c | 18 + 30 files changed, 2780 insertions(+), 17 deletions(-) diffs (truncated from 3226 to 300 lines): diff -r 4da6efa3fd91 -r 64c73e6bd397 .hgignore --- a/.hgignore Mon Aug 24 14:14:59 2015 +0300 +++ b/.hgignore Mon Aug 24 14:13:02 2015 +0300 @@ -68,6 +68,7 @@ src/dns/dns-client src/doveadm/doveadm src/doveadm/doveadm-server +src/imap-hibernate/imap-hibernate src/imap-login/imap-login src/imap-urlauth/imap-urlauth src/imap-urlauth/imap-urlauth-login diff -r 4da6efa3fd91 -r 64c73e6bd397 configure.ac --- a/configure.ac Mon Aug 24 14:14:59 2015 +0300 +++ b/configure.ac Mon Aug 24 14:13:02 2015 +0300 @@ -2931,6 +2931,7 @@ src/indexer/Makefile src/ipc/Makefile src/imap/Makefile +src/imap-hibernate/Makefile src/imap-login/Makefile src/imap-urlauth/Makefile src/login-common/Makefile diff -r 4da6efa3fd91 -r 64c73e6bd397 src/Makefile.am --- a/src/Makefile.am Mon Aug 24 14:14:59 2015 +0300 +++ b/src/Makefile.am Mon Aug 24 14:13:02 2015 +0300 @@ -37,6 +37,7 @@ ipc \ master \ login-common \ + imap-hibernate \ imap-login \ imap \ imap-urlauth \ diff -r 4da6efa3fd91 -r 64c73e6bd397 src/imap-hibernate/Makefile.am --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/imap-hibernate/Makefile.am Mon Aug 24 14:13:02 2015 +0300 @@ -0,0 +1,24 @@ +pkglibexecdir = $(libexecdir)/dovecot + +pkglibexec_PROGRAMS = imap-hibernate + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-master \ + -I$(top_srcdir)/src/lib-imap + +imap_hibernate_LDADD = $(LIBDOVECOT) +imap_hibernate_DEPENDENCIES = $(LIBDOVECOT_DEPS) + +imap_hibernate_SOURCES = \ + imap-client.c \ + imap-hibernate-client.c \ + imap-hibernate-settings.c \ + imap-master-connection.c \ + main.c + +noinst_HEADERS = \ + imap-client.h \ + imap-hibernate-client.h \ + imap-master-connection.h diff -r 4da6efa3fd91 -r 64c73e6bd397 src/imap-hibernate/imap-client.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/imap-hibernate/imap-client.c Mon Aug 24 14:13:02 2015 +0300 @@ -0,0 +1,530 @@ +/* Copyright (c) 2014-2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ioloop.h" +#include "fd-set-nonblock.h" +#include "fdpass.h" +#include "hostpid.h" +#include "connection.h" +#include "iostream.h" +#include "istream.h" +#include "ostream.h" +#include "llist.h" +#include "base64.h" +#include "str.h" +#include "strescape.h" +#include "var-expand.h" +#include "master-service.h" +#include "master-service-settings.h" +#include "imap-keepalive.h" +#include "imap-master-connection.h" +#include "imap-client.h" + +#include + +#define IMAP_MASTER_SOCKET_NAME "imap-master" + +/* we only need enough for "DONE\r\nIDLE\r\n" */ +#define IMAP_MAX_INBUF 12 + +enum imap_client_input_state { + IMAP_CLIENT_INPUT_STATE_UNKNOWN, + IMAP_CLIENT_INPUT_STATE_BAD, + IMAP_CLIENT_INPUT_STATE_DONE_LF, + IMAP_CLIENT_INPUT_STATE_DONE_CRLF, + IMAP_CLIENT_INPUT_STATE_DONEIDLE +}; + +struct imap_client_notify { + int fd; + struct io *io; +}; + +struct imap_client { + struct imap_client *prev, *next; + pool_t pool; + struct imap_client_state state; + ARRAY(struct imap_client_notify) notifys; + + int fd; + struct io *io; + struct istream *input; + struct timeout *to_keepalive; + struct imap_master_connection *master_conn; + struct ioloop_context *ioloop_ctx; + const char *log_prefix; + unsigned int imap_still_here_text_pos; + unsigned int next_read_threshold; + bool bad_done, idle_done; +}; + +static struct imap_client *imap_clients; +static const char imap_still_here_text[] = "* OK Still here\r\n"; + +static void imap_client_stop(struct imap_client *client); +void imap_client_destroy(struct imap_client **_client, const char *reason); +static void imap_client_add_idle_keepalive_timeout(struct imap_client *client); + +static void imap_client_disconnected(struct imap_client **_client) +{ + struct imap_client *client = *_client; + const char *reason; + + reason = io_stream_get_disconnect_reason(client->input, NULL); + imap_client_destroy(_client, reason); +} + +static void +imap_client_parse_userdb_fields(struct imap_client *client, + const char **auth_user_r) +{ + const char *const *field; + unsigned int i; + + *auth_user_r = NULL; + + if (client->state.userdb_fields == NULL) + return; + + field = t_strsplit_tabescaped(client->state.userdb_fields); + for (i = 0; field[i] != NULL; i++) { + if (strncmp(field[i], "auth_user=", 10) == 0) + *auth_user_r = field[i] + 10; + } +} + +static void +imap_client_move_back_send_callback(void *context, struct ostream *output) +{ + struct imap_client *client = context; + const struct imap_client_state *state = &client->state; + string_t *str = t_str_new(256); + const unsigned char *input_data; + size_t input_size; + ssize_t ret; + + str_append_tabescaped(str, state->username); + if (state->session_id != NULL) { + str_append(str, "\tsession="); + str_append_tabescaped(str, state->session_id); + } + if (state->local_ip.family != 0) + str_printfa(str, "\tlip=%s", net_ip2addr(&state->local_ip)); + if (state->remote_ip.family != 0) + str_printfa(str, "\trip=%s", net_ip2addr(&state->remote_ip)); + if (state->userdb_fields != NULL) { + str_append(str, "\tuserdb_fields="); + str_append_tabescaped(str, state->userdb_fields); + } + if (state->peer_ip.family != 0) + str_printfa(str, "\tpeer_ip=%s", net_ip2addr(&state->peer_ip)); + if (state->peer_port != 0) + str_printfa(str, "\tpeer_port=%u", state->peer_port); + if (state->state_size > 0) { + str_append(str, "\tstate="); + base64_encode(state->state, state->state_size, str); + } + input_data = i_stream_get_data(client->input, &input_size); + if (input_size > 0) { + str_append(str, "\tclient_input="); + base64_encode(input_data, input_size, str); + } + if (client->imap_still_here_text_pos != 0) { + str_append(str, "\tclient_output="); + base64_encode(imap_still_here_text + client->imap_still_here_text_pos, + sizeof(imap_still_here_text)-1 - client->imap_still_here_text_pos, + str); + } + if (client->idle_done) { + if (client->bad_done) + str_append(str, "\tbad-done"); + } else if (client->state.idle_cmd) { + /* IDLE continues after sending changes */ + str_append(str, "\tidle-continue"); + } + str_append_c(str, '\n'); + + /* send the fd first */ + ret = fd_send(o_stream_get_fd(output), client->fd, str_data(str), 1); + if (ret < 0) { + i_error("fd_send(%s) failed: %m", + o_stream_get_name(output)); + imap_client_destroy(&client, "Failed to recreate imap process"); + return; + } + i_assert(ret > 0); + o_stream_nsend(output, str_data(str) + 1, str_len(str) - 1); +} + +static void +imap_client_move_back_read_callback(void *context, const char *line) +{ + struct imap_client *client = context; + + if (line[0] != '+') { + /* failed - FIXME: retry later? */ + imap_client_destroy(&client, t_strdup_printf( + "Failed to recreate imap process: %s", line+1)); + } else { + imap_client_destroy(&client, NULL); + } +} + +static void imap_client_move_back(struct imap_client *client) +{ + const struct master_service_settings *master_set; + const char *path; + + imap_client_stop(client); + + master_set = master_service_settings_get(master_service); + path = t_strconcat(master_set->base_dir, + "/"IMAP_MASTER_SOCKET_NAME, NULL); + if (imap_master_connection_init(path, + imap_client_move_back_send_callback, + imap_client_move_back_read_callback, + client, &client->master_conn) < 0) { + /* failed to connect to the imap-master socket */ + imap_client_destroy(&client, "Failed to connect to master socket"); + } +} + +static enum imap_client_input_state +imap_client_input_parse(const unsigned char *data, size_t size) +{ + enum imap_client_input_state state = IMAP_CLIENT_INPUT_STATE_DONE_LF; + + /* skip over DONE[\r]\n */ + if (i_memcasecmp(data, "DONE", I_MIN(size, 4)) != 0) + return IMAP_CLIENT_INPUT_STATE_BAD; + if (size <= 4) + return IMAP_CLIENT_INPUT_STATE_UNKNOWN; + data += 4; size -= 4; + + if (data[0] == '\r') { + state = IMAP_CLIENT_INPUT_STATE_DONE_CRLF; + data++; size--; + } + if (size == 0) + return IMAP_CLIENT_INPUT_STATE_UNKNOWN; + if (data[0] != '\n') + return IMAP_CLIENT_INPUT_STATE_BAD; + data++; size--; + + /* skip over IDLE[\r]\n - checking this assumes that the DONE and IDLE + are sent in the same IP packet, otherwise we'll unnecessarily + recreate the imap process and immediately resume IDLE there. if this + becomes an issue we could add a small delay to the imap process + creation and wait for the IDLE command during it. */ + if (size <= 4 || i_memcasecmp(data, "IDLE", 4) != 0) + return state; + data += 4; size -= 4; + + if (data[0] == '\r') { + data++; size--; + } + return size == 1 && data[0] == '\n' ? + IMAP_CLIENT_INPUT_STATE_DONEIDLE : state; +} + +static void imap_client_input_idle_cmd(struct imap_client *client) +{ + const unsigned char *data; + size_t size; + int ret; From dovecot at dovecot.org Mon Aug 24 11:33:42 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 24 Aug 2015 11:33:42 +0000 Subject: dovecot-2.2: imap: Compiler warning fix to imap-hibernate commit Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/63ada4bc7508 changeset: 18997:63ada4bc7508 user: Timo Sirainen date: Mon Aug 24 14:32:52 2015 +0300 description: imap: Compiler warning fix to imap-hibernate commit diffstat: src/imap/imap-state.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 64c73e6bd397 -r 63ada4bc7508 src/imap/imap-state.c --- a/src/imap/imap-state.c Mon Aug 24 14:13:02 2015 +0300 +++ b/src/imap/imap-state.c Mon Aug 24 14:32:52 2015 +0300 @@ -798,7 +798,7 @@ }; static struct { - enum imap_state_type_public type; + enum imap_state_type_internal type; ssize_t (*import)(struct client *client, const unsigned char *data, size_t size, const char **error_r); } imap_states_internal[] = { From dovecot at dovecot.org Mon Aug 24 11:41:15 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 24 Aug 2015 11:41:15 +0000 Subject: dovecot-2.2: doveadm backup: Fixed assert-crash due to duplicate... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/196c76bdc11a changeset: 18998:196c76bdc11a user: Timo Sirainen date: Mon Aug 24 14:40:11 2015 +0300 description: doveadm backup: Fixed assert-crash due to duplicate 'R' in getopt args. diffstat: src/doveadm/doveadm-dsync.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 63ada4bc7508 -r 196c76bdc11a src/doveadm/doveadm-dsync.c --- a/src/doveadm/doveadm-dsync.c Mon Aug 24 14:32:52 2015 +0300 +++ b/src/doveadm/doveadm-dsync.c Mon Aug 24 14:40:11 2015 +0300 @@ -1036,7 +1036,7 @@ struct dsync_cmd_context *ctx; _ctx = cmd_dsync_alloc(); - _ctx->getopt_args = DSYNC_COMMON_GETOPT_ARGS"R"; + _ctx->getopt_args = DSYNC_COMMON_GETOPT_ARGS; ctx = (struct dsync_cmd_context *)_ctx; ctx->backup = TRUE; return _ctx; From dovecot at dovecot.org Mon Aug 24 12:01:04 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 24 Aug 2015 12:01:04 +0000 Subject: dovecot-2.2: dsync: Fixed memory leaks when importing attributes... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d5cddaa5ca92 changeset: 18999:d5cddaa5ca92 user: Timo Sirainen date: Mon Aug 24 14:59:52 2015 +0300 description: dsync: Fixed memory leaks when importing attributes whose values were in istreams. diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 23 ++++++++++++----------- 1 files changed, 12 insertions(+), 11 deletions(-) diffs (65 lines): diff -r 196c76bdc11a -r d5cddaa5ca92 src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Mon Aug 24 14:40:11 2015 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Mon Aug 24 14:59:52 2015 +0300 @@ -402,18 +402,15 @@ static int dsync_mailbox_import_attribute_real(struct dsync_mailbox_importer *importer, const struct dsync_mailbox_attribute *attr, + const struct dsync_mailbox_attribute *local_attr, const char **result_r) { - struct dsync_mailbox_attribute *local_attr; struct mail_attribute_value value; int cmp; bool ignore = FALSE; i_assert(DSYNC_ATTR_HAS_VALUE(attr) || attr->deleted); - if (dsync_mailbox_import_lookup_attr(importer, attr->type, - attr->key, &local_attr) < 0) - return -1; if (attr->deleted && (local_attr == NULL || !DSYNC_ATTR_HAS_VALUE(local_attr))) { /* attribute doesn't exist on either side -> ignore */ @@ -474,11 +471,8 @@ *result_r = "Value changed, but unknown which is newer - picking remote"; } } - if (ignore) { - if (local_attr->value_stream != NULL) - i_stream_unref(&local_attr->value_stream); + if (ignore) return 0; - } memset(&value, 0, sizeof(value)); value.value = attr->value; @@ -492,18 +486,25 @@ /* the attributes aren't vital, don't fail everything just because of them. */ } - if (local_attr != NULL && local_attr->value_stream != NULL) - i_stream_unref(&local_attr->value_stream); return 0; } int dsync_mailbox_import_attribute(struct dsync_mailbox_importer *importer, const struct dsync_mailbox_attribute *attr) { + struct dsync_mailbox_attribute *local_attr; const char *result; int ret; - ret = dsync_mailbox_import_attribute_real(importer, attr, &result); + if (dsync_mailbox_import_lookup_attr(importer, attr->type, + attr->key, &local_attr) < 0) + ret = -1; + else { + ret = dsync_mailbox_import_attribute_real(importer, attr, + local_attr, &result); + if (local_attr != NULL && local_attr->value_stream != NULL) + i_stream_unref(&local_attr->value_stream); + } imp_debug(importer, "Import attribute %s: %s", attr->key, ret < 0 ? "failed" : result); return ret; From dovecot at dovecot.org Mon Aug 24 12:34:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 24 Aug 2015 12:34:25 +0000 Subject: dovecot-2.2: dsync: Compiler warning fix to previous change. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/de8af4d857d3 changeset: 19000:de8af4d857d3 user: Timo Sirainen date: Mon Aug 24 15:33:36 2015 +0300 description: dsync: Compiler warning fix to previous change. diffstat: src/doveadm/dsync/dsync-mailbox-import.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r d5cddaa5ca92 -r de8af4d857d3 src/doveadm/dsync/dsync-mailbox-import.c --- a/src/doveadm/dsync/dsync-mailbox-import.c Mon Aug 24 14:59:52 2015 +0300 +++ b/src/doveadm/dsync/dsync-mailbox-import.c Mon Aug 24 15:33:36 2015 +0300 @@ -493,7 +493,7 @@ const struct dsync_mailbox_attribute *attr) { struct dsync_mailbox_attribute *local_attr; - const char *result; + const char *result = ""; int ret; if (dsync_mailbox_import_lookup_attr(importer, attr->type, From dovecot at dovecot.org Mon Aug 24 15:07:33 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 24 Aug 2015 15:07:33 +0000 Subject: dovecot-2.2: lib-master: struct master_service_connection.name m... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/acfe208a656d changeset: 19001:acfe208a656d user: Timo Sirainen date: Mon Aug 24 18:06:42 2015 +0300 description: lib-master: struct master_service_connection.name must never be NULL, use "" instead. Fixes crash at least in auth process when there are inet_listeners without names. diffstat: src/lib-master/master-service.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r de8af4d857d3 -r acfe208a656d src/lib-master/master-service.c --- a/src/lib-master/master-service.c Mon Aug 24 15:33:36 2015 +0300 +++ b/src/lib-master/master-service.c Mon Aug 24 18:06:42 2015 +0300 @@ -937,7 +937,7 @@ l->fd = -1; } conn.ssl = l->ssl; - conn.name = l->name; + conn.name = master_service_get_socket_name(service, conn.listen_fd); (void)net_getsockname(conn.fd, &conn.local_ip, &conn.local_port); conn.real_remote_ip = conn.remote_ip; From dovecot at dovecot.org Mon Aug 24 22:07:55 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 24 Aug 2015 22:07:55 +0000 Subject: dovecot-2.2: lib: Fixed io_loop_extract_notify_fd() to compile w... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f3e6256d3762 changeset: 19002:f3e6256d3762 user: Timo Sirainen date: Tue Aug 25 01:07:04 2015 +0300 description: lib: Fixed io_loop_extract_notify_fd() to compile with kqueue. diffstat: src/lib/ioloop-notify-kqueue.c | 11 ++++------- 1 files changed, 4 insertions(+), 7 deletions(-) diffs (23 lines): diff -r acfe208a656d -r f3e6256d3762 src/lib/ioloop-notify-kqueue.c --- a/src/lib/ioloop-notify-kqueue.c Mon Aug 24 18:06:42 2015 +0300 +++ b/src/lib/ioloop-notify-kqueue.c Tue Aug 25 01:07:04 2015 +0300 @@ -207,15 +207,12 @@ if (ctx->kq == -1) return -1; - new_kq = inotify_init(); - if (new_kq == -1) { - if (errno != EMFILE) - i_error("inotify_init() failed: %m"); - else - ioloop_inotify_user_limit_exceeded(); + new_kq = kqueue(); + if (new_kq < 0) { + i_error("kqueue(notify) failed: %m"); return -1; } - for (io = ctx->fd_ctx.notifies; io != NULL; io = io->next) + for (io = ctx->notifies; io != NULL; io = io->next) io->fd = -1; if (ctx->event_io != NULL) io_remove(&ctx->event_io); From dovecot at dovecot.org Tue Aug 25 13:51:10 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 25 Aug 2015 13:51:10 +0000 Subject: dovecot-2.2: imapc: If FETCH commands fail, have imapc_mail_fetc... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a98267f8a6e6 changeset: 19003:a98267f8a6e6 user: Timo Sirainen date: Tue Aug 25 16:50:17 2015 +0300 description: imapc: If FETCH commands fail, have imapc_mail_fetch() always return failure. Earlier it may have returned success and imapc would later complain about a missing FETCH field. diffstat: src/lib-storage/index/imapc/imapc-mail-fetch.c | 9 ++++++++- src/lib-storage/index/imapc/imapc-mail.h | 1 + 2 files changed, 9 insertions(+), 1 deletions(-) diffs (45 lines): diff -r f3e6256d3762 -r a98267f8a6e6 src/lib-storage/index/imapc/imapc-mail-fetch.c --- a/src/lib-storage/index/imapc/imapc-mail-fetch.c Tue Aug 25 01:07:04 2015 +0300 +++ b/src/lib-storage/index/imapc/imapc-mail-fetch.c Tue Aug 25 16:50:17 2015 +0300 @@ -28,6 +28,9 @@ struct imapc_mail *mail = *mailp; i_assert(mail->fetch_count > 0); + if (reply->state != IMAPC_COMMAND_STATE_OK && + reply->state != IMAPC_COMMAND_STATE_NO) + mail->fetch_failed = TRUE; if (--mail->fetch_count == 0) mail->fetching_fields = 0; pool_unref(&mail->imail.mail.pool); @@ -252,6 +255,7 @@ mail->fetching_fields |= fields; mail->fetch_count++; mail->fetch_sent = FALSE; + mail->fetch_failed = FALSE; imapc_mail_delayed_send_or_merge(mail, str); return 1; @@ -394,8 +398,11 @@ imapc_mail_fetch_flush(mbox); while (imail->fetch_count > 0 && (!imapc_mail_have_fields(imail, fields) || - !imail->header_list_fetched)) + !imail->header_list_fetched)) { imapc_mailbox_run_nofetch(mbox); + } + if (imail->fetch_failed) + return -1; return 0; } diff -r f3e6256d3762 -r a98267f8a6e6 src/lib-storage/index/imapc/imapc-mail.h --- a/src/lib-storage/index/imapc/imapc-mail.h Tue Aug 25 01:07:04 2015 +0300 +++ b/src/lib-storage/index/imapc/imapc-mail.h Tue Aug 25 16:50:17 2015 +0300 @@ -20,6 +20,7 @@ bool header_fetched; bool body_fetched; bool header_list_fetched; + bool fetch_failed; }; extern struct mail_vfuncs imapc_mail_vfuncs; From dovecot at dovecot.org Tue Aug 25 13:56:02 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 25 Aug 2015 13:56:02 +0000 Subject: dovecot-2.2: lib-imap-client: Removed unused pending_box_command... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e5bc6d90649b changeset: 19004:e5bc6d90649b user: Timo Sirainen date: Tue Aug 25 16:55:00 2015 +0300 description: lib-imap-client: Removed unused pending_box_command_count diffstat: src/lib-imap-client/imapc-client-private.h | 1 - src/lib-imap-client/imapc-connection.c | 1 - 2 files changed, 0 insertions(+), 2 deletions(-) diffs (22 lines): diff -r a98267f8a6e6 -r e5bc6d90649b src/lib-imap-client/imapc-client-private.h --- a/src/lib-imap-client/imapc-client-private.h Tue Aug 25 16:50:17 2015 +0300 +++ b/src/lib-imap-client/imapc-client-private.h Tue Aug 25 16:55:00 2015 +0300 @@ -35,7 +35,6 @@ void *reopen_context; void *untagged_box_context; - unsigned int pending_box_command_count; bool reconnect_ok; bool reconnecting; diff -r a98267f8a6e6 -r e5bc6d90649b src/lib-imap-client/imapc-connection.c --- a/src/lib-imap-client/imapc-connection.c Tue Aug 25 16:50:17 2015 +0300 +++ b/src/lib-imap-client/imapc-connection.c Tue Aug 25 16:55:00 2015 +0300 @@ -2059,7 +2059,6 @@ struct imapc_client_mailbox *box) { cmd->box = box; - box->pending_box_command_count++; } bool imapc_command_connection_is_selected(struct imapc_command *cmd) From dovecot at dovecot.org Tue Aug 25 13:59:05 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 25 Aug 2015 13:59:05 +0000 Subject: dovecot-2.2: lib-imap-client: Allow reconnecting to IMAP server ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/feef967b10fe changeset: 19005:feef967b10fe user: Timo Sirainen date: Tue Aug 25 16:57:57 2015 +0300 description: lib-imap-client: Allow reconnecting to IMAP server even if there had been no idle moments. diffstat: src/lib-imap-client/imapc-client.c | 8 ++++++++ src/lib-imap-client/imapc-connection.c | 17 +++++++++++++++++ src/lib-imap-client/imapc-connection.h | 1 + 3 files changed, 26 insertions(+), 0 deletions(-) diffs (87 lines): diff -r e5bc6d90649b -r feef967b10fe src/lib-imap-client/imapc-client.c --- a/src/lib-imap-client/imapc-client.c Tue Aug 25 16:55:00 2015 +0300 +++ b/src/lib-imap-client/imapc-client.c Tue Aug 25 16:57:57 2015 +0300 @@ -299,6 +299,7 @@ if (reply->state == IMAPC_COMMAND_STATE_OK) { /* reopen the mailbox */ box->reopen_callback(box->reopen_context); + imapc_connection_set_reconnected(box->conn); } else { imapc_connection_abort_commands(box->conn, NULL, FALSE); } @@ -306,6 +307,9 @@ bool imapc_client_mailbox_can_reconnect(struct imapc_client_mailbox *box) { + /* the reconnect_ok flag attempts to avoid infinite reconnection loops + to a server that keeps disconnecting us (e.g. some of the commands + we send keeps crashing it always) */ return box->reopen_callback != NULL && box->reconnect_ok; } @@ -322,6 +326,9 @@ imapc_connection_connect(box->conn, imapc_client_reconnect_cb, box); } + /* if we fail again, avoid reconnecting immediately. if the server is + broken we could just get into an infinitely failing reconnection + loop. */ box->reconnect_ok = FALSE; } @@ -393,6 +400,7 @@ timeout_add_short(IMAPC_CLIENT_IDLE_SEND_DELAY_MSECS, imapc_client_mailbox_idle_send, box); } + /* we're done with all work at this point. */ box->reconnect_ok = TRUE; } diff -r e5bc6d90649b -r feef967b10fe src/lib-imap-client/imapc-connection.c --- a/src/lib-imap-client/imapc-connection.c Tue Aug 25 16:55:00 2015 +0300 +++ b/src/lib-imap-client/imapc-connection.c Tue Aug 25 16:57:57 2015 +0300 @@ -110,6 +110,7 @@ ARRAY_TYPE(imapc_command) cmd_send_queue; /* commands that have been sent, waiting for their tagged reply */ ARRAY_TYPE(imapc_command) cmd_wait_list; + unsigned int reconnect_command_count; unsigned int ips_count, prev_connect_idx; struct ip_addr *ips; @@ -1303,6 +1304,16 @@ imapc_connection_unselect(conn->selected_box); } + if (conn->reconnect_command_count > 0) { + if (--conn->reconnect_command_count == 0) { + /* we've received replies for all the commands started + before reconnection. if we get disconnected now, we + can safely reconnect without worrying about infinite + reconnect loops. */ + if (conn->selected_box != NULL) + conn->selected_box->reconnect_ok = TRUE; + } + } imapc_connection_input_reset(conn); imapc_command_reply_free(cmd, &reply); imapc_command_send_more(conn); @@ -2216,3 +2227,9 @@ cmd->idle = TRUE; imapc_command_send(cmd, "IDLE"); } + +void imapc_connection_set_reconnected(struct imapc_connection *conn) +{ + conn->reconnect_command_count = array_count(&conn->cmd_wait_list) + + array_count(&conn->cmd_send_queue); +} diff -r e5bc6d90649b -r feef967b10fe src/lib-imap-client/imapc-connection.h --- a/src/lib-imap-client/imapc-connection.h Tue Aug 25 16:55:00 2015 +0300 +++ b/src/lib-imap-client/imapc-connection.h Tue Aug 25 16:57:57 2015 +0300 @@ -52,5 +52,6 @@ imapc_connection_get_mailbox(struct imapc_connection *conn); void imapc_connection_idle(struct imapc_connection *conn); +void imapc_connection_set_reconnected(struct imapc_connection *conn); #endif From dovecot at dovecot.org Tue Aug 25 14:04:17 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 25 Aug 2015 14:04:17 +0000 Subject: dovecot-2.2: lib-http: Added more debug logging. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5ec09852704a changeset: 19006:5ec09852704a user: Timo Sirainen date: Tue Aug 25 17:03:19 2015 +0300 description: lib-http: Added more debug logging. Patch by Stephan Bosch. diffstat: src/lib-http/http-client-peer.c | 11 ++++++++++- 1 files changed, 10 insertions(+), 1 deletions(-) diffs (38 lines): diff -r feef967b10fe -r 5ec09852704a src/lib-http/http-client-peer.c --- a/src/lib-http/http-client-peer.c Tue Aug 25 16:57:57 2015 +0300 +++ b/src/lib-http/http-client-peer.c Tue Aug 25 17:03:19 2015 +0300 @@ -220,6 +220,9 @@ /* disconnect if we're not linked to any queue anymore */ if (array_count(&peer->queues) == 0) { i_assert(peer->to_backoff != NULL); + http_client_peer_debug(peer, + "Peer no longer used; will now disconnect " + "(%u connections exist)", array_count(&peer->conns)); http_client_peer_disconnect(peer); return; } @@ -227,6 +230,9 @@ /* don't do anything unless we have pending requests */ num_pending = http_client_peer_requests_pending(peer, &num_urgent); if (num_pending == 0) { + http_client_peer_debug(peer, + "No requests to service for this peer " + "(%u connections exist)", array_count(&peer->conns)); http_client_peer_check_idle(peer); return; } @@ -296,10 +302,13 @@ } } } - + /* don't continue unless we have more pending requests */ num_pending = http_client_peer_requests_pending(peer, &num_urgent); if (num_pending == 0) { + http_client_peer_debug(peer, + "No more requests to service for this peer " + "(%u connections exist)", array_count(&peer->conns)); http_client_peer_check_idle(peer); return; } From dovecot at dovecot.org Tue Aug 25 15:28:37 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 25 Aug 2015 15:28:37 +0000 Subject: dovecot-2.2: imap: Fixed hibernation to work with non-TCP connec... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c3bea195a020 changeset: 19007:c3bea195a020 user: Timo Sirainen date: Tue Aug 25 18:27:44 2015 +0300 description: imap: Fixed hibernation to work with non-TCP connections. Mainly meaning UNIX socket connections from login processes, which are proxying TLS connections. diffstat: src/imap/imap-client-hibernate.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 5ec09852704a -r c3bea195a020 src/imap/imap-client-hibernate.c --- a/src/imap/imap-client-hibernate.c Tue Aug 25 17:03:19 2015 +0300 +++ b/src/imap/imap-client-hibernate.c Tue Aug 25 18:27:44 2015 +0300 @@ -49,7 +49,8 @@ str_append_tabescaped(cmd, client->user->set->mail_log_prefix); str_printfa(cmd, "\tidle_notify_interval=%u", client->set->imap_idle_notify_interval); - if (net_getpeername(client->fd_in, &peer_ip, &peer_port) == 0) { + if (net_getpeername(client->fd_in, &peer_ip, &peer_port) == 0 && + peer_port != 0) { str_printfa(cmd, "\tpeer_ip=%s\tpeer_port=%u", net_ip2addr(&peer_ip), peer_port); } From dovecot at dovecot.org Tue Aug 25 15:33:17 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 25 Aug 2015 15:33:17 +0000 Subject: dovecot-2.2: imap: Fail silently if selected mailbox format does... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/eeefd9b14eec changeset: 19008:eeefd9b14eec user: Timo Sirainen date: Tue Aug 25 18:32:23 2015 +0300 description: imap: Fail silently if selected mailbox format doesn't support GUIDs. diffstat: src/imap/imap-state.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (23 lines): diff -r c3bea195a020 -r eeefd9b14eec src/imap/imap-state.c --- a/src/imap/imap-state.c Tue Aug 25 18:27:44 2015 +0300 +++ b/src/imap/imap-state.c Tue Aug 25 18:32:23 2015 +0300 @@ -213,6 +213,7 @@ struct mailbox_status status; struct mailbox_metadata metadata; const char *vname = mailbox_get_vname(box); + enum mail_error mail_error; mailbox_get_open_status(box, STATUS_UIDVALIDITY | STATUS_UIDNEXT | STATUS_MESSAGES | STATUS_HIGHESTMODSEQ | @@ -224,8 +225,9 @@ } if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, &metadata) < 0) { - *error_r = mailbox_get_last_error(box, NULL); - return -1; + *error_r = mailbox_get_last_error(box, &mail_error); + /* if the selected mailbox can't have a GUID, fail silently */ + return mail_error == MAIL_ERROR_NOTPOSSIBLE ? 0 : -1; } buffer_append_c(dest, IMAP_STATE_TYPE_MAILBOX); From dovecot at dovecot.org Tue Aug 25 19:40:12 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 25 Aug 2015 19:40:12 +0000 Subject: dovecot-2.2: example-config: Updated mail_attribute_dict comment Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5f843b7da393 changeset: 19009:5f843b7da393 user: Timo Sirainen date: Tue Aug 25 22:39:16 2015 +0300 description: example-config: Updated mail_attribute_dict comment diffstat: doc/example-config/conf.d/10-mail.conf | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (14 lines): diff -r eeefd9b14eec -r 5f843b7da393 doc/example-config/conf.d/10-mail.conf --- a/doc/example-config/conf.d/10-mail.conf Tue Aug 25 18:32:23 2015 +0300 +++ b/doc/example-config/conf.d/10-mail.conf Tue Aug 25 22:39:16 2015 +0300 @@ -124,8 +124,8 @@ # or ~user/. #mail_full_filesystem_access = no -# Dictionary for key=value mailbox attributes. Currently used by URLAUTH, but -# soon intended to be used by METADATA as well. +# Dictionary for key=value mailbox attributes. This is used for example by +# URLAUTH and METADATA extensions. #mail_attribute_dict = ## From dovecot at dovecot.org Tue Aug 25 20:13:55 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 25 Aug 2015 20:13:55 +0000 Subject: dovecot-2.2: login_log_format_elements: Added %{listener} variab... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a919414e68d2 changeset: 19010:a919414e68d2 user: Timo Sirainen date: Tue Aug 25 23:12:51 2015 +0300 description: login_log_format_elements: Added %{listener} variable to expand to the listener socket name. diffstat: src/login-common/client-common.c | 5 ++++- src/login-common/client-common.h | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diffs (41 lines): diff -r 5f843b7da393 -r a919414e68d2 src/login-common/client-common.c --- a/src/login-common/client-common.c Tue Aug 25 22:39:16 2015 +0300 +++ b/src/login-common/client-common.c Tue Aug 25 23:12:51 2015 +0300 @@ -136,7 +136,8 @@ client->real_local_ip = conn->real_local_ip; client->real_local_port = conn->real_local_port; client->real_remote_ip = conn->real_remote_ip; - client->real_remote_port = conn->real_remote_port; + client->real_remote_port = conn->real_remote_port; + client->listener_name = p_strdup(client->pool, conn->name); client->trusted = client_is_trusted(client); client->secured = ssl || client->trusted || @@ -492,6 +493,7 @@ { '\0', NULL, "auth_user" }, { '\0', NULL, "auth_username" }, { '\0', NULL, "auth_domain" }, + { '\0', NULL, "listener" }, { '\0', NULL, NULL } }; @@ -565,6 +567,7 @@ tab[23].value = tab[20].value; tab[24].value = tab[21].value; } + tab[25].value = client->listener_name; return tab; } diff -r 5f843b7da393 -r a919414e68d2 src/login-common/client-common.h --- a/src/login-common/client-common.h Tue Aug 25 22:39:16 2015 +0300 +++ b/src/login-common/client-common.h Tue Aug 25 23:12:51 2015 +0300 @@ -113,7 +113,7 @@ struct ssl_proxy *ssl_proxy; const struct login_settings *set; const struct master_service_ssl_settings *ssl_set; - const char *session_id; + const char *session_id, *listener_name; int fd; struct istream *input; From dovecot at dovecot.org Tue Aug 25 20:21:39 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 25 Aug 2015 20:21:39 +0000 Subject: dovecot-2.2: imap: If mailbox state couldn't be exported for hib... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5220d361871e changeset: 19011:5220d361871e user: Timo Sirainen date: Tue Aug 25 23:20:49 2015 +0300 description: imap: If mailbox state couldn't be exported for hibernation, log also the mailbox name. diffstat: src/imap/imap-client-hibernate.c | 8 ++++++-- 1 files changed, 6 insertions(+), 2 deletions(-) diffs (20 lines): diff -r a919414e68d2 -r 5220d361871e src/imap/imap-client-hibernate.c --- a/src/imap/imap-client-hibernate.c Tue Aug 25 23:12:51 2015 +0300 +++ b/src/imap/imap-client-hibernate.c Tue Aug 25 23:20:49 2015 +0300 @@ -191,10 +191,14 @@ ret = imap_state_export_internal(client, state, &error); if (ret < 0) { i_error("Couldn't hibernate imap client: " - "Couldn't export state: %s", error); + "Couldn't export state: %s (mailbox=%s)", error, + client->mailbox == NULL ? "" : + mailbox_get_vname(client->mailbox)); } else if (ret == 0 && client->user->mail_debug) { i_debug("Couldn't hibernate imap client: " - "Couldn't export state: %s", error); + "Couldn't export state: %s (mailbox=%s)", error, + client->mailbox == NULL ? "" : + mailbox_get_vname(client->mailbox)); } if (ret > 0 && client->mailbox != NULL) { fd_notify = mailbox_watch_extract_notify_fd(client->mailbox, From dovecot at dovecot.org Wed Aug 26 11:41:35 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 26 Aug 2015 11:41:35 +0000 Subject: dovecot-2.2: dsync: Added assert to make sure end-of-list isn't ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/156aed1fc0fc changeset: 19012:156aed1fc0fc user: Timo Sirainen date: Wed Aug 26 13:38:52 2015 +0200 description: dsync: Added assert to make sure end-of-list isn't sent when brain has already failed. diffstat: src/doveadm/dsync/dsync-brain-mails.c | 15 +++++++++++---- 1 files changed, 11 insertions(+), 4 deletions(-) diffs (53 lines): diff -r 5220d361871e -r 156aed1fc0fc src/doveadm/dsync/dsync-brain-mails.c --- a/src/doveadm/dsync/dsync-brain-mails.c Tue Aug 25 23:20:49 2015 +0300 +++ b/src/doveadm/dsync/dsync-brain-mails.c Wed Aug 26 13:38:52 2015 +0200 @@ -90,6 +90,13 @@ return TRUE; } +static void dsync_brain_send_end_of_list(struct dsync_brain *brain, + enum dsync_ibc_eol_type type) +{ + i_assert(!brain->failed); + dsync_ibc_send_end_of_list(brain->ibc, type); +} + static void dsync_brain_send_mailbox_attribute(struct dsync_brain *brain) { const struct dsync_mailbox_attribute *attr; @@ -98,7 +105,7 @@ if (dsync_ibc_send_mailbox_attribute(brain->ibc, attr) == 0) return; } - dsync_ibc_send_end_of_list(brain->ibc, DSYNC_IBC_EOL_MAILBOX_ATTRIBUTE); + dsync_brain_send_end_of_list(brain, DSYNC_IBC_EOL_MAILBOX_ATTRIBUTE); brain->box_send_state = DSYNC_BOX_STATE_CHANGES; } @@ -130,7 +137,7 @@ if (dsync_ibc_send_change(brain->ibc, change) == 0) return; } - dsync_ibc_send_end_of_list(brain->ibc, DSYNC_IBC_EOL_MAIL_CHANGES); + dsync_brain_send_end_of_list(brain, DSYNC_IBC_EOL_MAIL_CHANGES); if (brain->mail_requests && brain->box_importer != NULL) brain->box_send_state = DSYNC_BOX_STATE_MAIL_REQUESTS; else @@ -170,7 +177,7 @@ if (brain->box_recv_state < DSYNC_BOX_STATE_MAIL_REQUESTS) return FALSE; - dsync_ibc_send_end_of_list(brain->ibc, DSYNC_IBC_EOL_MAIL_REQUESTS); + dsync_brain_send_end_of_list(brain, DSYNC_IBC_EOL_MAIL_REQUESTS); if (brain->box_exporter != NULL) brain->box_send_state = DSYNC_BOX_STATE_MAILS; else { @@ -274,7 +281,7 @@ } brain->box_send_state = DSYNC_BOX_STATE_DONE; - dsync_ibc_send_end_of_list(brain->ibc, DSYNC_IBC_EOL_MAILS); + dsync_brain_send_end_of_list(brain, DSYNC_IBC_EOL_MAILS); dsync_brain_sync_half_finished(brain); return TRUE; From dovecot at dovecot.org Wed Aug 26 13:06:17 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 26 Aug 2015 13:06:17 +0000 Subject: dovecot-2.2: dsync: If mailbox export deinit failed, the failure... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/adb69a13bc66 changeset: 19013:adb69a13bc66 user: Timo Sirainen date: Wed Aug 26 15:05:06 2015 +0200 description: dsync: If mailbox export deinit failed, the failure state may not have reached mailbox importer. The "end of list" was sent to the importer side dsync before export noticed the failure. This could have caused import to complain about missing mails. diffstat: src/doveadm/dsync/dsync-brain-mails.c | 25 ++++++++++++------------- 1 files changed, 12 insertions(+), 13 deletions(-) diffs (62 lines): diff -r 156aed1fc0fc -r adb69a13bc66 src/doveadm/dsync/dsync-brain-mails.c --- a/src/doveadm/dsync/dsync-brain-mails.c Wed Aug 26 13:38:52 2015 +0200 +++ b/src/doveadm/dsync/dsync-brain-mails.c Wed Aug 26 15:05:06 2015 +0200 @@ -190,25 +190,12 @@ static void dsync_brain_sync_half_finished(struct dsync_brain *brain) { struct dsync_mailbox_state state; - const char *errstr; - enum mail_error error; if (brain->box_recv_state < DSYNC_BOX_STATE_RECV_LAST_COMMON || brain->box_send_state < DSYNC_BOX_STATE_RECV_LAST_COMMON) return; /* finished with this mailbox */ - if (brain->box_exporter != NULL) { - if (dsync_mailbox_export_deinit(&brain->box_exporter, - &errstr, &error) < 0) { - i_error("Exporting mailbox %s failed: %s", - mailbox_get_vname(brain->box), errstr); - brain->mail_error = error; - brain->failed = TRUE; - return; - } - } - memset(&state, 0, sizeof(state)); memcpy(state.mailbox_guid, brain->local_dsync_box.mailbox_guid, sizeof(state.mailbox_guid)); @@ -249,6 +236,7 @@ return FALSE; if (ret == DSYNC_IBC_RECV_RET_FINISHED) { brain->box_recv_state = DSYNC_BOX_STATE_RECV_LAST_COMMON; + i_assert(brain->box_exporter == NULL); dsync_brain_sync_half_finished(brain); return TRUE; } @@ -265,6 +253,8 @@ static bool dsync_brain_send_mail(struct dsync_brain *brain) { const struct dsync_mail *mail; + const char *errstr; + enum mail_error error; if (brain->mail_requests && brain->box_recv_state < DSYNC_BOX_STATE_MAILS) { @@ -280,6 +270,15 @@ return TRUE; } + if (dsync_mailbox_export_deinit(&brain->box_exporter, + &errstr, &error) < 0) { + i_error("Exporting mailbox %s failed: %s", + mailbox_get_vname(brain->box), errstr); + brain->mail_error = error; + brain->failed = TRUE; + return TRUE; + } + brain->box_send_state = DSYNC_BOX_STATE_DONE; dsync_brain_send_end_of_list(brain, DSYNC_IBC_EOL_MAILS); From dovecot at dovecot.org Wed Aug 26 13:20:28 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 26 Aug 2015 13:20:28 +0000 Subject: dovecot-2.2: dsync: Added -T parameter to specify the I/O stall ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/11b107c8dc05 changeset: 19014:11b107c8dc05 user: Timo Sirainen date: Wed Aug 26 15:19:27 2015 +0200 description: dsync: Added -T parameter to specify the I/O stall timeout. diffstat: src/doveadm/doveadm-dsync.c | 20 ++++++++++++++++---- src/doveadm/dsync/dsync-ibc-stream.c | 10 ++++++---- src/doveadm/dsync/dsync-ibc.h | 3 ++- 3 files changed, 24 insertions(+), 9 deletions(-) diffs (154 lines): diff -r adb69a13bc66 -r 11b107c8dc05 src/doveadm/doveadm-dsync.c --- a/src/doveadm/doveadm-dsync.c Wed Aug 26 15:05:06 2015 +0200 +++ b/src/doveadm/doveadm-dsync.c Wed Aug 26 15:19:27 2015 +0200 @@ -38,7 +38,7 @@ #include #include -#define DSYNC_COMMON_GETOPT_ARGS "+1a:dEfg:l:m:n:NO:Pr:Rs:t:Ux:" +#define DSYNC_COMMON_GETOPT_ARGS "+1a:dEfg:l:m:n:NO:Pr:Rs:t:T:Ux:" #define DSYNC_REMOTE_CMD_EXIT_WAIT_SECS 30 /* The broken_char is mainly set to get a proper error message when trying to convert a mailbox with a name that can't be used properly translated between @@ -47,6 +47,8 @@ none of them are allowed to be created in regular mailbox names. */ #define DSYNC_LIST_BROKEN_CHAR '\003' +#define DSYNC_DEFAULT_IO_STREAM_TIMEOUT_SECS (60*10) + enum dsync_run_type { DSYNC_RUN_TYPE_LOCAL, DSYNC_RUN_TYPE_STREAM, @@ -64,6 +66,7 @@ ARRAY_TYPE(const_string) exclude_mailboxes; ARRAY_TYPE(const_string) namespace_prefixes; time_t sync_since_timestamp; + unsigned int io_timeout_secs; const char *remote_name; const char *local_location; @@ -485,7 +488,7 @@ i_stream_ref(ctx->input); o_stream_ref(ctx->output); return dsync_ibc_init_stream(ctx->input, ctx->output, - name, temp_prefix); + name, temp_prefix, ctx->io_timeout_secs); } static void @@ -1001,6 +1004,10 @@ if (mail_parse_human_timestamp(optarg, &ctx->sync_since_timestamp) < 0) i_fatal("Invalid -t parameter: %s", optarg); break; + case 'T': + if (str_to_uint(optarg, &ctx->io_timeout_secs) < 0) + i_fatal("Invalid -T parameter: %s", optarg); + break; case 'U': ctx->replicator_notify = TRUE; break; @@ -1015,6 +1022,7 @@ struct dsync_cmd_context *ctx; ctx = doveadm_mail_cmd_alloc(struct dsync_cmd_context); + ctx->io_timeout_secs = DSYNC_DEFAULT_IO_STREAM_TIMEOUT_SECS; ctx->ctx.getopt_args = DSYNC_COMMON_GETOPT_ARGS; ctx->ctx.v.parse_arg = cmd_mailbox_dsync_parse_arg; ctx->ctx.v.preinit = cmd_dsync_preinit; @@ -1036,7 +1044,6 @@ struct dsync_cmd_context *ctx; _ctx = cmd_dsync_alloc(); - _ctx->getopt_args = DSYNC_COMMON_GETOPT_ARGS; ctx = (struct dsync_cmd_context *)_ctx; ctx->backup = TRUE; return _ctx; @@ -1120,6 +1127,10 @@ case 'r': ctx->rawlog_path = optarg; break; + case 'T': + if (str_to_uint(optarg, &ctx->io_timeout_secs) < 0) + i_fatal("Invalid -T parameter: %s", optarg); + break; case 'U': ctx->replicator_notify = TRUE; break; @@ -1134,7 +1145,8 @@ struct dsync_cmd_context *ctx; ctx = doveadm_mail_cmd_alloc(struct dsync_cmd_context); - ctx->ctx.getopt_args = "Er:U"; + ctx->io_timeout_secs = DSYNC_DEFAULT_IO_STREAM_TIMEOUT_SECS; + ctx->ctx.getopt_args = "Er:T:U"; ctx->ctx.v.parse_arg = cmd_mailbox_dsync_server_parse_arg; ctx->ctx.v.run = cmd_dsync_server_run; ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_CHANGED; diff -r adb69a13bc66 -r 11b107c8dc05 src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Wed Aug 26 15:05:06 2015 +0200 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Wed Aug 26 15:19:27 2015 +0200 @@ -24,7 +24,6 @@ #include -#define DSYNC_IBC_STREAM_TIMEOUT_MSECS (60*10*1000) #define DSYNC_IBC_STREAM_OUTBUF_THROTTLE_SIZE (1024*128) #define DSYNC_PROTOCOL_VERSION_MAJOR 3 @@ -142,6 +141,7 @@ struct dsync_ibc ibc; char *name, *temp_path_prefix; + unsigned int timeout_secs; struct istream *input; struct ostream *output; struct io *io; @@ -291,7 +291,7 @@ static void dsync_ibc_stream_timeout(struct dsync_ibc_stream *ibc) { i_error("dsync(%s): I/O has stalled, no activity for %u seconds", - ibc->name, DSYNC_IBC_STREAM_TIMEOUT_MSECS/1000); + ibc->name, ibc->timeout_secs); ibc->ibc.timeout = TRUE; dsync_ibc_stream_stop(ibc); } @@ -303,7 +303,7 @@ ibc->io = io_add_istream(ibc->input, dsync_ibc_stream_input, ibc); o_stream_set_no_error_handling(ibc->output, TRUE); o_stream_set_flush_callback(ibc->output, dsync_ibc_stream_output, ibc); - ibc->to = timeout_add(DSYNC_IBC_STREAM_TIMEOUT_MSECS, + ibc->to = timeout_add(ibc->timeout_secs * 1000, dsync_ibc_stream_timeout, ibc); o_stream_cork(ibc->output); o_stream_nsend_str(ibc->output, DSYNC_HANDSHAKE_VERSION); @@ -1943,7 +1943,8 @@ struct dsync_ibc * dsync_ibc_init_stream(struct istream *input, struct ostream *output, - const char *name, const char *temp_path_prefix) + const char *name, const char *temp_path_prefix, + unsigned int timeout_secs) { struct dsync_ibc_stream *ibc; @@ -1953,6 +1954,7 @@ ibc->output = output; ibc->name = i_strdup(name); ibc->temp_path_prefix = i_strdup(temp_path_prefix); + ibc->timeout_secs = timeout_secs; ibc->ret_pool = pool_alloconly_create("ibc stream data", 2048); dsync_ibc_stream_init(ibc); return &ibc->ibc; diff -r adb69a13bc66 -r 11b107c8dc05 src/doveadm/dsync/dsync-ibc.h --- a/src/doveadm/dsync/dsync-ibc.h Wed Aug 26 15:05:06 2015 +0200 +++ b/src/doveadm/dsync/dsync-ibc.h Wed Aug 26 15:19:27 2015 +0200 @@ -70,7 +70,8 @@ struct dsync_ibc **ibc2_r); struct dsync_ibc * dsync_ibc_init_stream(struct istream *input, struct ostream *output, - const char *name, const char *temp_path_prefix); + const char *name, const char *temp_path_prefix, + unsigned int timeout_secs); void dsync_ibc_deinit(struct dsync_ibc **ibc); /* I/O callback is called whenever new data is available. It's also called on From dovecot at dovecot.org Wed Aug 26 13:45:37 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 26 Aug 2015 13:45:37 +0000 Subject: dovecot-2.2: dsync: Make sure we print all the stderr output fro... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/80efae622dc1 changeset: 19015:80efae622dc1 user: Timo Sirainen date: Wed Aug 26 15:44:45 2015 +0200 description: dsync: Make sure we print all the stderr output from remote processes at deinit. Earlier the final messages may have been lost, especially if debug logging was enabled. diffstat: src/doveadm/doveadm-dsync.c | 30 ++++++++++++++++++++---------- 1 files changed, 20 insertions(+), 10 deletions(-) diffs (62 lines): diff -r 11b107c8dc05 -r 80efae622dc1 src/doveadm/doveadm-dsync.c --- a/src/doveadm/doveadm-dsync.c Wed Aug 26 15:19:27 2015 +0200 +++ b/src/doveadm/doveadm-dsync.c Wed Aug 26 15:44:45 2015 +0200 @@ -118,6 +118,10 @@ case -1: if (ctx->io_err != NULL) io_remove(&ctx->io_err); + if (ctx->fd_in == -1) { + /* we're shutting down. */ + io_loop_stop(current_ioloop); + } break; default: while ((line = i_stream_next_line(ctx->err_stream)) != NULL) @@ -406,9 +410,18 @@ static void cmd_dsync_wait_remote(struct dsync_cmd_context *ctx, int *status_r) { - /* wait for the remote command to finish to see any final errors. - don't wait very long though. */ - alarm(DSYNC_REMOTE_CMD_EXIT_WAIT_SECS); + struct timeout *to; + + /* wait for stderr to close. this indicates that the remote process + has died. while we're running we're also reading and printing all + errors that still coming from it. */ + to = timeout_add(DSYNC_REMOTE_CMD_EXIT_WAIT_SECS*1000, + io_loop_stop, current_ioloop); + io_loop_run(current_ioloop); + timeout_remove(&to); + + /* unless we timed out, the process should be dead now or very soon. */ + alarm(1); if (waitpid(ctx->remote_pid, status_r, 0) == -1) { if (errno != EINTR) { i_error("waitpid(%ld) failed: %m", @@ -664,21 +677,18 @@ i_close_fd(&ctx->fd_out); i_close_fd(&ctx->fd_in); } - if (ctx->run_type == DSYNC_RUN_TYPE_CMD) - cmd_dsync_wait_remote(ctx, &status); - /* print any final errors after the process has died. not closing stdin/stdout before wait() may cause the process to hang, but stderr shouldn't (at least with ssh) and we need stderr to be open to be able to print the final errors */ - if (ctx->err_stream != NULL) { - remote_error_input(ctx); + if (ctx->run_type == DSYNC_RUN_TYPE_CMD) { + cmd_dsync_wait_remote(ctx, &status); remote_errors_logged = ctx->err_stream->v_offset > 0; i_stream_destroy(&ctx->err_stream); - } - if (ctx->run_type == DSYNC_RUN_TYPE_CMD) { cmd_dsync_log_remote_status(status, remote_errors_logged, ctx->remote_cmd_args); + } else { + i_assert(ctx->err_stream == NULL); } if (ctx->io_err != NULL) io_remove(&ctx->io_err); From dovecot at dovecot.org Wed Aug 26 14:59:59 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 26 Aug 2015 14:59:59 +0000 Subject: dovecot-2.2: dsync: Earlier end-of-list change broke single-proc... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ac062c048240 changeset: 19016:ac062c048240 user: Timo Sirainen date: Wed Aug 26 16:59:07 2015 +0200 description: dsync: Earlier end-of-list change broke single-process dsync. diffstat: src/doveadm/dsync/dsync-brain-mails.c | 31 +++++++++++++++++++++---------- 1 files changed, 21 insertions(+), 10 deletions(-) diffs (63 lines): diff -r 80efae622dc1 -r ac062c048240 src/doveadm/dsync/dsync-brain-mails.c --- a/src/doveadm/dsync/dsync-brain-mails.c Wed Aug 26 15:44:45 2015 +0200 +++ b/src/doveadm/dsync/dsync-brain-mails.c Wed Aug 26 16:59:07 2015 +0200 @@ -187,6 +187,22 @@ return TRUE; } +static int dsync_brain_export_deinit(struct dsync_brain *brain) +{ + const char *errstr; + enum mail_error error; + + if (dsync_mailbox_export_deinit(&brain->box_exporter, + &errstr, &error) < 0) { + i_error("Exporting mailbox %s failed: %s", + mailbox_get_vname(brain->box), errstr); + brain->mail_error = error; + brain->failed = TRUE; + return -1; + } + return 0; +} + static void dsync_brain_sync_half_finished(struct dsync_brain *brain) { struct dsync_mailbox_state state; @@ -236,7 +252,10 @@ return FALSE; if (ret == DSYNC_IBC_RECV_RET_FINISHED) { brain->box_recv_state = DSYNC_BOX_STATE_RECV_LAST_COMMON; - i_assert(brain->box_exporter == NULL); + if (brain->box_exporter != NULL) { + if (dsync_brain_export_deinit(brain) < 0) + return TRUE; + } dsync_brain_sync_half_finished(brain); return TRUE; } @@ -253,8 +272,6 @@ static bool dsync_brain_send_mail(struct dsync_brain *brain) { const struct dsync_mail *mail; - const char *errstr; - enum mail_error error; if (brain->mail_requests && brain->box_recv_state < DSYNC_BOX_STATE_MAILS) { @@ -270,14 +287,8 @@ return TRUE; } - if (dsync_mailbox_export_deinit(&brain->box_exporter, - &errstr, &error) < 0) { - i_error("Exporting mailbox %s failed: %s", - mailbox_get_vname(brain->box), errstr); - brain->mail_error = error; - brain->failed = TRUE; + if (dsync_brain_export_deinit(brain) < 0) return TRUE; - } brain->box_send_state = DSYNC_BOX_STATE_DONE; dsync_brain_send_end_of_list(brain, DSYNC_IBC_EOL_MAILS); From dovecot at dovecot.org Wed Aug 26 15:02:09 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 26 Aug 2015 15:02:09 +0000 Subject: dovecot-2.2: imapc: Fixed handling escape-char Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f363a7665a14 changeset: 19018:f363a7665a14 user: Timo Sirainen date: Wed Aug 26 17:00:55 2015 +0200 description: imapc: Fixed handling escape-char The current code should now handle all kinds of mailbox names correctly, including: ~/foo %7e/bar diffstat: src/lib-storage/index/imapc/imapc-list.c | 1 + src/lib-storage/index/imapc/imapc-save.c | 4 +++- src/lib-storage/index/imapc/imapc-storage.c | 22 ++++++++++++++-------- 3 files changed, 18 insertions(+), 9 deletions(-) diffs (71 lines): diff -r 0147397d9504 -r f363a7665a14 src/lib-storage/index/imapc/imapc-list.c --- a/src/lib-storage/index/imapc/imapc-list.c Wed Aug 26 16:59:47 2015 +0200 +++ b/src/lib-storage/index/imapc/imapc-list.c Wed Aug 26 17:00:55 2015 +0200 @@ -176,6 +176,7 @@ flags++; } + name = mailbox_list_escape_name(&list->list, name); T_BEGIN { const char *vname = mailbox_list_get_vname(&list->list, name); diff -r 0147397d9504 -r f363a7665a14 src/lib-storage/index/imapc/imapc-save.c --- a/src/lib-storage/index/imapc/imapc-save.c Wed Aug 26 16:59:47 2015 +0200 +++ b/src/lib-storage/index/imapc/imapc-save.c Wed Aug 26 17:00:55 2015 +0200 @@ -10,6 +10,7 @@ #include "index-mail.h" #include "mail-copy.h" #include "imapc-client.h" +#include "mailbox-list-private.h" #include "imapc-storage.h" #include "imapc-sync.h" #include "imapc-mail.h" @@ -245,7 +246,8 @@ cmd = imapc_client_cmd(ctx->mbox->storage->client->client, imapc_save_callback, &sctx); imapc_command_sendf(cmd, "APPEND %s%1s%1s %p", - ctx->mbox->box.name, flags, internaldate, input); + mailbox_list_unescape_name(ctx->mbox->box.list, ctx->mbox->box.name), + flags, internaldate, input); i_stream_unref(&input); while (sctx.ret == -2) imapc_mailbox_run(ctx->mbox); diff -r 0147397d9504 -r f363a7665a14 src/lib-storage/index/imapc/imapc-storage.c --- a/src/lib-storage/index/imapc/imapc-storage.c Wed Aug 26 16:59:47 2015 +0200 +++ b/src/lib-storage/index/imapc/imapc-storage.c Wed Aug 26 17:00:55 2015 +0200 @@ -471,10 +471,13 @@ cmd = imapc_client_mailbox_cmd(mbox->client_box, imapc_mailbox_reopen_callback, mbox); imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_SELECT); - if (imapc_mailbox_want_examine(mbox)) - imapc_command_sendf(cmd, "EXAMINE %s", mbox->box.name); - else - imapc_command_sendf(cmd, "SELECT %s", mbox->box.name); + if (imapc_mailbox_want_examine(mbox)) { + imapc_command_sendf(cmd, "EXAMINE %s", + mailbox_list_unescape_name(mbox->box.list, mbox->box.name)); + } else { + imapc_command_sendf(cmd, "SELECT %s", + mailbox_list_unescape_name(mbox->box.list, mbox->box.name)); + } mbox->storage->reopen_count++; if (mbox->syncing) @@ -546,10 +549,13 @@ cmd = imapc_client_mailbox_cmd(mbox->client_box, imapc_mailbox_open_callback, &ctx); imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_SELECT); - if (imapc_mailbox_want_examine(mbox)) - imapc_command_sendf(cmd, "EXAMINE %s", mbox->box.name); - else - imapc_command_sendf(cmd, "SELECT %s", mbox->box.name); + if (imapc_mailbox_want_examine(mbox)) { + imapc_command_sendf(cmd, "EXAMINE %s", + mailbox_list_unescape_name(mbox->box.list, mbox->box.name)); + } else { + imapc_command_sendf(cmd, "SELECT %s", + mailbox_list_unescape_name(mbox->box.list, mbox->box.name)); + } while (ctx.ret == -2) imapc_mailbox_run(mbox); From dovecot at dovecot.org Wed Aug 26 15:01:53 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 26 Aug 2015 15:01:53 +0000 Subject: dovecot-2.2: lib-storage: mailbox_list_[un]escape_name() can now... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0147397d9504 changeset: 19017:0147397d9504 user: Timo Sirainen date: Wed Aug 26 16:59:47 2015 +0200 description: lib-storage: mailbox_list_[un]escape_name() can now be called globally diffstat: src/lib-storage/mailbox-list-private.h | 4 ++++ src/lib-storage/mailbox-list.c | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diffs (35 lines): diff -r ac062c048240 -r 0147397d9504 src/lib-storage/mailbox-list-private.h --- a/src/lib-storage/mailbox-list-private.h Wed Aug 26 16:59:07 2015 +0200 +++ b/src/lib-storage/mailbox-list-private.h Wed Aug 26 16:59:47 2015 +0200 @@ -175,6 +175,10 @@ int mailbox_list_settings_parse(struct mail_user *user, const char *data, struct mailbox_list_settings *set_r, const char **error_r); +const char * +mailbox_list_escape_name(struct mailbox_list *list, const char *vname); +const char * +mailbox_list_unescape_name(struct mailbox_list *list, const char *src); const char *mailbox_list_default_get_storage_name(struct mailbox_list *list, const char *vname); const char *mailbox_list_default_get_vname(struct mailbox_list *list, diff -r ac062c048240 -r 0147397d9504 src/lib-storage/mailbox-list.c --- a/src/lib-storage/mailbox-list.c Wed Aug 26 16:59:07 2015 +0200 +++ b/src/lib-storage/mailbox-list.c Wed Aug 26 16:59:47 2015 +0200 @@ -426,7 +426,7 @@ return FALSE; } -static const char * +const char * mailbox_list_escape_name(struct mailbox_list *list, const char *vname) { char ns_sep = mail_namespace_get_sep(list->ns); @@ -594,7 +594,7 @@ return list->v.get_storage_name(list, vname); } -static const char * +const char * mailbox_list_unescape_name(struct mailbox_list *list, const char *src) { char ns_sep = mail_namespace_get_sep(list->ns); From dovecot at dovecot.org Thu Aug 27 08:41:28 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 27 Aug 2015 08:41:28 +0000 Subject: dovecot-2.2: ioloop: Delay actual start of a new normal timeout ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ec6e672a6e32 changeset: 19019:ec6e672a6e32 user: Stephan Bosch date: Thu Aug 27 10:39:26 2015 +0200 description: ioloop: Delay actual start of a new normal timeout until the next io_loop_run() cycle. This makes sure that timeouts will not expire before they get a chance to run. diffstat: src/lib/Makefile.am | 1 + src/lib/ioloop-private.h | 2 + src/lib/ioloop.c | 57 +++++++++++++++++++++++++++++++++++++++++++++-- src/lib/test-ioloop.c | 42 +++++++++++++++++++++++++++++++++++ src/lib/test-lib.c | 1 + src/lib/test-lib.h | 1 + 6 files changed, 101 insertions(+), 3 deletions(-) diffs (219 lines): diff -r f363a7665a14 -r ec6e672a6e32 src/lib/Makefile.am --- a/src/lib/Makefile.am Wed Aug 26 17:00:55 2015 +0200 +++ b/src/lib/Makefile.am Thu Aug 27 10:39:26 2015 +0200 @@ -304,6 +304,7 @@ test-hash-format.c \ test-hash-method.c \ test-hex-binary.c \ + test-ioloop.c \ test-iso8601-date.c \ test-istream.c \ test-istream-base64-decoder.c \ diff -r f363a7665a14 -r ec6e672a6e32 src/lib/ioloop-private.h --- a/src/lib/ioloop-private.h Wed Aug 26 17:00:55 2015 +0200 +++ b/src/lib/ioloop-private.h Thu Aug 27 10:39:26 2015 +0200 @@ -3,6 +3,7 @@ #include "priorityq.h" #include "ioloop.h" +#include "array-decl.h" #ifndef IOLOOP_INITIAL_FD_COUNT # define IOLOOP_INITIAL_FD_COUNT 128 @@ -16,6 +17,7 @@ struct io_file *io_files; struct io_file *next_io_file; struct priorityq *timeouts; + ARRAY(struct timeout *) timeouts_new; struct ioloop_handler_context *handler_context; struct ioloop_notify_handler_context *notify_handler_context; diff -r f363a7665a14 -r ec6e672a6e32 src/lib/ioloop.c --- a/src/lib/ioloop.c Wed Aug 26 17:00:55 2015 +0200 +++ b/src/lib/ioloop.c Thu Aug 27 10:39:26 2015 +0200 @@ -204,7 +204,8 @@ struct timeout *timeout; timeout = i_new(struct timeout, 1); - timeout->source_linenum = source_linenum; + timeout->item.idx = UINT_MAX; + timeout->source_linenum = source_linenum; timeout->ioloop = current_ioloop; timeout->callback = callback; @@ -227,9 +228,15 @@ timeout = timeout_add_common(source_linenum, callback, context); timeout->msecs = msecs; - timeout_update_next(timeout, timeout->ioloop->running ? + if (msecs > 0) { + /* start this timeout in the next run cycle */ + array_append(&timeout->ioloop->timeouts_new, &timeout, 1); + } else { + /* trigger zero timeouts as soon as possible */ + timeout_update_next(timeout, timeout->ioloop->running ? NULL : &ioloop_timeval); - priorityq_add(timeout->ioloop->timeouts, &timeout->item); + priorityq_add(timeout->ioloop->timeouts, &timeout->item); + } return timeout; } @@ -282,10 +289,21 @@ void timeout_remove(struct timeout **_timeout) { struct timeout *timeout = *_timeout; + struct ioloop *ioloop = timeout->ioloop; *_timeout = NULL; if (timeout->item.idx != UINT_MAX) priorityq_remove(timeout->ioloop->timeouts, &timeout->item); + else { + struct timeout *const *to_idx; + array_foreach(&ioloop->timeouts_new, to_idx) { + if (*to_idx == timeout) { + array_delete(&ioloop->timeouts_new, + array_foreach_idx(&ioloop->timeouts_new, to_idx), 1); + break; + } + } + } timeout_free(timeout); } @@ -393,6 +411,27 @@ } } +static void io_loop_timeouts_start_new(struct ioloop *ioloop) +{ + struct timeout *const *to_idx; + + if (array_count(&ioloop->timeouts_new) == 0) + return; + + io_loop_time_refresh(); + + array_foreach(&ioloop->timeouts_new, to_idx) { + struct timeout *timeout = *to_idx; + i_assert(timeout->next_run.tv_sec == 0 && + timeout->next_run.tv_usec == 0); + i_assert(!timeout->one_shot); + i_assert(timeout->msecs > 0); + timeout_update_next(timeout, &ioloop_timeval); + priorityq_add(ioloop->timeouts, &timeout->item); + } + array_clear(&ioloop->timeouts_new); +} + static void io_loop_timeouts_update(struct ioloop *ioloop, long diff_secs) { struct priorityq_item *const *items; @@ -545,6 +584,7 @@ void io_loop_handler_run(struct ioloop *ioloop) { + io_loop_timeouts_start_new(ioloop); io_loop_handler_run_internal(ioloop); io_loop_call_pending(ioloop); } @@ -587,6 +627,7 @@ ioloop = i_new(struct ioloop, 1); ioloop->timeouts = priorityq_init(timeout_cmp, 32); + i_array_init(&ioloop->timeouts_new, 8); ioloop->time_moved_callback = current_ioloop != NULL ? current_ioloop->time_moved_callback : @@ -600,6 +641,7 @@ void io_loop_destroy(struct ioloop **_ioloop) { struct ioloop *ioloop = *_ioloop; + struct timeout *const *to_idx; struct priorityq_item *item; *_ioloop = NULL; @@ -622,6 +664,15 @@ } i_assert(ioloop->io_pending_count == 0); + array_foreach(&ioloop->timeouts_new, to_idx) { + struct timeout *to = *to_idx; + + i_warning("Timeout leak: %p (line %u)", (void *)to->callback, + to->source_linenum); + timeout_free(to); + } + array_free(&ioloop->timeouts_new); + while ((item = priorityq_pop(ioloop->timeouts)) != NULL) { struct timeout *to = (struct timeout *)item; diff -r f363a7665a14 -r ec6e672a6e32 src/lib/test-ioloop.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/test-ioloop.c Thu Aug 27 10:39:26 2015 +0200 @@ -0,0 +1,42 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "test-lib.h" +#include "time-util.h" +#include "ioloop.h" + +#include + +static void timeout_callback(struct timeval *tv) +{ + if (gettimeofday(tv, NULL) < 0) + i_fatal("gettimeofday() failed: %m"); + io_loop_stop(current_ioloop); +} + +static void test_ioloop_timeout(void) +{ + struct ioloop *ioloop; + struct timeout *to; + struct timeval tv_start, tv_callback; + + test_begin("ioloop timeout"); + + ioloop = io_loop_create(); + sleep(1); + to = timeout_add(1000, timeout_callback, &tv_callback); + timeout_remove(&to); + to = timeout_add(1000, timeout_callback, &tv_callback); + if (gettimeofday(&tv_start, NULL) < 0) + i_fatal("gettimeofday() failed: %m"); + io_loop_run(ioloop); + test_assert(timeval_diff_msecs(&tv_callback, &tv_start) >= 500); + timeout_remove(&to); + io_loop_destroy(&ioloop); + + test_end(); +} + +void test_ioloop(void) +{ + test_ioloop_timeout(); +} diff -r f363a7665a14 -r ec6e672a6e32 src/lib/test-lib.c --- a/src/lib/test-lib.c Wed Aug 26 17:00:55 2015 +0200 +++ b/src/lib/test-lib.c Thu Aug 27 10:39:26 2015 +0200 @@ -20,6 +20,7 @@ test_hash_format, test_hash_method, test_hex_binary, + test_ioloop, test_iso8601_date, test_istream, test_istream_base64_decoder, diff -r f363a7665a14 -r ec6e672a6e32 src/lib/test-lib.h --- a/src/lib/test-lib.h Wed Aug 26 17:00:55 2015 +0200 +++ b/src/lib/test-lib.h Thu Aug 27 10:39:26 2015 +0200 @@ -21,6 +21,7 @@ void test_hash_format(void); void test_hash_method(void); void test_hex_binary(void); +void test_ioloop(void); void test_iso8601_date(void); void test_istream(void); void test_istream_base64_decoder(void); From dovecot at dovecot.org Thu Aug 27 10:34:39 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 27 Aug 2015 10:34:39 +0000 Subject: dovecot-2.2: dsync: Fixed again waiting for remote process wait ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/42d4da9ee7a9 changeset: 19020:42d4da9ee7a9 user: Timo Sirainen date: Thu Aug 27 12:33:47 2015 +0200 description: dsync: Fixed again waiting for remote process wait to die. We can't rely on stderr getting closed. It doesn't happen if the remote process crashes. Now waiting for SIGCHLD in ioloop should solve this and still log all the error messages at exit. diffstat: src/doveadm/doveadm-dsync.c | 55 ++++++++++++++++++++++++-------------------- 1 files changed, 30 insertions(+), 25 deletions(-) diffs (141 lines): diff -r ec6e672a6e32 -r 42d4da9ee7a9 src/doveadm/doveadm-dsync.c --- a/src/doveadm/doveadm-dsync.c Thu Aug 27 10:39:26 2015 +0200 +++ b/src/doveadm/doveadm-dsync.c Thu Aug 27 12:33:47 2015 +0200 @@ -5,6 +5,7 @@ #include "array.h" #include "execv-const.h" #include "fd-set-nonblock.h" +#include "child-wait.h" #include "istream.h" #include "ostream.h" #include "iostream-ssl.h" @@ -72,6 +73,8 @@ const char *local_location; pid_t remote_pid; const char *const *remote_cmd_args; + struct child_wait *child_wait; + int exit_status; int fd_in, fd_out, fd_err; struct io *io_err; @@ -99,6 +102,7 @@ unsigned int no_mail_sync:1; unsigned int local_location_from_arg:1; unsigned int replicator_notify:1; + unsigned int exited:1; }; static bool legacy_dsync = FALSE; @@ -118,10 +122,6 @@ case -1: if (ctx->io_err != NULL) io_remove(&ctx->io_err); - if (ctx->fd_in == -1) { - /* we're shutting down. */ - io_loop_stop(current_ioloop); - } break; default: while ((line = i_stream_next_line(ctx->err_stream)) != NULL) @@ -407,35 +407,33 @@ return doveadm_is_killed() ? -1 : 0; } -static void cmd_dsync_wait_remote(struct dsync_cmd_context *ctx, - int *status_r) +static void cmd_dsync_remote_exited(const struct child_wait_status *status, + struct dsync_cmd_context *ctx) +{ + ctx->exited = TRUE; + ctx->exit_status = status->status; + io_loop_stop(current_ioloop); +} + +static void cmd_dsync_wait_remote(struct dsync_cmd_context *ctx) { struct timeout *to; - /* wait for stderr to close. this indicates that the remote process - has died. while we're running we're also reading and printing all - errors that still coming from it. */ + /* wait in ioloop for the remote process to die. while we're running + we're also reading and printing all errors that still coming from + it. */ to = timeout_add(DSYNC_REMOTE_CMD_EXIT_WAIT_SECS*1000, io_loop_stop, current_ioloop); io_loop_run(current_ioloop); timeout_remove(&to); - /* unless we timed out, the process should be dead now or very soon. */ - alarm(1); - if (waitpid(ctx->remote_pid, status_r, 0) == -1) { - if (errno != EINTR) { - i_error("waitpid(%ld) failed: %m", + if (!ctx->exited) { + i_error("Remote command process isn't dying, killing it"); + if (kill(ctx->remote_pid, SIGKILL) < 0 && errno != ESRCH) { + i_error("kill(%ld, SIGKILL) failed: %m", (long)ctx->remote_pid); - } else { - i_error("Remote command process isn't dying, killing it"); - if (kill(ctx->remote_pid, SIGKILL) < 0 && errno != ESRCH) { - i_error("kill(%ld, SIGKILL) failed: %m", - (long)ctx->remote_pid); - } } - *status_r = -1; } - alarm(0); } static void cmd_dsync_log_remote_status(int status, bool remote_errors_logged, @@ -557,7 +555,7 @@ enum mail_error mail_error = 0, mail_error2; bool remote_errors_logged = FALSE; bool changes_during_sync = FALSE; - int status = 0, ret = 0; + int ret = 0; memset(&set, 0, sizeof(set)); if (_ctx->cur_client_ip.family != 0) { @@ -621,6 +619,7 @@ if (doveadm_debug) brain_flags |= DSYNC_BRAIN_FLAG_DEBUG; + child_wait_init(); brain = dsync_brain_master_init(user, ibc, ctx->sync_type, brain_flags, &set); @@ -629,6 +628,8 @@ &changes_during_sync, &mail_error) < 0) ret = -1; } else { + ctx->child_wait = child_wait_new_with_pid(ctx->remote_pid, + cmd_dsync_remote_exited, ctx); cmd_dsync_run_remote(user); } @@ -682,10 +683,11 @@ shouldn't (at least with ssh) and we need stderr to be open to be able to print the final errors */ if (ctx->run_type == DSYNC_RUN_TYPE_CMD) { - cmd_dsync_wait_remote(ctx, &status); + cmd_dsync_wait_remote(ctx); + remote_error_input(ctx); remote_errors_logged = ctx->err_stream->v_offset > 0; i_stream_destroy(&ctx->err_stream); - cmd_dsync_log_remote_status(status, remote_errors_logged, + cmd_dsync_log_remote_status(ctx->exit_status, remote_errors_logged, ctx->remote_cmd_args); } else { i_assert(ctx->err_stream == NULL); @@ -695,6 +697,9 @@ if (ctx->fd_err != -1) i_close_fd(&ctx->fd_err); + if (ctx->child_wait != NULL) + child_wait_free(&ctx->child_wait); + child_wait_deinit(); return ret; } From dovecot at dovecot.org Thu Aug 27 11:39:46 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 27 Aug 2015 11:39:46 +0000 Subject: dovecot-2.2: dsync: Added -D parameter to disable mailbox renaming. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3fc9658c9712 changeset: 19021:3fc9658c9712 user: Timo Sirainen date: Thu Aug 27 13:38:44 2015 +0200 description: dsync: Added -D parameter to disable mailbox renaming. The renaming logic is annoyingly complex and there are some bugs left in it. With this parameter renames are never even attempted, but instead a rename would be done (slowly) with mailbox delete + create + fill. Although with imapc protocol mailbox renames are rarely detected anyway. diffstat: src/doveadm/doveadm-dsync.c | 8 +++++- src/doveadm/dsync/dsync-brain-mailbox-tree.c | 3 +- src/doveadm/dsync/dsync-brain-private.h | 1 + src/doveadm/dsync/dsync-brain.c | 2 + src/doveadm/dsync/dsync-brain.h | 5 +++- src/doveadm/dsync/dsync-ibc-stream.c | 6 +++- src/doveadm/dsync/dsync-mailbox-tree-sync.c | 36 +++++++++++++++++---------- src/doveadm/dsync/dsync-mailbox-tree.h | 5 +++- 8 files changed, 47 insertions(+), 19 deletions(-) diffs (195 lines): diff -r 42d4da9ee7a9 -r 3fc9658c9712 src/doveadm/doveadm-dsync.c --- a/src/doveadm/doveadm-dsync.c Thu Aug 27 12:33:47 2015 +0200 +++ b/src/doveadm/doveadm-dsync.c Thu Aug 27 13:38:44 2015 +0200 @@ -39,7 +39,7 @@ #include #include -#define DSYNC_COMMON_GETOPT_ARGS "+1a:dEfg:l:m:n:NO:Pr:Rs:t:T:Ux:" +#define DSYNC_COMMON_GETOPT_ARGS "+1a:dDEfg:l:m:n:NO:Pr:Rs:t:T:Ux:" #define DSYNC_REMOTE_CMD_EXIT_WAIT_SECS 30 /* The broken_char is mainly set to get a proper error message when trying to convert a mailbox with a name that can't be used properly translated between @@ -100,6 +100,7 @@ unsigned int reverse_backup:1; unsigned int remote_user_prefix:1; unsigned int no_mail_sync:1; + unsigned int no_mailbox_renames:1; unsigned int local_location_from_arg:1; unsigned int replicator_notify:1; unsigned int exited:1; @@ -606,6 +607,8 @@ brain_flags |= DSYNC_BRAIN_FLAG_SYNC_VISIBLE_NAMESPACES; if (ctx->purge_remote) brain_flags |= DSYNC_BRAIN_FLAG_PURGE_REMOTE; + if (ctx->no_mailbox_renames) + brain_flags |= DSYNC_BRAIN_FLAG_NO_MAILBOX_RENAMES; if (ctx->reverse_backup) brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_RECV; @@ -952,6 +955,9 @@ case 'd': ctx->default_replica_location = TRUE; break; + case 'D': + ctx->no_mailbox_renames = TRUE; + break; case 'E': /* dsync wrapper detection flag */ legacy_dsync = TRUE; diff -r 42d4da9ee7a9 -r 3fc9658c9712 src/doveadm/dsync/dsync-brain-mailbox-tree.c --- a/src/doveadm/dsync/dsync-brain-mailbox-tree.c Thu Aug 27 12:33:47 2015 +0200 +++ b/src/doveadm/dsync/dsync-brain-mailbox-tree.c Thu Aug 27 13:38:44 2015 +0200 @@ -288,7 +288,8 @@ enum dsync_mailbox_trees_sync_type sync_type; enum dsync_mailbox_trees_sync_flags sync_flags = (brain->debug ? DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG : 0) | - (brain->master_brain ? DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN : 0); + (brain->master_brain ? DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN : 0) | + (brain->no_mailbox_renames ? DSYNC_MAILBOX_TREES_SYNC_FLAG_NO_RENAMES : 0); if (brain->no_backup_overwrite) sync_type = DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY; diff -r 42d4da9ee7a9 -r 3fc9658c9712 src/doveadm/dsync/dsync-brain-private.h --- a/src/doveadm/dsync/dsync-brain-private.h Thu Aug 27 12:33:47 2015 +0200 +++ b/src/doveadm/dsync/dsync-brain-private.h Thu Aug 27 13:38:44 2015 +0200 @@ -107,6 +107,7 @@ unsigned int no_mail_sync:1; unsigned int no_backup_overwrite:1; unsigned int no_mail_prefetch:1; + unsigned int no_mailbox_renames:1; unsigned int changes_during_sync:1; unsigned int require_full_resync:1; unsigned int verbose_proctitle:1; diff -r 42d4da9ee7a9 -r 3fc9658c9712 src/doveadm/dsync/dsync-brain.c --- a/src/doveadm/dsync/dsync-brain.c Thu Aug 27 12:33:47 2015 +0200 +++ b/src/doveadm/dsync/dsync-brain.c Thu Aug 27 13:38:44 2015 +0200 @@ -140,6 +140,8 @@ (flags & DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE) != 0; brain->no_mail_prefetch = (flags & DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH) != 0; + brain->no_mailbox_renames = + (flags & DSYNC_BRAIN_FLAG_NO_MAILBOX_RENAMES) != 0; } static void diff -r 42d4da9ee7a9 -r 3fc9658c9712 src/doveadm/dsync/dsync-brain.h --- a/src/doveadm/dsync/dsync-brain.h Thu Aug 27 12:33:47 2015 +0200 +++ b/src/doveadm/dsync/dsync-brain.h Thu Aug 27 13:38:44 2015 +0200 @@ -27,7 +27,10 @@ /* Don't prefetch mail bodies until they're actually needed. This works only with pipe ibc. It's useful if most of the mails can be copied directly within filesystem without having to read them. */ - DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH = 0x100 + DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH = 0x100, + /* Disable mailbox renaming logic. This is just a kludge that should + be removed once the renaming logic has no more bugs.. */ + DSYNC_BRAIN_FLAG_NO_MAILBOX_RENAMES = 0x200 }; enum dsync_brain_sync_type { diff -r 42d4da9ee7a9 -r 3fc9658c9712 src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Thu Aug 27 12:33:47 2015 +0200 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Thu Aug 27 13:38:44 2015 +0200 @@ -76,7 +76,7 @@ .optional_keys = "sync_ns_prefix sync_box sync_box_guid sync_type " "debug sync_visible_namespaces exclude_mailboxes " "send_mail_requests backup_send backup_recv lock_timeout " - "no_mail_sync no_backup_overwrite purge_remote " + "no_mail_sync no_mailbox_renames no_backup_overwrite purge_remote " "sync_since_timestamp sync_flags virtual_all_box" }, { .name = "mailbox_state", @@ -681,6 +681,8 @@ dsync_serializer_encode_add(encoder, "sync_visible_namespaces", ""); if ((set->brain_flags & DSYNC_BRAIN_FLAG_NO_MAIL_SYNC) != 0) dsync_serializer_encode_add(encoder, "no_mail_sync", ""); + if ((set->brain_flags & DSYNC_BRAIN_FLAG_NO_MAILBOX_RENAMES) != 0) + dsync_serializer_encode_add(encoder, "no_mailbox_renames", ""); if ((set->brain_flags & DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE) != 0) dsync_serializer_encode_add(encoder, "no_backup_overwrite", ""); if ((set->brain_flags & DSYNC_BRAIN_FLAG_PURGE_REMOTE) != 0) @@ -785,6 +787,8 @@ set->brain_flags |= DSYNC_BRAIN_FLAG_SYNC_VISIBLE_NAMESPACES; if (dsync_deserializer_decode_try(decoder, "no_mail_sync", &value)) set->brain_flags |= DSYNC_BRAIN_FLAG_NO_MAIL_SYNC; + if (dsync_deserializer_decode_try(decoder, "no_mailbox_renames", &value)) + set->brain_flags |= DSYNC_BRAIN_FLAG_NO_MAILBOX_RENAMES; if (dsync_deserializer_decode_try(decoder, "no_backup_overwrite", &value)) set->brain_flags |= DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE; if (dsync_deserializer_decode_try(decoder, "purge_remote", &value)) diff -r 42d4da9ee7a9 -r 3fc9658c9712 src/doveadm/dsync/dsync-mailbox-tree-sync.c --- a/src/doveadm/dsync/dsync-mailbox-tree-sync.c Thu Aug 27 12:33:47 2015 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-tree-sync.c Thu Aug 27 13:38:44 2015 +0200 @@ -1071,6 +1071,26 @@ return FALSE; } +static void +dsync_mailbox_tree_handle_renames(struct dsync_mailbox_tree_sync_ctx *ctx) +{ + bool changed; + + do { + T_BEGIN { + changed = sync_rename_mailboxes(ctx, &ctx->local_tree->root, + &ctx->remote_tree->root); + } T_END; + if ((ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG) != 0 && + changed) { + i_debug("brain %c: -- Mailbox renamed, restart sync --", + (ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN) != 0 ? 'M' : 'S'); + } + } while (changed); + while (sync_rename_temp_mailboxes(ctx, ctx->local_tree, &ctx->local_tree->root)) ; + while (sync_rename_temp_mailboxes(ctx, ctx->remote_tree, &ctx->remote_tree->root)) ; +} + static bool sync_is_wrong_mailbox(struct dsync_mailbox_node *node, const struct dsync_mailbox_node *wanted_node, const char **reason_r) @@ -1348,7 +1368,6 @@ { struct dsync_mailbox_tree_sync_ctx *ctx; pool_t pool; - bool changed; i_assert(hash_table_is_created(local_tree->guid_hash)); i_assert(hash_table_is_created(remote_tree->guid_hash)); @@ -1370,19 +1389,8 @@ dsync_mailbox_tree_update_child_timestamps(&local_tree->root, 0); dsync_mailbox_tree_update_child_timestamps(&remote_tree->root, 0); - do { - T_BEGIN { - changed = sync_rename_mailboxes(ctx, &local_tree->root, - &remote_tree->root); - } T_END; - if ((ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG) != 0 && - changed) { - i_debug("brain %c: -- Mailbox renamed, restart sync --", - (ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN) != 0 ? 'M' : 'S'); - } - } while (changed); - while (sync_rename_temp_mailboxes(ctx, local_tree, &local_tree->root)) ; - while (sync_rename_temp_mailboxes(ctx, remote_tree, &remote_tree->root)) ; + if ((sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_NO_RENAMES) == 0) + dsync_mailbox_tree_handle_renames(ctx); /* if we're not doing a two-way sync, delete now any mailboxes, which a) shouldn't exist, b) doesn't have a matching GUID/UIDVALIDITY, diff -r 42d4da9ee7a9 -r 3fc9658c9712 src/doveadm/dsync/dsync-mailbox-tree.h --- a/src/doveadm/dsync/dsync-mailbox-tree.h Thu Aug 27 12:33:47 2015 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-tree.h Thu Aug 27 13:38:44 2015 +0200 @@ -20,7 +20,10 @@ /* Enable debugging */ DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG = 0x01, /* Show ourself as "master brain" in the debug output */ - DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN = 0x02 + DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN = 0x02, + /* Disable mailbox renaming logic. This is just a kludge that should + be removed once the renaming logic has no more bugs.. */ + DSYNC_MAILBOX_TREES_SYNC_FLAG_NO_RENAMES = 0x04 }; enum dsync_mailbox_node_existence { From dovecot at dovecot.org Thu Aug 27 13:08:37 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 27 Aug 2015 13:08:37 +0000 Subject: dovecot-2.2: Reverted ec6e672a6e32 for now due to some bugs. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/79ad598d7856 changeset: 19022:79ad598d7856 user: Timo Sirainen date: Thu Aug 27 15:07:35 2015 +0200 description: Reverted ec6e672a6e32 for now due to some bugs. diffstat: src/lib/Makefile.am | 1 - src/lib/ioloop-private.h | 2 - src/lib/ioloop.c | 57 ++--------------------------------------------- src/lib/test-lib.c | 1 - src/lib/test-lib.h | 1 - 5 files changed, 3 insertions(+), 59 deletions(-) diffs (173 lines): diff -r 3fc9658c9712 -r 79ad598d7856 src/lib/Makefile.am --- a/src/lib/Makefile.am Thu Aug 27 13:38:44 2015 +0200 +++ b/src/lib/Makefile.am Thu Aug 27 15:07:35 2015 +0200 @@ -304,7 +304,6 @@ test-hash-format.c \ test-hash-method.c \ test-hex-binary.c \ - test-ioloop.c \ test-iso8601-date.c \ test-istream.c \ test-istream-base64-decoder.c \ diff -r 3fc9658c9712 -r 79ad598d7856 src/lib/ioloop-private.h --- a/src/lib/ioloop-private.h Thu Aug 27 13:38:44 2015 +0200 +++ b/src/lib/ioloop-private.h Thu Aug 27 15:07:35 2015 +0200 @@ -3,7 +3,6 @@ #include "priorityq.h" #include "ioloop.h" -#include "array-decl.h" #ifndef IOLOOP_INITIAL_FD_COUNT # define IOLOOP_INITIAL_FD_COUNT 128 @@ -17,7 +16,6 @@ struct io_file *io_files; struct io_file *next_io_file; struct priorityq *timeouts; - ARRAY(struct timeout *) timeouts_new; struct ioloop_handler_context *handler_context; struct ioloop_notify_handler_context *notify_handler_context; diff -r 3fc9658c9712 -r 79ad598d7856 src/lib/ioloop.c --- a/src/lib/ioloop.c Thu Aug 27 13:38:44 2015 +0200 +++ b/src/lib/ioloop.c Thu Aug 27 15:07:35 2015 +0200 @@ -204,8 +204,7 @@ struct timeout *timeout; timeout = i_new(struct timeout, 1); - timeout->item.idx = UINT_MAX; - timeout->source_linenum = source_linenum; + timeout->source_linenum = source_linenum; timeout->ioloop = current_ioloop; timeout->callback = callback; @@ -228,15 +227,9 @@ timeout = timeout_add_common(source_linenum, callback, context); timeout->msecs = msecs; - if (msecs > 0) { - /* start this timeout in the next run cycle */ - array_append(&timeout->ioloop->timeouts_new, &timeout, 1); - } else { - /* trigger zero timeouts as soon as possible */ - timeout_update_next(timeout, timeout->ioloop->running ? + timeout_update_next(timeout, timeout->ioloop->running ? NULL : &ioloop_timeval); - priorityq_add(timeout->ioloop->timeouts, &timeout->item); - } + priorityq_add(timeout->ioloop->timeouts, &timeout->item); return timeout; } @@ -289,21 +282,10 @@ void timeout_remove(struct timeout **_timeout) { struct timeout *timeout = *_timeout; - struct ioloop *ioloop = timeout->ioloop; *_timeout = NULL; if (timeout->item.idx != UINT_MAX) priorityq_remove(timeout->ioloop->timeouts, &timeout->item); - else { - struct timeout *const *to_idx; - array_foreach(&ioloop->timeouts_new, to_idx) { - if (*to_idx == timeout) { - array_delete(&ioloop->timeouts_new, - array_foreach_idx(&ioloop->timeouts_new, to_idx), 1); - break; - } - } - } timeout_free(timeout); } @@ -411,27 +393,6 @@ } } -static void io_loop_timeouts_start_new(struct ioloop *ioloop) -{ - struct timeout *const *to_idx; - - if (array_count(&ioloop->timeouts_new) == 0) - return; - - io_loop_time_refresh(); - - array_foreach(&ioloop->timeouts_new, to_idx) { - struct timeout *timeout = *to_idx; - i_assert(timeout->next_run.tv_sec == 0 && - timeout->next_run.tv_usec == 0); - i_assert(!timeout->one_shot); - i_assert(timeout->msecs > 0); - timeout_update_next(timeout, &ioloop_timeval); - priorityq_add(ioloop->timeouts, &timeout->item); - } - array_clear(&ioloop->timeouts_new); -} - static void io_loop_timeouts_update(struct ioloop *ioloop, long diff_secs) { struct priorityq_item *const *items; @@ -584,7 +545,6 @@ void io_loop_handler_run(struct ioloop *ioloop) { - io_loop_timeouts_start_new(ioloop); io_loop_handler_run_internal(ioloop); io_loop_call_pending(ioloop); } @@ -627,7 +587,6 @@ ioloop = i_new(struct ioloop, 1); ioloop->timeouts = priorityq_init(timeout_cmp, 32); - i_array_init(&ioloop->timeouts_new, 8); ioloop->time_moved_callback = current_ioloop != NULL ? current_ioloop->time_moved_callback : @@ -641,7 +600,6 @@ void io_loop_destroy(struct ioloop **_ioloop) { struct ioloop *ioloop = *_ioloop; - struct timeout *const *to_idx; struct priorityq_item *item; *_ioloop = NULL; @@ -664,15 +622,6 @@ } i_assert(ioloop->io_pending_count == 0); - array_foreach(&ioloop->timeouts_new, to_idx) { - struct timeout *to = *to_idx; - - i_warning("Timeout leak: %p (line %u)", (void *)to->callback, - to->source_linenum); - timeout_free(to); - } - array_free(&ioloop->timeouts_new); - while ((item = priorityq_pop(ioloop->timeouts)) != NULL) { struct timeout *to = (struct timeout *)item; diff -r 3fc9658c9712 -r 79ad598d7856 src/lib/test-lib.c --- a/src/lib/test-lib.c Thu Aug 27 13:38:44 2015 +0200 +++ b/src/lib/test-lib.c Thu Aug 27 15:07:35 2015 +0200 @@ -20,7 +20,6 @@ test_hash_format, test_hash_method, test_hex_binary, - test_ioloop, test_iso8601_date, test_istream, test_istream_base64_decoder, diff -r 3fc9658c9712 -r 79ad598d7856 src/lib/test-lib.h --- a/src/lib/test-lib.h Thu Aug 27 13:38:44 2015 +0200 +++ b/src/lib/test-lib.h Thu Aug 27 15:07:35 2015 +0200 @@ -21,7 +21,6 @@ void test_hash_format(void); void test_hash_method(void); void test_hex_binary(void); -void test_ioloop(void); void test_iso8601_date(void); void test_istream(void); void test_istream_base64_decoder(void); From dovecot at dovecot.org Thu Aug 27 13:29:27 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 27 Aug 2015 13:29:27 +0000 Subject: dovecot-2.2: dsync: Fixed another crash with recent end-of-list ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d6df577504cf changeset: 19023:d6df577504cf user: Timo Sirainen date: Thu Aug 27 15:28:21 2015 +0200 description: dsync: Fixed another crash with recent end-of-list changes. diffstat: src/doveadm/dsync/dsync-brain-mails.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 79ad598d7856 -r d6df577504cf src/doveadm/dsync/dsync-brain-mails.c --- a/src/doveadm/dsync/dsync-brain-mails.c Thu Aug 27 15:07:35 2015 +0200 +++ b/src/doveadm/dsync/dsync-brain-mails.c Thu Aug 27 15:28:21 2015 +0200 @@ -252,7 +252,8 @@ return FALSE; if (ret == DSYNC_IBC_RECV_RET_FINISHED) { brain->box_recv_state = DSYNC_BOX_STATE_RECV_LAST_COMMON; - if (brain->box_exporter != NULL) { + if (brain->box_exporter != NULL && + brain->box_send_state >= DSYNC_BOX_STATE_RECV_LAST_COMMON) { if (dsync_brain_export_deinit(brain) < 0) return TRUE; } From dovecot at dovecot.org Thu Aug 27 13:47:15 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 27 Aug 2015 13:47:15 +0000 Subject: dovecot-2.2: dsync: Fixed running with tcp/tcps destination. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/84b45106e143 changeset: 19024:84b45106e143 user: Timo Sirainen date: Thu Aug 27 15:46:23 2015 +0200 description: dsync: Fixed running with tcp/tcps destination. diffstat: src/doveadm/doveadm-dsync.c | 9 +++++++-- 1 files changed, 7 insertions(+), 2 deletions(-) diffs (25 lines): diff -r d6df577504cf -r 84b45106e143 src/doveadm/doveadm-dsync.c --- a/src/doveadm/doveadm-dsync.c Thu Aug 27 15:28:21 2015 +0200 +++ b/src/doveadm/doveadm-dsync.c Thu Aug 27 15:46:23 2015 +0200 @@ -626,14 +626,19 @@ brain = dsync_brain_master_init(user, ibc, ctx->sync_type, brain_flags, &set); - if (ctx->run_type == DSYNC_RUN_TYPE_LOCAL) { + switch (ctx->run_type) { + case DSYNC_RUN_TYPE_LOCAL: if (cmd_dsync_run_local(ctx, user, brain, ibc2, &changes_during_sync, &mail_error) < 0) ret = -1; - } else { + break; + case DSYNC_RUN_TYPE_CMD: ctx->child_wait = child_wait_new_with_pid(ctx->remote_pid, cmd_dsync_remote_exited, ctx); + /* fall through */ + case DSYNC_RUN_TYPE_STREAM: cmd_dsync_run_remote(user); + break; } if (ctx->state_input != NULL) { From dovecot at dovecot.org Thu Aug 27 15:35:46 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 27 Aug 2015 15:35:46 +0000 Subject: dovecot-2.2: lib: Try fixing ec6e672a6e32 (ioloop timeout fixing) Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5423ae859f15 changeset: 19025:5423ae859f15 user: Timo Sirainen date: Thu Aug 27 17:34:52 2015 +0200 description: lib: Try fixing ec6e672a6e32 (ioloop timeout fixing) Previous code was broken at least when moving a timeout between ioloops. This is now tested and works. diffstat: src/lib/Makefile.am | 1 + src/lib/ioloop-private.h | 2 + src/lib/ioloop.c | 68 +++++++++++++++++++++++++++++++++++++++++++++-- src/lib/test-ioloop.c | 18 +++++++++++- src/lib/test-lib.c | 1 + src/lib/test-lib.h | 1 + 6 files changed, 85 insertions(+), 6 deletions(-) diffs (239 lines): diff -r 84b45106e143 -r 5423ae859f15 src/lib/Makefile.am --- a/src/lib/Makefile.am Thu Aug 27 15:46:23 2015 +0200 +++ b/src/lib/Makefile.am Thu Aug 27 17:34:52 2015 +0200 @@ -304,6 +304,7 @@ test-hash-format.c \ test-hash-method.c \ test-hex-binary.c \ + test-ioloop.c \ test-iso8601-date.c \ test-istream.c \ test-istream-base64-decoder.c \ diff -r 84b45106e143 -r 5423ae859f15 src/lib/ioloop-private.h --- a/src/lib/ioloop-private.h Thu Aug 27 15:46:23 2015 +0200 +++ b/src/lib/ioloop-private.h Thu Aug 27 17:34:52 2015 +0200 @@ -3,6 +3,7 @@ #include "priorityq.h" #include "ioloop.h" +#include "array-decl.h" #ifndef IOLOOP_INITIAL_FD_COUNT # define IOLOOP_INITIAL_FD_COUNT 128 @@ -16,6 +17,7 @@ struct io_file *io_files; struct io_file *next_io_file; struct priorityq *timeouts; + ARRAY(struct timeout *) timeouts_new; struct ioloop_handler_context *handler_context; struct ioloop_notify_handler_context *notify_handler_context; diff -r 84b45106e143 -r 5423ae859f15 src/lib/ioloop.c --- a/src/lib/ioloop.c Thu Aug 27 15:46:23 2015 +0200 +++ b/src/lib/ioloop.c Thu Aug 27 17:34:52 2015 +0200 @@ -204,7 +204,8 @@ struct timeout *timeout; timeout = i_new(struct timeout, 1); - timeout->source_linenum = source_linenum; + timeout->item.idx = UINT_MAX; + timeout->source_linenum = source_linenum; timeout->ioloop = current_ioloop; timeout->callback = callback; @@ -227,9 +228,15 @@ timeout = timeout_add_common(source_linenum, callback, context); timeout->msecs = msecs; - timeout_update_next(timeout, timeout->ioloop->running ? + if (msecs > 0) { + /* start this timeout in the next run cycle */ + array_append(&timeout->ioloop->timeouts_new, &timeout, 1); + } else { + /* trigger zero timeouts as soon as possible */ + timeout_update_next(timeout, timeout->ioloop->running ? NULL : &ioloop_timeval); - priorityq_add(timeout->ioloop->timeouts, &timeout->item); + priorityq_add(timeout->ioloop->timeouts, &timeout->item); + } return timeout; } @@ -267,7 +274,13 @@ new_to->one_shot = old_to->one_shot; new_to->msecs = old_to->msecs; new_to->next_run = old_to->next_run; - priorityq_add(new_to->ioloop->timeouts, &new_to->item); + + if (old_to->item.idx != UINT_MAX) + priorityq_add(new_to->ioloop->timeouts, &new_to->item); + else if (!new_to->one_shot) { + i_assert(new_to->msecs > 0); + array_append(&new_to->ioloop->timeouts_new, &new_to, 1); + } return new_to; } @@ -282,16 +295,30 @@ void timeout_remove(struct timeout **_timeout) { struct timeout *timeout = *_timeout; + struct ioloop *ioloop = timeout->ioloop; *_timeout = NULL; if (timeout->item.idx != UINT_MAX) priorityq_remove(timeout->ioloop->timeouts, &timeout->item); + else if (!timeout->one_shot && timeout->msecs > 0) { + struct timeout *const *to_idx; + array_foreach(&ioloop->timeouts_new, to_idx) { + if (*to_idx == timeout) { + array_delete(&ioloop->timeouts_new, + array_foreach_idx(&ioloop->timeouts_new, to_idx), 1); + break; + } + } + } timeout_free(timeout); } static void ATTR_NULL(2) timeout_reset_timeval(struct timeout *timeout, struct timeval *tv_now) { + if (timeout->item.idx == UINT_MAX) + return; + timeout_update_next(timeout, tv_now); if (timeout->msecs <= 1) { /* if we came here from io_loop_handle_timeouts(), @@ -393,6 +420,27 @@ } } +static void io_loop_timeouts_start_new(struct ioloop *ioloop) +{ + struct timeout *const *to_idx; + + if (array_count(&ioloop->timeouts_new) == 0) + return; + + io_loop_time_refresh(); + + array_foreach(&ioloop->timeouts_new, to_idx) { + struct timeout *timeout= *to_idx; + i_assert(timeout->next_run.tv_sec == 0 && + timeout->next_run.tv_usec == 0); + i_assert(!timeout->one_shot); + i_assert(timeout->msecs > 0); + timeout_update_next(timeout, &ioloop_timeval); + priorityq_add(ioloop->timeouts, &timeout->item); + } + array_clear(&ioloop->timeouts_new); +} + static void io_loop_timeouts_update(struct ioloop *ioloop, long diff_secs) { struct priorityq_item *const *items; @@ -545,6 +593,7 @@ void io_loop_handler_run(struct ioloop *ioloop) { + io_loop_timeouts_start_new(ioloop); io_loop_handler_run_internal(ioloop); io_loop_call_pending(ioloop); } @@ -587,6 +636,7 @@ ioloop = i_new(struct ioloop, 1); ioloop->timeouts = priorityq_init(timeout_cmp, 32); + i_array_init(&ioloop->timeouts_new, 8); ioloop->time_moved_callback = current_ioloop != NULL ? current_ioloop->time_moved_callback : @@ -600,6 +650,7 @@ void io_loop_destroy(struct ioloop **_ioloop) { struct ioloop *ioloop = *_ioloop; + struct timeout *const *to_idx; struct priorityq_item *item; *_ioloop = NULL; @@ -622,6 +673,15 @@ } i_assert(ioloop->io_pending_count == 0); + array_foreach(&ioloop->timeouts_new, to_idx) { + struct timeout *to = *to_idx; + + i_warning("Timeout leak: %p (line %u)", (void *)to->callback, + to->source_linenum); + timeout_free(to); + } + array_free(&ioloop->timeouts_new); + while ((item = priorityq_pop(ioloop->timeouts)) != NULL) { struct timeout *to = (struct timeout *)item; diff -r 84b45106e143 -r 5423ae859f15 src/lib/test-ioloop.c --- a/src/lib/test-ioloop.c Thu Aug 27 15:46:23 2015 +0200 +++ b/src/lib/test-ioloop.c Thu Aug 27 17:34:52 2015 +0200 @@ -15,22 +15,36 @@ static void test_ioloop_timeout(void) { - struct ioloop *ioloop; - struct timeout *to; + struct ioloop *ioloop, *ioloop2; + struct timeout *to, *to2; struct timeval tv_start, tv_callback; test_begin("ioloop timeout"); ioloop = io_loop_create(); + + /* add a timeout by moving it from another ioloop */ + ioloop2 = io_loop_create(); + to2 = timeout_add(1000, timeout_callback, &tv_callback); + io_loop_set_current(ioloop); + to2 = io_loop_move_timeout(&to2); + io_loop_set_current(ioloop2); + io_loop_destroy(&ioloop2); + sleep(1); + + /* add & remove immediately */ to = timeout_add(1000, timeout_callback, &tv_callback); timeout_remove(&to); + + /* add the timeout we're actually testing below */ to = timeout_add(1000, timeout_callback, &tv_callback); if (gettimeofday(&tv_start, NULL) < 0) i_fatal("gettimeofday() failed: %m"); io_loop_run(ioloop); test_assert(timeval_diff_msecs(&tv_callback, &tv_start) >= 500); timeout_remove(&to); + timeout_remove(&to2); io_loop_destroy(&ioloop); test_end(); diff -r 84b45106e143 -r 5423ae859f15 src/lib/test-lib.c --- a/src/lib/test-lib.c Thu Aug 27 15:46:23 2015 +0200 +++ b/src/lib/test-lib.c Thu Aug 27 17:34:52 2015 +0200 @@ -20,6 +20,7 @@ test_hash_format, test_hash_method, test_hex_binary, + test_ioloop, test_iso8601_date, test_istream, test_istream_base64_decoder, diff -r 84b45106e143 -r 5423ae859f15 src/lib/test-lib.h --- a/src/lib/test-lib.h Thu Aug 27 15:46:23 2015 +0200 +++ b/src/lib/test-lib.h Thu Aug 27 17:34:52 2015 +0200 @@ -21,6 +21,7 @@ void test_hash_format(void); void test_hash_method(void); void test_hex_binary(void); +void test_ioloop(void); void test_iso8601_date(void); void test_istream(void); void test_istream_base64_decoder(void); From dovecot at dovecot.org Thu Aug 27 15:39:04 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 27 Aug 2015 15:39:04 +0000 Subject: dovecot-2.2: dsync: Include mailboxes' full path in debug messages. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7ecf4d436a46 changeset: 19027:7ecf4d436a46 user: Timo Sirainen date: Thu Aug 27 17:38:12 2015 +0200 description: dsync: Include mailboxes' full path in debug messages. diffstat: src/doveadm/dsync/dsync-mailbox-tree-sync.c | 20 +++++++++++--------- src/doveadm/dsync/dsync-mailbox-tree.c | 8 +++++++- src/doveadm/dsync/dsync-mailbox-tree.h | 3 +++ 3 files changed, 21 insertions(+), 10 deletions(-) diffs (103 lines): diff -r a55208061542 -r 7ecf4d436a46 src/doveadm/dsync/dsync-mailbox-tree-sync.c --- a/src/doveadm/dsync/dsync-mailbox-tree-sync.c Thu Aug 27 17:35:17 2015 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-tree-sync.c Thu Aug 27 17:38:12 2015 +0200 @@ -485,7 +485,7 @@ /* set the new name */ *reason_r = t_strdup_printf("%s + Renamed '%s' to '%s'", - *reason_r, node->name, other_node->name); + *reason_r, name, other_node->name); node->name = p_strdup(tree->pool, other_node->name); node->sync_temporary_name = other_node->sync_temporary_name; node->last_renamed_or_created = other_node->last_renamed_or_created; @@ -877,8 +877,9 @@ i_assert(strcmp(local_node->name, remote_node->name) == 0); if (debug != NULL) { str_truncate(debug, 0); - str_printfa(debug, "Mailbox %s: local=%s/%ld/%d, remote=%s/%ld/%d", - local_node->name, + str_append(debug, "Mailbox "); + dsync_mailbox_node_append_full_name(debug, ctx->local_tree, local_node); + str_printfa(debug, ": local=%s/%ld/%d, remote=%s/%ld/%d", guid_128_to_string(local_node->mailbox_guid), (long)local_node->last_renamed_or_created, local_node->existence, @@ -979,7 +980,7 @@ } static void -sync_rename_temp_mailbox_node(pool_t pool, +sync_rename_temp_mailbox_node(struct dsync_mailbox_tree *tree, struct dsync_mailbox_node *node, const char **reason_r) { @@ -1005,8 +1006,9 @@ while (node_has_child(node->parent, str_c(str))) suffix_inc(str); - *reason_r = t_strdup_printf("Renamed '%s' to '%s'", node->name, str_c(str)); - node->name = p_strdup(pool, str_c(str)); + *reason_r = t_strdup_printf("Renamed '%s' to '%s'", + dsync_mailbox_node_get_full_name(tree, node), str_c(str)); + node->name = p_strdup(tree->pool, str_c(str)); dsync_mailbox_tree_node_move_sorted(node, node->parent); node->sync_temporary_name = FALSE; @@ -1052,17 +1054,17 @@ i_debug("brain %c: %s mailbox %s: Delete directory-only tree", (ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN) != 0 ? 'M' : 'S', tree == ctx->local_tree ? "local" : "remote", - node->name); + dsync_mailbox_node_get_full_name(tree, node)); } sync_rename_delete_node_dirs(ctx, tree, node); } else { T_BEGIN { - sync_rename_temp_mailbox_node(tree->pool, node, &reason); + sync_rename_temp_mailbox_node(tree, node, &reason); if ((ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG) != 0) { i_debug("brain %c: %s mailbox %s: %s", (ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN) != 0 ? 'M' : 'S', tree == ctx->local_tree ? "local" : "remote", - node->name, reason); + dsync_mailbox_node_get_full_name(tree, node), reason); } } T_END; return TRUE; diff -r a55208061542 -r 7ecf4d436a46 src/doveadm/dsync/dsync-mailbox-tree.c --- a/src/doveadm/dsync/dsync-mailbox-tree.c Thu Aug 27 17:35:17 2015 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-tree.c Thu Aug 27 17:38:12 2015 +0200 @@ -138,13 +138,19 @@ const struct dsync_mailbox_node *node) { string_t *str = t_str_new(128); + dsync_mailbox_node_append_full_name(str, tree, node); + return str_c(str); +} +void dsync_mailbox_node_append_full_name(string_t *str, + const struct dsync_mailbox_tree *tree, + const struct dsync_mailbox_node *node) +{ i_assert(node->parent != NULL); node_get_full_name_recurse(tree, node, str); /* remove the trailing separator */ str_truncate(str, str_len(str)-1); - return str_c(str); } void dsync_mailbox_node_copy_data(struct dsync_mailbox_node *dest, diff -r a55208061542 -r 7ecf4d436a46 src/doveadm/dsync/dsync-mailbox-tree.h --- a/src/doveadm/dsync/dsync-mailbox-tree.h Thu Aug 27 17:35:17 2015 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-tree.h Thu Aug 27 17:38:12 2015 +0200 @@ -135,6 +135,9 @@ /* Returns full name for the given mailbox node. */ const char *dsync_mailbox_node_get_full_name(const struct dsync_mailbox_tree *tree, const struct dsync_mailbox_node *node); +void dsync_mailbox_node_append_full_name(string_t *str, + const struct dsync_mailbox_tree *tree, + const struct dsync_mailbox_node *node); /* Copy everything from src to dest, except name and hierarchy pointers */ void dsync_mailbox_node_copy_data(struct dsync_mailbox_node *dest, From dovecot at dovecot.org Thu Aug 27 15:39:04 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 27 Aug 2015 15:39:04 +0000 Subject: dovecot-2.2: imapc: Don't crash with mailbox_list_index=yes Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a55208061542 changeset: 19026:a55208061542 user: Timo Sirainen date: Thu Aug 27 17:35:17 2015 +0200 description: imapc: Don't crash with mailbox_list_index=yes diffstat: src/lib-storage/list/mailbox-list-index.c | 8 ++++++++ 1 files changed, 8 insertions(+), 0 deletions(-) diffs (25 lines): diff -r 5423ae859f15 -r a55208061542 src/lib-storage/list/mailbox-list-index.c --- a/src/lib-storage/list/mailbox-list-index.c Thu Aug 27 17:34:52 2015 +0200 +++ b/src/lib-storage/list/mailbox-list-index.c Thu Aug 27 17:35:17 2015 +0200 @@ -12,6 +12,8 @@ #define MAILBOX_LIST_INDEX_REFRESH_DELAY_MSECS 1000 +static void mailbox_list_index_init_finish(struct mailbox_list *list); + struct mailbox_list_index_module mailbox_list_index_module = MODULE_CONTEXT_INIT(&mailbox_list_module_register); @@ -656,6 +658,12 @@ v->notify_wait = mailbox_list_index_notify_wait; MODULE_CONTEXT_SET(list, mailbox_list_index_module, ilist); + + if ((list->flags & MAILBOX_LIST_FLAG_SECONDARY) != 0) { + /* secondary lists aren't accessible via namespaces, so we + need to finish them now. */ + mailbox_list_index_init_finish(list); + } } static void mailbox_list_index_init_finish(struct mailbox_list *list) From dovecot at dovecot.org Thu Aug 27 16:40:55 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 27 Aug 2015 16:40:55 +0000 Subject: dovecot-2.2: doveadm fs delete: Support giving multiple filename... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/17220b7e1378 changeset: 19028:17220b7e1378 user: Timo Sirainen date: Thu Aug 27 18:39:58 2015 +0200 description: doveadm fs delete: Support giving multiple filename parameters. diffstat: src/doveadm/doveadm-fs.c | 104 ++++++++++++++++++++++++++++------------------ 1 files changed, 64 insertions(+), 40 deletions(-) diffs (181 lines): diff -r 7ecf4d436a46 -r 17220b7e1378 src/doveadm/doveadm-fs.c --- a/src/doveadm/doveadm-fs.c Thu Aug 27 17:38:12 2015 +0200 +++ b/src/doveadm/doveadm-fs.c Thu Aug 27 18:39:58 2015 +0200 @@ -26,8 +26,13 @@ struct fs *fs; const char *error; - if (*argc != 3 + own_arg_count) - fs_cmd_help(cmd); + if (own_arg_count > 0) { + if (*argc != 3 + own_arg_count) + fs_cmd_help(cmd); + } else { + if (*argc <= 3) + fs_cmd_help(cmd); + } memset(&ssl_set, 0, sizeof(ssl_set)); ssl_set.ca_dir = doveadm_settings->ssl_client_ca_dir; @@ -242,6 +247,10 @@ else if (errno == EAGAIN) { if (ret == 0) ret = 1; + } else if (errno == ENOENT) { + i_error("%s doesn't exist", fs_file_path(ctx->files[i])); + doveadm_exit_code = DOVEADM_EX_NOTFOUND; + ret = -1; } else { i_error("fs_delete(%s) failed: %s", fs_file_path(ctx->files[i]), @@ -253,8 +262,8 @@ return ret; } -static int doveadm_fs_delete_recursive_fname(struct fs_delete_ctx *ctx, - const char *fname) +static int doveadm_fs_delete_async_fname(struct fs_delete_ctx *ctx, + const char *fname) { unsigned int i; int ret; @@ -278,11 +287,28 @@ doveadm_exit_code = EX_TEMPFAIL; return -1;; } - return doveadm_fs_delete_recursive_fname(ctx, fname); + return doveadm_fs_delete_async_fname(ctx, fname); } return 0; } +static void doveadm_fs_delete_async_finish(struct fs_delete_ctx *ctx) +{ + unsigned int i; + + while (doveadm_exit_code == 0 && cmd_fs_delete_ctx_run(ctx) > 0) { + if (fs_wait_async(ctx->fs) < 0) { + i_error("fs_wait_async() failed: %s", fs_last_error(ctx->fs)); + doveadm_exit_code = EX_TEMPFAIL; + break; + } + } + for (i = 0; i < ctx->files_count; i++) { + if (ctx->files[i] != NULL) + fs_file_deinit(&ctx->files[i]); + } +} + static void cmd_fs_delete_dir_recursive(struct fs *fs, unsigned int async_count, const char *path_prefix) @@ -291,7 +317,6 @@ ARRAY_TYPE(const_string) fnames; struct fs_delete_ctx ctx; const char *fname, *const *fnamep; - unsigned int i; int ret; memset(&ctx, 0, sizeof(ctx)); @@ -341,22 +366,12 @@ array_foreach(&fnames, fnamep) { T_BEGIN { - ret = doveadm_fs_delete_recursive_fname(&ctx, *fnamep); + ret = doveadm_fs_delete_async_fname(&ctx, *fnamep); } T_END; if (ret < 0) break; } - while (doveadm_exit_code == 0 && cmd_fs_delete_ctx_run(&ctx) > 0) { - if (fs_wait_async(fs) < 0) { - i_error("fs_wait_async() failed: %s", fs_last_error(fs)); - doveadm_exit_code = EX_TEMPFAIL; - break; - } - } - for (i = 0; i < ctx.files_count; i++) { - if (ctx.files[i] != NULL) - fs_file_deinit(&ctx.files[i]); - } + doveadm_fs_delete_async_finish(&ctx); } static void @@ -387,10 +402,35 @@ fs_deinit(&fs); } +static void cmd_fs_delete_paths(int argc, char *argv[], + unsigned int async_count) +{ + struct fs *fs; + struct fs_delete_ctx ctx; + unsigned int i; + int ret; + + fs = cmd_fs_init(&argc, &argv, 0, cmd_fs_delete); + + memset(&ctx, 0, sizeof(ctx)); + ctx.fs = fs; + ctx.path_prefix = ""; + ctx.files_count = I_MAX(async_count, 1); + ctx.files = t_new(struct fs_file *, ctx.files_count); + + for (i = 0; argv[i] != NULL; i++) { + T_BEGIN { + ret = doveadm_fs_delete_async_fname(&ctx, argv[i]); + } T_END; + if (ret < 0) + break; + } + doveadm_fs_delete_async_finish(&ctx); + fs_deinit(&fs); +} + static void cmd_fs_delete(int argc, char *argv[]) { - struct fs *fs; - struct fs_file *file; bool recursive = FALSE; unsigned int async_count = 0; int c; @@ -410,26 +450,10 @@ } argc -= optind-1; argv += optind-1; - if (recursive) { + if (recursive) cmd_fs_delete_recursive(argc, argv, async_count); - return; - } - - fs = cmd_fs_init(&argc, &argv, 1, cmd_fs_delete); - - file = fs_file_init(fs, argv[0], FS_OPEN_MODE_READONLY); - if (fs_delete(file) == 0) - ; - else if (errno == ENOENT) { - i_error("%s doesn't exist", fs_file_path(file)); - doveadm_exit_code = DOVEADM_EX_NOTFOUND; - } else { - i_error("fs_delete(%s) failed: %s", - fs_file_path(file), fs_file_last_error(file)); - doveadm_exit_code = EX_TEMPFAIL; - } - fs_file_deinit(&file); - fs_deinit(&fs); + else + cmd_fs_delete_paths(argc, argv, async_count); } static void cmd_fs_iter_full(int argc, char *argv[], enum fs_iter_flags flags, @@ -468,7 +492,7 @@ { cmd_fs_copy, "fs copy", " " }, { cmd_fs_stat, "fs stat", " " }, { cmd_fs_metadata, "fs metadata", " " }, - { cmd_fs_delete, "fs delete", "[-R] [-n ] " }, + { cmd_fs_delete, "fs delete", "[-R] [-n ] [ ...]" }, { cmd_fs_iter, "fs iter", " " }, { cmd_fs_iter_dirs, "fs iter-dirs", " " }, }; From dovecot at dovecot.org Fri Aug 28 12:44:30 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 28 Aug 2015 12:44:30 +0000 Subject: dovecot-2.2: *-login: Added %{passdb:*} fields to login_log_form... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/170f2b9f4977 changeset: 19029:170f2b9f4977 user: Timo Sirainen date: Fri Aug 28 14:43:35 2015 +0200 description: *-login: Added %{passdb:*} fields to login_log_format_elements diffstat: src/login-common/client-common.c | 29 +++++++++++++++++++++++++++-- src/login-common/client-common.h | 1 + src/login-common/sasl-server.c | 1 + 3 files changed, 29 insertions(+), 2 deletions(-) diffs (74 lines): diff -r 17220b7e1378 -r 170f2b9f4977 src/login-common/client-common.c --- a/src/login-common/client-common.c Thu Aug 27 18:39:58 2015 +0200 +++ b/src/login-common/client-common.c Fri Aug 28 14:43:35 2015 +0200 @@ -587,13 +587,37 @@ } static const char * +client_var_expand_func_passdb(const char *data, void *context) +{ + struct client *client = context; + const char *field_name = data; + unsigned int i, field_name_len; + + if (client->auth_passdb_args == NULL) + return NULL; + + field_name_len = strlen(field_name); + for (i = 0; client->auth_passdb_args[i] != NULL; i++) { + if (strncmp(client->auth_passdb_args[i], field_name, + field_name_len) == 0 && + client->auth_passdb_args[i][field_name_len] == '=') + return client->auth_passdb_args[i] + field_name_len+1; + } + return NULL; +} + +static const char * client_get_log_str(struct client *client, const char *msg) { - static struct var_expand_table static_tab[3] = { + static const struct var_expand_table static_tab[3] = { { 's', NULL, NULL }, { '$', NULL, NULL }, { '\0', NULL, NULL } }; + static const struct var_expand_func_table func_table[] = { + { "passdb", client_var_expand_func_passdb }, + { NULL, NULL } + }; const struct var_expand_table *var_expand_table; struct var_expand_table *tab; char *const *e; @@ -609,7 +633,8 @@ str2 = t_str_new(128); for (e = client->set->log_format_elements_split; *e != NULL; e++) { pos = str_len(str); - var_expand(str, *e, var_expand_table); + var_expand_with_funcs(str, *e, var_expand_table, + func_table, client); if (have_username_key(*e)) { /* username is added even if it's empty */ } else { diff -r 17220b7e1378 -r 170f2b9f4977 src/login-common/client-common.h --- a/src/login-common/client-common.h Thu Aug 27 18:39:58 2015 +0200 +++ b/src/login-common/client-common.h Fri Aug 28 14:43:35 2015 +0200 @@ -137,6 +137,7 @@ string_t *auth_response; time_t auth_first_started, auth_finished; const char *sasl_final_resp; + const char *const *auth_passdb_args; unsigned int master_auth_id; unsigned int master_tag; diff -r 17220b7e1378 -r 170f2b9f4977 src/login-common/sasl-server.c --- a/src/login-common/sasl-server.c Thu Aug 27 18:39:58 2015 +0200 +++ b/src/login-common/sasl-server.c Fri Aug 28 14:43:35 2015 +0200 @@ -234,6 +234,7 @@ case AUTH_REQUEST_STATUS_OK: client->auth_request = NULL; client->auth_successes++; + client->auth_passdb_args = p_strarray_dup(client->pool, args); nologin = FALSE; for (i = 0; args[i] != NULL; i++) { From dovecot at dovecot.org Fri Aug 28 13:08:38 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 28 Aug 2015 13:08:38 +0000 Subject: dovecot-2.2: lib: If var_expand_with_funcs() function returns NU... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/061c21b5c5e4 changeset: 19030:061c21b5c5e4 user: Timo Sirainen date: Fri Aug 28 15:07:35 2015 +0200 description: lib: If var_expand_with_funcs() function returns NULL, it should be treated the same as "" The previous behavior was to return "%{foo:bar}" as "foo:bar}". diffstat: src/lib/test-var-expand.c | 49 +++++++++++++++++++++++++++++++++++++++++++++++ src/lib/var-expand.c | 14 +++++++++--- 2 files changed, 59 insertions(+), 4 deletions(-) diffs (90 lines): diff -r 170f2b9f4977 -r 061c21b5c5e4 src/lib/test-var-expand.c --- a/src/lib/test-var-expand.c Fri Aug 28 14:43:35 2015 +0200 +++ b/src/lib/test-var-expand.c Fri Aug 28 15:07:35 2015 +0200 @@ -100,9 +100,58 @@ test_end(); } +static const char *test_var_expand_func1(const char *data, void *context) +{ + test_assert(*(int *)context == 0xabcdef); + return t_strdup_printf("<%s>", data); +} + +static const char *test_var_expand_func2(const char *data ATTR_UNUSED, + void *context ATTR_UNUSED) +{ + return ""; +} + +static const char *test_var_expand_func3(const char *data ATTR_UNUSED, + void *context ATTR_UNUSED) +{ + return NULL; +} + +static void test_var_expand_with_funcs(void) +{ + static struct var_expand_test tests[] = { + { "%{func1}", "<>" }, + { "%{func1:foo}", "" }, + { "%{func2}", "" }, + { "%{func3}", "" } + }; + static struct var_expand_table table[] = { + { '\0', NULL, NULL } + }; + static const struct var_expand_func_table func_table[] = { + { "func1", test_var_expand_func1 }, + { "func2", test_var_expand_func2 }, + { "func3", test_var_expand_func3 }, + { NULL, NULL } + }; + string_t *str = t_str_new(128); + unsigned int i; + int ctx = 0xabcdef; + + test_begin("var_expand_with_funcs"); + for (i = 0; i < N_ELEMENTS(tests); i++) { + str_truncate(str, 0); + var_expand_with_funcs(str, tests[i].in, table, func_table, &ctx); + test_assert_idx(strcmp(tests[i].out, str_c(str)) == 0, i); + } + test_end(); +} + void test_var_expand(void) { test_var_expand_ranges(); test_var_expand_builtin(); test_var_get_key_range(); + test_var_expand_with_funcs(); } diff -r 170f2b9f4977 -r 061c21b5c5e4 src/lib/var-expand.c --- a/src/lib/var-expand.c Fri Aug 28 14:43:35 2015 +0200 +++ b/src/lib/var-expand.c Fri Aug 28 15:07:35 2015 +0200 @@ -170,14 +170,20 @@ var_expand_func(const struct var_expand_func_table *func_table, const char *key, const char *data, void *context) { - if (strcmp(key, "env") == 0) - return getenv(data); + const char *value; + + if (strcmp(key, "env") == 0) { + value = getenv(data); + return value != NULL ? value : ""; + } if (func_table == NULL) return NULL; for (; func_table->key != NULL; func_table++) { - if (strcmp(func_table->key, key) == 0) - return func_table->func(data, context); + if (strcmp(func_table->key, key) == 0) { + value = func_table->func(data, context); + return value != NULL ? value : ""; + } } return NULL; } From dovecot at dovecot.org Fri Aug 28 13:46:24 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 28 Aug 2015 13:46:24 +0000 Subject: dovecot-2.2: lib-settings: Added settings_var_expand_with_funcs() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/224f0f61d03f changeset: 19031:224f0f61d03f user: Timo Sirainen date: Fri Aug 28 15:43:45 2015 +0200 description: lib-settings: Added settings_var_expand_with_funcs() diffstat: src/lib-settings/settings-parser.c | 25 +++++++++++++++++++------ src/lib-settings/settings-parser.h | 6 ++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diffs (90 lines): diff -r 061c21b5c5e4 -r 224f0f61d03f src/lib-settings/settings-parser.c --- a/src/lib-settings/settings-parser.c Fri Aug 28 15:07:35 2015 +0200 +++ b/src/lib-settings/settings-parser.c Fri Aug 28 15:43:45 2015 +0200 @@ -1211,7 +1211,9 @@ static void ATTR_NULL(3, 4, 5) settings_var_expand_info(const struct setting_parser_info *info, void *set, pool_t pool, - const struct var_expand_table *table, string_t *str) + const struct var_expand_table *table, + const struct var_expand_func_table *func_table, + void *func_context, string_t *str) { const struct setting_define *def; void *value, *const *children; @@ -1242,7 +1244,8 @@ *val += 1; } else if (**val == SETTING_STRVAR_UNEXPANDED[0]) { str_truncate(str, 0); - var_expand(str, *val + 1, table); + var_expand_with_funcs(str, *val + 1, table, + func_table, func_context); *val = p_strdup(pool, str_c(str)); } else { i_assert(**val == SETTING_STRVAR_EXPANDED[0]); @@ -1260,8 +1263,8 @@ children = array_get(val, &count); for (i = 0; i < count; i++) { settings_var_expand_info(def->list_info, - children[i], pool, - table, str); + children[i], pool, table, func_table, + func_context, str); } break; } @@ -1273,11 +1276,21 @@ void *set, pool_t pool, const struct var_expand_table *table) { + return settings_var_expand_with_funcs(info, set, pool, table, NULL, NULL); +} + +void settings_var_expand_with_funcs(const struct setting_parser_info *info, + void *set, pool_t pool, + const struct var_expand_table *table, + const struct var_expand_func_table *func_table, + void *func_context) +{ string_t *str; T_BEGIN { str = t_str_new(256); - settings_var_expand_info(info, set, pool, table, str); + settings_var_expand_info(info, set, pool, table, + func_table, func_context, str); } T_END; } @@ -1288,7 +1301,7 @@ for (i = 0; i < ctx->root_count; i++) { settings_var_expand_info(ctx->roots[i].info, ctx->roots[i].set_struct, - NULL, NULL, NULL); + NULL, NULL, NULL, NULL, NULL); } } diff -r 061c21b5c5e4 -r 224f0f61d03f src/lib-settings/settings-parser.h --- a/src/lib-settings/settings-parser.h Fri Aug 28 15:07:35 2015 +0200 +++ b/src/lib-settings/settings-parser.h Fri Aug 28 15:43:45 2015 +0200 @@ -2,6 +2,7 @@ #define SETTINGS_PARSER_H struct var_expand_table; +struct var_expand_func_table; #define SETTINGS_SEPARATOR '/' #define SETTINGS_SEPARATOR_S "/" @@ -178,6 +179,11 @@ void settings_var_expand(const struct setting_parser_info *info, void *set, pool_t pool, const struct var_expand_table *table); +void settings_var_expand_with_funcs(const struct setting_parser_info *info, + void *set, pool_t pool, + const struct var_expand_table *table, + const struct var_expand_func_table *func_table, + void *func_context); /* Go through all the settings and return the first one that has an unexpanded setting containing the given %key. */ bool settings_vars_have_key(const struct setting_parser_info *info, void *set, From dovecot at dovecot.org Fri Aug 28 13:46:27 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 28 Aug 2015 13:46:27 +0000 Subject: dovecot-2.2: lib-storage: Added %{userdb:*} expansion to mail se... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/65b88128c7eb changeset: 19032:65b88128c7eb user: Timo Sirainen date: Fri Aug 28 15:44:34 2015 +0200 description: lib-storage: Added %{userdb:*} expansion to mail settings. diffstat: src/lib-storage/mail-storage-service.c | 50 ++++++++++++++++++++++++++++++--- src/lib-storage/mail-user.c | 36 ++++++++++++++++++++++-- src/lib-storage/mail-user.h | 2 + 3 files changed, 80 insertions(+), 8 deletions(-) diffs (171 lines): diff -r 224f0f61d03f -r 65b88128c7eb src/lib-storage/mail-storage-service.c --- a/src/lib-storage/mail-storage-service.c Fri Aug 28 15:43:45 2015 +0200 +++ b/src/lib-storage/mail-storage-service.c Fri Aug 28 15:44:34 2015 +0200 @@ -651,6 +651,8 @@ mail_user->auth_user = p_strdup(mail_user->pool, user->auth_user); mail_user->session_id = p_strdup(mail_user->pool, user->input.session_id); + mail_user->userdb_fields = user->input.userdb_fields == NULL ? NULL : + p_strarray_dup(mail_user->pool, user->input.userdb_fields); mail_set = mail_user_set_get_storage_set(mail_user); @@ -713,6 +715,42 @@ i_set_failure_prefix("%s", ctx->default_log_prefix); } +static const char * +mail_storage_service_input_var_userdb(const char *data, void *context) +{ + struct mail_storage_service_user *user = context; + const char *field_name = data; + unsigned int i, field_name_len; + + if (user == NULL || user->input.userdb_fields == NULL) + return NULL; + + field_name_len = strlen(field_name); + for (i = 0; user->input.userdb_fields[i] != NULL; i++) { + if (strncmp(user->input.userdb_fields[i], field_name, + field_name_len) == 0 && + user->input.userdb_fields[i][field_name_len] == '=') + return user->input.userdb_fields[i] + field_name_len+1; + } + return NULL; +} + +static void +mail_storage_service_var_expand(struct mail_storage_service_ctx *ctx, + string_t *str, const char *format, + struct mail_storage_service_user *user, + const struct mail_storage_service_input *input, + const struct mail_storage_service_privileges *priv) +{ + static const struct var_expand_func_table func_table[] = { + { "userdb", mail_storage_service_input_var_userdb }, + { NULL, NULL } + }; + var_expand_with_funcs(str, format, + get_var_expand_table(ctx->service, user, input, priv), + func_table, user); +} + static void mail_storage_service_init_log(struct mail_storage_service_ctx *ctx, struct mail_storage_service_user *user, @@ -723,8 +761,9 @@ string_t *str; str = t_str_new(256); - var_expand(str, user->user_set->mail_log_prefix, - get_var_expand_table(ctx->service, user, &user->input, priv)); + mail_storage_service_var_expand(ctx, str, + user->user_set->mail_log_prefix, + user, &user->input, priv); user->log_prefix = p_strdup(user->pool, str_c(str)); } T_END; @@ -1020,8 +1059,8 @@ string_t *str; str = t_str_new(256); - var_expand(str, user_set->mail_log_prefix, - get_var_expand_table(ctx->service, user, input, priv)); + mail_storage_service_var_expand(ctx, str, user_set->mail_log_prefix, + user, input, priv); i_set_failure_prefix("%s", str_c(str)); } @@ -1135,7 +1174,8 @@ user->service_ctx = ctx; user->pool = user_pool; user->input = *input; - user->input.userdb_fields = NULL; + user->input.userdb_fields = userdb_fields == NULL ? NULL : + p_strarray_dup(user_pool, userdb_fields); user->input.username = p_strdup(user_pool, username); user->input.session_id = p_strdup(user_pool, input->session_id); if (user->input.session_id == NULL) { diff -r 224f0f61d03f -r 65b88128c7eb src/lib-storage/mail-user.c --- a/src/lib-storage/mail-user.c Fri Aug 28 15:43:45 2015 +0200 +++ b/src/lib-storage/mail-user.c Fri Aug 28 15:44:34 2015 +0200 @@ -99,7 +99,9 @@ return; } str_truncate(str, 0); - var_expand(str, envs[i+1], mail_user_var_expand_table(user)); + var_expand_with_funcs(str, envs[i+1], + mail_user_var_expand_table(user), + mail_user_var_expand_func_table, user); envs[i+1] = p_strdup(user->pool, str_c(str)); } } @@ -115,8 +117,9 @@ 'h', "home", &key, &value); /* expand mail_home setting before calling mail_user_get_home() */ - settings_var_expand(user->set_info, user->set, - user->pool, mail_user_var_expand_table(user)); + settings_var_expand_with_funcs(user->set_info, user->set, + user->pool, mail_user_var_expand_table(user), + mail_user_var_expand_func_table, user); if (need_home_dir && mail_user_get_home(user, &home) <= 0) { user->error = p_strdup_printf(user->pool, @@ -258,6 +261,26 @@ return user->var_expand_table; } +static const char * +mail_user_var_expand_func_userdb(const char *data, void *context) +{ + struct mail_user *user = context; + const char *field_name = data; + unsigned int i, field_name_len; + + if (user->userdb_fields == NULL) + return NULL; + + field_name_len = strlen(field_name); + for (i = 0; user->userdb_fields[i] != NULL; i++) { + if (strncmp(user->userdb_fields[i], field_name, + field_name_len) == 0 && + user->userdb_fields[i][field_name_len] == '=') + return user->userdb_fields[i] + field_name_len+1; + } + return NULL; +} + void mail_user_set_home(struct mail_user *user, const char *home) { user->_home = p_strdup(user->pool, home); @@ -534,3 +557,10 @@ { user->v.stats_fill(user, stats); } + +static const struct var_expand_func_table mail_user_var_expand_func_table_arr[] = { + { "userdb", mail_user_var_expand_func_userdb }, + { NULL, NULL } +}; +const struct var_expand_func_table *mail_user_var_expand_func_table = + mail_user_var_expand_func_table_arr; diff -r 224f0f61d03f -r 65b88128c7eb src/lib-storage/mail-user.h --- a/src/lib-storage/mail-user.h Fri Aug 28 15:43:45 2015 +0200 +++ b/src/lib-storage/mail-user.h Fri Aug 28 15:44:34 2015 +0200 @@ -34,6 +34,7 @@ const char *session_id; struct ip_addr *local_ip, *remote_ip; const char *auth_token, *auth_user; + const char *const *userdb_fields; const struct var_expand_table *var_expand_table; /* If non-NULL, fail the user initialization with this error. @@ -92,6 +93,7 @@ }; extern struct mail_user_module_register mail_user_module_register; extern struct auth_master_connection *mail_user_auth_master_conn; +extern const struct var_expand_func_table *mail_user_var_expand_func_table; struct mail_user *mail_user_alloc(const char *username, const struct setting_parser_info *set_info, From pigeonhole at rename-it.nl Sat Aug 29 10:53:34 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sat, 29 Aug 2015 12:53:34 +0200 Subject: dovecot-2.2-pigeonhole: Merged concurrent changes. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/72e423848263 changeset: 2091:72e423848263 user: Stephan Bosch date: Sat Aug 29 12:53:29 2015 +0200 description: Merged concurrent changes. diffstat: Makefile.am | 2 + configure.ac | 1 + src/lib-sieve/Makefile.am | 1 + src/lib-sieve/plugins/environment/ext-environment-common.c | 180 +++++--- src/lib-sieve/plugins/environment/ext-environment-common.h | 7 +- src/lib-sieve/plugins/environment/ext-environment.c | 16 +- src/lib-sieve/plugins/environment/sieve-ext-environment.h | 33 +- src/lib-sieve/plugins/environment/tst-environment.c | 2 +- src/lib-sieve/plugins/variables/ext-variables-common.c | 13 +- src/lib-sieve/plugins/variables/ext-variables-dump.c | 6 +- src/lib-sieve/plugins/variables/ext-variables-namespaces.c | 1 + src/lib-sieve/plugins/variables/ext-variables-operands.c | 3 + src/lib-sieve/plugins/vnd.dovecot/Makefile.am | 2 +- src/lib-sieve/plugins/vnd.dovecot/environment/Makefile.am | 16 + src/lib-sieve/plugins/vnd.dovecot/environment/ext-vnd-environment-common.h | 40 + src/lib-sieve/plugins/vnd.dovecot/environment/ext-vnd-environment-items.c | 66 +++ src/lib-sieve/plugins/vnd.dovecot/environment/ext-vnd-environment-variables.c | 207 ++++++++++ src/lib-sieve/plugins/vnd.dovecot/environment/ext-vnd-environment.c | 102 ++++ src/lib-sieve/sieve-code.c | 15 +- src/lib-sieve/sieve-extensions.c | 3 +- src/sieve-tools/sieve-filter.c | 21 +- tests/extensions/vnd.dovecot/environment/basic.svtest | 19 + tests/extensions/vnd.dovecot/environment/variables.svtest | 19 + 23 files changed, 681 insertions(+), 94 deletions(-) diffs (truncated from 1100 to 300 lines): diff -r a5743e2bfef6 -r 72e423848263 Makefile.am --- a/Makefile.am Thu Aug 06 22:23:29 2015 +0200 +++ b/Makefile.am Sat Aug 29 12:53:29 2015 +0200 @@ -162,6 +162,8 @@ tests/extensions/metadata/execute.svtest \ tests/extensions/metadata/errors.svtest \ tests/extensions/vnd.dovecot/debug/execute.svtest \ + tests/extensions/vnd.dovecot/environment/basic.svtest \ + tests/extensions/vnd.dovecot/environment/variables.svtest \ tests/deprecated/notify/basic.svtest \ tests/deprecated/notify/mailto.svtest \ tests/deprecated/notify/errors.svtest \ diff -r a5743e2bfef6 -r 72e423848263 configure.ac --- a/configure.ac Thu Aug 06 22:23:29 2015 +0200 +++ b/configure.ac Sat Aug 29 12:53:29 2015 +0200 @@ -211,6 +211,7 @@ src/lib-sieve/plugins/index/Makefile src/lib-sieve/plugins/vnd.dovecot/Makefile src/lib-sieve/plugins/vnd.dovecot/debug/Makefile +src/lib-sieve/plugins/vnd.dovecot/environment/Makefile src/lib-sieve-tool/Makefile src/lib-managesieve/Makefile src/plugins/Makefile diff -r a5743e2bfef6 -r 72e423848263 src/lib-sieve/Makefile.am --- a/src/lib-sieve/Makefile.am Thu Aug 06 22:23:29 2015 +0200 +++ b/src/lib-sieve/Makefile.am Sat Aug 29 12:53:29 2015 +0200 @@ -78,6 +78,7 @@ $(extdir)/index/libsieve_ext_index.la \ $(extdir)/metadata/libsieve_ext_metadata.la \ $(extdir)/vnd.dovecot/debug/libsieve_ext_debug.la \ + $(extdir)/vnd.dovecot/environment/libsieve_ext_vnd_environment.la \ $(unfinished_plugins) libdovecot_sieve_la_DEPENDENCIES = \ diff -r a5743e2bfef6 -r 72e423848263 src/lib-sieve/plugins/environment/ext-environment-common.c --- a/src/lib-sieve/plugins/environment/ext-environment-common.c Thu Aug 06 22:23:29 2015 +0200 +++ b/src/lib-sieve/plugins/environment/ext-environment-common.c Sat Aug 29 12:53:29 2015 +0200 @@ -6,14 +6,10 @@ #include "sieve-common.h" #include "sieve-extensions.h" +#include "sieve-interpreter.h" #include "ext-environment-common.h" -struct ext_environment_context { - HASH_TABLE(const char *, - const struct sieve_environment_item *) environment_items; -}; - /* * Core environment items */ @@ -30,71 +26,121 @@ static unsigned int core_env_items_count = N_ELEMENTS(core_env_items); /* + * Validator context + */ + +struct ext_environment_interpreter_context { + HASH_TABLE(const char *, + const struct sieve_environment_item *) environment_items; + + unsigned int active:1; +}; + +static void ext_environment_interpreter_extension_free + (const struct sieve_extension *ext, struct sieve_interpreter *interp, + void *context); + +struct sieve_interpreter_extension environment_interpreter_extension = { + .ext_def = &environment_extension, + .free = ext_environment_interpreter_extension_free, +}; + +static struct ext_environment_interpreter_context * +ext_environment_interpreter_context_create +(const struct sieve_extension *this_ext, struct sieve_interpreter *interp) +{ + pool_t pool = sieve_interpreter_pool(interp); + struct ext_environment_interpreter_context *ctx; + + ctx = p_new(pool, struct ext_environment_interpreter_context, 1); + + hash_table_create + (&ctx->environment_items, default_pool, 0, str_hash, strcmp); + + sieve_interpreter_extension_register + (interp, this_ext, &environment_interpreter_extension, (void *)ctx); + return ctx; +} + +static void ext_environment_interpreter_extension_free +(const struct sieve_extension *ext ATTR_UNUSED, + struct sieve_interpreter *interp ATTR_UNUSED, void *context) +{ + struct ext_environment_interpreter_context *ctx = + (struct ext_environment_interpreter_context *)context; + + hash_table_destroy(&ctx->environment_items); +} + +static struct ext_environment_interpreter_context * +ext_environment_interpreter_context_get +(const struct sieve_extension *this_ext, struct sieve_interpreter *interp) +{ + struct ext_environment_interpreter_context *ctx = + (struct ext_environment_interpreter_context *) + sieve_interpreter_extension_get_context(interp, this_ext); + + if ( ctx == NULL ) + ctx = ext_environment_interpreter_context_create(this_ext, interp); + + return ctx; +} + +void ext_environment_interpreter_init +(const struct sieve_extension *this_ext, struct sieve_interpreter *interp) +{ + struct ext_environment_interpreter_context *ctx; + unsigned int i; + + /* Create our context */ + ctx = ext_environment_interpreter_context_get(this_ext, interp); + + for ( i = 0; i < core_env_items_count; i++ ) { + const struct sieve_environment_item *item = core_env_items[i]; + hash_table_insert(ctx->environment_items, item->name, item); + } + + ctx->active = TRUE; +} + +bool sieve_ext_environment_is_active +(const struct sieve_extension *env_ext, struct sieve_interpreter *interp) +{ + struct ext_environment_interpreter_context *ctx = + ext_environment_interpreter_context_get(env_ext, interp); + + return ( ctx != NULL && ctx->active ); +} + +/* * Registration */ -static void ext_environment_item_register -(struct ext_environment_context *ectx, +void sieve_environment_item_register +(const struct sieve_extension *env_ext, struct sieve_interpreter *interp, const struct sieve_environment_item *item) { - hash_table_insert(ectx->environment_items, item->name, item); + struct ext_environment_interpreter_context *ctx; + + i_assert( sieve_extension_is(env_ext, environment_extension) ); + ctx = ext_environment_interpreter_context_get(env_ext, interp); + hash_table_insert(ctx->environment_items, item->name, item); } -void sieve_ext_environment_item_register -(const struct sieve_extension *ext, const struct sieve_environment_item *item) -{ - struct ext_environment_context *ectx = - (struct ext_environment_context *) ext->context; - - ext_environment_item_register(ectx, item); -} - -/* - * Initialization - */ - -bool ext_environment_init -(const struct sieve_extension *ext ATTR_UNUSED, void **context) -{ - struct ext_environment_context *ectx = - i_new(struct ext_environment_context, 1); - - unsigned int i; - - hash_table_create - (&ectx->environment_items, default_pool, 0, str_hash, strcmp); - - for ( i = 0; i < core_env_items_count; i++ ) { - ext_environment_item_register(ectx, core_env_items[i]); - } - - *context = (void *) ectx; - - return TRUE; -} - -void ext_environment_deinit(const struct sieve_extension *ext) -{ - struct ext_environment_context *ectx = - (struct ext_environment_context *) ext->context; - - hash_table_destroy(&ectx->environment_items); - i_free(ectx); -} - - /* * Retrieval */ const char *ext_environment_item_get_value -(const struct sieve_extension *ext, const char *name, - const struct sieve_script_env *senv) +(const struct sieve_extension *env_ext, + const struct sieve_runtime_env *renv, const char *name) { - struct ext_environment_context *ectx = - (struct ext_environment_context *) ext->context; + struct ext_environment_interpreter_context *ctx = + ext_environment_interpreter_context_get(env_ext, renv->interp); const struct sieve_environment_item *item = - hash_table_lookup(ectx->environment_items, name); + hash_table_lookup(ctx->environment_items, name); + + i_assert( sieve_extension_is(env_ext, environment_extension) ); if ( item == NULL ) return NULL; @@ -103,7 +149,7 @@ return item->value; if ( item->get_value != NULL ) - return item->get_value(ext->svinst, senv); + return item->get_value(renv); return NULL; } @@ -119,10 +165,9 @@ */ static const char *envit_domain_get_value -(struct sieve_instance *svinst, - const struct sieve_script_env *senv ATTR_UNUSED) +(const struct sieve_runtime_env *renv) { - return svinst->domainname; + return renv->svinst->domainname; } const struct sieve_environment_item domain_env_item = { @@ -137,10 +182,9 @@ */ static const char *envit_host_get_value -(struct sieve_instance *svinst, - const struct sieve_script_env *senv ATTR_UNUSED) +(const struct sieve_runtime_env *renv) { - return svinst->hostname; + return renv->svinst->hostname; } const struct sieve_environment_item host_env_item = { @@ -160,10 +204,9 @@ */ static const char *envit_location_get_value -(struct sieve_instance *svinst, - const struct sieve_script_env *senv ATTR_UNUSED) +(const struct sieve_runtime_env *renv) { - switch ( svinst->env_location ) { + switch ( renv->svinst->env_location ) { case SIEVE_ENV_LOCATION_MDA: return "MDA"; case SIEVE_ENV_LOCATION_MTA: @@ -190,10 +233,9 @@ */ static const char *envit_phase_get_value -(struct sieve_instance *svinst, - const struct sieve_script_env *senv ATTR_UNUSED) +(const struct sieve_runtime_env *renv) { - switch ( svinst->delivery_phase ) { + switch ( renv->svinst->delivery_phase ) { case SIEVE_DELIVERY_PHASE_PRE: return "pre"; case SIEVE_DELIVERY_PHASE_DURING: diff -r a5743e2bfef6 -r 72e423848263 src/lib-sieve/plugins/environment/ext-environment-common.h --- a/src/lib-sieve/plugins/environment/ext-environment-common.h Thu Aug 06 22:23:29 2015 +0200 +++ b/src/lib-sieve/plugins/environment/ext-environment-common.h Sat Aug 29 12:53:29 2015 +0200 @@ -47,11 +47,10 @@ void ext_environment_deinit(const struct sieve_extension *ext); /* - * Environment item retrieval + * Validator context */ -const char *ext_environment_item_get_value - (const struct sieve_extension *ext, const char *name, From pigeonhole at rename-it.nl Sat Aug 29 10:53:34 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sat, 29 Aug 2015 12:53:34 +0200 Subject: dovecot-2.2-pigeonhole: doveadm sieve plugin: Fixed incorrect in... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/a5743e2bfef6 changeset: 2090:a5743e2bfef6 user: Stephan Bosch date: Thu Aug 06 22:23:29 2015 +0200 description: doveadm sieve plugin: Fixed incorrect initialization of mail user. diffstat: src/plugins/doveadm-sieve/doveadm-sieve-sync.c | 33 +++++++++++++++++-------- 1 files changed, 22 insertions(+), 11 deletions(-) diffs (78 lines): diff -r 49ac5b0000fd -r a5743e2bfef6 src/plugins/doveadm-sieve/doveadm-sieve-sync.c --- a/src/plugins/doveadm-sieve/doveadm-sieve-sync.c Thu Aug 06 22:23:03 2015 +0200 +++ b/src/plugins/doveadm-sieve/doveadm-sieve-sync.c Thu Aug 06 22:23:29 2015 +0200 @@ -60,9 +60,11 @@ { struct sieve_mail_user *suser = SIEVE_USER_CONTEXT(user); - if (suser->sieve_storage != NULL) - sieve_storage_unref(&suser->sieve_storage); - sieve_deinit(&suser->svinst); + if ( suser->svinst != NULL ) { + if (suser->sieve_storage != NULL) + sieve_storage_unref(&suser->sieve_storage); + sieve_deinit(&suser->svinst); + } suser->module_ctx.super.deinit(user); } @@ -75,10 +77,11 @@ enum sieve_storage_flags storage_flags = SIEVE_STORAGE_FLAG_READWRITE | SIEVE_STORAGE_FLAG_SYNCHRONIZING; - struct mail_user_vfuncs *v = user->vlast; struct sieve_environment svenv; - if (suser != NULL) { + i_assert( suser != NULL ); + + if ( suser->svinst != NULL ) { *svstorage_r = suser->sieve_storage; return suser->sieve_storage != NULL ? 1 : 0; } @@ -90,17 +93,11 @@ svenv.base_dir = user->set->base_dir; svenv.flags = SIEVE_FLAG_HOME_RELATIVE; - suser = p_new(user->pool, struct sieve_mail_user, 1); - suser->module_ctx.super = *v; - user->vlast = &suser->module_ctx.super; - v->deinit = mail_sieve_user_deinit; - suser->svinst = sieve_init(&svenv, &mail_sieve_callbacks, user, user->mail_debug); suser->sieve_storage = sieve_storage_create_main (suser->svinst, user, storage_flags, NULL); - MODULE_CONTEXT_SET(user, sieve_user_module, suser); *svstorage_r = suser->sieve_storage; return suser->sieve_storage != NULL ? 1 : 0; } @@ -703,6 +700,19 @@ } static void +sieve_mail_user_created(struct mail_user *user) +{ + struct sieve_mail_user *suser; + struct mail_user_vfuncs *v = user->vlast; + + suser = p_new(user->pool, struct sieve_mail_user, 1); + suser->module_ctx.super = *v; + user->vlast = &suser->module_ctx.super; + v->deinit = mail_sieve_user_deinit; + MODULE_CONTEXT_SET(user, sieve_user_module, suser); +} + +static void sieve_mailbox_allocated(struct mailbox *box) { struct mailbox_vfuncs *v = box->vlast; @@ -724,6 +734,7 @@ } static struct mail_storage_hooks doveadm_sieve_mail_storage_hooks = { + .mail_user_created = sieve_mail_user_created, .mailbox_allocated = sieve_mailbox_allocated }; From dovecot at dovecot.org Sat Aug 29 11:50:11 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 29 Aug 2015 11:50:11 +0000 Subject: dovecot-2.2: lib-http: client: Added proper handling of 408 resp... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a8e01e385e90 changeset: 19033:a8e01e385e90 user: Stephan Bosch date: Sat Aug 29 14:20:57 2015 +0300 description: lib-http: client: Added proper handling of 408 response status. This is treated as a special server connection close event, rather than a response to the last issued request. diffstat: src/lib-http/http-client-connection.c | 18 +++++++++++++++++- 1 files changed, 17 insertions(+), 1 deletions(-) diffs (35 lines): diff -r 65b88128c7eb -r a8e01e385e90 src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Fri Aug 28 15:44:34 2015 +0200 +++ b/src/lib-http/http-client-connection.c Sat Aug 29 14:20:57 2015 +0300 @@ -657,7 +657,15 @@ if (req == NULL) { /* server sent response without any requests in the wait list */ - http_client_connection_debug(conn, "Got unexpected input from server"); + if (response.status == 408) { + http_client_connection_debug(conn, + "Server explicitly closed connection: 408 %s", + response.reason); + } else { + http_client_connection_debug(conn, + "Got unexpected input from server: %u %s", + response.status, response.reason); + } http_client_connection_close(&conn); return; } @@ -780,6 +788,14 @@ http_client_request_delay_from_response(req, &response) > 0 && http_client_request_try_retry(req)) handled = TRUE; + /* request timeout (by server) */ + } else if (response.status == 408) { + /* automatically retry */ + if (http_client_request_try_retry(req)) + handled = TRUE; + /* connection close is implicit, although server should indicate + that explicitly */ + conn->close_indicated = TRUE; } } From dovecot at dovecot.org Sat Aug 29 11:50:14 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 29 Aug 2015 11:50:14 +0000 Subject: dovecot-2.2: ioloop-epoll: Fix fatal epoll_wait() error occurrin... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f7c1c1dac689 changeset: 19034:f7c1c1dac689 user: Stephan Bosch date: Sat Aug 29 14:21:27 2015 +0300 description: ioloop-epoll: Fix fatal epoll_wait() error occurring when there are only ios with no fd. diffstat: src/lib/ioloop-epoll.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r a8e01e385e90 -r f7c1c1dac689 src/lib/ioloop-epoll.c --- a/src/lib/ioloop-epoll.c Sat Aug 29 14:20:57 2015 +0300 +++ b/src/lib/ioloop-epoll.c Sat Aug 29 14:21:27 2015 +0300 @@ -177,7 +177,7 @@ msecs = io_loop_get_wait_time(ioloop, &tv); events = array_get_modifiable(&ctx->events, &events_count); - if (ioloop->io_files != NULL) { + if (ioloop->io_files != NULL && events_count > ctx->deleted_count) { ret = epoll_wait(ctx->epfd, events, events_count, msecs); if (ret < 0 && errno != EINTR) i_fatal("epoll_wait(): %m"); From dovecot at dovecot.org Sat Aug 29 11:50:15 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 29 Aug 2015 11:50:15 +0000 Subject: dovecot-2.2: Changed type of internet port values to in_port_t e... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/aabfe48db1cf changeset: 19035:aabfe48db1cf user: Stephan Bosch date: Sat Aug 29 14:26:30 2015 +0300 description: Changed type of internet port values to in_port_t everywhere. Created special SET_IN_PORT setting type for internet port values. Created net_str2port() for parsing internet port values. Removed several atoi() invocations in the process. diffstat: src/auth/auth-request.c | 8 +- src/auth/auth-request.h | 2 +- src/auth/passdb-imap.c | 3 +- src/config/config-request.c | 8 +++ src/director/director-connection.c | 20 +++++---- src/director/director-host.c | 11 ++-- src/director/director-host.h | 8 +- src/director/director-settings.c | 2 +- src/director/director-settings.h | 4 +- src/director/director-test.c | 2 +- src/director/director.c | 4 +- src/director/director.h | 6 +- src/director/doveadm-connection.c | 8 +- src/director/login-connection.c | 8 +- src/director/main.c | 15 +++--- src/doveadm/client-connection.h | 2 +- src/doveadm/doveadm-auth.c | 6 +- src/doveadm/doveadm-director.c | 8 +- src/doveadm/doveadm-mail-server.c | 5 +- src/doveadm/doveadm-settings.c | 2 +- src/doveadm/doveadm-settings.h | 4 +- src/doveadm/doveadm-util.c | 14 +++--- src/doveadm/doveadm-util.h | 6 +- src/doveadm/doveadm-zlib.c | 5 +- src/doveadm/server-connection.c | 2 +- src/imap-hibernate/imap-client.h | 2 +- src/imap-hibernate/imap-hibernate-client.c | 4 +- src/imap-login/client.c | 16 +++--- src/imap-urlauth/imap-urlauth-worker-settings.c | 2 +- src/imap-urlauth/imap-urlauth-worker-settings.h | 2 +- src/imap/imap-client-hibernate.c | 2 +- src/imap/imap-master-client.c | 7 +- src/imap/imap-settings.c | 2 +- src/imap/imap-settings.h | 4 +- src/lib-auth/auth-client.h | 2 +- src/lib-auth/auth-master.h | 2 +- src/lib-dict/dict-memcached-ascii.c | 4 +- src/lib-dict/dict-memcached.c | 4 +- src/lib-dict/dict-redis.c | 5 +- src/lib-http/http-client-connection.c | 2 +- src/lib-http/http-server-connection.c | 2 +- src/lib-http/test-http-server.c | 4 +- src/lib-imap-client/imapc-client.h | 4 +- src/lib-imap-urlauth/imap-urlauth-private.h | 2 +- src/lib-imap-urlauth/imap-urlauth.h | 4 +- src/lib-lda/lmtp-client.c | 4 +- src/lib-lda/lmtp-client.h | 4 +- src/lib-lda/smtp-client.c | 5 +- src/lib-master/master-service-haproxy.c | 8 +-- src/lib-master/master-service.h | 4 +- src/lib-master/service-settings.h | 4 +- src/lib-settings/settings-parser.c | 26 ++++++++++++ src/lib-settings/settings-parser.h | 5 ++ src/lib-sql/driver-mysql.c | 11 +++- src/lib-storage/index/imapc/imapc-settings.c | 6 +-- src/lib-storage/index/imapc/imapc-settings.h | 4 +- src/lib-storage/index/pop3c/pop3c-client.h | 4 +- src/lib-storage/index/pop3c/pop3c-settings.c | 19 +-------- src/lib-storage/index/pop3c/pop3c-settings.h | 4 +- src/lib-storage/mail-storage-service.h | 2 +- src/lib/connection.c | 2 +- src/lib/connection.h | 4 +- src/lib/fd-close-on-exec.c | 2 +- src/lib/iostream-rawlog.c | 5 +- src/lib/net.c | 45 ++++++++++++++------- src/lib/net.h | 21 +++++---- src/lib/uri-util.c | 30 ++++++------- src/lmtp/client.h | 2 +- src/lmtp/commands.c | 11 +++- src/lmtp/lmtp-proxy.h | 4 +- src/login-common/client-common-auth.c | 9 ++- src/login-common/client-common.h | 6 +- src/login-common/login-common.h | 5 +- src/login-common/login-proxy-state.c | 2 +- src/login-common/login-proxy-state.h | 4 +- src/login-common/login-proxy.c | 8 +- src/login-common/login-proxy.h | 6 +- src/master/master-settings.c | 4 +- src/master/service-listen.c | 10 ++- src/master/service.c | 14 +----- src/pop3-login/client.c | 5 +- src/replication/aggregator/aggregator-settings.c | 2 +- src/replication/aggregator/aggregator-settings.h | 2 +- src/replication/aggregator/replicator-connection.c | 5 +- src/replication/aggregator/replicator-connection.h | 2 +- 85 files changed, 310 insertions(+), 249 deletions(-) diffs (truncated from 2165 to 300 lines): diff -r f7c1c1dac689 -r aabfe48db1cf src/auth/auth-request.c --- a/src/auth/auth-request.c Sat Aug 29 14:21:27 2015 +0300 +++ b/src/auth/auth-request.c Sat Aug 29 14:26:30 2015 +0300 @@ -276,11 +276,11 @@ if (request->real_remote_ip.family == 0) request->real_remote_ip = request->remote_ip; } else if (strcmp(key, "lport") == 0) { - request->local_port = atoi(value); + (void)net_str2port(value, &request->local_port); if (request->real_local_port == 0) request->real_local_port = request->local_port; } else if (strcmp(key, "rport") == 0) { - request->remote_port = atoi(value); + (void)net_str2port(value, &request->remote_port); if (request->real_remote_port == 0) request->real_remote_port = request->remote_port; } @@ -289,9 +289,9 @@ else if (strcmp(key, "real_rip") == 0) (void)net_addr2ip(value, &request->real_remote_ip); else if (strcmp(key, "real_lport") == 0) - request->real_local_port = atoi(value); + (void)net_str2port(value, &request->real_local_port); else if (strcmp(key, "real_rport") == 0) - request->real_remote_port = atoi(value); + (void)net_str2port(value, &request->real_remote_port); else if (strcmp(key, "session") == 0) request->session_id = p_strdup(request->pool, value); else diff -r f7c1c1dac689 -r aabfe48db1cf src/auth/auth-request.h --- a/src/auth/auth-request.h Sat Aug 29 14:21:27 2015 +0300 +++ b/src/auth/auth-request.h Sat Aug 29 14:26:30 2015 +0300 @@ -73,7 +73,7 @@ const char *service, *mech_name, *session_id; struct ip_addr local_ip, remote_ip, real_local_ip, real_remote_ip; - unsigned int local_port, remote_port, real_local_port, real_remote_port; + in_port_t local_port, remote_port, real_local_port, real_remote_port; struct timeout *to_abort, *to_penalty; unsigned int last_penalty; diff -r f7c1c1dac689 -r aabfe48db1cf src/auth/passdb-imap.c --- a/src/auth/passdb-imap.c Sat Aug 29 14:21:27 2015 +0300 +++ b/src/auth/passdb-imap.c Sat Aug 29 14:26:30 2015 +0300 @@ -133,8 +133,7 @@ if (strcmp(key, "host") == 0) module->set.host = value; else if (strcmp(key, "port") == 0) { - if (str_to_uint(value, &module->set.port) < 0 || - module->set.port == 0 || module->set.port > 65535) + if (net_str2port(value, &module->set.port) < 0) i_fatal("passdb imap: Invalid port: %s", value); port_set = TRUE; } else if (strcmp(key, "username") == 0) diff -r f7c1c1dac689 -r aabfe48db1cf src/config/config-request.c --- a/src/config/config-request.c Sat Aug 29 14:21:27 2015 +0300 +++ b/src/config/config-request.c Sat Aug 29 14:26:30 2015 +0300 @@ -116,6 +116,13 @@ } break; } + case SET_IN_PORT: { + const in_port_t *val = value, *dval = default_value; + + if (dump_default || dval == NULL || *val != *dval) + str_printfa(str, "%u", *val); + break; + } case SET_STR_VARS: { const char *const *val = value, *sval; const char *const *_dval = default_value; @@ -245,6 +252,7 @@ case SET_UINT: case SET_UINT_OCT: case SET_TIME: + case SET_IN_PORT: case SET_STR_VARS: case SET_STR: case SET_ENUM: diff -r f7c1c1dac689 -r aabfe48db1cf src/director/director-connection.c --- a/src/director/director-connection.c Sat Aug 29 14:21:27 2015 +0300 +++ b/src/director/director-connection.c Sat Aug 29 14:26:30 2015 +0300 @@ -345,7 +345,7 @@ static bool director_args_parse_ip_port(struct director_connection *conn, const char *const *args, - struct ip_addr *ip_r, unsigned int *port_r) + struct ip_addr *ip_r, in_port_t *port_r) { if (args[0] == NULL || args[1] == NULL) { director_cmd_error(conn, "Missing IP+port parameters"); @@ -355,7 +355,7 @@ director_cmd_error(conn, "Invalid IP address: %s", args[0]); return FALSE; } - if (str_to_uint(args[1], port_r) < 0) { + if (net_str2port(args[1], port_r) < 0) { director_cmd_error(conn, "Invalid port: %s", args[1]); return FALSE; } @@ -368,7 +368,7 @@ struct director *dir = conn->dir; const char *connect_str; struct ip_addr ip; - unsigned int port; + in_port_t port; time_t next_comm_attempt; if (!director_args_parse_ip_port(conn, args, &ip, &port)) @@ -650,7 +650,7 @@ { struct director_host *host; struct ip_addr ip; - unsigned int port; + in_port_t port; if (!director_args_parse_ip_port(conn, args, &ip, &port)) return FALSE; @@ -699,7 +699,7 @@ { struct director_host *host; struct ip_addr ip; - unsigned int port; + in_port_t port; if (!director_args_parse_ip_port(conn, args, &ip, &port)) return FALSE; @@ -746,12 +746,13 @@ { const char *const *args = *_args; struct ip_addr ip; - unsigned int port, seq; + in_port_t port; + unsigned int seq; struct director_host *host; if (str_array_length(args) < 3 || net_addr2ip(args[0], &ip) < 0 || - str_to_uint(args[1], &port) < 0 || + net_str2port(args[1], &port) < 0 || str_to_uint(args[2], &seq) < 0) { director_cmd_error(conn, "Invalid parameters"); return -1; @@ -1295,7 +1296,8 @@ struct director *dir = conn->dir; struct director_host *host; struct ip_addr ip; - unsigned int port, seq, minor_version = 0, timestamp = ioloop_time; + in_port_t port; + unsigned int seq, minor_version = 0, timestamp = ioloop_time; if (str_array_length(args) < 3 || !director_args_parse_ip_port(conn, args, &ip, &port) || @@ -1332,7 +1334,7 @@ struct director *dir = conn->dir; struct director_host *host; struct ip_addr ip; - unsigned int port; + in_port_t port; if (str_array_length(args) != 2 || !director_args_parse_ip_port(conn, args, &ip, &port)) { diff -r f7c1c1dac689 -r aabfe48db1cf src/director/director-host.c --- a/src/director/director-host.c Sat Aug 29 14:21:27 2015 +0300 +++ b/src/director/director-host.c Sat Aug 29 14:26:30 2015 +0300 @@ -24,7 +24,7 @@ struct director_host * director_host_add(struct director *dir, - const struct ip_addr *ip, unsigned int port) + const struct ip_addr *ip, in_port_t port) { struct director_host *host; @@ -90,7 +90,7 @@ struct director_host * director_host_get(struct director *dir, const struct ip_addr *ip, - unsigned int port) + in_port_t port) { struct director_host *host; @@ -102,7 +102,7 @@ struct director_host * director_host_lookup(struct director *dir, const struct ip_addr *ip, - unsigned int port) + in_port_t port) { struct director_host *const *hostp; @@ -150,12 +150,13 @@ static void director_host_add_string(struct director *dir, const char *host) { struct ip_addr *ips; - unsigned int i, port, ips_count; + in_port_t port; + unsigned int i, ips_count; const char *p; p = strrchr(host, ':'); if (p != NULL) { - if (str_to_uint(p + 1, &port) < 0 || port == 0 || port > 65535) + if (net_str2port(p + 1, &port) < 0) i_fatal("Invalid director port in %s", host); host = t_strdup_until(host, p); } else { diff -r f7c1c1dac689 -r aabfe48db1cf src/director/director-host.h --- a/src/director/director-host.h Sat Aug 29 14:21:27 2015 +0300 +++ b/src/director/director-host.h Sat Aug 29 14:26:30 2015 +0300 @@ -10,7 +10,7 @@ int refcount; struct ip_addr ip; - unsigned int port; + in_port_t port; /* name contains "ip:port" */ char *name; @@ -33,7 +33,7 @@ struct director_host * director_host_add(struct director *dir, const struct ip_addr *ip, - unsigned int port); + in_port_t port); void director_host_free(struct director_host **host); void director_host_ref(struct director_host *host); @@ -43,10 +43,10 @@ struct director_host * director_host_get(struct director *dir, const struct ip_addr *ip, - unsigned int port); + in_port_t port); struct director_host * director_host_lookup(struct director *dir, const struct ip_addr *ip, - unsigned int port); + in_port_t port); struct director_host * director_host_lookup_ip(struct director *dir, const struct ip_addr *ip); diff -r f7c1c1dac689 -r aabfe48db1cf src/director/director-settings.c --- a/src/director/director-settings.c Sat Aug 29 14:21:27 2015 +0300 +++ b/src/director/director-settings.c Sat Aug 29 14:26:30 2015 +0300 @@ -72,7 +72,7 @@ DEF(SET_STR, director_mail_servers), DEF(SET_STR, director_username_hash), DEF(SET_TIME, director_user_expire), - DEF(SET_UINT, director_doveadm_port), + DEF(SET_IN_PORT, director_doveadm_port), DEF(SET_BOOL, director_consistent_hashing), SETTING_DEFINE_LIST_END diff -r f7c1c1dac689 -r aabfe48db1cf src/director/director-settings.h --- a/src/director/director-settings.h Sat Aug 29 14:21:27 2015 +0300 +++ b/src/director/director-settings.h Sat Aug 29 14:26:30 2015 +0300 @@ -1,6 +1,8 @@ #ifndef DIRECTOR_SETTINGS_H #define DIRECTOR_SETTINGS_H +#include "net.h" + struct director_settings { const char *master_user_separator; @@ -8,7 +10,7 @@ const char *director_mail_servers; const char *director_username_hash; unsigned int director_user_expire; - unsigned int director_doveadm_port; + in_port_t director_doveadm_port; bool director_consistent_hashing; }; diff -r f7c1c1dac689 -r aabfe48db1cf src/director/director-test.c --- a/src/director/director-test.c Sat Aug 29 14:21:27 2015 +0300 +++ b/src/director/director-test.c Sat Aug 29 14:26:30 2015 +0300 @@ -388,7 +388,7 @@ static void client_connected(struct master_service_connection *conn) { struct ip_addr local_ip, remote_ip; - unsigned int local_port; + in_port_t local_port; if (net_getsockname(conn->fd, &local_ip, &local_port) < 0) i_fatal("net_getsockname() failed: %m"); diff -r f7c1c1dac689 -r aabfe48db1cf src/director/director.c --- a/src/director/director.c Sat Aug 29 14:21:27 2015 +0300 +++ b/src/director/director.c Sat Aug 29 14:26:30 2015 +0300 @@ -103,7 +103,7 @@ int director_connect_host(struct director *dir, struct director_host *host) { - unsigned int port; + in_port_t port; int fd; if (director_has_outgoing_connection(dir, host)) @@ -928,7 +928,7 @@ From dovecot at dovecot.org Sat Aug 29 11:50:15 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 29 Aug 2015 11:50:15 +0000 Subject: dovecot-2.2: Removed all invocations of atoi(). Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f78e38c7cba2 changeset: 19036:f78e38c7cba2 user: Stephan Bosch date: Sat Aug 29 14:30:37 2015 +0300 description: Removed all invocations of atoi(). diffstat: src/auth/db-ldap.c | 10 +++++++--- src/auth/mech-digest-md5.c | 9 ++++++++- src/auth/passdb-blocking.c | 27 ++++++++++++++++----------- src/dict/dict-connection.c | 7 ++++++- src/director/director-connection.c | 18 +++++++++++++----- src/doveadm/doveadm-dump-index.c | 6 ++++-- src/doveadm/doveadm-who.c | 3 ++- src/imap-login/client.c | 6 +++--- src/lib-dict/dict.h | 3 ++- src/lib-master/master-service.c | 7 ++++--- src/lib-sql/driver-mysql.c | 7 ++++--- src/lib-sql/driver-pgsql.c | 14 +++++++++----- src/lib-storage/mail-storage-service.c | 8 +++++--- src/lmtp/commands.c | 12 +++++++----- src/login-common/client-common-auth.c | 19 ++++++++++++------- src/master/main.c | 11 +++++++---- src/util/script.c | 5 ++++- 17 files changed, 113 insertions(+), 59 deletions(-) diffs (truncated from 434 to 300 lines): diff -r aabfe48db1cf -r f78e38c7cba2 src/auth/db-ldap.c --- a/src/auth/db-ldap.c Sat Aug 29 14:26:30 2015 +0300 +++ b/src/auth/db-ldap.c Sat Aug 29 14:30:37 2015 +0300 @@ -1103,8 +1103,7 @@ db_ldap_set_opt(conn, conn->ld, LDAP_OPT_DEREF, &conn->set.ldap_deref, "deref", conn->set.deref); #ifdef LDAP_OPT_DEBUG_LEVEL - value = atoi(conn->set.debug_level); - if (value != 0) { + if (str_to_int(conn->set.debug_level, &value) >= 0 && value != 0) { db_ldap_set_opt(conn, NULL, LDAP_OPT_DEBUG_LEVEL, &value, "debug_level", conn->set.debug_level); } @@ -1143,10 +1142,15 @@ int db_ldap_connect(struct ldap_connection *conn) { - bool debug = atoi(conn->set.debug_level) > 0; + int debug_level; + bool debug; struct timeval start, end; int ret; + debug = FALSE; + if (str_to_int(conn->set.debug_level, &debug_level) >= 0) + debug = debug_level > 0; + if (conn->conn_state != LDAP_CONN_STATE_DISCONNECTED) return 0; diff -r aabfe48db1cf -r f78e38c7cba2 src/auth/mech-digest-md5.c --- a/src/auth/mech-digest-md5.c Sat Aug 29 14:26:30 2015 +0300 +++ b/src/auth/mech-digest-md5.c Sat Aug 29 14:30:37 2015 +0300 @@ -334,12 +334,19 @@ } if (strcmp(key, "nc") == 0) { + unsigned int nc; + if (request->nonce_count != NULL) { *error = "nonce-count must not exist more than once"; return FALSE; } - if (atoi(value) != 1) { + if (str_to_uint(value, &nc) < 0) { + *error = "nonce-count value invalid"; + return FALSE; + } + + if (nc != 1) { *error = "re-auth not supported currently"; return FALSE; } diff -r aabfe48db1cf -r f78e38c7cba2 src/auth/passdb-blocking.c --- a/src/auth/passdb-blocking.c Sat Aug 29 14:26:30 2015 +0300 +++ b/src/auth/passdb-blocking.c Sat Aug 29 14:30:37 2015 +0300 @@ -38,20 +38,25 @@ } if (strcmp(*args, "FAIL") == 0 && args[1] != NULL) { + int result; /* FAIL \t result [\t user \t password [\t extra]] */ - ret = atoi(args[1]); - if (ret == PASSDB_RESULT_OK) { + if (str_to_int(args[1], &result) < 0) { /* shouldn't happen */ - } else if (args[2] == NULL) { - /* internal failure most likely */ - return ret; - } else if (args[3] != NULL) { - if (*args[2] != '\0') { - auth_request_set_field(request, "user", - args[2], NULL); + } else { + ret = (enum passdb_result)result; + if (ret == PASSDB_RESULT_OK) { + /* shouldn't happen */ + } else if (args[2] == NULL) { + /* internal failure most likely */ + return ret; + } else if (args[3] != NULL) { + if (*args[2] != '\0') { + auth_request_set_field(request, "user", + args[2], NULL); + } + auth_worker_reply_parse_args(request, args + 3); + return ret; } - auth_worker_reply_parse_args(request, args + 3); - return ret; } } diff -r aabfe48db1cf -r f78e38c7cba2 src/dict/dict-connection.c --- a/src/dict/dict-connection.c Sat Aug 29 14:26:30 2015 +0300 +++ b/src/dict/dict-connection.c Sat Aug 29 14:30:37 2015 +0300 @@ -21,6 +21,7 @@ const char *line) { const char *username, *name, *value_type; + unsigned int value_type_num; if (*line++ != DICT_PROTOCOL_CMD_HELLO) return -1; @@ -42,7 +43,11 @@ if (*line++ != '\t') return -1; - conn->value_type = atoi(t_strdup_until(value_type, line - 1)); + if (str_to_uint(t_strdup_until(value_type, line - 1), &value_type_num) < 0) + return -1; + if (value_type_num >= DICT_DATA_TYPE_LAST) + return -1; + conn->value_type = (enum dict_data_type)value_type_num; /* get username */ username = line; diff -r aabfe48db1cf -r f78e38c7cba2 src/director/director-connection.c --- a/src/director/director-connection.c Sat Aug 29 14:26:30 2015 +0300 +++ b/src/director/director-connection.c Sat Aug 29 14:30:37 2015 +0300 @@ -44,7 +44,6 @@ #include "user-directory.h" #include "director-connection.h" -#include #include #define MAX_INBUF_SIZE 1024 @@ -1145,6 +1144,8 @@ director_connection_handle_handshake(struct director_connection *conn, const char *cmd, const char *const *args) { + unsigned int major_version; + /* both incoming and outgoing connections get VERSION and ME */ if (strcmp(cmd, "VERSION") == 0 && str_array_length(args) >= 3) { if (strcmp(args[0], DIRECTOR_VERSION_NAME) != 0) { @@ -1152,13 +1153,17 @@ "(%s vs %s)", conn->name, args[0], DIRECTOR_VERSION_NAME); return -1; - } else if (atoi(args[1]) != DIRECTOR_VERSION_MAJOR) { + } else if (str_to_uint(args[1], &major_version) < 0 || + str_to_uint(args[2], &conn->minor_version) < 0) { + i_error("director(%s): Invalid protocol version: " + "%s.%s", conn->name, args[1], args[2]); + return -1; + } else if (major_version != DIRECTOR_VERSION_MAJOR) { i_error("director(%s): Incompatible protocol version: " - "%u vs %u", conn->name, atoi(args[1]), + "%u vs %u", conn->name, major_version, DIRECTOR_VERSION_MAJOR); return -1; } - conn->minor_version = atoi(args[2]); conn->version_received = TRUE; if (conn->done_pending) { if (director_connection_send_done(conn) < 0) @@ -1306,7 +1311,10 @@ return FALSE; } if (args[3] != NULL) { - minor_version = atoi(args[3]); + if (str_to_uint(args[3], &minor_version) < 0) { + director_cmd_error(conn, "Invalid parameters"); + return FALSE; + } if (args[4] != NULL && str_to_uint(args[4], ×tamp) < 0) { director_cmd_error(conn, "Invalid parameters"); return FALSE; diff -r aabfe48db1cf -r f78e38c7cba2 src/doveadm/doveadm-dump-index.c --- a/src/doveadm/doveadm-dump-index.c Sat Aug 29 14:26:30 2015 +0300 +++ b/src/doveadm/doveadm-dump-index.c Sat Aug 29 14:30:37 2015 +0300 @@ -668,8 +668,10 @@ if (index == NULL || mail_index_open(index, MAIL_INDEX_OPEN_FLAG_READONLY) <= 0) i_fatal("Couldn't open index %s", argv[1]); - if (argv[2] != NULL) - uid = atoi(argv[2]); + if (argv[2] != NULL) { + if (str_to_uint(argv[2], &uid) < 0) + i_fatal("Invalid uid number %s", argv[2]); + } view = mail_index_view_open(index); cache_view = mail_cache_view_open(index->cache, view); diff -r aabfe48db1cf -r f78e38c7cba2 src/doveadm/doveadm-who.c --- a/src/doveadm/doveadm-who.c Sat Aug 29 14:26:30 2015 +0300 +++ b/src/doveadm/doveadm-who.c Sat Aug 29 14:30:37 2015 +0300 @@ -68,7 +68,8 @@ line_r->username = strchr(p, '/'); if (line_r->username == NULL) return -1; - line_r->refcount = atoi(refcount_str); + if (str_to_uint(refcount_str, &line_r->refcount) < 0) + return -1; ip_str = t_strdup_until(p, line_r->username++); (void)net_addr2ip(ip_str, &line_r->ip); return 0; diff -r aabfe48db1cf -r f78e38c7cba2 src/imap-login/client.c --- a/src/imap-login/client.c Sat Aug 29 14:26:30 2015 +0300 +++ b/src/imap-login/client.c Sat Aug 29 14:30:37 2015 +0300 @@ -21,8 +21,6 @@ #include "imap-proxy.h" #include "imap-login-settings.h" -#include - #if LOGIN_MAX_INBUF_SIZE < 1024+2 # error LOGIN_MAX_INBUF_SIZE too short to fit all ID command parameters #endif @@ -158,7 +156,9 @@ } else if (strcasecmp(key, "x-connected-port") == 0) { (void)net_str2port(value, &client->common.local_port); } else if (strcasecmp(key, "x-proxy-ttl") == 0) { - client->common.proxy_ttl = atoi(value); + if (str_to_uint(value, &client->common.proxy_ttl) < 0) { + /* nothing */ + } } else if (strcasecmp(key, "x-session-id") == 0 || strcasecmp(key, "x-session-ext-id") == 0) { if (strlen(value) <= LOGIN_MAX_SESSION_ID_LEN) { diff -r aabfe48db1cf -r f78e38c7cba2 src/lib-dict/dict.h --- a/src/lib-dict/dict.h Sat Aug 29 14:26:30 2015 +0300 +++ b/src/lib-dict/dict.h Sat Aug 29 14:30:37 2015 +0300 @@ -23,7 +23,8 @@ enum dict_data_type { DICT_DATA_TYPE_STRING = 0, - DICT_DATA_TYPE_UINT32 + DICT_DATA_TYPE_UINT32, + DICT_DATA_TYPE_LAST }; struct dict_settings { diff -r aabfe48db1cf -r f78e38c7cba2 src/lib-master/master-service.c --- a/src/lib-master/master-service.c Sat Aug 29 14:26:30 2015 +0300 +++ b/src/lib-master/master-service.c Sat Aug 29 14:30:37 2015 +0300 @@ -164,7 +164,8 @@ int count; value = getenv("SOCKET_COUNT"); - count = value == NULL ? 0 : atoi(value); + if (value == NULL || str_to_uint(value, &count) < 0) + count = 0; fd_debug_verify_leaks(MASTER_LISTEN_FD_FIRST + count, 1024); } #endif @@ -222,8 +223,8 @@ /* listener configuration */ value = getenv("SOCKET_COUNT"); - if (value != NULL) - service->socket_count = atoi(value); + if (value != NULL && str_to_uint(value, &service->socket_count) < 0) + i_fatal("Invalid SOCKET_COUNT environment"); T_BEGIN { master_service_init_socket_listeners(service); } T_END; diff -r aabfe48db1cf -r f78e38c7cba2 src/lib-sql/driver-mysql.c --- a/src/lib-sql/driver-mysql.c Sat Aug 29 14:26:30 2015 +0300 +++ b/src/lib-sql/driver-mysql.c Sat Aug 29 14:30:37 2015 +0300 @@ -184,9 +184,10 @@ else if (strcmp(name, "port") == 0) { if (net_str2port(value, &db->port) < 0) i_fatal("mysql: Invalid port number: %s", value); - } else if (strcmp(name, "client_flags") == 0) - db->client_flags = atoi(value); - else if (strcmp(name, "ssl_cert") == 0) + } else if (strcmp(name, "client_flags") == 0) { + if (str_to_uint(value, &db->client_flags) < 9) + i_fatal("mysql: Invalid client flags: %s", value); + } else if (strcmp(name, "ssl_cert") == 0) field = &db->ssl_cert; else if (strcmp(name, "ssl_key") == 0) field = &db->ssl_key; diff -r aabfe48db1cf -r f78e38c7cba2 src/lib-sql/driver-pgsql.c --- a/src/lib-sql/driver-pgsql.c Sat Aug 29 14:26:30 2015 +0300 +++ b/src/lib-sql/driver-pgsql.c Sat Aug 29 14:30:37 2015 +0300 @@ -898,7 +898,9 @@ } else if (query->affected_rows != NULL) { struct pgsql_result *pg_result = (struct pgsql_result *)result; - *query->affected_rows = atoi(PQcmdTuples(pg_result->pgres)); + if (str_to_uint(PQcmdTuples(pg_result->pgres), + query->affected_rows) < 0) + i_unreached(); } driver_pgsql_transaction_unref(ctx); } @@ -969,8 +971,9 @@ struct pgsql_result *pg_result = (struct pgsql_result *)result; - *query->affected_rows = - atoi(PQcmdTuples(pg_result->pgres)); + if (str_to_uint(PQcmdTuples(pg_result->pgres), + query->affected_rows) < 0) + i_unreached(); From dovecot at dovecot.org Sat Aug 29 11:50:16 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 29 Aug 2015 11:50:16 +0000 Subject: dovecot-2.2: strnum: Implemented many more numeric string parsin... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/de73e7121676 changeset: 19037:de73e7121676 user: Stephan Bosch date: Sat Aug 29 14:31:51 2015 +0300 description: strnum: Implemented many more numeric string parsing functions. Created macros for most of the implementation to avoid further code duplication. diffstat: src/lib/strnum.c | 337 +++++++++++++++++++++++++++++++++++++++--------------- src/lib/strnum.h | 184 ++++++++++++++++++++++++++--- 2 files changed, 401 insertions(+), 120 deletions(-) diffs (truncated from 611 to 300 lines): diff -r f78e38c7cba2 -r de73e7121676 src/lib/strnum.c --- a/src/lib/strnum.c Sat Aug 29 14:30:37 2015 +0300 +++ b/src/lib/strnum.c Sat Aug 29 14:31:51 2015 +0300 @@ -17,72 +17,44 @@ return TRUE; } -int str_to_uint(const char *str, unsigned int *num_r) -{ - uintmax_t l; +/* + * Unsigned decimal + */ - if (str_to_uintmax(str, &l) < 0) - return -1; - - if (l > UINT_MAX) - return -1; - *num_r = (unsigned int)l; - return 0; +#define STR_PARSE_U__TEMPLATE(name, type) \ +int name(const char *str, type *num_r, const char **endp_r) \ +{ \ + uintmax_t l; \ + if (str_parse_uintmax(str, &l, endp_r) < 0 || l > (type)-1) \ + return -1; \ + *num_r = (type)l; \ + return 0; \ } -int str_to_ulong(const char *str, unsigned long *num_r) -{ - uintmax_t l; +STR_PARSE_U__TEMPLATE(str_parse_uint, unsigned int) +STR_PARSE_U__TEMPLATE(str_parse_ulong, unsigned long) +STR_PARSE_U__TEMPLATE(str_parse_ullong, unsigned long long) +STR_PARSE_U__TEMPLATE(str_parse_uint32, uint32_t) +STR_PARSE_U__TEMPLATE(str_parse_uint64, uint64_t) - if (str_to_uintmax(str, &l) < 0) - return -1; - - if (l > (unsigned long)-1) - return -1; - *num_r = (unsigned long)l; - return 0; +#define STR_TO_U__TEMPLATE(name, type) \ +int name(const char *str, type *num_r) \ +{ \ + uintmax_t l; \ + if (str_to_uintmax(str, &l) < 0 || l > (type)-1) \ + return -1; \ + *num_r = (type)l; \ + return 0; \ } -int str_to_ullong(const char *str, unsigned long long *num_r) -{ - uintmax_t l; +STR_TO_U__TEMPLATE(str_to_uint, unsigned int) +STR_TO_U__TEMPLATE(str_to_ulong, unsigned long) +STR_TO_U__TEMPLATE(str_to_ullong, unsigned long long) +STR_TO_U__TEMPLATE(str_to_uint32, uint32_t) +STR_TO_U__TEMPLATE(str_to_uint64, uint64_t) - if (str_to_uintmax(str, &l) < 0) - return -1; - - if (l > (unsigned long long)-1) - return -1; - *num_r = (unsigned long long)l; - return 0; -} - -int str_to_uint32(const char *str, uint32_t *num_r) -{ - uintmax_t l; - - if (str_to_uintmax(str, &l) < 0) - return -1; - - if (l > (uint32_t)-1) - return -1; - *num_r = (uint32_t)l; - return 0; -} - -int str_to_uint64(const char *str, uint64_t *num_r) -{ - uintmax_t l; - - if (str_to_uintmax(str, &l) < 0) - return -1; - - if (l > (uint64_t)-1) - return -1; - *num_r = (uint64_t)l; - return 0; -} - -int str_parse_uintmax(const char *str, uintmax_t *num_r, const char **endp_r) +int str_parse_uintmax(const char *str, uintmax_t *num_r, + const char **endp_r) { uintmax_t n = 0; @@ -114,58 +86,216 @@ return 0; } -#define STR_TO_U__TEMPLATE(name, type) \ -int name(const char *str, type *num_r, const char **endp_r) \ -{ \ - uintmax_t l; \ - if (str_parse_uintmax(str, &l, endp_r) < 0 || l > (type)-1)\ - return -1; \ - *num_r = l; \ - return 0; \ +bool str_uint_equals(const char *str, uintmax_t num) +{ + uintmax_t l; + + if (str_to_uintmax(str, &l) < 0) + return FALSE; + return l == num; } -STR_TO_U__TEMPLATE(str_parse_uoff, uoff_t) -STR_TO_U__TEMPLATE(str_parse_uint, unsigned int) -int str_to_int(const char *str, int *num_r) +/* + * Unsigned hexadecimal + */ + +#define STR_PARSE_UHEX__TEMPLATE(name, type) \ +int name(const char *str, type *num_r, const char **endp_r) \ +{ \ + uintmax_t l; \ + if (str_parse_uintmax_hex(str, &l, endp_r) < 0 || l > (type)-1) \ + return -1; \ + *num_r = (type)l; \ + return 0; \ +} + +STR_PARSE_UHEX__TEMPLATE(str_parse_uint_hex, unsigned int) +STR_PARSE_UHEX__TEMPLATE(str_parse_ulong_hex, unsigned long) +STR_PARSE_UHEX__TEMPLATE(str_parse_ullong_hex, unsigned long long) +STR_PARSE_UHEX__TEMPLATE(str_parse_uint32_hex, uint32_t) +STR_PARSE_UHEX__TEMPLATE(str_parse_uint64_hex, uint64_t) + +#define STR_TO_UHEX__TEMPLATE(name, type) \ +int name(const char *str, type *num_r) \ +{ \ + uintmax_t l; \ + if (str_to_uintmax_hex(str, &l) < 0 || l > (type)-1) \ + return -1; \ + *num_r = (type)l; \ + return 0; \ +} + +STR_TO_UHEX__TEMPLATE(str_to_uint_hex, unsigned int) +STR_TO_UHEX__TEMPLATE(str_to_ulong_hex, unsigned long) +STR_TO_UHEX__TEMPLATE(str_to_ullong_hex, unsigned long long) +STR_TO_UHEX__TEMPLATE(str_to_uint32_hex, uint32_t) +STR_TO_UHEX__TEMPLATE(str_to_uint64_hex, uint64_t) + +static inline int _str_parse_hex(const char ch, + unsigned int *hex_r) { - intmax_t l; + switch (ch) { + case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': + *hex_r = (unsigned int)(ch - 'a' + 10); + return 0; + case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': + *hex_r = (unsigned int)(ch - 'A' + 10); + return 0; + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + *hex_r = (unsigned int)(ch - '0'); + return 0; + default: + break; + } + return -1; +} +int str_parse_uintmax_hex(const char *str, uintmax_t *num_r, + const char **endp_r) +{ + unsigned int hex; + uintmax_t n = 0; - if (str_to_intmax(str, &l) < 0) + if (_str_parse_hex(*str, &hex) < 0) return -1; - if (l < INT_MIN || l > INT_MAX) + do { + if (n >= ((uintmax_t)-1 >> 4)) { + if (n > (uintmax_t)-1 >> 4) + return -1; + if ((uintmax_t)hex > ((uintmax_t)-1 & 0x0f)) + return -1; + } + n = (n << 4) + hex; + str++; + } while (_str_parse_hex(*str, &hex) >= 0); + if (endp_r != NULL) + *endp_r = str; + *num_r = n; + return 0; +} +int str_to_uintmax_hex(const char *str, uintmax_t *num_r) +{ + const char *endp; + uintmax_t n; + int ret = str_parse_uintmax_hex(str, &n, &endp); + if ((ret != 0) || (*endp != '\0')) return -1; - *num_r = (int)l; + *num_r = n; return 0; } -int str_to_long(const char *str, long *num_r) +/* + * Unsigned octal + */ + +#define STR_PARSE_UOCT__TEMPLATE(name, type) \ +int name(const char *str, type *num_r, const char **endp_r) \ +{ \ + uintmax_t l; \ + if (str_parse_uintmax_oct(str, &l, endp_r) < 0 || l > (type)-1) \ + return -1; \ + *num_r = (type)l; \ + return 0; \ +} + +STR_PARSE_UOCT__TEMPLATE(str_parse_uint_oct, unsigned int) +STR_PARSE_UOCT__TEMPLATE(str_parse_ulong_oct, unsigned long) +STR_PARSE_UOCT__TEMPLATE(str_parse_ullong_oct, unsigned long long) +STR_PARSE_UOCT__TEMPLATE(str_parse_uint32_oct, uint32_t) +STR_PARSE_UOCT__TEMPLATE(str_parse_uint64_oct, uint64_t) + +#define STR_TO_UOCT__TEMPLATE(name, type) \ +int name(const char *str, type *num_r) \ +{ \ + uintmax_t l; \ + if (str_to_uintmax_oct(str, &l) < 0 || l > (type)-1) \ + return -1; \ + *num_r = (type)l; \ + return 0; \ +} + +STR_TO_UOCT__TEMPLATE(str_to_uint_oct, unsigned int) +STR_TO_UOCT__TEMPLATE(str_to_ulong_oct, unsigned long) +STR_TO_UOCT__TEMPLATE(str_to_ullong_oct, unsigned long long) +STR_TO_UOCT__TEMPLATE(str_to_uint32_oct, uint32_t) +STR_TO_UOCT__TEMPLATE(str_to_uint64_oct, uint64_t) + +int str_parse_uintmax_oct(const char *str, uintmax_t *num_r, + const char **endp_r) { - intmax_t l; + uintmax_t n = 0; - if (str_to_intmax(str, &l) < 0) + if (*str < '0' || *str > '7') return -1; - if (l < LONG_MIN || l > LONG_MAX) + for (; *str >= '0' && *str <= '7'; str++) { + if (n >= ((uintmax_t)-1 >> 3)) { + if (n > (uintmax_t)-1 >> 3) + return -1; + if ((uintmax_t)(*str - '0') > ((uintmax_t)-1 & 0x03)) + return -1; + } + n = (n << 3) + (*str - '0'); + } + if (endp_r != NULL) + *endp_r = str; + *num_r = n; + return 0; +} +int str_to_uintmax_oct(const char *str, uintmax_t *num_r) +{ + const char *endp; + uintmax_t n; + int ret = str_parse_uintmax_oct(str, &n, &endp); + if ((ret != 0) || (*endp != '\0')) return -1; - *num_r = (long)l; + *num_r = n; return 0; } -int str_to_llong(const char *str, long long *num_r) From dovecot at dovecot.org Sat Aug 29 11:50:16 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 29 Aug 2015 11:50:16 +0000 Subject: dovecot-2.2: Removed all invocations of strtoll() and friends. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f8ab4f979e92 changeset: 19038:f8ab4f979e92 user: Stephan Bosch date: Sat Aug 29 14:42:49 2015 +0300 description: Removed all invocations of strtoll() and friends. diffstat: configure.ac | 40 ++--------------------- src/anvil/anvil-connection.c | 15 +++++++- src/auth/checkpassword-reply.c | 12 ++++-- src/doveadm/doveadm-penalty.c | 7 ++- src/doveadm/doveadm-who.c | 3 +- src/doveadm/dsync/dsync-ibc-stream.c | 34 +++++++++++++++++--- src/lib-dict/dict-file.c | 5 +- src/lib-fs/fs-posix.c | 7 +++- src/lib-otp/otp-parse.c | 4 +- src/lib-settings/settings-parser.c | 24 ++++++++------ src/lib-storage/index/dbox-common/dbox-file.c | 21 +++++++++--- src/lib-storage/index/dbox-common/dbox-mail.c | 12 ++++++- src/lib-storage/index/maildir/maildir-uidlist.c | 12 ++++++- src/lib-storage/mailbox-uidvalidity.c | 9 +--- src/lib/compat.c | 41 ------------------------- src/lib/compat.h | 9 ----- src/lib/rand.c | 5 +- src/lib/var-expand.c | 3 +- src/log/log-connection.c | 8 +++- src/login-common/sasl-server.c | 8 +++- src/master/main.c | 7 +++- src/plugins/quota/quota-dict.c | 5 +- src/plugins/quota/quota-maildir.c | 7 +++- src/plugins/quota/quota-util.c | 27 +++++++++++---- src/util/maildirlock.c | 3 +- 25 files changed, 172 insertions(+), 156 deletions(-) diffs (truncated from 774 to 300 lines): diff -r de73e7121676 -r f8ab4f979e92 configure.ac --- a/configure.ac Sat Aug 29 14:31:51 2015 +0300 +++ b/configure.ac Sat Aug 29 14:42:49 2015 +0300 @@ -453,10 +453,10 @@ AC_CHECK_FUNCS(fcntl flock lockf inet_aton sigaction getpagesize madvise \ strcasecmp stricmp vsyslog writev pread uname unsetenv \ setrlimit setproctitle seteuid setreuid setegid setresgid \ - strtoull strtoll strtouq strtoq getmntinfo \ - setpriority quotactl getmntent kqueue kevent backtrace_symbols \ - walkcontext dirfd clearenv malloc_usable_size glob fallocate \ - posix_fadvise getpeereid getpeerucred inotify_init) + getmntinfo setpriority quotactl getmntent kqueue kevent \ + backtrace_symbols walkcontext dirfd clearenv \ + malloc_usable_size glob fallocate posix_fadvise \ + getpeereid getpeerucred inotify_init) AC_CHECK_TYPES([struct sockpeercred],,,[ #include @@ -482,38 +482,6 @@ AC_DEFINE(HAVE_TYPEOF,, [Define if you have typeof()]) fi -dnl strtoimax and strtoumax are macros in HP-UX, so inttypes.h must be included -dnl Link instead of just compiling since there's something wrong with Tru64 -AC_CACHE_CHECK([for strtoimax],i_cv_have_strtoimax,[ - AC_TRY_LINK([ - #include - ], [ - strtoimax(0, 0, 0); - ], [ - i_cv_have_strtoimax=yes - ], [ - i_cv_have_strtoimax=no - ]) -]) -if test $i_cv_have_strtoimax = yes; then - AC_DEFINE(HAVE_STRTOIMAX,, [Define if you have strtoimax function]) -fi - -AC_CACHE_CHECK([for strtoumax],i_cv_have_strtoumax,[ - AC_TRY_LINK([ - #include - ], [ - strtoumax(0, 0, 0); - ], [ - i_cv_have_strtoumax=yes - ], [ - i_cv_have_strtoumax=no - ]) -]) -if test $i_cv_have_strtoumax = yes; then - AC_DEFINE(HAVE_STRTOUMAX,, [Define if you have strtoumax function]) -fi - dnl * I/O loop function have_ioloop=no diff -r de73e7121676 -r f8ab4f979e92 src/anvil/anvil-connection.c --- a/src/anvil/anvil-connection.c Sat Aug 29 14:31:51 2015 +0300 +++ b/src/anvil/anvil-connection.c Sat Aug 29 14:42:49 2015 +0300 @@ -58,14 +58,20 @@ *error_r = "CONNECT: Not enough parameters"; return -1; } - pid = strtol(args[0], NULL, 10); + if (str_to_pid(args[0], &pid) < 0) { + *error_r = "CONNECT: Invalid pid"; + return -1; + } connect_limit_connect(connect_limit, pid, args[1]); } else if (strcmp(cmd, "DISCONNECT") == 0) { if (args[0] == NULL || args[1] == NULL) { *error_r = "DISCONNECT: Not enough parameters"; return -1; } - pid = strtol(args[0], NULL, 10); + if (str_to_pid(args[0], &pid) < 0) { + *error_r = "DISCONNECT: Invalid pid"; + return -1; + } connect_limit_disconnect(connect_limit, pid, args[1]); } else if (strcmp(cmd, "CONNECT-DUMP") == 0) { connect_limit_dump(connect_limit, conn->output); @@ -78,7 +84,10 @@ *error_r = "KILL sent by a non-master connection"; return -1; } - pid = strtol(args[0], NULL, 10); + if (str_to_pid(args[0], &pid) < 0) { + *error_r = "KILL: Invalid pid"; + return -1; + } connect_limit_disconnect_pid(connect_limit, pid); } else if (strcmp(cmd, "LOOKUP") == 0) { if (args[0] == NULL) { diff -r de73e7121676 -r f8ab4f979e92 src/auth/checkpassword-reply.c --- a/src/auth/checkpassword-reply.c Sat Aug 29 14:31:51 2015 +0300 +++ b/src/auth/checkpassword-reply.c Sat Aug 29 14:42:49 2015 +0300 @@ -11,14 +11,18 @@ int main(void) { string_t *str; - const char *user, *home, *authorized, *orig_uid; + const char *user, *home, *authorized, *orig_uid_env; const char *extra_env, *key, *value, *const *tmp; bool uid_found = FALSE, gid_found = FALSE; + uid_t orig_uid; lib_init(); str = t_str_new(1024); - orig_uid = getenv("ORIG_UID"); + orig_uid_env = getenv("ORIG_UID"); + if (orig_uid_env == NULL || str_to_uid(orig_uid_env, &orig_uid) < 0) + orig_uid = (uid_t)-1; + /* ORIG_UID should have the auth process's UID that forked us. if the checkpassword changed the UID, this could be a security hole because the UID's other processes can ptrace this process and write @@ -30,9 +34,9 @@ userdb_uid instead) */ if (getenv("INSECURE_SETUID") == NULL && - (orig_uid == NULL || strtoul(orig_uid, NULL, 10) != getuid()) && + (orig_uid == (uid_t)-1 || orig_uid != getuid()) && getuid() == geteuid() && getgid() == getegid()) { - if (orig_uid == NULL) { + if (orig_uid_env == NULL) { i_error("checkpassword: ORIG_UID environment was dropped by checkpassword. " "Can't verify if we're safe to run. See " "http://wiki2.dovecot.org/AuthDatabase/CheckPassword#Security"); diff -r de73e7121676 -r f8ab4f979e92 src/doveadm/doveadm-penalty.c --- a/src/doveadm/doveadm-penalty.c Sat Aug 29 14:31:51 2015 +0300 +++ b/src/doveadm/doveadm-penalty.c Sat Aug 29 14:42:49 2015 +0300 @@ -36,9 +36,10 @@ memset(line_r, 0, sizeof(*line_r)); (void)net_addr2ip(ident, &line_r->ip); - line_r->penalty = strtoul(penalty_str, NULL, 10); - line_r->last_penalty = strtoul(last_penalty_str, NULL, 10); - line_r->last_update = strtoul(last_update_str, NULL, 10); + if (str_to_uint(penalty_str, &line_r->penalty) < 0 || + str_to_time(last_penalty_str, &line_r->last_penalty) < 0 || + str_to_time(last_update_str, &line_r->last_update) < 0) + i_fatal("Read invalid penalty line: %s", line); } static void diff -r de73e7121676 -r f8ab4f979e92 src/doveadm/doveadm-who.c --- a/src/doveadm/doveadm-who.c Sat Aug 29 14:31:51 2015 +0300 +++ b/src/doveadm/doveadm-who.c Sat Aug 29 14:42:49 2015 +0300 @@ -63,7 +63,8 @@ p = strchr(ident, '/'); if (p == NULL) return -1; - line_r->pid = strtoul(pid_str, NULL, 10); + if (str_to_pid(pid_str, &line_r->pid) < 0) + return -1; line_r->service = t_strdup_until(ident, p++); line_r->username = strchr(p, '/'); if (line_r->username == NULL) diff -r de73e7121676 -r f8ab4f979e92 src/doveadm/dsync/dsync-ibc-stream.c --- a/src/doveadm/dsync/dsync-ibc-stream.c Sat Aug 29 14:31:51 2015 +0300 +++ b/src/doveadm/dsync/dsync-ibc-stream.c Sat Aug 29 14:42:49 2015 +0300 @@ -1593,6 +1593,7 @@ struct dsync_deserializer_decoder *decoder; struct dsync_mail_change *change; const char *value; + unsigned int uintval; enum dsync_ibc_recv_ret ret; p_clear(pool); @@ -1639,12 +1640,33 @@ return DSYNC_IBC_RECV_RET_TRYAGAIN; } - if (dsync_deserializer_decode_try(decoder, "add_flags", &value)) - change->add_flags = strtoul(value, NULL, 16); - if (dsync_deserializer_decode_try(decoder, "remove_flags", &value)) - change->remove_flags = strtoul(value, NULL, 16); - if (dsync_deserializer_decode_try(decoder, "final_flags", &value)) - change->final_flags = strtoul(value, NULL, 16); + if (dsync_deserializer_decode_try(decoder, "add_flags", &value)) { + if (str_to_uint_hex(value, &uintval) < 0 || + uintval > (uint8_t)-1) { + dsync_ibc_input_error(ibc, decoder, + "Invalid add_flags: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + change->add_flags = uintval; + } + if (dsync_deserializer_decode_try(decoder, "remove_flags", &value)) { + if (str_to_uint_hex(value, &uintval) < 0 || + uintval > (uint8_t)-1) { + dsync_ibc_input_error(ibc, decoder, + "Invalid remove_flags: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + change->remove_flags = uintval; + } + if (dsync_deserializer_decode_try(decoder, "final_flags", &value)) { + if (str_to_uint_hex(value, &uintval) < 0 || + uintval > (uint8_t)-1) { + dsync_ibc_input_error(ibc, decoder, + "Invalid final_flags: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + change->final_flags = uintval; + } if (dsync_deserializer_decode_try(decoder, "keywords_reset", &value)) change->keywords_reset = TRUE; diff -r de73e7121676 -r f8ab4f979e92 src/lib-dict/dict-file.c --- a/src/lib-dict/dict-file.c Sat Aug 29 14:31:51 2015 +0300 +++ b/src/lib-dict/dict-file.c Sat Aug 29 14:42:49 2015 +0300 @@ -329,8 +329,9 @@ *atomic_inc_not_found_r = TRUE; break; } - diff = strtoll(old_value, NULL, 10) + - change->value.diff; + if (str_to_llong(old_value, &diff) < 0) + i_unreached(); + diff += change->value.diff; tmp = t_strdup_printf("%lld", diff); new_len = strlen(tmp); if (old_value == NULL || new_len > strlen(old_value)) diff -r de73e7121676 -r f8ab4f979e92 src/lib-fs/fs-posix.c --- a/src/lib-fs/fs-posix.c Sat Aug 29 14:31:51 2015 +0300 +++ b/src/lib-fs/fs-posix.c Sat Aug 29 14:42:49 2015 +0300 @@ -104,7 +104,12 @@ else fs->path_prefix = i_strdup(arg + 7); } else if (strncmp(arg, "mode=", 5) == 0) { - fs->mode = strtoul(arg+5, NULL, 8) & 0666; + unsigned int mode; + if (str_to_uint_oct(arg+5, &mode) < 0) { + fs_set_error(_fs, "Invalid mode value: %s", arg+5); + return -1; + } + fs->mode = mode & 0666; if (fs->mode == 0) { fs_set_error(_fs, "Invalid mode: %s", arg+5); return -1; diff -r de73e7121676 -r f8ab4f979e92 src/lib-otp/otp-parse.c --- a/src/lib-otp/otp-parse.c Sat Aug 29 14:31:51 2015 +0300 +++ b/src/lib-otp/otp-parse.c Sat Aug 29 14:42:49 2015 +0300 @@ -148,7 +148,6 @@ struct otp_state *state) { const char *p, *s; - char *end; unsigned int i = 0; int algo; @@ -164,8 +163,7 @@ state->algo = algo; s = p; - state->seq = strtol(s, &end, 10); p = end; - if ((p == s) || !IS_LWS(*p)) + if (str_parse_int(s, &state->seq, &p) < 0 || !IS_LWS(*p)) return -3; p++; diff -r de73e7121676 -r f8ab4f979e92 src/lib-settings/settings-parser.c --- a/src/lib-settings/settings-parser.c Sat Aug 29 14:31:51 2015 +0300 +++ b/src/lib-settings/settings-parser.c Sat Aug 29 14:42:49 2015 +0300 @@ -338,13 +338,11 @@ unsigned int *result_r) { unsigned long long octal; - char *p; if (*value != '0') return get_uint(ctx, value, result_r); - octal = strtoull(value + 1, &p, 8); - if (*p != '\0' || octal > UINT_MAX) { + if (str_to_ullong_oct(value+1, &octal) < 0) { ctx->error = p_strconcat(ctx->parser_pool, "Invalid number: ", value, NULL); } @@ -355,10 +353,13 @@ int settings_get_time(const char *str, unsigned int *secs_r, const char **error_r) { - unsigned int num, multiply = 1; - char *p; + uintmax_t num, multiply = 1; + const char *p; - num = strtoull(str, &p, 10); + if (str_parse_uintmax(str, &num, &p) < 0) { + *error_r = t_strconcat("Invalid time interval: ", str, NULL); + return -1; + } while (*p == ' ') p++; switch (i_toupper(*p)) { case 'S': From dovecot at dovecot.org Sat Aug 29 11:50:16 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 29 Aug 2015 11:50:16 +0000 Subject: dovecot-2.2: http-client: Added support for using an HTTP proxy ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8f8f768937f5 changeset: 19039:8f8f768937f5 user: Stephan Bosch date: Sat Apr 25 11:42:06 2015 +0200 description: http-client: Added support for using an HTTP proxy running on a unix socket. diffstat: src/lib-http/http-client-connection.c | 31 +++++++--- src/lib-http/http-client-host.c | 71 ++++++++++++++++++------- src/lib-http/http-client-peer.c | 58 +++++++++++++++----- src/lib-http/http-client-private.h | 96 +++++++++++++++++++++++----------- src/lib-http/http-client-queue.c | 97 ++++++++++++++++++++++------------ src/lib-http/http-client-request.c | 48 +++++++++++++++-- src/lib-http/http-client.h | 6 +- 7 files changed, 286 insertions(+), 121 deletions(-) diffs (truncated from 781 to 300 lines): diff -r f8ab4f979e92 -r 8f8f768937f5 src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Sat Aug 29 14:42:49 2015 +0300 +++ b/src/lib-http/http-client-connection.c Sat Apr 25 11:42:06 2015 +0200 @@ -1015,7 +1015,7 @@ http_client_connection_ssl_handshaked(const char **error_r, void *context) { struct http_client_connection *conn = context; - const char *error, *host = conn->peer->addr.https_name; + const char *error, *host = conn->peer->addr.a.tcp.https_name; if (ssl_iostream_check_cert_validity(conn->ssl_iostream, host, &error) == 0) http_client_connection_debug(conn, "SSL handshake successful"); @@ -1049,7 +1049,7 @@ http_client_connection_debug(conn, "Starting SSL handshake"); if (io_stream_create_ssl_client(conn->client->ssl_ctx, - conn->peer->addr.https_name, &ssl_set, + conn->peer->addr.a.tcp.https_name, &ssl_set, &conn->conn.input, &conn->conn.output, &conn->ssl_iostream, &error) < 0) { *error_r = t_strdup_printf( @@ -1089,7 +1089,7 @@ } else { conn->connected_timestamp = ioloop_timeval; http_client_connection_debug(conn, "Connected"); - if (conn->peer->addr.https_name != NULL) { + if (http_client_peer_addr_is_https(&conn->peer->addr)) { if (http_client_connection_ssl_init(conn, &error) < 0) { http_client_peer_connection_failure(conn->peer, error); http_client_connection_debug(conn, "%s", error); @@ -1104,7 +1104,8 @@ static const struct connection_settings http_client_connection_set = { .input_max_size = (size_t)-1, .output_max_size = (size_t)-1, - .client = TRUE + .client = TRUE, + .delayed_unix_client_connected_callback = TRUE }; static const struct connection_vfuncs http_client_connection_vfuncs = { @@ -1248,6 +1249,9 @@ case HTTP_CLIENT_PEER_ADDR_RAW: conn_type = "Raw"; break; + case HTTP_CLIENT_PEER_ADDR_UNIX: + conn_type = "Unix"; + break; } conn = i_new(struct http_client_connection, 1); @@ -1258,11 +1262,20 @@ if (peer->addr.type != HTTP_CLIENT_PEER_ADDR_RAW) i_array_init(&conn->request_wait_list, 16); - if (peer->addr.type == HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL) { - http_client_connection_connect_tunnel(conn, &addr->ip, addr->port); - } else { - connection_init_client_ip - (peer->client->conn_list, &conn->conn, &addr->ip, addr->port); + switch (peer->addr.type) { + case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL: + http_client_connection_connect_tunnel + (conn, &addr->a.tcp.ip, addr->a.tcp.port); + break; + case HTTP_CLIENT_PEER_ADDR_UNIX: + connection_init_client_unix(peer->client->conn_list, &conn->conn, + addr->a.un.path); + conn->connect_initialized = TRUE; + http_client_connection_connect(conn); + break; + default: + connection_init_client_ip(peer->client->conn_list, &conn->conn, + &addr->a.tcp.ip, addr->a.tcp.port); conn->connect_initialized = TRUE; http_client_connection_connect(conn); } diff -r f8ab4f979e92 -r 8f8f768937f5 src/lib-http/http-client-host.c --- a/src/lib-http/http-client-host.c Sat Aug 29 14:42:49 2015 +0300 +++ b/src/lib-http/http-client-host.c Sat Apr 25 11:42:06 2015 +0200 @@ -137,31 +137,55 @@ } } +static struct http_client_host *http_client_host_create +(struct http_client *client) +{ + struct http_client_host *host; + + // FIXME: limit the maximum number of inactive cached hosts + host = i_new(struct http_client_host, 1); + host->client = client; + i_array_init(&host->queues, 4); + DLLIST_PREPEND(&client->hosts_list, host); + + return host; +} + struct http_client_host *http_client_host_get (struct http_client *client, const struct http_url *host_url) { struct http_client_host *host; - const char *hostname = host_url->host_name; - host = hash_table_lookup(client->hosts, hostname); - if (host == NULL) { - // FIXME: limit the maximum number of inactive cached hosts - host = i_new(struct http_client_host, 1); - host->client = client; - host->name = i_strdup(hostname); - i_array_init(&host->queues, 4); + if (host_url == NULL) { + host = client->unix_host; + if (host == NULL) { + host = http_client_host_create(client); + host->name = i_strdup("[unix]"); + host->unix_local = TRUE; - hostname = host->name; - hash_table_insert(client->hosts, hostname, host); - DLLIST_PREPEND(&client->hosts_list, host); + client->unix_host = host; - if (host_url->have_host_ip) { - host->ips_count = 1; - host->ips = i_new(struct ip_addr, host->ips_count); - host->ips[0] = host_url->host_ip; + http_client_host_debug(host, "Unix host created"); } - http_client_host_debug(host, "Host created"); + } else { + const char *hostname = host_url->host_name; + + host = hash_table_lookup(client->hosts, hostname); + if (host == NULL) { + host = http_client_host_create(client); + host->name = i_strdup(hostname); + hostname = host->name; + hash_table_insert(client->hosts, hostname, host); + + if (host_url->have_host_ip) { + host->ips_count = 1; + host->ips = i_new(struct ip_addr, host->ips_count); + host->ips[0] = host_url->host_ip; + } + + http_client_host_debug(host, "Host created"); + } } return host; } @@ -170,13 +194,14 @@ struct http_client_request *req) { struct http_client_queue *queue; - const struct http_url *host_url = req->host_url; struct http_client_peer_addr addr; const char *error; req->host = host; - if (host_url->have_ssl && host->client->ssl_ctx == NULL) { + http_client_request_get_peer_addr(req, &addr); + if (http_client_peer_addr_is_https(&addr) && + host->client->ssl_ctx == NULL) { if (http_client_init_ssl_ctx(host->client, &error) < 0) { http_client_request_error(req, HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED, error); @@ -184,12 +209,15 @@ } } - http_client_request_get_peer_addr(req, &addr); - /* add request to queue (grouped by tcp port) */ queue = http_client_queue_create(host, &addr); http_client_queue_submit_request(queue, req); + if (host->unix_local) { + http_client_queue_connection_setup(queue); + return; + } + /* start DNS lookup if necessary */ if (host->ips_count == 0 && host->dns_lookup == NULL) http_client_host_lookup(host); @@ -210,7 +238,8 @@ http_client_host_debug(host, "Host destroy"); DLLIST_REMOVE(&host->client->hosts_list, host); - hash_table_remove(host->client->hosts, hostname); + if (host != host->client->unix_host) + hash_table_remove(host->client->hosts, hostname); if (host->dns_lookup != NULL) dns_lookup_abort(&host->dns_lookup); diff -r f8ab4f979e92 -r 8f8f768937f5 src/lib-http/http-client-peer.c --- a/src/lib-http/http-client-peer.c Sat Aug 29 14:42:49 2015 +0300 +++ b/src/lib-http/http-client-peer.c Sat Apr 25 11:42:06 2015 +0200 @@ -45,13 +45,16 @@ { switch (peer->type) { case HTTP_CLIENT_PEER_ADDR_RAW: - return net_ip_hash(&peer->ip) + peer->port + 1; + return net_ip_hash(&peer->a.tcp.ip) + peer->a.tcp.port + 1; case HTTP_CLIENT_PEER_ADDR_HTTP: - return net_ip_hash(&peer->ip) + peer->port; + return net_ip_hash(&peer->a.tcp.ip) + peer->a.tcp.port; case HTTP_CLIENT_PEER_ADDR_HTTPS: case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL: - return net_ip_hash(&peer->ip) + peer->port + - (peer->https_name == NULL ? 0 : str_hash(peer->https_name)); + return net_ip_hash(&peer->a.tcp.ip) + peer->a.tcp.port + + (peer->a.tcp.https_name == NULL ? + 0 : str_hash(peer->a.tcp.https_name)); + case HTTP_CLIENT_PEER_ADDR_UNIX: + return str_hash(peer->a.un.path); } i_unreached(); return 0; @@ -65,13 +68,24 @@ if (peer1->type != peer2->type) return (peer1->type > peer2->type ? 1 : -1); - if ((ret=net_ip_cmp(&peer1->ip, &peer2->ip)) != 0) - return ret; - if (peer1->port != peer2->port) - return (peer1->port > peer2->port ? 1 : -1); - if (peer1->type != HTTP_CLIENT_PEER_ADDR_HTTPS) - return 0; - return null_strcmp(peer1->https_name, peer2->https_name); + switch (peer1->type) { + case HTTP_CLIENT_PEER_ADDR_RAW: + case HTTP_CLIENT_PEER_ADDR_HTTP: + case HTTP_CLIENT_PEER_ADDR_HTTPS: + case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL: + if ((ret=net_ip_cmp(&peer1->a.tcp.ip, &peer2->a.tcp.ip)) != 0) + return ret; + if (peer1->a.tcp.port != peer2->a.tcp.port) + return (peer1->a.tcp.port > peer2->a.tcp.port ? 1 : -1); + if (peer1->type != HTTP_CLIENT_PEER_ADDR_HTTPS) + return 0; + return null_strcmp + (peer1->a.tcp.https_name, peer2->a.tcp.https_name); + case HTTP_CLIENT_PEER_ADDR_UNIX: + return null_strcmp(peer1->a.un.path, peer2->a.un.path); + } + i_unreached(); + return 0; } /* @@ -438,13 +452,25 @@ { struct http_client_peer *peer; - i_assert(addr->https_name == NULL || client->ssl_ctx != NULL); - peer = i_new(struct http_client_peer, 1); peer->client = client; peer->addr = *addr; - peer->https_name = i_strdup(addr->https_name); - peer->addr.https_name = peer->https_name; + + switch (addr->type) { + case HTTP_CLIENT_PEER_ADDR_HTTPS: + case HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL: + i_assert(client->ssl_ctx != NULL); + peer->addr_name = i_strdup(addr->a.tcp.https_name); + peer->addr.a.tcp.https_name = peer->addr_name; + break; + case HTTP_CLIENT_PEER_ADDR_UNIX: + peer->addr_name = i_strdup(addr->a.un.path); + peer->addr.a.un.path = peer->addr_name; + break; + default: + break; + } + i_array_init(&peer->queues, 16); i_array_init(&peer->conns, 16); @@ -479,7 +505,7 @@ (peer->client->peers, (const struct http_client_peer_addr *)&peer->addr); DLLIST_REMOVE(&peer->client->peers_list, peer); - i_free(peer->https_name); + i_free(peer->addr_name); i_free(peer); *_peer = NULL; } diff -r f8ab4f979e92 -r 8f8f768937f5 src/lib-http/http-client-private.h --- a/src/lib-http/http-client-private.h Sat Aug 29 14:42:49 2015 +0300 +++ b/src/lib-http/http-client-private.h Sat Apr 25 11:42:06 2015 +0200 @@ -38,14 +38,22 @@ HTTP_CLIENT_PEER_ADDR_HTTP = 0, HTTP_CLIENT_PEER_ADDR_HTTPS, HTTP_CLIENT_PEER_ADDR_HTTPS_TUNNEL, - HTTP_CLIENT_PEER_ADDR_RAW + HTTP_CLIENT_PEER_ADDR_RAW, From pigeonhole at rename-it.nl Sat Aug 29 13:04:47 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sat, 29 Aug 2015 15:04:47 +0200 Subject: dovecot-2.2-pigeonhole: Removed all invocations of strtoll() and... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/45854bc3a431 changeset: 2094:45854bc3a431 user: Stephan Bosch date: Sat Aug 29 13:12:58 2015 +0200 description: Removed all invocations of strtoll() and friends. diffstat: src/lib-sieve/sieve-settings.c | 81 ++++++++++------------------------------- 1 files changed, 20 insertions(+), 61 deletions(-) diffs (166 lines): diff -r a2e0882300c9 -r 45854bc3a431 src/lib-sieve/sieve-settings.c --- a/src/lib-sieve/sieve-settings.c Sat Aug 29 13:12:58 2015 +0200 +++ b/src/lib-sieve/sieve-settings.c Sat Aug 29 13:12:58 2015 +0200 @@ -38,60 +38,18 @@ * Access to settings */ -static bool sieve_setting_parse_uint -(struct sieve_instance *svinst, const char *setting, const char *str_value, - char **endptr, unsigned long long int *value_r) -{ - if ( (*value_r = strtoull(str_value, endptr, 10)) == ULLONG_MAX - && errno == ERANGE ) { - sieve_sys_warning(svinst, - "overflowing unsigned integer value for setting '%s': '%s'", - setting, str_value); - return FALSE; - } - - return TRUE; -} - -static bool sieve_setting_parse_int -(struct sieve_instance *svinst, const char *setting, const char *str_value, - char **endptr, long long int *value_r) -{ - *value_r = strtoll(str_value, endptr, 10); - - if ( *value_r == LLONG_MIN && errno == ERANGE ) { - sieve_sys_warning(svinst, - "underflowing integer value for setting '%s': '%s'", - setting, str_value); - return FALSE; - } - - if ( *value_r == LLONG_MAX && errno == ERANGE ) { - sieve_sys_warning(svinst, - "overflowing integer value for setting '%s': '%s'", - setting, str_value); - return FALSE; - } - - return TRUE; -} - bool sieve_setting_get_uint_value (struct sieve_instance *svinst, const char *setting, unsigned long long int *value_r) { const char *str_value; - char *endp; str_value = sieve_setting_get(svinst, setting); if ( str_value == NULL || *str_value == '\0' ) return FALSE; - if ( !sieve_setting_parse_uint(svinst, setting, str_value, &endp, value_r) ) - return FALSE; - - if ( *endp != '\0' ) { + if ( str_to_ullong(str_value, value_r) < 0 ) { sieve_sys_warning(svinst, "invalid unsigned integer value for setting '%s': '%s'", setting, str_value); @@ -106,20 +64,16 @@ long long int *value_r) { const char *str_value; - char *endp; str_value = sieve_setting_get(svinst, setting); if ( str_value == NULL || *str_value == '\0' ) return FALSE; - if ( !sieve_setting_parse_int(svinst, setting, str_value, &endp, value_r) ) - return FALSE; - - if ( *endp != '\0' ) { - sieve_sys_warning(svinst, "invalid integer value for setting '%s': '%s'", + if ( str_to_llong(str_value, value_r) < 0 ) { + sieve_sys_warning(svinst, + "invalid integer value for setting '%s': '%s'", setting, str_value); - return FALSE; } @@ -131,17 +85,20 @@ size_t *value_r) { const char *str_value; - unsigned long long int value, multiply = 1; - char *endp; + uintmax_t value, multiply = 1; + const char *endp; str_value = sieve_setting_get(svinst, setting); if ( str_value == NULL || *str_value == '\0' ) return FALSE; - if ( !sieve_setting_parse_uint(svinst, setting, str_value, &endp, &value) ) + if ( str_parse_uintmax(str_value, &value, &endp) < 0 ) { + sieve_sys_warning(svinst, + "invalid size value for setting '%s': '%s'", + setting, str_value); return FALSE; - + } switch (i_toupper(*endp)) { case '\0': /* default */ case 'B': /* byte (useless) */ @@ -174,7 +131,6 @@ } *value_r = (size_t) (value * multiply); - return TRUE; } @@ -202,7 +158,8 @@ return TRUE; } - sieve_sys_warning(svinst, "invalid boolean value for setting '%s': '%s'", + sieve_sys_warning(svinst, + "invalid boolean value for setting '%s': '%s'", setting, str_value); return FALSE; } @@ -212,8 +169,8 @@ sieve_number_t *value_r) { const char *str_value; - unsigned long long int value, multiply = 1; - char *endp; + uintmax_t value, multiply = 1; + const char *endp; str_value = sieve_setting_get(svinst, setting); if ( str_value == NULL ) @@ -223,9 +180,12 @@ if ( *str_value == '\0' ) return FALSE; - if ( !sieve_setting_parse_uint(svinst, setting, str_value, &endp, &value) ) + if ( str_parse_uintmax(str_value, &value, &endp) < 0 ) { + sieve_sys_warning(svinst, + "invalid duration value for setting '%s': '%s'", + setting, str_value); return FALSE; - + } switch (i_tolower(*endp)) { case '\0': /* default */ case 's': /* seconds */ @@ -255,7 +215,6 @@ } *value_r = (unsigned int) (value * multiply); - return TRUE; } From pigeonhole at rename-it.nl Sat Aug 29 13:04:47 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sat, 29 Aug 2015 15:04:47 +0200 Subject: dovecot-2.2-pigeonhole: Removed all invocations of atoi(). Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/a2e0882300c9 changeset: 2093:a2e0882300c9 user: Stephan Bosch date: Sat Aug 29 13:12:58 2015 +0200 description: Removed all invocations of atoi(). diffstat: src/lib-sieve/storage/ldap/sieve-ldap-db.c | 11 +++++++---- 1 files changed, 7 insertions(+), 4 deletions(-) diffs (33 lines): diff -r f84fbac570f5 -r a2e0882300c9 src/lib-sieve/storage/ldap/sieve-ldap-db.c --- a/src/lib-sieve/storage/ldap/sieve-ldap-db.c Sat Aug 29 13:12:58 2015 +0200 +++ b/src/lib-sieve/storage/ldap/sieve-ldap-db.c Sat Aug 29 13:12:58 2015 +0200 @@ -764,8 +764,7 @@ "deref", set->deref) < 0) return -1; #ifdef LDAP_OPT_DEBUG_LEVEL - value = atoi(set->debug_level); - if (value != 0) { + if (str_to_int(set->debug_level, &value) >= 0 && value != 0) { if (db_ldap_set_opt(NULL, LDAP_OPT_DEBUG_LEVEL, &value, "debug_level", set->debug_level) < 0) return -1; @@ -798,13 +797,17 @@ { const struct sieve_ldap_storage_settings *set = &conn->lstorage->set; struct sieve_storage *storage = &conn->lstorage->storage; - bool debug = atoi(set->debug_level) > 0; struct timeval start, end; - int ret; + int debug_level, ret; + bool debug; if (conn->conn_state != LDAP_CONN_STATE_DISCONNECTED) return 0; + debug = FALSE; + if (str_to_int(set->debug_level, &debug_level) >= 0) + debug = debug_level > 0; + if (debug) { if (gettimeofday(&start, NULL) < 0) memset(&start, 0, sizeof(start)); From pigeonhole at rename-it.nl Sat Aug 29 13:04:47 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sat, 29 Aug 2015 15:04:47 +0200 Subject: dovecot-2.2-pigeonhole: Changed type of internet port values to ... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/f84fbac570f5 changeset: 2092:f84fbac570f5 user: Stephan Bosch date: Sat Aug 29 13:12:58 2015 +0200 description: Changed type of internet port values to in_port_t everywhere. diffstat: src/managesieve-login/client.c | 6 +----- 1 files changed, 1 insertions(+), 5 deletions(-) diffs (24 lines): diff -r 72e423848263 -r f84fbac570f5 src/managesieve-login/client.c --- a/src/managesieve-login/client.c Sat Aug 29 12:53:29 2015 +0200 +++ b/src/managesieve-login/client.c Sat Aug 29 13:12:58 2015 +0200 @@ -159,7 +159,6 @@ (struct managesieve_client *client, const struct managesieve_arg *args) { - unsigned int remote_port; const char *arg; bool args_ok = TRUE; @@ -174,11 +173,8 @@ if (net_addr2ip(arg + 5, &client->common.ip) < 0) args_ok = FALSE; } else if ( strncasecmp(arg, "PORT=", 5) == 0 ) { - if (str_to_uint(arg + 5, &remote_port) < 0 || - remote_port == 0 || remote_port > 65535) + if (net_str2port(arg + 5, &client->common.remote_port) < 0) args_ok = FALSE; - else - client->common.remote_port = remote_port; } else if ( strncasecmp(arg, "SESSION=", 8) == 0 ) { const char *value = arg + 8; From dovecot at dovecot.org Sat Aug 29 21:43:12 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 29 Aug 2015 21:43:12 +0000 Subject: dovecot-2.2: lib-master: Compiler warning fix Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/512a4792d9f5 changeset: 19040:512a4792d9f5 user: Timo Sirainen date: Sun Aug 30 00:42:16 2015 +0300 description: lib-master: Compiler warning fix diffstat: src/lib-master/master-service.c | 2 -- 1 files changed, 0 insertions(+), 2 deletions(-) diffs (12 lines): diff -r 8f8f768937f5 -r 512a4792d9f5 src/lib-master/master-service.c --- a/src/lib-master/master-service.c Sat Apr 25 11:42:06 2015 +0200 +++ b/src/lib-master/master-service.c Sun Aug 30 00:42:16 2015 +0300 @@ -161,8 +161,6 @@ #ifdef DEBUG if (getenv("GDB") == NULL && (flags & MASTER_SERVICE_FLAG_STANDALONE) == 0) { - int count; - value = getenv("SOCKET_COUNT"); if (value == NULL || str_to_uint(value, &count) < 0) count = 0; From dovecot at dovecot.org Sat Aug 29 21:47:40 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 29 Aug 2015 21:47:40 +0000 Subject: dovecot-2.2: Earlier in_port_t fix created problems with service... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c53ccafc4a8e changeset: 19041:c53ccafc4a8e user: Stephan Bosch date: Sun Aug 30 00:46:32 2015 +0300 description: Earlier in_port_t fix created problems with service listener configuration. Listeners are disabled with port=0, which was not allowed anymore. diffstat: src/config/config-request.c | 4 +++- src/lib-settings/settings-parser.c | 22 +++++++++++++++++++++- src/lib-settings/settings-parser.h | 1 + src/lib/net.c | 13 +++++++++++++ src/lib/net.h | 2 ++ src/master/master-settings.c | 2 +- 6 files changed, 41 insertions(+), 3 deletions(-) diffs (146 lines): diff -r 512a4792d9f5 -r c53ccafc4a8e src/config/config-request.c --- a/src/config/config-request.c Sun Aug 30 00:42:16 2015 +0300 +++ b/src/config/config-request.c Sun Aug 30 00:46:32 2015 +0300 @@ -116,7 +116,8 @@ } break; } - case SET_IN_PORT: { + case SET_IN_PORT: + case SET_IN_PORT_ZERO: { const in_port_t *val = value, *dval = default_value; if (dump_default || dval == NULL || *val != *dval) @@ -253,6 +254,7 @@ case SET_UINT_OCT: case SET_TIME: case SET_IN_PORT: + case SET_IN_PORT_ZERO: case SET_STR_VARS: case SET_STR: case SET_ENUM: diff -r 512a4792d9f5 -r c53ccafc4a8e src/lib-settings/settings-parser.c --- a/src/lib-settings/settings-parser.c Sun Aug 30 00:42:16 2015 +0300 +++ b/src/lib-settings/settings-parser.c Sun Aug 30 00:46:32 2015 +0300 @@ -595,6 +595,18 @@ } static int +get_in_port_zero(struct setting_parser_context *ctx, const char *value, + in_port_t *result_r) +{ + if (net_str2port(value, result_r) < 0) { + ctx->error = p_strdup_printf(ctx->parser_pool, + "Invalid port number %s", value); + return -1; + } + return 0; +} + +static int settings_parse(struct setting_parser_context *ctx, struct setting_link *link, const struct setting_define *def, const char *key, const char *value) @@ -646,6 +658,10 @@ if (get_in_port(ctx, value, (in_port_t *)ptr) < 0) return -1; break; + case SET_IN_PORT_ZERO: + if (get_in_port_zero(ctx, value, (in_port_t *)ptr) < 0) + return -1; + break; case SET_STR: *((char **)ptr) = p_strdup(ctx->set_pool, value); break; @@ -1248,6 +1264,7 @@ case SET_TIME: case SET_SIZE: case SET_IN_PORT: + case SET_IN_PORT_ZERO: case SET_STR: case SET_ENUM: case SET_STRLIST: @@ -1344,6 +1361,7 @@ case SET_TIME: case SET_SIZE: case SET_IN_PORT: + case SET_IN_PORT_ZERO: case SET_STR: case SET_ENUM: case SET_STRLIST: @@ -1428,7 +1446,8 @@ *dest_size = *src_size; break; } - case SET_IN_PORT: { + case SET_IN_PORT: + case SET_IN_PORT_ZERO: { const in_port_t *src_size = src; in_port_t *dest_size = dest; @@ -1549,6 +1568,7 @@ case SET_TIME: case SET_SIZE: case SET_IN_PORT: + case SET_IN_PORT_ZERO: case SET_STR_VARS: case SET_STR: case SET_ENUM: diff -r 512a4792d9f5 -r c53ccafc4a8e src/lib-settings/settings-parser.h --- a/src/lib-settings/settings-parser.h Sun Aug 30 00:42:16 2015 +0300 +++ b/src/lib-settings/settings-parser.h Sun Aug 30 00:46:32 2015 +0300 @@ -24,6 +24,7 @@ SET_TIME, SET_SIZE, SET_IN_PORT, /* internet port */ + SET_IN_PORT_ZERO, /* internet port */ SET_STR, SET_STR_VARS, /* string with %variables */ SET_ENUM, diff -r 512a4792d9f5 -r c53ccafc4a8e src/lib/net.c --- a/src/lib/net.c Sun Aug 30 00:42:16 2015 +0300 +++ b/src/lib/net.c Sun Aug 30 00:46:32 2015 +0300 @@ -957,6 +957,19 @@ return 0; } +int net_str2port_zero(const char *str, in_port_t *port_r) +{ + uintmax_t l; + + if (str_to_uintmax(str, &l) < 0) + return -1; + + if (l > (in_port_t)-1) + return -1; + *port_r = (in_port_t)l; + return 0; +} + int net_ipv6_mapped_ipv4_convert(const struct ip_addr *src, struct ip_addr *dest) { diff -r 512a4792d9f5 -r c53ccafc4a8e src/lib/net.h --- a/src/lib/net.h Sun Aug 30 00:42:16 2015 +0300 +++ b/src/lib/net.h Sun Aug 30 00:46:32 2015 +0300 @@ -150,6 +150,8 @@ int net_addr2ip(const char *addr, struct ip_addr *ip); /* char* -> net_port_t translation */ int net_str2port(const char *str, in_port_t *port_r); +/* char* -> net_port_t translation (allows port zero) */ +int net_str2port_zero(const char *str, in_port_t *port_r); /* Convert IPv6 mapped IPv4 address to an actual IPv4 address. Returns 0 if successful, -1 if the source address isn't IPv6 mapped IPv4 address. */ diff -r 512a4792d9f5 -r c53ccafc4a8e src/master/master-settings.c --- a/src/master/master-settings.c Sun Aug 30 00:42:16 2015 +0300 +++ b/src/master/master-settings.c Sun Aug 30 00:46:32 2015 +0300 @@ -62,7 +62,7 @@ static const struct setting_define inet_listener_setting_defines[] = { DEF(SET_STR, name), DEF(SET_STR, address), - DEF(SET_IN_PORT, port), + DEF(SET_IN_PORT_ZERO, port), DEF(SET_BOOL, ssl), DEF(SET_BOOL, reuse_port), DEF(SET_BOOL, haproxy), From dovecot at dovecot.org Sat Aug 29 22:01:24 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 29 Aug 2015 22:01:24 +0000 Subject: dovecot-2.2: aggregator: Allow replicator_port=0 setting (fix to... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e1c7fcd62813 changeset: 19042:e1c7fcd62813 user: Timo Sirainen date: Sun Aug 30 01:00:30 2015 +0300 description: aggregator: Allow replicator_port=0 setting (fix to previous in_port_t changes) diffstat: src/replication/aggregator/aggregator-settings.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r c53ccafc4a8e -r e1c7fcd62813 src/replication/aggregator/aggregator-settings.c --- a/src/replication/aggregator/aggregator-settings.c Sun Aug 30 00:46:32 2015 +0300 +++ b/src/replication/aggregator/aggregator-settings.c Sun Aug 30 01:00:30 2015 +0300 @@ -61,7 +61,7 @@ static const struct setting_define aggregator_setting_defines[] = { DEF(SET_STR, replicator_host), - DEF(SET_IN_PORT, replicator_port), + DEF(SET_IN_PORT_ZERO, replicator_port), SETTING_DEFINE_LIST_END }; From dovecot at dovecot.org Mon Aug 31 10:35:15 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 31 Aug 2015 10:35:15 +0000 Subject: dovecot-2.2: lib-fts: Add UTF-8 unit test for lowercase filter. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b5dfafc0b235 changeset: 19044:b5dfafc0b235 user: Teemu Huovila date: Mon Aug 31 13:33:26 2015 +0300 description: lib-fts: Add UTF-8 unit test for lowercase filter. diffstat: src/lib-fts/test-fts-filter.c | 32 ++++++++++++++++++++++++++++++++ 1 files changed, 32 insertions(+), 0 deletions(-) diffs (49 lines): diff -r 4b8c7440cf4f -r b5dfafc0b235 src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Mon Aug 31 13:33:26 2015 +0300 +++ b/src/lib-fts/test-fts-filter.c Mon Aug 31 13:33:26 2015 +0300 @@ -110,6 +110,35 @@ test_end(); } +#ifdef HAVE_LIBICU +static void test_fts_filter_lowercase_utf8(void) +{ + struct { + const char *input; + const char *output; + } tests[] = { + { "f\xC3\x85\xC3\x85", "f\xC3\xA5\xC3\xA5" }, + { "F\xC3\x85\xC3\x85", "f\xC3\xA5\xC3\xA5" }, + { "F\xC3\x85\xC3\xA5", "f\xC3\xA5\xC3\xA5" } + }; + struct fts_filter *filter; + const char *error; + const char *token; + unsigned int i; + + test_begin("fts filter lowercase, UTF8"); + test_assert(fts_filter_create(fts_filter_lowercase, NULL, &english_language, NULL, &filter, &error) == 0); + + for (i = 0; i < N_ELEMENTS(tests); i++) { + token = tests[i].input; + test_assert_idx(fts_filter_filter(filter, &token, &error) > 0 && + strcmp(token, tests[i].output) == 0, 0); + } + fts_filter_unref(&filter); + test_end(); +} +#endif + static void test_fts_filter_stopwords_eng(void) { struct fts_filter *filter; @@ -688,6 +717,9 @@ test_fts_filter_contractions_fail, test_fts_filter_contractions_fr, test_fts_filter_lowercase, +#ifdef HAVE_LIBICU + test_fts_filter_lowercase_utf8, +#endif test_fts_filter_stopwords_eng, test_fts_filter_stopwords_fin, test_fts_filter_stopwords_fra, From dovecot at dovecot.org Mon Aug 31 10:35:14 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 31 Aug 2015 10:35:14 +0000 Subject: dovecot-2.2: lib-fts: Add prefixing contraction filter. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4b8c7440cf4f changeset: 19043:4b8c7440cf4f user: Teemu Huovila date: Mon Aug 31 13:33:26 2015 +0300 description: lib-fts: Add prefixing contraction filter. Filters away prefixing contracted words, e.g. "l'homme" -> "homme". Tokens to be filtered must be lower case. Only supports French in this initial version. diffstat: src/lib-fts/Makefile.am | 1 + src/lib-fts/fts-filter-contractions.c | 84 +++++++++++++++++++++++++++++++++++ src/lib-fts/fts-filter-private.h | 2 +- src/lib-fts/fts-filter.c | 1 + src/lib-fts/fts-filter.h | 3 + src/lib-fts/test-fts-filter.c | 68 ++++++++++++++++++++++++++- 6 files changed, 154 insertions(+), 5 deletions(-) diffs (255 lines): diff -r e1c7fcd62813 -r 4b8c7440cf4f src/lib-fts/Makefile.am --- a/src/lib-fts/Makefile.am Sun Aug 30 01:00:30 2015 +0300 +++ b/src/lib-fts/Makefile.am Mon Aug 31 13:33:26 2015 +0300 @@ -62,6 +62,7 @@ libfts_la_SOURCES = \ fts-filter.c \ + fts-filter-contractions.c \ fts-filter-english-possessive.c \ fts-filter-lowercase.c \ fts-filter-normalizer-icu.c \ diff -r e1c7fcd62813 -r 4b8c7440cf4f src/lib-fts/fts-filter-contractions.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-fts/fts-filter-contractions.c Mon Aug 31 13:33:26 2015 +0300 @@ -0,0 +1,84 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "fts-language.h" +#include "fts-filter-private.h" +#include "fts-common.h" +#include "unichar.h" + +static int +fts_filter_contractions_create(const struct fts_language *lang, + const char *const *settings, + struct fts_filter **filter_r, + const char **error_r) +{ + struct fts_filter *filter; + + if (settings[0] != NULL) { + *error_r = t_strdup_printf("Unknown setting: %s", settings[0]); + return -1; + } + if (strcmp(lang->name, "fr") != 0) { + *error_r = t_strdup_printf("Unsupported language: %s", lang->name); + return -1; + } + + filter = i_new(struct fts_filter, 1); + *filter = *fts_filter_contractions; + filter->token = str_new(default_pool, 64); + *filter_r = filter; + return 0; +} + +static int +fts_filter_contractions_filter(struct fts_filter *filter ATTR_UNUSED, + const char **_token, + const char **error_r ATTR_UNUSED) +{ + int char_size, pos = 0; + unichar_t apostrophe; + const char *token = *_token; + + switch (token[pos]) { + case 'q': + pos++; + if (token[pos] == '\0' || token[pos] != 'u') + break; + /* otherwise fall through */ + case 'c': + case 'd': + case 'l': + case 'm': + case 'n': + case 's': + case 't': + pos++; + if (token[pos] == '\0') + break; + char_size = uni_utf8_get_char(token + pos, &apostrophe); + if (IS_APOSTROPHE(apostrophe)) { + pos += char_size; + *_token = token + pos; + } + if (token[pos] == '\0') /* nothing left */ + return 0; + break; + default: + /* do nothing */ + break; + } + + return 1; +} + +static const struct fts_filter fts_filter_contractions_real = { + .class_name = "contractions", + .v = { + fts_filter_contractions_create, + fts_filter_contractions_filter, + NULL + } +}; + +const struct fts_filter *fts_filter_contractions = &fts_filter_contractions_real; diff -r e1c7fcd62813 -r 4b8c7440cf4f src/lib-fts/fts-filter-private.h --- a/src/lib-fts/fts-filter-private.h Sun Aug 30 01:00:30 2015 +0300 +++ b/src/lib-fts/fts-filter-private.h Mon Aug 31 13:33:26 2015 +0300 @@ -3,7 +3,7 @@ #include "fts-filter.h" -#define FTS_FILTER_CLASSES_NR 3 +#define FTS_FILTER_CLASSES_NR 6 /* API that stemming providers (classes) must provide: The create() diff -r e1c7fcd62813 -r 4b8c7440cf4f src/lib-fts/fts-filter.c --- a/src/lib-fts/fts-filter.c Sun Aug 30 01:00:30 2015 +0300 +++ b/src/lib-fts/fts-filter.c Mon Aug 31 13:33:26 2015 +0300 @@ -21,6 +21,7 @@ fts_filter_register(fts_filter_normalizer_icu); fts_filter_register(fts_filter_lowercase); fts_filter_register(fts_filter_english_possessive); + fts_filter_register(fts_filter_contractions); } void fts_filters_deinit(void) diff -r e1c7fcd62813 -r 4b8c7440cf4f src/lib-fts/fts-filter.h --- a/src/lib-fts/fts-filter.h Sun Aug 30 01:00:30 2015 +0300 +++ b/src/lib-fts/fts-filter.h Mon Aug 31 13:33:26 2015 +0300 @@ -35,6 +35,9 @@ /* Removes <'s> suffix from words. */ extern const struct fts_filter *fts_filter_english_possessive; +/* Removes prefixing contractions from words. */ +extern const struct fts_filter *fts_filter_contractions; + /* Register all built-in filters. */ void fts_filters_init(void); void fts_filters_deinit(void); diff -r e1c7fcd62813 -r 4b8c7440cf4f src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Sun Aug 30 01:00:30 2015 +0300 +++ b/src/lib-fts/test-fts-filter.c Mon Aug 31 13:33:26 2015 +0300 @@ -12,6 +12,7 @@ static const char *const stopword_settings[] = {"stopwords_dir", TEST_STOPWORDS_DIR, NULL}; static struct fts_language english_language = { .name = "en" }; +static struct fts_language french_language = { .name = "fr" }; static void test_fts_filter_find(void) { @@ -20,6 +21,65 @@ test_assert(fts_filter_find("snowball") == fts_filter_stemmer_snowball); test_assert(fts_filter_find("normalizer-icu") == fts_filter_normalizer_icu); test_assert(fts_filter_find("lowercase") == fts_filter_lowercase); + test_assert(fts_filter_find("contractions") == fts_filter_contractions); + test_end(); +} + + +static void test_fts_filter_contractions_fail(void) +{ + + struct fts_filter *filter; + const char *error; + + test_begin("fts filter contractions, unsupported language"); + test_assert(fts_filter_create(fts_filter_contractions, NULL, &english_language, NULL, &filter, &error) != 0); + test_assert(error != NULL); + test_end(); +} + +static void test_fts_filter_contractions_fr(void) +{ + struct { + const char *input; + const char *output; + } tests[] = { + { "foo", "foo" }, + { "you're", "you're" }, + { "l'homme", "homme" }, + { "l\xE2\x80\x99homme", "homme" }, + { "aujourd'hui", "aujourd'hui" }, + { "qu\xE2\x80\x99il", "il" }, + { "qu'il", "il" }, + { "du'il", "du'il" }, + { "que", "que" }, + { "'foobar'", "'foobar'" }, + { "foo'bar", "foo'bar" }, + { "a'foo", "a'foo" }, + { "cu'", "cu'" }, + { "qu", "qu" }, + { "d", "d" }, + { "qu'", NULL } + }; + struct fts_filter *filter; + const char *error; + const char *token; + unsigned int i; + int ret; + + test_begin("fts filter contractions, French"); + test_assert(fts_filter_create(fts_filter_contractions, NULL, &french_language, NULL, &filter, &error) == 0); + + for (i = 0; i < N_ELEMENTS(tests); i++) { + token = tests[i].input; + ret = fts_filter_filter(filter, &token, &error); + test_assert(ret >= 0); + if (ret > 0) + test_assert_idx(strcmp(token, tests[i].output) == 0, i); + else if (ret == 0) + test_assert_idx(token == NULL && tests[i].output == NULL, i); + } + fts_filter_unref(&filter); test_end(); } @@ -151,7 +211,6 @@ static void test_fts_filter_stopwords_fra(void) { - const struct fts_language french = { .name = "fr" }; struct fts_filter *filter; const char *error; int ret; @@ -167,7 +226,7 @@ const char *token; test_begin("fts filter stopwords, French"); - test_assert(fts_filter_create(fts_filter_stopwords, NULL, &french, stopword_settings, &filter, &error) == 0); + test_assert(fts_filter_create(fts_filter_stopwords, NULL, &french_language, stopword_settings, &filter, &error) == 0); ip = input; op = output; @@ -245,7 +304,6 @@ { struct fts_filter *stemmer; const char *error; - struct fts_language language = { .name = "fr" }; const char *token = NULL; const char * const tokens[] = { "Tous", "les", "\xC3\xAAtres", "humains", "naissent", @@ -258,7 +316,7 @@ const char * const *bpp; test_begin("fts filter stem French"); - test_assert(fts_filter_create(fts_filter_stemmer_snowball, NULL, &language, NULL, &stemmer, &error) == 0); + test_assert(fts_filter_create(fts_filter_stemmer_snowball, NULL, &french_language, NULL, &stemmer, &error) == 0); bpp = bases; for (tpp=tokens; *tpp != NULL; tpp++) { token = *tpp; @@ -627,6 +685,8 @@ { static void (*test_functions[])(void) = { test_fts_filter_find, + test_fts_filter_contractions_fail, + test_fts_filter_contractions_fr, test_fts_filter_lowercase, test_fts_filter_stopwords_eng, test_fts_filter_stopwords_fin, From dovecot at dovecot.org Mon Aug 31 11:25:53 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 31 Aug 2015 11:25:53 +0000 Subject: dovecot-2.2: Earlier in_port_t fix was ineffective due to one sm... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/08b29b4b5e96 changeset: 19045:08b29b4b5e96 user: Stephan Bosch date: Mon Aug 31 14:24:54 2015 +0300 description: Earlier in_port_t fix was ineffective due to one small detail. The new SET_IN_PORT_ZERO did not actually use the new net_str2port_zero() function. diffstat: src/lib-settings/settings-parser.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r b5dfafc0b235 -r 08b29b4b5e96 src/lib-settings/settings-parser.c --- a/src/lib-settings/settings-parser.c Mon Aug 31 13:33:26 2015 +0300 +++ b/src/lib-settings/settings-parser.c Mon Aug 31 14:24:54 2015 +0300 @@ -598,7 +598,7 @@ get_in_port_zero(struct setting_parser_context *ctx, const char *value, in_port_t *result_r) { - if (net_str2port(value, result_r) < 0) { + if (net_str2port_zero(value, result_r) < 0) { ctx->error = p_strdup_printf(ctx->parser_pool, "Invalid port number %s", value); return -1; From dovecot at dovecot.org Mon Aug 31 17:24:34 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 31 Aug 2015 17:24:34 +0000 Subject: dovecot-2.2: lib-fts: Add missing "j'" to French contractions. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7316c0b61cb9 changeset: 19046:7316c0b61cb9 user: Teemu Huovila date: Mon Aug 31 20:23:32 2015 +0300 description: lib-fts: Add missing "j'" to French contractions. diffstat: src/lib-fts/fts-filter-contractions.c | 1 + src/lib-fts/test-fts-filter.c | 4 +++- 2 files changed, 4 insertions(+), 1 deletions(-) diffs (25 lines): diff -r 08b29b4b5e96 -r 7316c0b61cb9 src/lib-fts/fts-filter-contractions.c --- a/src/lib-fts/fts-filter-contractions.c Mon Aug 31 14:24:54 2015 +0300 +++ b/src/lib-fts/fts-filter-contractions.c Mon Aug 31 20:23:32 2015 +0300 @@ -48,6 +48,7 @@ /* otherwise fall through */ case 'c': case 'd': + case 'j': case 'l': case 'm': case 'n': diff -r 08b29b4b5e96 -r 7316c0b61cb9 src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Mon Aug 31 14:24:54 2015 +0300 +++ b/src/lib-fts/test-fts-filter.c Mon Aug 31 20:23:32 2015 +0300 @@ -59,7 +59,9 @@ { "cu'", "cu'" }, { "qu", "qu" }, { "d", "d" }, - { "qu'", NULL } + { "qu'", NULL }, + { "j'adore", "adore" }, + { "quelqu'un", "quelqu'un" } }; struct fts_filter *filter; const char *error; From dovecot at dovecot.org Mon Aug 31 18:32:43 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 31 Aug 2015 18:32:43 +0000 Subject: dovecot-2.2: cassandra: Split consistency setting to read_consis... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c33868915764 changeset: 19047:c33868915764 user: Timo Sirainen date: Mon Aug 31 21:31:46 2015 +0300 description: cassandra: Split consistency setting to read_consistency and write_consistency. diffstat: src/lib-sql/driver-cassandra.c | 100 ++++++++++++++++------------------------ 1 files changed, 40 insertions(+), 60 deletions(-) diffs (181 lines): diff -r 7316c0b61cb9 -r c33868915764 src/lib-sql/driver-cassandra.c --- a/src/lib-sql/driver-cassandra.c Mon Aug 31 20:23:32 2015 +0300 +++ b/src/lib-sql/driver-cassandra.c Mon Aug 31 21:31:46 2015 +0300 @@ -29,7 +29,7 @@ struct sql_db api; char *hosts, *keyspace; - CassConsistency consistency; + CassConsistency read_consistency, write_consistency; CassLogLevel log_level; CassCluster *cluster; @@ -46,7 +46,8 @@ char *error; - unsigned int set_consistency:1; + unsigned int set_read_consistency:1; + unsigned int set_write_consistency:1; }; struct cassandra_result { @@ -64,6 +65,7 @@ void *context; unsigned int finished:1; + unsigned int write_query:1; }; struct cassandra_transaction_context { @@ -352,10 +354,14 @@ strcmp(key, "keyspace") == 0) { i_free(db->keyspace); db->keyspace = i_strdup(value); - } else if (strcmp(key, "consistency") == 0) { - if (consistency_parse(value, &db->consistency) < 0) - i_fatal("cassandra: Unknown consistency: %s", value); - db->set_consistency = TRUE; + } else if (strcmp(key, "read_consistency") == 0) { + if (consistency_parse(value, &db->read_consistency) < 0) + i_fatal("cassandra: Unknown read_consistency: %s", value); + db->set_read_consistency = TRUE; + } else if (strcmp(key, "write_consistency") == 0) { + if (consistency_parse(value, &db->write_consistency) < 0) + i_fatal("cassandra: Unknown write_consistency: %s", value); + db->set_write_consistency = TRUE; } else if (strcmp(key, "log_level") == 0) { if (log_level_parse(value, &db->log_level) < 0) i_fatal("cassandra: Unknown log_level: %s", value); @@ -518,8 +524,13 @@ result->query = i_strdup(query); result->row_pool = pool_alloconly_create("cassandra result", 512); result->statement = cass_statement_new(query, 0); - if (db->set_consistency) - cass_statement_set_consistency(result->statement, db->consistency); + if (result->write_query) { + if (db->set_write_consistency) + cass_statement_set_consistency(result->statement, db->write_consistency); + } else { + if (db->set_read_consistency) + cass_statement_set_consistency(result->statement, db->read_consistency); + } future = cass_session_execute(db->session, result->statement); driver_cassandra_set_callback(future, db, query_callback, result); } @@ -529,20 +540,9 @@ { } -static void driver_cassandra_exec(struct sql_db *db, const char *query) -{ - struct cassandra_result *result; - - result = i_new(struct cassandra_result, 1); - result->api = driver_cassandra_result; - result->api.db = db; - result->api.refcount = 1; - result->callback = exec_callback; - do_query(result, query); -} - -static void driver_cassandra_query(struct sql_db *db, const char *query, - sql_query_callback_t *callback, void *context) +static void +driver_cassandra_query_full(struct sql_db *db, const char *query, bool write_query, + sql_query_callback_t *callback, void *context) { struct cassandra_result *result; @@ -552,9 +552,21 @@ result->api.refcount = 1; result->callback = callback; result->context = context; + result->write_query = write_query; do_query(result, query); } +static void driver_cassandra_exec(struct sql_db *db, const char *query) +{ + driver_cassandra_query_full(db, query, TRUE, exec_callback, NULL); +} + +static void driver_cassandra_query(struct sql_db *db, const char *query, + sql_query_callback_t *callback, void *context) +{ + driver_cassandra_query_full(db, query, FALSE, callback, context); +} + static void cassandra_query_s_callback(struct sql_result *result, void *context) { struct cassandra_db *db = context; @@ -797,22 +809,10 @@ } static void -transaction_begin_callback(struct sql_result *result, - struct cassandra_transaction_context *ctx) +transaction_commit_callback(struct sql_result *result, void *context) { - if (sql_result_next_row(result) < 0) { - ctx->begin_failed = TRUE; - transaction_set_failed(ctx, sql_result_get_error(result)); - } else { - ctx->begin_succeeded = TRUE; - } - driver_cassandra_transaction_unref(&ctx); -} + struct cassandra_transaction_context *ctx = context; -static void -transaction_commit_callback(struct sql_result *result, - struct cassandra_transaction_context *ctx) -{ if (sql_result_next_row(result) < 0) ctx->callback(sql_result_get_error(result), ctx->context); else @@ -821,20 +821,8 @@ } static void -transaction_update_callback(struct sql_result *result, - struct sql_transaction_query *query) -{ - struct cassandra_transaction_context *ctx = - (struct cassandra_transaction_context *)query->trans; - - if (sql_result_next_row(result) < 0) - transaction_set_failed(ctx, sql_result_get_error(result)); - driver_cassandra_transaction_unref(&ctx); -} - -static void driver_cassandra_transaction_commit(struct sql_transaction_context *_ctx, - sql_commit_callback_t *callback, void *context) + sql_commit_callback_t *callback, void *context) { struct cassandra_transaction_context *ctx = (struct cassandra_transaction_context *)_ctx; @@ -847,19 +835,11 @@ driver_cassandra_transaction_unref(&ctx); } else if (_ctx->head->next == NULL) { /* just a single query, send it */ - sql_query(_ctx->db, _ctx->head->query, + driver_cassandra_query_full(_ctx->db, _ctx->head->query, TRUE, transaction_commit_callback, ctx); } else { - /* multiple queries, use a transaction */ - ctx->refcount++; - sql_query(_ctx->db, "BEGIN", transaction_begin_callback, ctx); - while (_ctx->head != NULL) { - ctx->refcount++; - sql_query(_ctx->db, _ctx->head->query, - transaction_update_callback, _ctx->head); - _ctx->head = _ctx->head->next; - } - sql_query(_ctx->db, "COMMIT", transaction_commit_callback, ctx); + /* multiple queries - we don't actually have a transaction though */ + callback("Multiple changes in transaction not supported", context); } } From dovecot at dovecot.org Mon Aug 31 19:27:02 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 31 Aug 2015 19:27:02 +0000 Subject: dovecot-2.2: cassandra: Changed default consistency levels to lo... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7648b127ca37 changeset: 19048:7648b127ca37 user: Timo Sirainen date: Mon Aug 31 22:25:57 2015 +0300 description: cassandra: Changed default consistency levels to local-quorum. It is a much safer default than "one". diffstat: src/lib-sql/driver-cassandra.c | 18 ++++++------------ 1 files changed, 6 insertions(+), 12 deletions(-) diffs (52 lines): diff -r c33868915764 -r 7648b127ca37 src/lib-sql/driver-cassandra.c --- a/src/lib-sql/driver-cassandra.c Mon Aug 31 21:31:46 2015 +0300 +++ b/src/lib-sql/driver-cassandra.c Mon Aug 31 22:25:57 2015 +0300 @@ -45,9 +45,6 @@ struct sql_result *sync_result; char *error; - - unsigned int set_read_consistency:1; - unsigned int set_write_consistency:1; }; struct cassandra_result { @@ -336,6 +333,8 @@ string_t *hosts = t_str_new(64); db->log_level = CASS_LOG_WARN; + db->read_consistency = CASS_CONSISTENCY_LOCAL_QUORUM; + db->write_consistency = CASS_CONSISTENCY_LOCAL_QUORUM; args = t_strsplit_spaces(connect_string, " "); for (; *args != NULL; args++) { @@ -357,11 +356,9 @@ } else if (strcmp(key, "read_consistency") == 0) { if (consistency_parse(value, &db->read_consistency) < 0) i_fatal("cassandra: Unknown read_consistency: %s", value); - db->set_read_consistency = TRUE; } else if (strcmp(key, "write_consistency") == 0) { if (consistency_parse(value, &db->write_consistency) < 0) i_fatal("cassandra: Unknown write_consistency: %s", value); - db->set_write_consistency = TRUE; } else if (strcmp(key, "log_level") == 0) { if (log_level_parse(value, &db->log_level) < 0) i_fatal("cassandra: Unknown log_level: %s", value); @@ -524,13 +521,10 @@ result->query = i_strdup(query); result->row_pool = pool_alloconly_create("cassandra result", 512); result->statement = cass_statement_new(query, 0); - if (result->write_query) { - if (db->set_write_consistency) - cass_statement_set_consistency(result->statement, db->write_consistency); - } else { - if (db->set_read_consistency) - cass_statement_set_consistency(result->statement, db->read_consistency); - } + if (result->write_query) + cass_statement_set_consistency(result->statement, db->write_consistency); + else + cass_statement_set_consistency(result->statement, db->read_consistency); future = cass_session_execute(db->session, result->statement); driver_cassandra_set_callback(future, db, query_callback, result); } From dovecot at dovecot.org Mon Aug 31 20:12:16 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 31 Aug 2015 20:12:16 +0000 Subject: dovecot-2.2: *-login: mail_max_userip_connections=0 was broken b... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8f96bbd1691f changeset: 19049:8f96bbd1691f user: Timo Sirainen date: Mon Aug 31 22:45:17 2015 +0300 description: *-login: mail_max_userip_connections=0 was broken by f8ab4f979e92 diffstat: src/login-common/sasl-server.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (14 lines): diff -r 7648b127ca37 -r 8f96bbd1691f src/login-common/sasl-server.c --- a/src/login-common/sasl-server.c Mon Aug 31 22:25:57 2015 +0300 +++ b/src/login-common/sasl-server.c Mon Aug 31 22:45:17 2015 +0300 @@ -169,7 +169,9 @@ if (reply != NULL && str_to_uint(reply, &conn_count) < 0) i_fatal("Received invalid reply from anvil: %s", reply); - if (conn_count < set->mail_max_userip_connections) + /* reply=NULL if we didn't need to do anvil lookup, + or if the anvil lookup failed. allow failed anvil lookups in. */ + if (reply == NULL || conn_count < set->mail_max_userip_connections) master_send_request(req); else { client->authenticating = FALSE; From dovecot at dovecot.org Mon Aug 31 20:12:17 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 31 Aug 2015 20:12:17 +0000 Subject: dovecot-2.2: imap: NOTIFY SET STATUS didn't send HIGHESTMODSEQ i... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/238a34ad1ab0 changeset: 19050:238a34ad1ab0 user: Timo Sirainen date: Mon Aug 31 23:11:17 2015 +0300 description: imap: NOTIFY SET STATUS didn't send HIGHESTMODSEQ in STATUS reponses when needed. diffstat: src/imap/cmd-notify.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (13 lines): diff -r 8f96bbd1691f -r 238a34ad1ab0 src/imap/cmd-notify.c --- a/src/imap/cmd-notify.c Mon Aug 31 22:45:17 2015 +0300 +++ b/src/imap/cmd-notify.c Mon Aug 31 23:11:17 2015 +0300 @@ -391,6 +391,9 @@ items.status = STATUS_UIDVALIDITY | STATUS_UIDNEXT | STATUS_MESSAGES | STATUS_UNSEEN; + if ((ctx->global_used_events & (IMAP_NOTIFY_EVENT_FLAG_CHANGE | + IMAP_NOTIFY_EVENT_ANNOTATION_CHANGE)) != 0) + items.status |= STATUS_HIGHESTMODSEQ; box = mailbox_alloc(info->ns->list, info->vname, MAILBOX_FLAG_READONLY); if (ctx->client->enabled_features != 0) From dovecot at dovecot.org Mon Aug 31 20:26:09 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 31 Aug 2015 20:26:09 +0000 Subject: dovecot-2.2: lib: Added str_to_ino() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9dd0f155731c changeset: 19051:9dd0f155731c user: Timo Sirainen date: Mon Aug 31 23:25:12 2015 +0300 description: lib: Added str_to_ino() diffstat: src/lib/strnum.c | 13 +++++++++++++ src/lib/strnum.h | 3 +++ 2 files changed, 16 insertions(+), 0 deletions(-) diffs (36 lines): diff -r 238a34ad1ab0 -r 9dd0f155731c src/lib/strnum.c --- a/src/lib/strnum.c Mon Aug 31 23:11:17 2015 +0300 +++ b/src/lib/strnum.c Mon Aug 31 23:25:12 2015 +0300 @@ -387,6 +387,19 @@ return 0; } +int str_to_ino(const char *str, ino_t *num_r) +{ + uintmax_t l; + + if (str_to_uintmax(str, &l) < 0) + return -1; + + if (verify_xid(l, sizeof(*num_r)) < 0) + return -1; + *num_r = (ino_t)l; + return 0; +} + int str_to_uoff(const char *str, uoff_t *num_r) { uintmax_t l; diff -r 238a34ad1ab0 -r 9dd0f155731c src/lib/strnum.h --- a/src/lib/strnum.h Mon Aug 31 23:11:17 2015 +0300 +++ b/src/lib/strnum.h Mon Aug 31 23:25:12 2015 +0300 @@ -161,6 +161,9 @@ int str_to_pid(const char *str, pid_t *num_r) ATTR_WARN_UNUSED_RESULT; +int str_to_ino(const char *str, ino_t *num_r) + ATTR_WARN_UNUSED_RESULT; + int str_to_uoff(const char *str, uoff_t *num_r) ATTR_WARN_UNUSED_RESULT; int str_parse_uoff(const char *str, uoff_t *num_r, From dovecot at dovecot.org Mon Aug 31 20:48:03 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 31 Aug 2015 20:48:03 +0000 Subject: dovecot-2.2: imap-hibernate: Use sockets' st_dev and st_ino to v... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/55cab04ef2af changeset: 19052:55cab04ef2af user: Timo Sirainen date: Mon Aug 31 23:46:05 2015 +0300 description: imap-hibernate: Use sockets' st_dev and st_ino to verify that the passed fd is correct. diffstat: src/imap-hibernate/imap-client.c | 16 ++++----- src/imap-hibernate/imap-client.h | 4 +- src/imap-hibernate/imap-hibernate-client.c | 21 ++++++++--- src/imap/imap-client-hibernate.c | 13 ++++--- src/imap/imap-master-client.c | 51 ++++++++++++++++++----------- 5 files changed, 62 insertions(+), 43 deletions(-) diffs (235 lines): diff -r 9dd0f155731c -r 55cab04ef2af src/imap-hibernate/imap-client.c --- a/src/imap-hibernate/imap-client.c Mon Aug 31 23:25:12 2015 +0300 +++ b/src/imap-hibernate/imap-client.c Mon Aug 31 23:46:05 2015 +0300 @@ -117,10 +117,12 @@ str_append(str, "\tuserdb_fields="); str_append_tabescaped(str, state->userdb_fields); } - if (state->peer_ip.family != 0) - str_printfa(str, "\tpeer_ip=%s", net_ip2addr(&state->peer_ip)); - if (state->peer_port != 0) - str_printfa(str, "\tpeer_port=%u", state->peer_port); + if (major(state->peer_dev) != 0 || minor(state->peer_dev) != 0) { + str_printfa(str, "\tpeer_dev_major=%u\tpeer_dev_minor=%u", + major(state->peer_dev), minor(state->peer_dev)); + } + if (state->peer_ino != 0) + str_printfa(str, "\tpeer_ino=%llu", (unsigned long long)state->peer_ino); if (state->state_size > 0) { str_append(str, "\tstate="); base64_encode(state->state, state->state_size, str); @@ -411,15 +413,11 @@ client->fd = fd; client->input = i_stream_create_fd(fd, IMAP_MAX_INBUF, FALSE); + client->state = *state; client->state.username = p_strdup(pool, state->username); client->state.session_id = p_strdup(pool, state->session_id); client->state.userdb_fields = p_strdup(pool, state->userdb_fields); client->state.stats = p_strdup(pool, state->stats); - client->state.local_ip = state->local_ip; - client->state.remote_ip = state->remote_ip; - client->state.imap_idle_notify_interval = - state->imap_idle_notify_interval; - client->state.idle_cmd = state->idle_cmd; if (state->state_size > 0) { client->state.state = statebuf = p_malloc(pool, state->state_size); diff -r 9dd0f155731c -r 55cab04ef2af src/imap-hibernate/imap-client.h --- a/src/imap-hibernate/imap-client.h Mon Aug 31 23:25:12 2015 +0300 +++ b/src/imap-hibernate/imap-client.h Mon Aug 31 23:46:05 2015 +0300 @@ -13,8 +13,8 @@ uid_t uid; gid_t gid; - struct ip_addr peer_ip; - in_port_t peer_port; + dev_t peer_dev; + ino_t peer_ino; const unsigned char *state; size_t state_size; diff -r 9dd0f155731c -r 55cab04ef2af src/imap-hibernate/imap-hibernate-client.c --- a/src/imap-hibernate/imap-hibernate-client.c Mon Aug 31 23:25:12 2015 +0300 +++ b/src/imap-hibernate/imap-hibernate-client.c Mon Aug 31 23:46:05 2015 +0300 @@ -44,6 +44,7 @@ const char **error_r) { const char *key, *value; + unsigned int peer_dev_major = 0, peer_dev_minor = 0; memset(state_r, 0, sizeof(*state_r)); if (args[0] == NULL) { @@ -77,16 +78,22 @@ "Invalid rip value: %s", value); return -1; } - } else if (strcmp(key, "peer_ip") == 0) { - if (net_addr2ip(value, &state_r->peer_ip) < 0) { + } else if (strcmp(key, "peer_dev_major") == 0) { + if (str_to_uint(value, &peer_dev_major) < 0) { *error_r = t_strdup_printf( - "Invalid peer_ip value: %s", value); + "Invalid peer_dev_major value: %s", value); return -1; } - } else if (strcmp(key, "peer_port") == 0) { - if (net_str2port(value, &state_r->peer_port) < 0) { + } else if (strcmp(key, "peer_dev_minor") == 0) { + if (str_to_uint(value, &peer_dev_minor) < 0) { *error_r = t_strdup_printf( - "Invalid peer_port value: %s", value); + "Invalid peer_dev_minor value: %s", value); + return -1; + } + } else if (strcmp(key, "peer_ino") == 0) { + if (str_to_ino(value, &state_r->peer_ino) < 0) { + *error_r = t_strdup_printf( + "Invalid peer_ino value: %s", value); return -1; } } else if (strcmp(key, "uid") == 0) { @@ -131,6 +138,8 @@ state_r->state_size = state_buf->used; } } + if (peer_dev_major != 0 || peer_dev_minor != 0) + state_r->peer_dev = makedev(peer_dev_major, peer_dev_minor); return 0; } diff -r 9dd0f155731c -r 55cab04ef2af src/imap/imap-client-hibernate.c --- a/src/imap/imap-client-hibernate.c Mon Aug 31 23:25:12 2015 +0300 +++ b/src/imap/imap-client-hibernate.c Mon Aug 31 23:46:05 2015 +0300 @@ -13,6 +13,8 @@ #include "imap-state.h" #include "imap-client.h" +#include + #define IMAP_HIBERNATE_SOCKET_NAME "imap-hibernate" #define IMAP_HIBERNATE_SEND_TIMEOUT_SECS 10 #define IMAP_HIBERNATE_HANDSHAKE "VERSION\timap-hibernate\t1\t0\n" @@ -41,18 +43,17 @@ static void imap_hibernate_write_cmd(struct client *client, string_t *cmd, const buffer_t *state, int fd_notify) { - struct ip_addr peer_ip; - in_port_t peer_port; + struct stat peer_st; str_append_tabescaped(cmd, client->user->username); str_append_c(cmd, '\t'); str_append_tabescaped(cmd, client->user->set->mail_log_prefix); str_printfa(cmd, "\tidle_notify_interval=%u", client->set->imap_idle_notify_interval); - if (net_getpeername(client->fd_in, &peer_ip, &peer_port) == 0 && - peer_port != 0) { - str_printfa(cmd, "\tpeer_ip=%s\tpeer_port=%u", - net_ip2addr(&peer_ip), peer_port); + if (fstat(client->fd_in, &peer_st) == 0) { + str_printfa(cmd, "\tpeer_dev_major=%u\tpeer_dev_minor=%u\tpeer_ino=%llu", + major(peer_st.st_dev), minor(peer_st.st_dev), + (unsigned long long)peer_st.st_ino); } if (client->session_id != NULL) { diff -r 9dd0f155731c -r 55cab04ef2af src/imap/imap-master-client.c --- a/src/imap/imap-master-client.c Mon Aug 31 23:25:12 2015 +0300 +++ b/src/imap/imap-master-client.c Mon Aug 31 23:46:05 2015 +0300 @@ -27,8 +27,8 @@ /* IMAP connection state */ buffer_t *state; - struct ip_addr peer_ip; - in_port_t peer_port; + dev_t peer_dev; + ino_t peer_ino; bool state_import_bad_idle_done; bool state_import_idle_continue; @@ -53,6 +53,7 @@ const char **error_r) { const char *key, *value; + unsigned int peer_dev_major = 0, peer_dev_minor = 0; memset(input_r, 0, sizeof(*input_r)); memset(master_input_r, 0, sizeof(*master_input_r)); @@ -91,16 +92,22 @@ "Invalid rip value: %s", value); return -1; } - } else if (strcmp(key, "peer_ip") == 0) { - if (net_addr2ip(value, &master_input_r->peer_ip) < 0) { + } else if (strcmp(key, "peer_dev_major") == 0) { + if (str_to_uint(value, &peer_dev_major) < 0) { *error_r = t_strdup_printf( - "Invalid peer_ip value: %s", value); + "Invalid peer_dev_major value: %s", value); return -1; } - } else if (strcmp(key, "peer_port") == 0) { - if (net_str2port(value, &master_input_r->peer_port) < 0) { + } else if (strcmp(key, "peer_dev_minor") == 0) { + if (str_to_uint(value, &peer_dev_minor) < 0) { *error_r = t_strdup_printf( - "Invalid peer_port value: %s", value); + "Invalid peer_dev_minor value: %s", value); + return -1; + } + } else if (strcmp(key, "peer_ino") == 0) { + if (str_to_ino(value, &master_input_r->peer_ino) < 0) { + *error_r = t_strdup_printf( + "Invalid peer_ino value: %s", value); return -1; } } else if (strcmp(key, "session") == 0) { @@ -135,31 +142,35 @@ master_input_r->state_import_idle_continue = TRUE; } } + if (peer_dev_major != 0 || peer_dev_minor != 0) { + master_input_r->peer_dev = + makedev(peer_dev_major, peer_dev_minor); + } return 0; } static int imap_master_client_verify(const struct imap_master_input *master_input, int fd_client, const char **error_r) { - struct ip_addr peer_ip; - in_port_t peer_port; + struct stat peer_st; - if (master_input->peer_port == 0) + if (master_input->peer_ino == 0) return 0; /* make sure we have the right fd */ - if (net_getpeername(fd_client, &peer_ip, &peer_port) < 0) { - *error_r = t_strdup_printf("net_getpeername() failed: %m"); + if (fstat(fd_client, &peer_st) < 0) { + *error_r = t_strdup_printf("fstat(peer) failed: %m"); return -1; } - if (!net_ip_compare(&peer_ip, &master_input->peer_ip) || - peer_port != master_input->peer_port) { + if (peer_st.st_ino != master_input->peer_ino || + !CMP_DEV_T(peer_st.st_dev, master_input->peer_dev)) { *error_r = t_strdup_printf( - "BUG: Expected peer_ip=%s peer_port=%u doesn't match " - "client fd's actual ip=%s port=%u", - net_ip2addr(&master_input->peer_ip), - master_input->peer_port, - net_ip2addr(&peer_ip), peer_port); + "BUG: Expected peer device=%u,%u inode=%s doesn't match " + "client fd's actual device=%u,%u inode=%s", + major(peer_st.st_dev), minor(peer_st.st_dev), dec2str(peer_st.st_ino), + major(master_input->peer_dev), + minor(master_input->peer_dev), + dec2str(master_input->peer_ino)); return -1; } return 0; From dovecot at dovecot.org Mon Aug 31 20:55:20 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 31 Aug 2015 20:55:20 +0000 Subject: dovecot-2.2: lib-fs: Added "dict" backend, which is a wrapper to... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6adcfddd354d changeset: 19053:6adcfddd354d user: Timo Sirainen date: Mon Aug 31 23:54:24 2015 +0300 description: lib-fs: Added "dict" backend, which is a wrapper to using lib-dict. diffstat: src/lib-fs/Makefile.am | 2 + src/lib-fs/fs-api-private.h | 1 + src/lib-fs/fs-api.c | 1 + src/lib-fs/fs-dict.c | 312 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 316 insertions(+), 0 deletions(-) diffs (truncated from 355 to 300 lines): diff -r 55cab04ef2af -r 6adcfddd354d src/lib-fs/Makefile.am --- a/src/lib-fs/Makefile.am Mon Aug 31 23:46:05 2015 +0300 +++ b/src/lib-fs/Makefile.am Mon Aug 31 23:54:24 2015 +0300 @@ -2,11 +2,13 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-dict \ -I$(top_srcdir)/src/lib-ssl-iostream \ -DMODULE_DIR=\""$(moduledir)"\" libfs_la_SOURCES = \ fs-api.c \ + fs-dict.c \ fs-metawrap.c \ fs-randomfail.c \ fs-posix.c \ diff -r 55cab04ef2af -r 6adcfddd354d src/lib-fs/fs-api-private.h --- a/src/lib-fs/fs-api-private.h Mon Aug 31 23:46:05 2015 +0300 +++ b/src/lib-fs/fs-api-private.h Mon Aug 31 23:54:24 2015 +0300 @@ -151,6 +151,7 @@ void *async_context; }; +extern const struct fs fs_class_dict; extern const struct fs fs_class_posix; extern const struct fs fs_class_randomfail; extern const struct fs fs_class_metawrap; diff -r 55cab04ef2af -r 6adcfddd354d src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Mon Aug 31 23:46:05 2015 +0300 +++ b/src/lib-fs/fs-api.c Mon Aug 31 23:54:24 2015 +0300 @@ -63,6 +63,7 @@ static void fs_classes_init(void) { i_array_init(&fs_classes, 8); + fs_class_register(&fs_class_dict); fs_class_register(&fs_class_posix); fs_class_register(&fs_class_randomfail); fs_class_register(&fs_class_metawrap); diff -r 55cab04ef2af -r 6adcfddd354d src/lib-fs/fs-dict.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-fs/fs-dict.c Mon Aug 31 23:54:24 2015 +0300 @@ -0,0 +1,312 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "buffer.h" +#include "str.h" +#include "guid.h" +#include "base64.h" +#include "istream.h" +#include "ostream.h" +#include "dict.h" +#include "fs-api-private.h" + +enum fs_dict_value_encoding { + FS_DICT_VALUE_ENCODING_RAW, + FS_DICT_VALUE_ENCODING_BASE64 +}; + +struct dict_fs { + struct fs fs; + struct dict *dict; + char *path_prefix; + enum fs_dict_value_encoding encoding; +}; + +struct dict_fs_file { + struct fs_file file; + pool_t pool; + const char *key, *value; + buffer_t *write_buffer; +}; + +struct dict_fs_iter { + struct fs_iter iter; + struct dict_iterate_context *dict_iter; +}; + +static struct fs *fs_dict_alloc(void) +{ + struct dict_fs *fs; + + fs = i_new(struct dict_fs, 1); + fs->fs = fs_class_dict; + return &fs->fs; +} + +static int +fs_dict_init(struct fs *_fs, const char *args, const struct fs_settings *set) +{ + struct dict_fs *fs = (struct dict_fs *)_fs; + struct dict_settings dict_set; + const char *p, *encoding_str, *error; + + p = strchr(args, ':'); + if (p == NULL) { + fs_set_error(_fs, "':' missing in args"); + return -1; + } + encoding_str = t_strdup_until(args, p++); + if (strcmp(encoding_str, "raw") == 0) + fs->encoding = FS_DICT_VALUE_ENCODING_RAW; + else if (strcmp(encoding_str, "base64") == 0) + fs->encoding = FS_DICT_VALUE_ENCODING_BASE64; + else { + fs_set_error(_fs, "Unknown value encoding '%s'", encoding_str); + return -1; + } + + memset(&dict_set, 0, sizeof(dict_set)); + dict_set.username = set->username; + dict_set.base_dir = set->base_dir; + + if (dict_init_full(p, &dict_set, &fs->dict, &error) < 0) { + fs_set_error(_fs, "dict_init(%s) failed: %s", args, error); + return -1; + } + return 0; +} + +static void fs_dict_deinit(struct fs *_fs) +{ + struct dict_fs *fs = (struct dict_fs *)_fs; + + dict_deinit(&fs->dict); + i_free(fs); +} + +static enum fs_properties fs_dict_get_properties(struct fs *fs ATTR_UNUSED) +{ + return FS_PROPERTY_ITER | FS_PROPERTY_RELIABLEITER; +} + +static struct fs_file * +fs_dict_file_init(struct fs *_fs, const char *path, + enum fs_open_mode mode, enum fs_open_flags flags ATTR_UNUSED) +{ + struct dict_fs *fs = (struct dict_fs *)_fs; + struct dict_fs_file *file; + guid_128_t guid; + pool_t pool; + + i_assert(mode != FS_OPEN_MODE_APPEND); /* not supported */ + i_assert(mode != FS_OPEN_MODE_CREATE); /* not supported */ + + pool = pool_alloconly_create("fs dict file", 128); + file = p_new(pool, struct dict_fs_file, 1); + file->pool = pool; + file->file.fs = _fs; + if (mode != FS_OPEN_MODE_CREATE_UNIQUE_128) + file->file.path = p_strdup(pool, path); + else { + guid_128_generate(guid); + file->file.path = p_strdup_printf(pool, "%s/%s", path, + guid_128_to_string(guid)); + } + file->key = fs->path_prefix == NULL ? + p_strdup(pool, file->file.path) : + p_strconcat(pool, fs->path_prefix, file->file.path, NULL); + return &file->file; +} + +static void fs_dict_file_deinit(struct fs_file *_file) +{ + struct dict_fs_file *file = (struct dict_fs_file *)_file; + + i_assert(_file->output == NULL); + + pool_unref(&file->pool); +} + +static bool fs_dict_prefetch(struct fs_file *_file ATTR_UNUSED, + uoff_t length ATTR_UNUSED) +{ + /* once async dict_lookup() is implemented, we want to start it here */ + return TRUE; +} + +static int fs_dict_lookup(struct dict_fs_file *file) +{ + struct dict_fs *fs = (struct dict_fs *)file->file.fs; + int ret; + + if (file->value != NULL) + return 0; + + ret = dict_lookup(fs->dict, file->pool, file->key, &file->value); + if (ret > 0) + return 0; + else if (ret < 0) { + errno = EIO; + fs_set_error(&fs->fs, "Dict lookup failed"); + return -1; + } else { + errno = ENOENT; + fs_set_error(&fs->fs, "Dict key doesn't exist"); + return -1; + } +} + +static struct istream * +fs_dict_read_stream(struct fs_file *_file, size_t max_buffer_size ATTR_UNUSED) +{ + struct dict_fs_file *file = (struct dict_fs_file *)_file; + struct istream *input; + + if (fs_dict_lookup(file) < 0) + input = i_stream_create_error_str(errno, "%s", fs_last_error(_file->fs)); + else + input = i_stream_create_from_data(file->value, strlen(file->value)); + i_stream_set_name(input, file->key); + return input; +} + +static void fs_dict_write_stream(struct fs_file *_file) +{ + struct dict_fs_file *file = (struct dict_fs_file *)_file; + + i_assert(_file->output == NULL); + + file->write_buffer = buffer_create_dynamic(file->pool, 128); + _file->output = o_stream_create_buffer(file->write_buffer); + o_stream_set_name(_file->output, file->key); +} + +static int fs_dict_write_stream_finish(struct fs_file *_file, bool success) +{ + struct dict_fs_file *file = (struct dict_fs_file *)_file; + struct dict_fs *fs = (struct dict_fs *)_file->fs; + struct dict_transaction_context *trans; + + o_stream_destroy(&_file->output); + if (!success) + return -1; + + trans = dict_transaction_begin(fs->dict); + switch (fs->encoding) { + case FS_DICT_VALUE_ENCODING_RAW: + dict_set(trans, file->key, str_c(file->write_buffer)); + break; + case FS_DICT_VALUE_ENCODING_BASE64: { + const unsigned int base64_size = + MAX_BASE64_ENCODED_SIZE(file->write_buffer->used); + string_t *base64 = t_str_new(base64_size); + base64_encode(file->write_buffer->data, + file->write_buffer->used, base64); + dict_set(trans, file->key, str_c(base64)); + } + } + if (dict_transaction_commit(&trans) < 0) { + errno = EIO; + fs_set_error(_file->fs, "Dict transaction commit failed"); + return -1; + } + return 1; +} + +static int fs_dict_stat(struct fs_file *_file, struct stat *st_r) +{ + struct dict_fs_file *file = (struct dict_fs_file *)_file; + + memset(st_r, 0, sizeof(*st_r)); + + if (fs_dict_lookup(file) < 0) + return -1; + st_r->st_size = strlen(file->value); + return 0; +} + +static int fs_dict_delete(struct fs_file *_file) +{ + struct dict_fs_file *file = (struct dict_fs_file *)_file; + struct dict_fs *fs = (struct dict_fs *)_file->fs; + struct dict_transaction_context *trans; + + trans = dict_transaction_begin(fs->dict); + dict_unset(trans, file->key); + if (dict_transaction_commit(&trans) < 0) { + errno = EIO; + fs_set_error(_file->fs, "Dict transaction commit failed"); + return -1; + } + return 0; +} + +static struct fs_iter * +fs_dict_iter_init(struct fs *_fs, const char *path, enum fs_iter_flags flags) +{ + struct dict_fs *fs = (struct dict_fs *)_fs; + struct dict_fs_iter *iter; + + iter = i_new(struct dict_fs_iter, 1); + iter->iter.fs = _fs; + iter->iter.flags = flags; + if (fs->path_prefix != NULL) + path = t_strconcat(fs->path_prefix, path, NULL); + + iter->dict_iter = dict_iterate_init(fs->dict, path, 0); + return &iter->iter; From dovecot at dovecot.org Mon Aug 31 21:12:34 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 31 Aug 2015 21:12:34 +0000 Subject: dovecot-2.2: auth: Added ":protected" suffix to passdb and userd... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c0e61e4f47ec changeset: 19054:c0e61e4f47ec user: Timo Sirainen date: Tue Sep 01 00:11:37 2015 +0300 description: auth: Added ":protected" suffix to passdb and userdb field names. This means that if the field is set only if it hasn't already been set. Usually an earlier passdb/userdb would have set the field and this is setting a default (e.g. per-user settings override per-domain settings). diffstat: src/auth/auth-request.c | 31 +++++++++++++++++++++++-------- 1 files changed, 23 insertions(+), 8 deletions(-) diffs (65 lines): diff -r 6adcfddd354d -r c0e61e4f47ec src/auth/auth-request.c --- a/src/auth/auth-request.c Mon Aug 31 23:54:24 2015 +0300 +++ b/src/auth/auth-request.c Tue Sep 01 00:11:37 2015 +0300 @@ -1447,11 +1447,20 @@ const char *name, const char *value, const char *default_scheme) { + unsigned int name_len = strlen(name); + i_assert(*name != '\0'); i_assert(value != NULL); i_assert(request->passdb != NULL); + if (name_len > 10 && strcmp(name+name_len-10, ":protected") == 0) { + /* set this field only if it hasn't been set before */ + name = t_strndup(name, name_len-10); + if (auth_fields_exists(request->extra_fields, name)) + return; + } + if (strcmp(name, "password") == 0) { auth_request_set_password(request, value, default_scheme, FALSE); @@ -1510,14 +1519,12 @@ return; } - if ((passdb_cache != NULL && - request->passdb->passdb->cache_key != NULL) || worker) { - /* we'll need to get this field stored into cache, - or we're a worker and we'll need to send this to the main - auth process that can store it in the cache. */ - auth_fields_add(request->extra_fields, name, value, - AUTH_FIELD_FLAG_HIDDEN); - } + /* add the field unconditionally to extra_fields. this is required if + a) auth cache is used, b) if we're a worker and we'll need to send + this to the main auth process that can store it in the cache, + c) for easily checking :protected fields' existence. */ + auth_fields_add(request->extra_fields, name, value, + AUTH_FIELD_FLAG_HIDDEN); } void auth_request_set_null_field(struct auth_request *request, const char *name) @@ -1605,11 +1612,19 @@ void auth_request_set_userdb_field(struct auth_request *request, const char *name, const char *value) { + unsigned int name_len = strlen(name); uid_t uid; gid_t gid; i_assert(value != NULL); + if (name_len > 10 && strcmp(name+name_len-10, ":protected") == 0) { + /* set this field only if it hasn't been set before */ + name = t_strndup(name, name_len-10); + if (auth_fields_exists(request->userdb_reply, name)) + return; + } + if (strcmp(name, "uid") == 0) { uid = userdb_parse_uid(request, value); if (uid == (uid_t)-1) {