From dovecot at dovecot.org Sat Nov 1 00:06:33 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 01 Nov 2014 00:06:33 +0000 Subject: dovecot-2.2: lib-ssl-iostream: Added dh_length parameter to ssl_... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f7ebc677fdb9 changeset: 18050:f7ebc677fdb9 user: Timo Sirainen date: Fri Oct 31 17:04:58 2014 -0700 description: lib-ssl-iostream: Added dh_length parameter to ssl_iostream_generate_params() This is an API change, but nobody was actually using this function. diffstat: src/lib-ssl-iostream/iostream-openssl-params.c | 14 ++++++-------- src/lib-ssl-iostream/iostream-openssl.h | 3 ++- src/lib-ssl-iostream/iostream-ssl-private.h | 3 ++- src/lib-ssl-iostream/iostream-ssl.c | 5 +++-- src/lib-ssl-iostream/iostream-ssl.h | 3 ++- 5 files changed, 15 insertions(+), 13 deletions(-) diffs (91 lines): diff -r 6efd7ab25b71 -r f7ebc677fdb9 src/lib-ssl-iostream/iostream-openssl-params.c --- a/src/lib-ssl-iostream/iostream-openssl-params.c Fri Oct 31 16:54:07 2014 -0700 +++ b/src/lib-ssl-iostream/iostream-openssl-params.c Fri Oct 31 17:04:58 2014 -0700 @@ -8,8 +8,6 @@ default.. */ #define DH_GENERATOR 2 -static int dh_param_bitsizes[] = { 512, 1024 }; - static int generate_dh_parameters(int bitsize, buffer_t *output, const char **error_r) { @@ -43,13 +41,13 @@ return 0; } -int openssl_iostream_generate_params(buffer_t *output, const char **error_r) +int openssl_iostream_generate_params(buffer_t *output, unsigned int dh_length, + const char **error_r) { - unsigned int i; - - for (i = 0; i < N_ELEMENTS(dh_param_bitsizes); i++) { - if (generate_dh_parameters(dh_param_bitsizes[i], - output, error_r) < 0) + if (generate_dh_parameters(512, output, error_r) < 0) + return -1; + if (dh_length != 512) { + if (generate_dh_parameters(dh_length, output, error_r) < 0) return -1; } buffer_append_zero(output, sizeof(int)); diff -r 6efd7ab25b71 -r f7ebc677fdb9 src/lib-ssl-iostream/iostream-openssl.h --- a/src/lib-ssl-iostream/iostream-openssl.h Fri Oct 31 16:54:07 2014 -0700 +++ b/src/lib-ssl-iostream/iostream-openssl.h Fri Oct 31 17:04:58 2014 -0700 @@ -93,7 +93,8 @@ const char *openssl_iostream_error(void); const char *openssl_iostream_key_load_error(void); -int openssl_iostream_generate_params(buffer_t *output, const char **error_r); +int openssl_iostream_generate_params(buffer_t *output, unsigned int dh_length, + const char **error_r); int openssl_iostream_context_import_params(struct ssl_iostream_context *ctx, const buffer_t *input); void openssl_iostream_context_free_params(struct ssl_iostream_context *ctx); diff -r 6efd7ab25b71 -r f7ebc677fdb9 src/lib-ssl-iostream/iostream-ssl-private.h --- a/src/lib-ssl-iostream/iostream-ssl-private.h Fri Oct 31 16:54:07 2014 -0700 +++ b/src/lib-ssl-iostream/iostream-ssl-private.h Fri Oct 31 17:04:58 2014 -0700 @@ -13,7 +13,8 @@ const char **error_r); void (*context_deinit)(struct ssl_iostream_context *ctx); - int (*generate_params)(buffer_t *output, const char **error_r); + int (*generate_params)(buffer_t *output, unsigned int dh_length, + const char **error_r); int (*context_import_params)(struct ssl_iostream_context *ctx, const buffer_t *input); diff -r 6efd7ab25b71 -r f7ebc677fdb9 src/lib-ssl-iostream/iostream-ssl.c --- a/src/lib-ssl-iostream/iostream-ssl.c Fri Oct 31 16:54:07 2014 -0700 +++ b/src/lib-ssl-iostream/iostream-ssl.c Fri Oct 31 17:04:58 2014 -0700 @@ -82,13 +82,14 @@ ssl_vfuncs->context_deinit(ctx); } -int ssl_iostream_generate_params(buffer_t *output, const char **error_r) +int ssl_iostream_generate_params(buffer_t *output, unsigned int dh_length, + const char **error_r) { if (!ssl_module_loaded) { if (ssl_module_load(error_r) < 0) return -1; } - return ssl_vfuncs->generate_params(output, error_r); + return ssl_vfuncs->generate_params(output, dh_length, error_r); } int ssl_iostream_context_import_params(struct ssl_iostream_context *ctx, diff -r 6efd7ab25b71 -r f7ebc677fdb9 src/lib-ssl-iostream/iostream-ssl.h --- a/src/lib-ssl-iostream/iostream-ssl.h Fri Oct 31 16:54:07 2014 -0700 +++ b/src/lib-ssl-iostream/iostream-ssl.h Fri Oct 31 17:04:58 2014 -0700 @@ -64,7 +64,8 @@ const char *ssl_iostream_get_security_string(struct ssl_iostream *ssl_io); const char *ssl_iostream_get_last_error(struct ssl_iostream *ssl_io); -int ssl_iostream_generate_params(buffer_t *output, const char **error_r); +int ssl_iostream_generate_params(buffer_t *output, unsigned int dh_length, + const char **error_r); int ssl_iostream_context_import_params(struct ssl_iostream_context *ctx, const buffer_t *input); From dovecot at dovecot.org Sat Nov 1 00:06:38 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 01 Nov 2014 00:06:38 +0000 Subject: dovecot-2.2: ssl-params: Use lib-ssl-iostream's ssl_iostream_gen... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8b3ae8a07f31 changeset: 18051:8b3ae8a07f31 user: Timo Sirainen date: Fri Oct 31 17:05:31 2014 -0700 description: ssl-params: Use lib-ssl-iostream's ssl_iostream_generate_params() instead of OpenSSL directly diffstat: src/ssl-params/Makefile.am | 6 +- src/ssl-params/ssl-params-openssl.c | 71 ------------------------------------- src/ssl-params/ssl-params.c | 17 ++++++-- src/ssl-params/ssl-params.h | 2 - 4 files changed, 16 insertions(+), 80 deletions(-) diffs (154 lines): diff -r f7ebc677fdb9 -r 8b3ae8a07f31 src/ssl-params/Makefile.am --- a/src/ssl-params/Makefile.am Fri Oct 31 17:04:58 2014 -0700 +++ b/src/ssl-params/Makefile.am Fri Oct 31 17:05:31 2014 -0700 @@ -6,14 +6,14 @@ -I$(top_srcdir)/src/lib \ -I$(top_srcdir)/src/lib-master \ -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-ssl-iostream \ -DPKG_STATEDIR=\""$(statedir)"\" -ssl_params_LDADD = $(LIBDOVECOT) $(SSL_LIBS) -ssl_params_DEPENDENCIES = $(LIBDOVECOT_DEPS) +ssl_params_LDADD = $(LIBDOVECOT) ../lib-ssl-iostream/libssl_iostream.la +ssl_params_DEPENDENCIES = $(LIBDOVECOT_DEPS) ../lib-ssl-iostream/libssl_iostream.la ssl_params_SOURCES = \ main.c \ ssl-params.c \ - ssl-params-openssl.c \ ssl-params-settings.c noinst_HEADERS = \ diff -r f7ebc677fdb9 -r 8b3ae8a07f31 src/ssl-params/ssl-params-openssl.c --- a/src/ssl-params/ssl-params-openssl.c Fri Oct 31 17:04:58 2014 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,71 +0,0 @@ -/* Copyright (c) 2002-2014 Dovecot authors, see the included COPYING file */ - -#include "lib.h" -#include "write-full.h" -#include "ssl-params.h" - -#ifdef HAVE_OPENSSL - -#include -#include - -/* 2 or 5. Haven't seen their difference explained anywhere, but 2 is the - default.. */ -#define DH_GENERATOR 2 - -static const char *ssl_last_error(void) -{ - unsigned long err; - char *buf; - size_t err_size = 256; - - err = ERR_get_error(); - if (err == 0) - return strerror(errno); - - buf = t_malloc(err_size); - buf[err_size-1] = '\0'; - ERR_error_string_n(err, buf, err_size-1); - return buf; -} - -static bool generate_dh_parameters(int bitsize, int fd, const char *fname) -{ - DH *dh = DH_generate_parameters(bitsize, DH_GENERATOR, NULL, NULL); - unsigned char *buf, *p; - int len; - - if (dh == NULL) - return FALSE; - - len = i2d_DHparams(dh, NULL); - if (len < 0) - i_fatal("i2d_DHparams() failed: %s", ssl_last_error()); - - buf = p = i_malloc(len); - len = i2d_DHparams(dh, &p); - - if (write_full(fd, &bitsize, sizeof(bitsize)) < 0 || - write_full(fd, &len, sizeof(len)) < 0 || - write_full(fd, buf, len) < 0) - i_fatal("write_full() failed for file %s: %m", fname); - i_free(buf); - return TRUE; -} - -void ssl_generate_parameters(int fd, unsigned int dh_length, const char *fname) -{ - int bits; - - /* this fails in FIPS mode */ - (void)generate_dh_parameters(512, fd, fname); - if (!generate_dh_parameters(dh_length, fd, fname)) { - i_fatal("DH_generate_parameters(bits=%d, gen=%d) failed: %s", - dh_length, DH_GENERATOR, ssl_last_error()); - } - bits = 0; - if (write_full(fd, &bits, sizeof(bits)) < 0) - i_fatal("write_full() failed for file %s: %m", fname); -} - -#endif diff -r f7ebc677fdb9 -r 8b3ae8a07f31 src/ssl-params/ssl-params.c --- a/src/ssl-params/ssl-params.c Fri Oct 31 17:04:58 2014 -0700 +++ b/src/ssl-params/ssl-params.c Fri Oct 31 17:05:31 2014 -0700 @@ -5,9 +5,11 @@ #include "buffer.h" #include "file-lock.h" #include "read-full.h" +#include "write-full.h" #include "master-interface.h" #include "master-service.h" #include "master-service-settings.h" +#include "iostream-ssl.h" #include "ssl-params-settings.h" #include "ssl-params.h" @@ -38,11 +40,12 @@ ssl_params_if_unchanged(const char *path, time_t mtime, unsigned int ssl_dh_parameters_length ATTR_UNUSED) { - const char *temp_path; + const char *temp_path, *error; struct file_lock *lock; struct stat st, st2; mode_t old_mask; int fd, ret; + buffer_t *buf; #ifdef HAVE_SETPRIORITY if (setpriority(PRIO_PROCESS, 0, SSL_PARAMS_PRIORITY) < 0) @@ -99,9 +102,15 @@ i_fatal("ftruncate(%s) failed: %m", temp_path); i_info("Generating SSL parameters"); -#ifdef HAVE_SSL - ssl_generate_parameters(fd, ssl_dh_parameters_length, temp_path); -#endif + + buf = buffer_create_dynamic(pool_datastack_create(), 1024); + if (ssl_iostream_generate_params(buf, ssl_dh_parameters_length, + &error) < 0) { + i_fatal("ssl_iostream_generate_params(%u) failed: %s", + ssl_dh_parameters_length, error); + } + if (write_full(fd, buf->data, buf->used) < 0) + i_fatal("write(%s) failed: %m", temp_path); if (rename(temp_path, path) < 0) i_fatal("rename(%s, %s) failed: %m", temp_path, path); diff -r f7ebc677fdb9 -r 8b3ae8a07f31 src/ssl-params/ssl-params.h --- a/src/ssl-params/ssl-params.h Fri Oct 31 17:04:58 2014 -0700 +++ b/src/ssl-params/ssl-params.h Fri Oct 31 17:05:31 2014 -0700 @@ -12,6 +12,4 @@ void ssl_params_refresh(struct ssl_params *param); -void ssl_generate_parameters(int fd, unsigned int dh_length, const char *fname); - #endif From pigeonhole at rename-it.nl Sun Nov 2 17:32:55 2014 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 02 Nov 2014 18:32:55 +0100 Subject: dovecot-2.2-pigeonhole: Released v0.4.6 for Dovecot v2.2.15. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/99d796a8cd26 changeset: 1956:99d796a8cd26 user: Stephan Bosch date: Sun Nov 02 18:32:06 2014 +0100 description: Released v0.4.6 for Dovecot v2.2.15. diffstat: NEWS | 5 +++++ configure.ac | 2 +- 2 files changed, 6 insertions(+), 1 deletions(-) diffs (21 lines): diff -r 3e924b1b6c5c -r 99d796a8cd26 NEWS --- a/NEWS Fri Oct 31 00:50:28 2014 +0100 +++ b/NEWS Sun Nov 02 18:32:06 2014 +0100 @@ -1,3 +1,8 @@ +v0.4.6 02-11-2014 Stephan Bosch + + - After make distclean the distributed tarball would fail to recompile. + This causes problems for some distribution builds. + v0.4.5 30-10-2014 Stephan Bosch + Added a Pigeonhole version banner to doveconf output. This way, future diff -r 3e924b1b6c5c -r 99d796a8cd26 configure.ac --- a/configure.ac Fri Oct 31 00:50:28 2014 +0100 +++ b/configure.ac Sun Nov 02 18:32:06 2014 +0100 @@ -1,4 +1,4 @@ -AC_INIT([Pigeonhole], [0.4.5], [dovecot at dovecot.org], [dovecot-2.2-pigeonhole]) +AC_INIT([Pigeonhole], [0.4.6], [dovecot at dovecot.org], [dovecot-2.2-pigeonhole]) AC_CONFIG_AUX_DIR([.]) AC_CONFIG_SRCDIR([src]) AC_CONFIG_MACRO_DIR([m4]) From pigeonhole at rename-it.nl Sun Nov 2 17:32:55 2014 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 02 Nov 2014 18:32:55 +0100 Subject: dovecot-2.2-pigeonhole: Added tag 0.4.6 for changeset 99d796a8cd26 Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/7b48b996cd01 changeset: 1957:7b48b996cd01 user: Stephan Bosch date: Sun Nov 02 18:32:40 2014 +0100 description: Added tag 0.4.6 for changeset 99d796a8cd26 diffstat: .hgtags | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 99d796a8cd26 -r 7b48b996cd01 .hgtags --- a/.hgtags Sun Nov 02 18:32:06 2014 +0100 +++ b/.hgtags Sun Nov 02 18:32:40 2014 +0100 @@ -22,3 +22,4 @@ 1c6130ff5dd6933f03a40666b7f73ae51c9f90fe 0.4.3 b3303f675157c954d2de08599e2c83db5a772450 0.4.4 f5e3ef477a32498584ac924963ff2e11c75cd477 0.4.5 +99d796a8cd26f5f64d56b02c1f0e720ffd6131e7 0.4.6 From pigeonhole at rename-it.nl Sun Nov 2 17:35:01 2014 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 02 Nov 2014 18:35:01 +0100 Subject: dovecot-2.2-pigeonhole: Added signature for changeset 99d796a8cd26 Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/be2f902b41fb changeset: 1958:be2f902b41fb user: Stephan Bosch date: Sun Nov 02 18:35:34 2014 +0100 description: Added signature for changeset 99d796a8cd26 diffstat: .hgsigs | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (8 lines): diff -r 7b48b996cd01 -r be2f902b41fb .hgsigs --- a/.hgsigs Sun Nov 02 18:32:40 2014 +0100 +++ b/.hgsigs Sun Nov 02 18:35:34 2014 +0100 @@ -16,3 +16,4 @@ 1c6130ff5dd6933f03a40666b7f73ae51c9f90fe 0 iQEcBAABAgAGBQJTcR2fAAoJEATWKx49+7T0tcwIANUB68pzhOoVHY/BhReqVfFLJCZLdDpsTFYkPhS99lcI4LwTNrNDe4OjpNNdk3UESOvUi19pIzDCewXSkk1gVAK4CGFbYNgvLpaX+8xjfBiPv8Ct47ALCISpT74TCMXR8t+tQ3jqaPFD/0j7tAt3sP9vYDNKt1QJ8YLSH1z9aSEYng+LJT9rZxMkw6/SnZw4GVFmQyuPWFBEe7oBE/RdURvQ+i+luuBIs7BaJWe3HkMD9giEafxHzBtib4QDSUOeloUOl+7ezeBJDxdcp+CzbjU4J0QfpokuU8YhEEnnzZ+9x+Tmmnv04qzB5BP8msVVCldxfymQdGx/qxCLj26H6Uk= b3303f675157c954d2de08599e2c83db5a772450 0 iQEcBAABAgAGBQJUTukTAAoJEATWKx49+7T0nWQH/3enKf5GbOe8eDBU83MvlYARFAbGEGb70coH3sVFhC50SjPSyIAU+Am048jVmLKvy2r/ABYXHwVn73rRbgR+X92iH4Q+ivYI7ZtUEr7iUoEUVetwks6YkUezthnzCVyWk+Hy9BnkoTEN08zPGkLnVt/e5SPAqQ1tZttpKOO+HZp3JxzGtMFBY/k+ZX8fU6PZKM6FZEPu8S8m2PnQUGaCgREZrt+ikMAzgwqgpEtYoiT6M6yQLyAVRpDvFsIU3q56vA3Zc3rl6vdJjBY9AMflyd1fEYv1IfUkVPjKZg/9JL42qRRQLccEGU2W+xpd7HYi1WfI+60buVmjPNTNr/GYPfo= f5e3ef477a32498584ac924963ff2e11c75cd477 0 iQEcBAABAgAGBQJUUrtaAAoJEATWKx49+7T0C+AH+QGUi15egTON1XwJ0aZEuI3qQScDif04EoteSwjRN+RYFLSkMPnkjG+4kx0tAG2RahlZ9gdyr4TW5a95aQhAxYS7j0bbEyJNIAektYtVniWhuxxxo5m1LjNbI6yYIYjqAqnyfzI55E15tNT2O2VkUBxgExdWACMcv0YiMGidQ+7JVox1QFwFMrJPK17NiCsboByZN9J0xj4tg0hibkt2MZ1TaYj10TMmyYDxVvZPedwa71YRNz/T3T+reroqhUYqAL2XszSX+PSMwdcxyo8+qoErBQyacD4zNAcXhwUmhTjw2g4QtwSk1WK0FDC/BunIlb6ccFoZ6FpQhO/8lu74Wjc= +99d796a8cd26f5f64d56b02c1f0e720ffd6131e7 0 iQEcBAABAgAGBQJUVmtZAAoJEATWKx49+7T0/wgH/0GyApuqHfn/PtKhZUhX78wvZ5t6IdU7Rs905qC/j4Y1u5JRg3EhvXs/JJUDlDsJBAVU9EwQRi37erF+LPIm+1W9ZuNr4H4kgea2QM+lnEFhmmt7tqgXUhPnpufkPP1pfnyjDOq5oeMeA8i/Ecic9pdBIIY+YEU80knf0PbqFxGzpA4gbX2dXOPQlqs+9h0hEUjg8t8hUXtqxSkO2qoJIbWQIFqGaqXCUCnLQ4LZYxLOGMki3I8Nrjaa/objgDlH+bsNe2zGyUdICQgcyKc4ZeeZcEpnJBZvVhlhfzDlCaJIXxGmZjRF1lt382dvObNhlDEOi0NzapCwF0xE4WtOPIw= From dovecot at dovecot.org Tue Nov 4 12:50:47 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 04 Nov 2014 12:50:47 +0000 Subject: dovecot-2.2: imap: If GETMETADATA with wildcards can't open some... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/46b1b81ad90d changeset: 18052:46b1b81ad90d user: Timo Sirainen date: Tue Nov 04 14:49:44 2014 +0200 description: imap: If GETMETADATA with wildcards can't open some mailbox, don't assert-crash. Just send an untagged NO for the mailbox. diffstat: src/imap/cmd-getmetadata.c | 50 ++++++++++++++++++++++++++++++++++----------- 1 files changed, 38 insertions(+), 12 deletions(-) diffs (81 lines): diff -r 8b3ae8a07f31 -r 46b1b81ad90d src/imap/cmd-getmetadata.c --- a/src/imap/cmd-getmetadata.c Fri Oct 31 17:05:31 2014 -0700 +++ b/src/imap/cmd-getmetadata.c Tue Nov 04 14:49:44 2014 +0200 @@ -373,18 +373,15 @@ return TRUE; } -static bool -cmd_getmetadata_mailbox(struct imap_getmetadata_context *ctx, - struct mail_namespace *ns, const char *mailbox) +static int +cmd_getmetadata_try_mailbox(struct imap_getmetadata_context *ctx, + struct mail_namespace *ns, const char *mailbox) { struct client_command_context *cmd = ctx->cmd; ctx->box = mailbox_alloc(ns->list, mailbox, MAILBOX_FLAG_READONLY); - if (mailbox_open(ctx->box) < 0) { - client_send_box_error(cmd, ctx->box); - mailbox_free(&ctx->box); - return TRUE; - } + if (mailbox_open(ctx->box) < 0) + return -1; ctx->trans = mailbox_transaction_begin(ctx->box, 0); if (ctx->depth > 0) @@ -393,20 +390,49 @@ if (!cmd_getmetadata_continue(cmd)) { cmd->state = CLIENT_COMMAND_STATE_WAIT_OUTPUT; cmd->func = cmd_getmetadata_continue; - return FALSE; + return 0; } - return TRUE; + return 1; +} + +static bool +cmd_getmetadata_mailbox(struct imap_getmetadata_context *ctx, + struct mail_namespace *ns, const char *mailbox) +{ + int ret; + + ret = cmd_getmetadata_try_mailbox(ctx, ns, mailbox); + if (ret < 0) { + client_send_box_error(ctx->cmd, ctx->box); + mailbox_free(&ctx->box); + } + return ret != 0; } static bool cmd_getmetadata_iter_next(struct imap_getmetadata_context *ctx) { const struct mailbox_info *info; + int ret; while ((info = mailbox_list_iter_next(ctx->list_iter)) != NULL) { if ((info->flags & (MAILBOX_NOSELECT | MAILBOX_NONEXISTENT)) != 0) continue; - /* we'll get back here recursively */ - return cmd_getmetadata_mailbox(ctx, info->ns, info->vname); + ret = cmd_getmetadata_try_mailbox(ctx, info->ns, info->vname); + if (ret > 0) { + /* we'll already recursively went through + all the mailboxes (FIXME: ugly and potentially + stack consuming) */ + return TRUE; + } else if (ret == 0) { + /* need to send more data later */ + return FALSE; + } + T_BEGIN { + client_send_line(ctx->cmd->client, t_strdup_printf( + "* NO Failed to open mailbox %s: %s", + info->vname, mailbox_get_last_error(ctx->box, NULL))); + } T_END; + mailbox_free(&ctx->box); } cmd_getmetadata_deinit(ctx); return TRUE; From dovecot at dovecot.org Wed Nov 5 11:01:25 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 05 Nov 2014 11:01:25 +0000 Subject: dovecot-2.2: doveadm: Removed duplicate subcommands from list of... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8fe127697587 changeset: 18053:8fe127697587 user: Timo Sirainen date: Wed Nov 05 13:00:14 2014 +0200 description: doveadm: Removed duplicate subcommands from list of commands. For example don't show "mailbox ..|metadata|metadata|metadata|.." diffstat: src/doveadm/doveadm.c | 10 ++++++++-- 1 files changed, 8 insertions(+), 2 deletions(-) diffs (28 lines): diff -r 46b1b81ad90d -r 8fe127697587 src/doveadm/doveadm.c --- a/src/doveadm/doveadm.c Tue Nov 04 14:49:44 2014 +0200 +++ b/src/doveadm/doveadm.c Wed Nov 05 13:00:14 2014 +0200 @@ -54,7 +54,8 @@ static void doveadm_usage_compress_lines(FILE *out, const char *str, const char *prefix) { - const char *cmd, *args, *p, *short_name, *prev_name = ""; + const char *cmd, *args, *p, *short_name, *sub_name; + const char *prev_name = "", *prev_sub_name = ""; const char **lines; unsigned int i, count, prefix_len = strlen(prefix); @@ -97,8 +98,13 @@ fprintf(out, USAGE_CMDNAME_FMT" %s", short_name, t_strcut(p + 1, ' ')); prev_name = short_name; + prev_sub_name = ""; } else { - fprintf(out, "|%s", t_strcut(p + 1, ' ')); + sub_name = t_strcut(p + 1, ' '); + if (strcmp(prev_sub_name, sub_name) != 0) { + fprintf(out, "|%s", sub_name); + prev_sub_name = sub_name; + } } } } From dovecot at dovecot.org Tue Nov 11 01:12:50 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 11 Nov 2014 01:12:50 +0000 Subject: dovecot-2.2: lib-storage: Mailbox list index now detects duplica... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e5cf0209bc9e changeset: 18054:e5cf0209bc9e user: Timo Sirainen date: Fri Oct 31 04:27:27 2014 +0200 description: lib-storage: Mailbox list index now detects duplicate mailbox names. With LAYOUT=index the duplicates are renamed. diffstat: src/lib-storage/list/mailbox-list-index.c | 48 +++++++++++++++++++++++++++++- 1 files changed, 45 insertions(+), 3 deletions(-) diffs (103 lines): diff -r 8fe127697587 -r e5cf0209bc9e src/lib-storage/list/mailbox-list-index.c --- a/src/lib-storage/list/mailbox-list-index.c Wed Nov 05 13:00:14 2014 +0200 +++ b/src/lib-storage/list/mailbox-list-index.c Fri Oct 31 04:27:27 2014 +0200 @@ -238,11 +238,27 @@ ilist->highest_name_id = node->name_id; } +static int mailbox_list_index_node_cmp(const struct mailbox_list_index_node *n1, + const struct mailbox_list_index_node *n2) +{ + return n1->parent == n2->parent && + strcmp(n1->name, n2->name) == 0 ? 0 : -1; +} + +static unsigned int +mailbox_list_index_node_hash(const struct mailbox_list_index_node *node) +{ + return str_hash(node->name) ^ + POINTER_CAST_TO(node->parent, unsigned int); +} + static int mailbox_list_index_parse_records(struct mailbox_list_index *ilist, struct mail_index_view *view, const char **error_r) { struct mailbox_list_index_node *node; + HASH_TABLE(struct mailbox_list_index_node *, + struct mailbox_list_index_node *) duplicate_hash; const struct mail_index_record *rec; const struct mailbox_list_index_record *irec; const void *data; @@ -251,6 +267,9 @@ *error_r = NULL; + hash_table_create(&duplicate_hash, default_pool, 0, + mailbox_list_index_node_hash, + mailbox_list_index_node_cmp); count = mail_index_view_get_messages_count(view); for (seq = 1; seq <= count; seq++) { node = p_new(ilist->mailbox_pool, @@ -263,7 +282,7 @@ &data, &expunged); if (data == NULL) { *error_r = "Missing list extension data"; - return -1; + break; } irec = data; @@ -273,7 +292,7 @@ if (node->name == NULL) { *error_r = "name_id not in index header"; if (ilist->has_backing_store) - return -1; + break; /* generate a new name and use it */ mailbox_list_index_generate_name(ilist, node); } @@ -284,6 +303,8 @@ /* do a second scan to create the actual mailbox tree hierarchy. this is needed because the parent_uid may be smaller or higher than the current node's uid */ + if (*error_r != NULL) + count = 0; for (seq = 1; seq <= count; seq++) { mail_index_lookup_uid(view, seq, &uid); mail_index_lookup_ext(view, seq, ilist->ext_id, @@ -304,12 +325,33 @@ } *error_r = "parent_uid points to nonexistent record"; if (ilist->has_backing_store) - return -1; + break; /* just place it under the root */ } + if (hash_table_lookup(duplicate_hash, node) == NULL) + hash_table_insert(duplicate_hash, node, node); + else { + guid_128_t guid; + + if (ilist->has_backing_store) { + *error_r = "Duplicate mailbox in index"; + break; + } + + /* we have only the mailbox list index and this node + may have a different GUID, so rename it. */ + guid_128_generate(guid); + node->name = p_strdup_printf(ilist->mailbox_pool, + "%s-duplicate-%s", node->name, + guid_128_to_string(guid)); + *error_r = t_strdup_printf( + "Duplicate mailbox in index, renaming to %s", + node->name); + } node->next = ilist->mailbox_tree; ilist->mailbox_tree = node; } + hash_table_destroy(&duplicate_hash); return *error_r == NULL ? 0 : -1; } From dovecot at dovecot.org Tue Nov 11 01:15:40 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 11 Nov 2014 01:15:40 +0000 Subject: dovecot-2.2: lib: file_wait/try_lock() treats EINTR differently ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4cc4d7b03b3b changeset: 18055:4cc4d7b03b3b user: Timo Sirainen date: Tue Nov 11 03:13:16 2014 +0200 description: lib: file_wait/try_lock() treats EINTR differently now for timeouts and for actual signals. diffstat: src/lib/file-lock.c | 18 +++++++++++++----- 1 files changed, 13 insertions(+), 5 deletions(-) diffs (54 lines): diff -r e5cf0209bc9e -r 4cc4d7b03b3b src/lib/file-lock.c --- a/src/lib/file-lock.c Fri Oct 31 04:27:27 2014 +0200 +++ b/src/lib/file-lock.c Tue Nov 11 03:13:16 2014 +0200 @@ -4,6 +4,7 @@ #include "istream.h" #include "file-lock.h" +#include #include #ifdef HAVE_FLOCK # include @@ -144,11 +145,21 @@ return file_lock_find_proc_locks(lock_fd); } +static bool err_is_lock_timeout(time_t started, unsigned int timeout_secs) +{ + /* if EINTR took at least timeout_secs-1 number of seconds, + assume it was the alarm. otherwise log EINTR failure. + (We most likely don't want to retry EINTR since a signal + means somebody wants us to stop blocking). */ + return errno == EINTR && time(NULL) - started + 1 >= timeout_secs; +} + static int file_lock_do(int fd, const char *path, int lock_type, enum file_lock_method lock_method, unsigned int timeout_secs, const char **error_r) { const char *lock_type_str; + time_t started = time(NULL); int ret; i_assert(fd != -1); @@ -188,10 +199,7 @@ return 0; } - if (errno == EINTR) { - /* most likely alarm hit, meaning we timeouted. - even if not, we probably want to be killed - so stop blocking. */ + if (err_is_lock_timeout(started, timeout_secs)) { errno = EAGAIN; *error_r = t_strdup_printf( "fcntl(%s, %s, F_SETLKW) locking failed: " @@ -238,7 +246,7 @@ "(File is already locked)", path, lock_type_str); return 0; } - if (errno == EINTR) { + if (err_is_lock_timeout(started, timeout_secs)) { errno = EAGAIN; *error_r = t_strdup_printf("flock(%s, %s) failed: " "Timed out after %u seconds%s", From dovecot at dovecot.org Tue Nov 11 03:07:22 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 11 Nov 2014 03:07:22 +0000 Subject: dovecot-2.2: lib: Compiler warning fix in 32bit systems to previ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/062230bdbc4d changeset: 18056:062230bdbc4d user: Timo Sirainen date: Tue Nov 11 05:06:09 2014 +0200 description: lib: Compiler warning fix in 32bit systems to previous change diffstat: src/lib/file-lock.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 4cc4d7b03b3b -r 062230bdbc4d src/lib/file-lock.c --- a/src/lib/file-lock.c Tue Nov 11 03:13:16 2014 +0200 +++ b/src/lib/file-lock.c Tue Nov 11 05:06:09 2014 +0200 @@ -151,7 +151,8 @@ assume it was the alarm. otherwise log EINTR failure. (We most likely don't want to retry EINTR since a signal means somebody wants us to stop blocking). */ - return errno == EINTR && time(NULL) - started + 1 >= timeout_secs; + return errno == EINTR && + (unsigned long)(time(NULL) - started + 1) >= timeout_secs; } static int file_lock_do(int fd, const char *path, int lock_type, From dovecot at dovecot.org Tue Nov 11 23:08:52 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 11 Nov 2014 23:08:52 +0000 Subject: dovecot-2.2: doveadm: Code cleanups to prepare server code for n... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b8a7fec1beeb changeset: 18057:b8a7fec1beeb user: Timo Sirainen date: Wed Nov 12 00:20:40 2014 +0200 description: doveadm: Code cleanups to prepare server code for non-mail commands. diffstat: src/doveadm/client-connection.c | 48 ++++++++++++++-------- src/doveadm/doveadm.c | 85 ++++++++++++++++++++++++---------------- 2 files changed, 81 insertions(+), 52 deletions(-) diffs (218 lines): diff -r 062230bdbc4d -r b8a7fec1beeb src/doveadm/client-connection.c --- a/src/doveadm/client-connection.c Tue Nov 11 05:06:09 2014 +0200 +++ b/src/doveadm/client-connection.c Wed Nov 12 00:20:40 2014 +0200 @@ -26,24 +26,18 @@ static void client_connection_input(struct client_connection *conn); -static struct doveadm_mail_cmd_context * -doveadm_mail_cmd_server_parse(const char *cmd_name, +static int +doveadm_mail_cmd_server_parse(const struct doveadm_mail_cmd *cmd, const struct doveadm_settings *set, const struct mail_storage_service_input *input, - int argc, char *argv[]) + int argc, char *argv[], + struct doveadm_mail_cmd_context **ctx_r) { struct doveadm_mail_cmd_context *ctx; - const struct doveadm_mail_cmd *cmd; const char *getopt_args; bool add_username_header = FALSE; int c; - cmd = doveadm_mail_cmd_find(cmd_name); - if (cmd == NULL) { - i_error("doveadm: Client sent unknown command: %s", cmd_name); - return NULL; - } - ctx = doveadm_mail_cmd_init(cmd, set); ctx->full_args = (const void *)(argv + 1); ctx->proxying = TRUE; @@ -77,7 +71,7 @@ cmd->name, c); ctx->v.deinit(ctx); pool_unref(&ctx->pool); - return NULL; + return -1; } } } @@ -88,7 +82,7 @@ cmd->name, argv[0]); ctx->v.deinit(ctx); pool_unref(&ctx->pool); - return NULL; + return -1; } ctx->args = (const void *)argv; @@ -98,7 +92,8 @@ DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); doveadm_print_sticky("username", input->username); } - return ctx; + *ctx_r = ctx; + return 0; } static void @@ -172,10 +167,30 @@ return ret; } +static int doveadm_cmd_handle(struct client_connection *conn, + const char *cmd_name, + const struct mail_storage_service_input *input, + int argc, char *argv[]) +{ + const struct doveadm_mail_cmd *cmd; + struct doveadm_mail_cmd_context *ctx; + + cmd = doveadm_mail_cmd_find(cmd_name); + if (cmd == NULL) { + i_error("doveadm: Client sent unknown command: %s", cmd_name); + return -1; + } + + if (doveadm_mail_cmd_server_parse(cmd, conn->set, input, + argc, argv, &ctx) < 0) + return -1; + doveadm_mail_cmd_server_run(conn, ctx, input); + return 0; +} + static bool client_handle_command(struct client_connection *conn, char **args) { struct mail_storage_service_input input; - struct doveadm_mail_cmd_context *ctx; const char *flags, *cmd_name; unsigned int argc; @@ -225,11 +240,8 @@ } o_stream_cork(conn->output); - ctx = doveadm_mail_cmd_server_parse(cmd_name, conn->set, &input, argc, args); - if (ctx == NULL) + if (doveadm_cmd_handle(conn, cmd_name, &input, argc, args) < 0) o_stream_nsend(conn->output, "\n-\n", 3); - else - doveadm_mail_cmd_server_run(conn, ctx, &input); o_stream_uncork(conn->output); /* flush the output and disconnect */ diff -r 062230bdbc4d -r b8a7fec1beeb src/doveadm/doveadm.c --- a/src/doveadm/doveadm.c Tue Nov 11 05:06:09 2014 +0200 +++ b/src/doveadm/doveadm.c Wed Nov 12 00:20:40 2014 +0200 @@ -199,58 +199,75 @@ i_fatal("execv(%s) failed: %m", argv[0]); } -static bool -doveadm_try_run_multi_word(const struct doveadm_cmd *cmd, - const char *cmdname, int argc, char *argv[]) +static const struct doveadm_cmd * +doveadm_cmd_find_multi_word(const struct doveadm_cmd *cmd, + const char *cmdname, int *_argc, char **_argv[]) { + int argc = *_argc; + char **argv = *_argv; + const struct doveadm_cmd *subcmd; unsigned int len; if (argc < 2) - return FALSE; + return NULL; len = strlen(argv[1]); if (strncmp(cmdname, argv[1], len) != 0) - return FALSE; + return NULL; + argc--; argv++; if (cmdname[len] == ' ') { /* more args */ - return doveadm_try_run_multi_word(cmd, cmdname + len + 1, - argc - 1, argv + 1); + subcmd = doveadm_cmd_find_multi_word(cmd, cmdname + len + 1, + &argc, &argv); + if (subcmd == NULL) + return NULL; + } else { + if (cmdname[len] != '\0') + return NULL; } - if (cmdname[len] != '\0') - return FALSE; - /* match */ - cmd->cmd(argc - 1, argv + 1); - return TRUE; + *_argc = argc; + *_argv = argv; + return cmd; +} + +static const struct doveadm_cmd * +doveadm_cmd_find(const char *cmd_name, int *argc, char **argv[]) +{ + const struct doveadm_cmd *cmd, *subcmd; + unsigned int cmd_name_len; + + i_assert(*argc > 0); + + cmd_name_len = strlen(cmd_name); + array_foreach(&doveadm_cmds, cmd) { + if (strcmp(cmd->name, cmd_name) == 0) + return cmd; + + /* see if it matches a multi-word command */ + if (strncmp(cmd->name, cmd_name, cmd_name_len) == 0 && + cmd->name[cmd_name_len] == ' ') { + const char *subcmd_name = cmd->name + cmd_name_len + 1; + + subcmd = doveadm_cmd_find_multi_word(cmd, subcmd_name, + argc, argv); + if (subcmd != NULL) + return subcmd; + } + } + return NULL; } static bool doveadm_try_run(const char *cmd_name, int argc, char *argv[]) { const struct doveadm_cmd *cmd; - unsigned int cmd_name_len; - i_assert(argc > 0); - - cmd_name_len = strlen(cmd_name); - array_foreach(&doveadm_cmds, cmd) { - if (strcmp(cmd->name, cmd_name) == 0) { - cmd->cmd(argc, argv); - return TRUE; - } - - /* see if it matches a multi-word command */ - if (strncmp(cmd->name, cmd_name, cmd_name_len) == 0 && - cmd->name[cmd_name_len] == ' ') { - const char *subcmd = cmd->name + cmd_name_len + 1; - - if (doveadm_try_run_multi_word(cmd, subcmd, - argc, argv)) - return TRUE; - } - } - - return FALSE; + cmd = doveadm_cmd_find(cmd_name, &argc, &argv); + if (cmd == NULL) + return FALSE; + cmd->cmd(argc, argv); + return TRUE; } static bool doveadm_has_subcommands(const char *cmd_name) From dovecot at dovecot.org Tue Nov 11 23:08:53 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 11 Nov 2014 23:08:53 +0000 Subject: dovecot-2.2: doveadm who: If parameters are invalid, print help ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e4a3e8554663 changeset: 18058:e4a3e8554663 user: Timo Sirainen date: Wed Nov 12 00:52:44 2014 +0200 description: doveadm who: If parameters are invalid, print help text for who command instead of the whole doveadm usage. diffstat: src/doveadm/doveadm-who.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (18 lines): diff -r b8a7fec1beeb -r e4a3e8554663 src/doveadm/doveadm-who.c --- a/src/doveadm/doveadm-who.c Wed Nov 12 00:20:40 2014 +0200 +++ b/src/doveadm/doveadm-who.c Wed Nov 12 00:52:44 2014 +0200 @@ -119,12 +119,12 @@ while (args[1] != NULL) { if (net_parse_range(args[1], &net_ip, &net_bits) == 0) { if (ctx->filter.net_bits != 0) - usage(); + help(&doveadm_cmd_who); ctx->filter.net_ip = net_ip; ctx->filter.net_bits = net_bits; } else { if (ctx->filter.username != NULL) - usage(); + help(&doveadm_cmd_who); ctx->filter.username = args[1]; } args++; From dovecot at dovecot.org Tue Nov 11 23:08:54 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 11 Nov 2014 23:08:54 +0000 Subject: dovecot-2.2: doveadm: Moved commands related code to doveadm-cmd... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c54b6bbf215a changeset: 18059:c54b6bbf215a user: Timo Sirainen date: Wed Nov 12 01:04:27 2014 +0200 description: doveadm: Moved commands related code to doveadm-cmd.[ch] Commands are already split for command line-only commands and commands that can be run via doveadm-server also. diffstat: src/doveadm/Makefile.am | 2 + src/doveadm/doveadm-cmd.c | 108 ++++++++++++++++++++++++++++++++++++++++++++++ src/doveadm/doveadm-cmd.h | 46 +++++++++++++++++++ src/doveadm/doveadm.c | 96 ++-------------------------------------- src/doveadm/doveadm.h | 35 +-------------- 5 files changed, 162 insertions(+), 125 deletions(-) diffs (truncated from 390 to 300 lines): diff -r e4a3e8554663 -r c54b6bbf215a src/doveadm/Makefile.am --- a/src/doveadm/Makefile.am Wed Nov 12 00:52:44 2014 +0200 +++ b/src/doveadm/Makefile.am Wed Nov 12 01:04:27 2014 +0200 @@ -86,6 +86,7 @@ $(common) \ doveadm.c \ doveadm-auth.c \ + doveadm-cmd.c \ doveadm-dict.c \ doveadm-director.c \ doveadm-dump.c \ @@ -123,6 +124,7 @@ pkginc_libdir = $(pkgincludedir) pkginc_lib_HEADERS = \ doveadm.h \ + doveadm-cmd.h \ doveadm-dsync.h \ doveadm-dump.h \ doveadm-mail.h \ diff -r e4a3e8554663 -r c54b6bbf215a src/doveadm/doveadm-cmd.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/doveadm-cmd.c Wed Nov 12 01:04:27 2014 +0200 @@ -0,0 +1,108 @@ +/* Copyright (c) 2009-2014 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "doveadm-cmd.h" + +static struct doveadm_cmd *doveadm_commands[] = { + &doveadm_cmd_stop, + &doveadm_cmd_reload, + &doveadm_cmd_who, + &doveadm_cmd_penalty, + &doveadm_cmd_kick, + &doveadm_cmd_mailbox_mutf7, + &doveadm_cmd_sis_deduplicate, + &doveadm_cmd_sis_find, + &doveadm_cmd_stats_dump +}; + +ARRAY_TYPE(doveadm_cmd) doveadm_cmds; + +void doveadm_register_cmd(const struct doveadm_cmd *cmd) +{ + array_append(&doveadm_cmds, cmd, 1); +} + +static const struct doveadm_cmd * +doveadm_cmd_find_multi_word(const struct doveadm_cmd *cmd, + const char *cmdname, int *_argc, char **_argv[]) +{ + int argc = *_argc; + char **argv = *_argv; + const struct doveadm_cmd *subcmd; + unsigned int len; + + if (argc < 2) + return NULL; + + len = strlen(argv[1]); + if (strncmp(cmdname, argv[1], len) != 0) + return NULL; + + argc--; argv++; + if (cmdname[len] == ' ') { + /* more args */ + subcmd = doveadm_cmd_find_multi_word(cmd, cmdname + len + 1, + &argc, &argv); + if (subcmd == NULL) + return NULL; + } else { + if (cmdname[len] != '\0') + return NULL; + } + + *_argc = argc; + *_argv = argv; + return cmd; +} + +const struct doveadm_cmd * +doveadm_cmd_find(const char *cmd_name, int *argc, char **argv[]) +{ + const struct doveadm_cmd *cmd, *subcmd; + unsigned int cmd_name_len; + + i_assert(*argc > 0); + + cmd_name_len = strlen(cmd_name); + array_foreach(&doveadm_cmds, cmd) { + if (strcmp(cmd->name, cmd_name) == 0) + return cmd; + + /* see if it matches a multi-word command */ + if (strncmp(cmd->name, cmd_name, cmd_name_len) == 0 && + cmd->name[cmd_name_len] == ' ') { + const char *subcmd_name = cmd->name + cmd_name_len + 1; + + subcmd = doveadm_cmd_find_multi_word(cmd, subcmd_name, + argc, argv); + if (subcmd != NULL) + return subcmd; + } + } + return NULL; +} + +void doveadm_cmds_init(void) +{ + unsigned int i; + + i_array_init(&doveadm_cmds, 32); + for (i = 0; i < N_ELEMENTS(doveadm_commands); i++) + doveadm_register_cmd(doveadm_commands[i]); + + doveadm_register_auth_commands(); + doveadm_register_director_commands(); + doveadm_register_instance_commands(); + doveadm_register_mount_commands(); + doveadm_register_proxy_commands(); + doveadm_register_log_commands(); + doveadm_register_replicator_commands(); + doveadm_register_dict_commands(); + doveadm_register_fs_commands(); +} + +void doveadm_cmds_deinit(void) +{ + array_free(&doveadm_cmds); +} diff -r e4a3e8554663 -r c54b6bbf215a src/doveadm/doveadm-cmd.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/doveadm/doveadm-cmd.h Wed Nov 12 01:04:27 2014 +0200 @@ -0,0 +1,46 @@ +#ifndef DOVEADM_CMD_H +#define DOVEADM_CMD_H + +typedef void doveadm_command_t(int argc, char *argv[]); + +struct doveadm_cmd { + doveadm_command_t *cmd; + const char *name; + const char *short_usage; +}; +ARRAY_DEFINE_TYPE(doveadm_cmd, struct doveadm_cmd); +extern ARRAY_TYPE(doveadm_cmd) doveadm_cmds; + +extern struct doveadm_cmd doveadm_cmd_stop; +extern struct doveadm_cmd doveadm_cmd_reload; +extern struct doveadm_cmd doveadm_cmd_dump; +extern struct doveadm_cmd doveadm_cmd_pw; +extern struct doveadm_cmd doveadm_cmd_who; +extern struct doveadm_cmd doveadm_cmd_penalty; +extern struct doveadm_cmd doveadm_cmd_kick; +extern struct doveadm_cmd doveadm_cmd_mailbox_mutf7; +extern struct doveadm_cmd doveadm_cmd_sis_deduplicate; +extern struct doveadm_cmd doveadm_cmd_sis_find; +extern struct doveadm_cmd doveadm_cmd_stats_dump; +extern struct doveadm_cmd doveadm_cmd_stats_top; +extern struct doveadm_cmd doveadm_cmd_zlibconnect; + +void doveadm_register_cmd(const struct doveadm_cmd *cmd); + +const struct doveadm_cmd * +doveadm_cmd_find(const char *cmd_name, int *argc, char **argv[]); + +void doveadm_register_auth_commands(void); +void doveadm_register_director_commands(void); +void doveadm_register_proxy_commands(void); +void doveadm_register_log_commands(void); +void doveadm_register_instance_commands(void); +void doveadm_register_mount_commands(void); +void doveadm_register_replicator_commands(void); +void doveadm_register_dict_commands(void); +void doveadm_register_fs_commands(void); + +void doveadm_cmds_init(void); +void doveadm_cmds_deinit(void); + +#endif diff -r e4a3e8554663 -r c54b6bbf215a src/doveadm/doveadm.c --- a/src/doveadm/doveadm.c Wed Nov 12 00:52:44 2014 +0200 +++ b/src/doveadm/doveadm.c Wed Nov 12 01:04:27 2014 +0200 @@ -28,8 +28,6 @@ int doveadm_exit_code = 0; -static ARRAY(struct doveadm_cmd) doveadm_cmds; - static void failure_exit_callback(int *status) { enum fatal_exit_status fatal_status = *status; @@ -46,11 +44,6 @@ } } -void doveadm_register_cmd(const struct doveadm_cmd *cmd) -{ - array_append(&doveadm_cmds, cmd, 1); -} - static void doveadm_usage_compress_lines(FILE *out, const char *str, const char *prefix) { @@ -199,66 +192,6 @@ i_fatal("execv(%s) failed: %m", argv[0]); } -static const struct doveadm_cmd * -doveadm_cmd_find_multi_word(const struct doveadm_cmd *cmd, - const char *cmdname, int *_argc, char **_argv[]) -{ - int argc = *_argc; - char **argv = *_argv; - const struct doveadm_cmd *subcmd; - unsigned int len; - - if (argc < 2) - return NULL; - - len = strlen(argv[1]); - if (strncmp(cmdname, argv[1], len) != 0) - return NULL; - - argc--; argv++; - if (cmdname[len] == ' ') { - /* more args */ - subcmd = doveadm_cmd_find_multi_word(cmd, cmdname + len + 1, - &argc, &argv); - if (subcmd == NULL) - return NULL; - } else { - if (cmdname[len] != '\0') - return NULL; - } - - *_argc = argc; - *_argv = argv; - return cmd; -} - -static const struct doveadm_cmd * -doveadm_cmd_find(const char *cmd_name, int *argc, char **argv[]) -{ - const struct doveadm_cmd *cmd, *subcmd; - unsigned int cmd_name_len; - - i_assert(*argc > 0); - - cmd_name_len = strlen(cmd_name); - array_foreach(&doveadm_cmds, cmd) { - if (strcmp(cmd->name, cmd_name) == 0) - return cmd; - - /* see if it matches a multi-word command */ - if (strncmp(cmd->name, cmd_name, cmd_name_len) == 0 && - cmd->name[cmd_name_len] == ' ') { - const char *subcmd_name = cmd->name + cmd_name_len + 1; - - subcmd = doveadm_cmd_find_multi_word(cmd, subcmd_name, - argc, argv); - if (subcmd != NULL) - return subcmd; - } - } - return NULL; -} - static bool doveadm_try_run(const char *cmd_name, int argc, char *argv[]) { const struct doveadm_cmd *cmd; @@ -312,21 +245,12 @@ pool_datastack_create()); } -static struct doveadm_cmd *doveadm_commands[] = { +static struct doveadm_cmd *doveadm_cmdline_commands[] = { &doveadm_cmd_help, &doveadm_cmd_config, &doveadm_cmd_exec, - &doveadm_cmd_stop, - &doveadm_cmd_reload, &doveadm_cmd_dump, &doveadm_cmd_pw, - &doveadm_cmd_who, - &doveadm_cmd_penalty, - &doveadm_cmd_kick, - &doveadm_cmd_mailbox_mutf7, - &doveadm_cmd_sis_deduplicate, - &doveadm_cmd_sis_find, - &doveadm_cmd_stats_dump, &doveadm_cmd_stats_top, &doveadm_cmd_zlibconnect }; @@ -374,10 +298,9 @@ doveadm_read_settings(); } - i_array_init(&doveadm_cmds, 32); From dovecot at dovecot.org Tue Nov 11 23:08:55 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 11 Nov 2014 23:08:55 +0000 Subject: dovecot-2.2: doveadm: Support running non-mail commands via dove... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a4fbe94952ea changeset: 18060:a4fbe94952ea user: Timo Sirainen date: Wed Nov 12 01:06:02 2014 +0200 description: doveadm: Support running non-mail commands via doveadm server. There are still some commands that print their output via printf though, so they need to be fixed separately. diffstat: src/doveadm/Makefile.am | 44 ++++++++++------- src/doveadm/client-connection.c | 100 ++++++++++++++++++++++++++++++--------- src/doveadm/main.c | 8 +++ 3 files changed, 109 insertions(+), 43 deletions(-) diffs (260 lines): diff -r c54b6bbf215a -r a4fbe94952ea src/doveadm/Makefile.am --- a/src/doveadm/Makefile.am Wed Nov 12 01:04:27 2014 +0200 +++ b/src/doveadm/Makefile.am Wed Nov 12 01:06:02 2014 +0200 @@ -58,7 +58,25 @@ $(LIBDOVECOT_STORAGE_DEPS) \ $(LIBDOVECOT_DEPS) -common = \ +doveadm_common_cmds = \ + doveadm-auth.c \ + doveadm-dict.c \ + doveadm-director.c \ + doveadm-fs.c \ + doveadm-instance.c \ + doveadm-kick.c \ + doveadm-log.c \ + doveadm-master.c \ + doveadm-mount.c \ + doveadm-mutf7.c \ + doveadm-penalty.c \ + doveadm-proxy.c \ + doveadm-replicator.c \ + doveadm-sis.c \ + doveadm-stats.c \ + doveadm-who.c + +doveadm_common_mail_cmds = \ doveadm-dsync.c \ doveadm-mail.c \ doveadm-mail-altmove.c \ @@ -76,7 +94,12 @@ doveadm-mail-copymove.c \ doveadm-mailbox-list-iter.c \ doveadm-mail-search.c \ - doveadm-mail-server.c \ + doveadm-mail-server.c + +common = \ + $(doveadm_common_cmds) \ + $(doveadm_common_mail_cmds) \ + doveadm-cmd.c \ doveadm-print.c \ doveadm-settings.c \ doveadm-util.c \ @@ -85,34 +108,17 @@ doveadm_SOURCES = \ $(common) \ doveadm.c \ - doveadm-auth.c \ - doveadm-cmd.c \ - doveadm-dict.c \ - doveadm-director.c \ doveadm-dump.c \ doveadm-dump-dbox.c \ doveadm-dump-index.c \ doveadm-dump-log.c \ doveadm-dump-mailboxlog.c \ doveadm-dump-thread.c \ - doveadm-fs.c \ - doveadm-instance.c \ - doveadm-kick.c \ - doveadm-log.c \ - doveadm-master.c \ - doveadm-mount.c \ - doveadm-mutf7.c \ - doveadm-penalty.c \ doveadm-print-flow.c \ doveadm-print-pager.c \ doveadm-print-tab.c \ doveadm-print-table.c \ - doveadm-proxy.c \ doveadm-pw.c \ - doveadm-replicator.c \ - doveadm-sis.c \ - doveadm-stats.c \ - doveadm-who.c \ doveadm-zlib.c doveadm_server_SOURCES = \ diff -r c54b6bbf215a -r a4fbe94952ea src/doveadm/client-connection.c --- a/src/doveadm/client-connection.c Wed Nov 12 01:04:27 2014 +0200 +++ b/src/doveadm/client-connection.c Wed Nov 12 01:06:02 2014 +0200 @@ -24,8 +24,52 @@ #define MAX_INBUF_SIZE (1024*1024) +static struct { + int code; + const char *str; +} exit_code_strings[] = { + { EX_TEMPFAIL, "TEMPFAIL" }, + { EX_USAGE, "USAGE" }, + { EX_NOUSER, "NOUSER" }, + { EX_NOPERM, "NOPERM" }, + { EX_PROTOCOL, "PROTOCOL" }, + { EX_DATAERR, "DATAERR" }, + { DOVEADM_EX_NOTFOUND, "NOTFOUND" } +}; + static void client_connection_input(struct client_connection *conn); +static void +doveadm_cmd_server_run(struct client_connection *conn, + const struct doveadm_cmd *cmd, int argc, char *argv[]) +{ + const char *str = NULL; + unsigned int i; + + doveadm_exit_code = 0; + cmd->cmd(argc, argv); + + if (doveadm_exit_code == 0) { + o_stream_nsend(conn->output, "\n+\n", 3); + return; + } + + for (i = 0; i < N_ELEMENTS(exit_code_strings); i++) { + if (exit_code_strings[i].code == doveadm_exit_code) { + str = exit_code_strings[i].str; + break; + } + } + if (str != NULL) { + o_stream_nsend_str(conn->output, + t_strdup_printf("\n-%s\n", str)); + } else { + o_stream_nsend_str(conn->output, "\n-\n"); + i_error("BUG: Command '%s' returned unknown error code %d", + cmd->name, doveadm_exit_code); + } +} + static int doveadm_mail_cmd_server_parse(const struct doveadm_mail_cmd *cmd, const struct doveadm_settings *set, @@ -102,16 +146,10 @@ const struct mail_storage_service_input *input) { const char *error; - struct ioloop *ioloop, *prev_ioloop = current_ioloop; int ret; ctx->conn = conn; - /* some commands will want to call io_loop_run(), but we're already - running one and we can't call the original one recursively, so - create a new ioloop. */ - ioloop = io_loop_create(); - lib_signals_reset_ioloop(); if (ctx->v.preinit != NULL) ctx->v.preinit(ctx); @@ -121,12 +159,6 @@ doveadm_print_flush(); mail_storage_service_deinit(&ctx->storage_service); - io_loop_set_current(prev_ioloop); - lib_signals_reset_ioloop(); - o_stream_switch_ioloop(conn->output); - io_loop_set_current(ioloop); - io_loop_destroy(&ioloop); - if (ret < 0) { i_error("%s: %s", ctx->cmd->name, error); o_stream_nsend(conn->output, "\n-\n", 3); @@ -140,10 +172,6 @@ o_stream_nsend(conn->output, "\n+\n", 3); } pool_unref(&ctx->pool); - - /* clear all headers */ - doveadm_print_deinit(); - doveadm_print_init(DOVEADM_PRINT_TYPE_SERVER); } static bool client_is_allowed_command(const struct doveadm_settings *set, @@ -172,19 +200,43 @@ const struct mail_storage_service_input *input, int argc, char *argv[]) { - const struct doveadm_mail_cmd *cmd; + struct ioloop *ioloop, *prev_ioloop = current_ioloop; + const struct doveadm_cmd *cmd; + const struct doveadm_mail_cmd *mail_cmd; struct doveadm_mail_cmd_context *ctx; - cmd = doveadm_mail_cmd_find(cmd_name); + cmd = doveadm_cmd_find(cmd_name, &argc, &argv); if (cmd == NULL) { - i_error("doveadm: Client sent unknown command: %s", cmd_name); - return -1; + mail_cmd = doveadm_mail_cmd_find(cmd_name); + if (mail_cmd == NULL) { + i_error("doveadm: Client sent unknown command: %s", cmd_name); + return -1; + } + if (doveadm_mail_cmd_server_parse(mail_cmd, conn->set, input, + argc, argv, &ctx) < 0) + return -1; } - if (doveadm_mail_cmd_server_parse(cmd, conn->set, input, - argc, argv, &ctx) < 0) - return -1; - doveadm_mail_cmd_server_run(conn, ctx, input); + /* some commands will want to call io_loop_run(), but we're already + running one and we can't call the original one recursively, so + create a new ioloop. */ + ioloop = io_loop_create(); + lib_signals_reset_ioloop(); + + if (cmd != NULL) + doveadm_cmd_server_run(conn, cmd, argc, argv); + else + doveadm_mail_cmd_server_run(conn, ctx, input); + + io_loop_set_current(prev_ioloop); + lib_signals_reset_ioloop(); + o_stream_switch_ioloop(conn->output); + io_loop_set_current(ioloop); + io_loop_destroy(&ioloop); + + /* clear all headers */ + doveadm_print_deinit(); + doveadm_print_init(DOVEADM_PRINT_TYPE_SERVER); return 0; } diff -r c54b6bbf215a -r a4fbe94952ea src/doveadm/main.c --- a/src/doveadm/main.c Wed Nov 12 01:04:27 2014 +0200 +++ b/src/doveadm/main.c Wed Nov 12 01:06:02 2014 +0200 @@ -36,6 +36,12 @@ conn->ssl); } +void help(const struct doveadm_cmd *cmd) +{ + i_fatal("Client sent invalid command. Usage: %s %s", + cmd->name, cmd->short_usage); +} + static void main_preinit(void) { restrict_access_by_env(NULL, FALSE); @@ -50,6 +56,7 @@ doveadm_settings, pool_datastack_create()); + doveadm_cmds_init(); doveadm_mail_init(); doveadm_load_modules(); doveadm_print_init(DOVEADM_PRINT_TYPE_SERVER); @@ -62,6 +69,7 @@ doveadm_mail_deinit(); doveadm_unload_modules(); doveadm_print_deinit(); + doveadm_cmds_deinit(); } int main(int argc, char *argv[]) From dovecot at dovecot.org Tue Nov 11 23:27:22 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 11 Nov 2014 23:27:22 +0000 Subject: dovecot-2.2: lmtp: Show more information in process title. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/aca0b7142046 changeset: 18061:aca0b7142046 user: Timo Sirainen date: Wed Nov 12 01:26:04 2014 +0200 description: lmtp: Show more information in process title. Most importantly show the username while messages are being delivered. diffstat: src/lmtp/client.c | 34 ++++++++++++++++++++++++---------- src/lmtp/client.h | 4 ++-- src/lmtp/commands.c | 21 ++++++++++++--------- 3 files changed, 38 insertions(+), 21 deletions(-) diffs (208 lines): diff -r a4fbe94952ea -r aca0b7142046 src/lmtp/client.c --- a/src/lmtp/client.c Wed Nov 12 01:06:02 2014 +0200 +++ b/src/lmtp/client.c Wed Nov 12 01:26:04 2014 +0200 @@ -34,18 +34,32 @@ static struct client *clients = NULL; unsigned int clients_count = 0; -void client_state_set(struct client *client, const char *name) +void client_state_set(struct client *client, const char *name, const char *args) { + string_t *str; + client->state.name = name; if (!client->service_set->verbose_proctitle) return; - if (clients_count == 0) + + if (clients_count == 0) { process_title_set("[idling]"); - else if (clients_count > 1) + return; + } + if (clients_count > 1) { process_title_set(t_strdup_printf("[%u clients]", clients_count)); - else - process_title_set(t_strdup_printf("[%s]", client->state.name)); + return; + } + + str = t_str_new(128); + str_printfa(str, "[%s", client->state.name); + if (client->remote_ip.family != 0) + str_printfa(str, " %s", net_ip2addr(&client->remote_ip)); + if (args[0] != '\0') + str_printfa(str, " %s", args); + str_append_c(str, ']'); + process_title_set(str_c(str)); } static void client_idle_timeout(struct client *client) @@ -253,7 +267,7 @@ DLLIST_PREPEND(&clients, client); clients_count++; - client_state_set(client, "banner"); + client_state_set(client, "banner", ""); client_send_line(client, "220 %s %s", client->my_domain, client->lmtp_set->login_greeting); i_info("Connect from %s", client_remote_id(client)); @@ -269,7 +283,7 @@ clients_count--; DLLIST_REMOVE(&clients, client); - client_state_set(client, "destroyed"); + client_state_set(client, "destroyed", ""); if (client->raw_mail_user != NULL) mail_user_unref(&client->raw_mail_user); @@ -287,7 +301,7 @@ net_disconnect(client->fd_in); if (client->fd_in != client->fd_out) net_disconnect(client->fd_out); - client_state_reset(client); + client_state_reset(client, "destroyed"); i_free(client->lhlo); pool_unref(&client->state_pool); pool_unref(&client->pool); @@ -333,7 +347,7 @@ client->disconnected = TRUE; } -void client_state_reset(struct client *client) +void client_state_reset(struct client *client, const char *state_name) { struct mail_recipient *rcpt; @@ -369,7 +383,7 @@ client->state.mail_data_fd = -1; client_generate_session_id(client); - client_state_set(client, "reset"); + client_state_set(client, state_name, ""); } void client_send_line(struct client *client, const char *fmt, ...) diff -r a4fbe94952ea -r aca0b7142046 src/lmtp/client.h --- a/src/lmtp/client.h Wed Nov 12 01:06:02 2014 +0200 +++ b/src/lmtp/client.h Wed Nov 12 01:26:04 2014 +0200 @@ -79,8 +79,8 @@ void client_disconnect(struct client *client, const char *prefix, const char *reason); void client_io_reset(struct client *client); -void client_state_reset(struct client *client); -void client_state_set(struct client *client, const char *name); +void client_state_reset(struct client *client, const char *state_name); +void client_state_set(struct client *client, const char *name, const char *args); const char *client_remote_id(struct client *client); void client_input_handle(struct client *client); diff -r a4fbe94952ea -r aca0b7142046 src/lmtp/commands.c --- a/src/lmtp/commands.c Wed Nov 12 01:06:02 2014 +0200 +++ b/src/lmtp/commands.c Wed Nov 12 01:26:04 2014 +0200 @@ -70,7 +70,7 @@ str_append(domain, "invalid"); } - client_state_reset(client); + client_state_reset(client, "LHLO"); client_send_line(client, "250-%s", client->my_domain); if (master_service_ssl_is_enabled(master_service) && client->ssl_iostream == NULL) @@ -83,7 +83,7 @@ i_free(client->lhlo); client->lhlo = i_strdup(str_c(domain)); - client_state_set(client, "LHLO"); + client_state_set(client, "LHLO", ""); return 0; } @@ -180,7 +180,7 @@ client->state.mail_from = p_strdup(client->state_pool, addr); p_array_init(&client->state.rcpt_to, client->state_pool, 64); client_send_line(client, "250 2.1.0 OK"); - client_state_set(client, "MAIL FROM"); + client_state_set(client, "MAIL FROM", client->state.mail_from); return 0; } @@ -541,8 +541,6 @@ const char *error = NULL; int ret = 0; - client_state_set(client, "RCPT TO"); - if (client->state.mail_from == NULL) { client_send_line(client, "503 5.5.1 MAIL needed first"); return 0; @@ -563,6 +561,8 @@ } rcpt_address_parse(client, address, &username, &detail); + client_state_set(client, "RCPT TO", address); + if (client->lmtp_set->lmtp_proxy) { if (client_proxy_rcpt(client, address, username, detail)) return 0; @@ -636,7 +636,7 @@ int cmd_rset(struct client *client, const char *args ATTR_UNUSED) { - client_state_reset(client); + client_state_reset(client, "RSET"); client_send_line(client, "250 2.0.0 OK"); return 0; } @@ -683,6 +683,7 @@ i_unreached(); } + client_state_set(client, "DATA", username); i_set_failure_prefix("lmtp(%s, %s): ", my_pid, username); if (mail_storage_service_next(storage_service, rcpt->service_user, &client->state.dest_user) < 0) { @@ -772,6 +773,7 @@ while (client->state.rcpt_idx < count) { ret = client_deliver(client, &rcpts[client->state.rcpt_idx], src_mail, session); + client_state_set(client, "DATA", ""); i_set_failure_prefix("lmtp(%s): ", my_pid); client->state.rcpt_idx++; @@ -917,7 +919,7 @@ static void client_input_data_finish(struct client *client) { client_io_reset(client); - client_state_reset(client); + client_state_reset(client, "DATA finished"); if (i_stream_have_bytes_left(client->input)) client_input_handle(client); } @@ -979,6 +981,7 @@ if (array_count(&client->state.rcpt_to) != 0) client_input_data_write_local(client, input); if (client->proxy != NULL) { + client_state_set(client, "DATA", "proxying"); lmtp_proxy_start(client->proxy, input, client_proxy_finish, client); ret = FALSE; @@ -1107,7 +1110,7 @@ o_stream_uncork(client->output); io_remove(&client->io); - client_state_set(client, "DATA"); + client_state_set(client, "DATA", ""); client->io = io_add(client->fd_in, IO_READ, client_input_data, client); client_input_data_handle(client); return -1; @@ -1147,7 +1150,7 @@ } /* args ok, set them and reset the state */ - client_state_reset(client); + client_state_reset(client, "XCLIENT"); if (remote_ip.family != 0) client->remote_ip = remote_ip; if (remote_port != 0) From dovecot at dovecot.org Tue Nov 11 23:52:46 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 11 Nov 2014 23:52:46 +0000 Subject: dovecot-2.2: doveadm director ring status shows now more informa... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ade6dcace5cd changeset: 18062:ade6dcace5cd user: Timo Sirainen date: Wed Nov 12 01:51:29 2014 +0200 description: doveadm director ring status shows now more information diffstat: src/director/director-connection.c | 5 +++ src/director/director-connection.h | 1 + src/director/doveadm-connection.c | 60 +++++++++++++++++++++++++++++++++++++- src/doveadm/doveadm-director.c | 4 +- 4 files changed, 68 insertions(+), 2 deletions(-) diffs (133 lines): diff -r aca0b7142046 -r ade6dcace5cd src/director/director-connection.c --- a/src/director/director-connection.c Wed Nov 12 01:26:04 2014 +0200 +++ b/src/director/director-connection.c Wed Nov 12 01:51:29 2014 +0200 @@ -1841,6 +1841,11 @@ return conn->handshake_received; } +bool director_connection_is_synced(struct director_connection *conn) +{ + return conn->synced; +} + bool director_connection_is_incoming(struct director_connection *conn) { return conn->in; diff -r aca0b7142046 -r ade6dcace5cd src/director/director-connection.h --- a/src/director/director-connection.h Wed Nov 12 01:26:04 2014 +0200 +++ b/src/director/director-connection.h Wed Nov 12 01:51:29 2014 +0200 @@ -23,6 +23,7 @@ struct director_host * director_connection_get_host(struct director_connection *conn); bool director_connection_is_handshaked(struct director_connection *conn); +bool director_connection_is_synced(struct director_connection *conn); bool director_connection_is_incoming(struct director_connection *conn); unsigned int director_connection_get_minor_version(struct director_connection *conn); diff -r aca0b7142046 -r ade6dcace5cd src/director/doveadm-connection.c --- a/src/director/doveadm-connection.c Wed Nov 12 01:26:04 2014 +0200 +++ b/src/director/doveadm-connection.c Wed Nov 12 01:51:29 2014 +0200 @@ -92,6 +92,59 @@ mail_hosts_deinit(&orig_hosts_list); } +static void doveadm_director_append_status(struct director *dir, string_t *str) +{ + if (!dir->ring_handshaked) + str_append(str, "handshaking"); + else if (dir->ring_synced) + str_append(str, "synced"); + else { + str_printfa(str, "syncing - last sync %d secs ago", + (int)(ioloop_time - dir->ring_last_sync_time)); + } +} + +static void +doveadm_director_connection_append_status(struct director_connection *conn, + string_t *str) +{ + if (!director_connection_is_handshaked(conn)) + str_append(str, "handshaking"); + else if (director_connection_is_synced(conn)) + str_append(str, "synced"); + else + str_append(str, "syncing"); +} + +static void +doveadm_director_host_append_status(struct director *dir, + const struct director_host *host, + string_t *str) +{ + struct director_connection *conn = NULL; + + if (dir->left != NULL && + director_connection_get_host(dir->left) == host) + conn = dir->left; + else if (dir->right != NULL && + director_connection_get_host(dir->right) == host) + conn = dir->right; + else { + /* we might have a connection that is being connected */ + struct director_connection *const *connp; + + array_foreach(&dir->connections, connp) { + if (director_connection_get_host(*connp) == host) { + conn = *connp; + break; + } + } + } + + if (conn != NULL) + doveadm_director_connection_append_status(conn, str); +} + static void doveadm_cmd_director_list(struct doveadm_connection *conn) { struct director *dir = conn->dir; @@ -122,9 +175,14 @@ last_failed = I_MAX(host->last_network_failure, host->last_protocol_failure); - str_printfa(str, "%s\t%u\t%s\t%lu\n", + str_printfa(str, "%s\t%u\t%s\t%lu\t", net_ip2addr(&host->ip), host->port, type, (unsigned long)last_failed); + if (dir->self_host == host) + doveadm_director_append_status(dir, str); + else + doveadm_director_host_append_status(dir, host, str); + str_append_c(str, '\n'); } str_append_c(str, '\n'); o_stream_nsend(conn->output, str_data(str), str_len(str)); diff -r aca0b7142046 -r ade6dcace5cd src/doveadm/doveadm-director.c --- a/src/doveadm/doveadm-director.c Wed Nov 12 01:26:04 2014 +0200 +++ b/src/doveadm/doveadm-director.c Wed Nov 12 01:51:29 2014 +0200 @@ -718,6 +718,7 @@ doveadm_print_header_simple("port"); doveadm_print_header_simple("type"); doveadm_print_header_simple("last failed"); + doveadm_print_header_simple("status"); director_send(ctx, "DIRECTOR-LIST\n"); while ((line = i_stream_read_next_line(ctx->input)) != NULL) { @@ -725,7 +726,7 @@ break; T_BEGIN { args = t_strsplit_tab(line); - if (str_array_length(args) >= 4 && + if (str_array_length(args) >= 5 && str_to_ulong(args[3], &l) == 0) { doveadm_print(args[0]); doveadm_print(args[1]); @@ -734,6 +735,7 @@ doveadm_print("never"); else doveadm_print(unixdate2str(l)); + doveadm_print(args[4]); } } T_END; } From dovecot at dovecot.org Wed Nov 12 00:01:03 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 12 Nov 2014 00:01:03 +0000 Subject: dovecot-2.2: director: Always log a message when director handsh... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/91c525000811 changeset: 18063:91c525000811 user: Timo Sirainen date: Wed Nov 12 01:59:48 2014 +0200 description: director: Always log a message when director handshake finishes. diffstat: src/director/director-connection.c | 20 +++++++++----------- 1 files changed, 9 insertions(+), 11 deletions(-) diffs (30 lines): diff -r ade6dcace5cd -r 91c525000811 src/director/director-connection.c --- a/src/director/director-connection.c Wed Nov 12 01:51:29 2014 +0200 +++ b/src/director/director-connection.c Wed Nov 12 01:59:48 2014 +0200 @@ -1036,17 +1036,15 @@ user_directory_sort(conn->dir->users); } - if (handshake_secs >= DIRECTOR_HANDSHAKE_WARN_SECS || director_debug) { - str = t_str_new(128); - str_printfa(str, "director(%s): Handshake took %u secs, " - "bytes in=%"PRIuUOFF_T" out=%"PRIuUOFF_T, - conn->name, handshake_secs, conn->input->v_offset, - conn->output->offset); - if (handshake_secs >= DIRECTOR_HANDSHAKE_WARN_SECS) - i_warning("%s", str_c(str)); - else - i_debug("%s", str_c(str)); - } + str = t_str_new(128); + str_printfa(str, "director(%s): Handshake finished in %u secs " + "(bytes in=%"PRIuUOFF_T" out=%"PRIuUOFF_T")", + conn->name, handshake_secs, conn->input->v_offset, + conn->output->offset); + if (handshake_secs >= DIRECTOR_HANDSHAKE_WARN_SECS) + i_warning("%s", str_c(str)); + else + i_info("%s", str_c(str)); /* the host is up now, make sure we can connect to it immediately if needed */ From dovecot at dovecot.org Wed Nov 12 00:05:28 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 12 Nov 2014 00:05:28 +0000 Subject: dovecot-2.2: director: When logging a warning about ring being s... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/389f084b3d2a changeset: 18064:389f084b3d2a user: Timo Sirainen date: Wed Nov 12 02:04:14 2014 +0200 description: director: When logging a warning about ring being synced again, log how many seconds it took diffstat: src/director/director.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (14 lines): diff -r 91c525000811 -r 389f084b3d2a src/director/director.c --- a/src/director/director.c Wed Nov 12 01:59:48 2014 +0200 +++ b/src/director/director.c Wed Nov 12 02:04:14 2014 +0200 @@ -274,7 +274,9 @@ if (dir->to_handshake_warning != NULL) timeout_remove(&dir->to_handshake_warning); if (dir->ring_handshake_warning_sent) { - i_warning("Ring is synced, continuing delayed requests"); + i_warning("Ring is synced, continuing delayed requests " + "(syncing took %d secs)", + (int)(ioloop_time - dir->ring_last_sync_time)); dir->ring_handshake_warning_sent = FALSE; } From dovecot at dovecot.org Wed Nov 12 01:46:25 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 12 Nov 2014 01:46:25 +0000 Subject: dovecot-2.2: director: Added director_consistent_hashing setting. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b9df3d654710 changeset: 18065:b9df3d654710 user: Timo Sirainen date: Wed Nov 12 03:29:04 2014 +0200 description: director: Added director_consistent_hashing setting. This should have been the default since the beginning. I didn't thik it was worth the trouble originally because in theory it shouldn't matter much if users jump between multiple backends. But this makes caching behavior worse, especially for systems which are using local cache files. diffstat: src/director/director-connection.c | 55 ++++++++++++++- src/director/director-settings.c | 1 + src/director/director-settings.h | 1 + src/director/director.c | 2 +- src/director/director.h | 4 +- src/director/doveadm-connection.c | 2 +- src/director/mail-host.c | 135 +++++++++++++++++++++++++++++++++--- src/director/mail-host.h | 2 +- 8 files changed, 184 insertions(+), 18 deletions(-) diffs (truncated from 396 to 300 lines): diff -r 389f084b3d2a -r b9df3d654710 src/director/director-connection.c --- a/src/director/director-connection.c Wed Nov 12 02:04:14 2014 +0200 +++ b/src/director/director-connection.c Wed Nov 12 03:29:04 2014 +0200 @@ -90,6 +90,8 @@ #define CMD_IS_USER_HANDHAKE(args) \ (str_array_length(args) > 2) +#define DIRECTOR_OPT_CONSISTENT_HASHING "consistent-hashing" + struct director_connection { struct director *dir; char *name; @@ -125,6 +127,7 @@ unsigned int wrong_host:1; unsigned int verifying_left:1; unsigned int users_unsorted:1; + unsigned int done_pending:1; }; static void director_connection_disconnected(struct director_connection **conn); @@ -132,6 +135,7 @@ const char *reason); static void director_connection_log_disconnect(struct director_connection *conn, int err); +static int director_connection_send_done(struct director_connection *conn); static void ATTR_FORMAT(2, 3) director_cmd_error(struct director_connection *conn, const char *fmt, ...) @@ -1070,6 +1074,25 @@ } static int +director_handshake_cmd_options(struct director_connection *conn, + const char *const *args) +{ + bool consistent_hashing = FALSE; + unsigned int i; + + for (i = 0; args[i] != NULL; i++) { + if (strcmp(args[i], DIRECTOR_OPT_CONSISTENT_HASHING) == 0) + consistent_hashing = TRUE; + } + if (consistent_hashing != conn->dir->set->director_consistent_hashing) { + i_error("director(%s): director_consistent_hashing settings differ between directors", + conn->name); + return -1; + } + return 1; +} + +static int director_connection_handle_handshake(struct director_connection *conn, const char *cmd, const char *const *args) { @@ -1088,6 +1111,10 @@ } conn->minor_version = atoi(args[2]); conn->version_received = TRUE; + if (conn->done_pending) { + if (director_connection_send_done(conn) < 0) + return -1; + } return 1; } if (!conn->version_received) { @@ -1114,6 +1141,8 @@ director_cmd_error(conn, "Unexpected command during host list"); return -1; } + if (strcmp(cmd, "OPTIONS") == 0) + return director_handshake_cmd_options(conn, args); if (strcmp(cmd, "HOST-HAND-START") == 0) { if (!conn->in) { director_cmd_error(conn, @@ -1518,6 +1547,25 @@ str_printfa(str, "HOST-HAND-END\t%u\n", conn->dir->ring_handshaked); } +static int director_connection_send_done(struct director_connection *conn) +{ + i_assert(conn->version_received); + + if (!conn->dir->set->director_consistent_hashing) + ; + else if (conn->minor_version >= DIRECTOR_VERSION_OPTIONS) { + director_connection_send(conn, + "OPTIONS\t"DIRECTOR_OPT_CONSISTENT_HASHING"\n"); + } else { + i_error("director(%s): Director version is too old for supporting director_consistent_hashing=yes", + conn->name); + return -1; + } + director_connection_send(conn, "DONE\n"); + conn->done_pending = FALSE; + return 0; +} + static int director_connection_send_users(struct director_connection *conn) { struct user *user; @@ -1546,7 +1594,12 @@ } } user_directory_iter_deinit(&conn->user_iter); - director_connection_send(conn, "DONE\n"); + if (!conn->version_received) + conn->done_pending = TRUE; + else { + if (director_connection_send_done(conn) < 0) + return -1; + } if (conn->users_unsorted && conn->handshake_received) { /* we received remote's list of users before sending ours */ diff -r 389f084b3d2a -r b9df3d654710 src/director/director-settings.c --- a/src/director/director-settings.c Wed Nov 12 02:04:14 2014 +0200 +++ b/src/director/director-settings.c Wed Nov 12 03:29:04 2014 +0200 @@ -73,6 +73,7 @@ DEF(SET_STR, director_username_hash), DEF(SET_TIME, director_user_expire), DEF(SET_UINT, director_doveadm_port), + DEF(SET_BOOL, director_consistent_hashing), SETTING_DEFINE_LIST_END }; diff -r 389f084b3d2a -r b9df3d654710 src/director/director-settings.h --- a/src/director/director-settings.h Wed Nov 12 02:04:14 2014 +0200 +++ b/src/director/director-settings.h Wed Nov 12 03:29:04 2014 +0200 @@ -9,6 +9,7 @@ const char *director_username_hash; unsigned int director_user_expire; unsigned int director_doveadm_port; + bool director_consistent_hashing; }; extern const struct setting_parser_info director_setting_parser_info; diff -r 389f084b3d2a -r b9df3d654710 src/director/director.c --- a/src/director/director.c Wed Nov 12 02:04:14 2014 +0200 +++ b/src/director/director.c Wed Nov 12 03:29:04 2014 +0200 @@ -909,7 +909,7 @@ i_array_init(&dir->connections, 8); dir->users = user_directory_init(set->director_user_expire, set->director_username_hash); - dir->mail_hosts = mail_hosts_init(); + dir->mail_hosts = mail_hosts_init(set->director_consistent_hashing); dir->ipc_proxy = ipc_client_init(DIRECTOR_IPC_PROXY_PATH); dir->ring_min_version = DIRECTOR_VERSION_MINOR; diff -r 389f084b3d2a -r b9df3d654710 src/director/director.h --- a/src/director/director.h Wed Nov 12 02:04:14 2014 +0200 +++ b/src/director/director.h Wed Nov 12 03:29:04 2014 +0200 @@ -6,7 +6,7 @@ #define DIRECTOR_VERSION_NAME "director" #define DIRECTOR_VERSION_MAJOR 1 -#define DIRECTOR_VERSION_MINOR 4 +#define DIRECTOR_VERSION_MINOR 5 /* weak users supported in protocol */ #define DIRECTOR_VERSION_WEAK_USERS 1 @@ -16,6 +16,8 @@ #define DIRECTOR_VERSION_QUIT 3 /* user-kick supported */ #define DIRECTOR_VERSION_USER_KICK 4 +/* options supported in handshake */ +#define DIRECTOR_VERSION_OPTIONS 5 /* Minimum time between even attempting to communicate with a director that failed due to a protocol error. */ diff -r 389f084b3d2a -r b9df3d654710 src/director/doveadm-connection.c --- a/src/director/doveadm-connection.c Wed Nov 12 02:04:14 2014 +0200 +++ b/src/director/doveadm-connection.c Wed Nov 12 03:29:04 2014 +0200 @@ -62,7 +62,7 @@ string_t *str = t_str_new(1024); int ret; - orig_hosts_list = mail_hosts_init(); + orig_hosts_list = mail_hosts_init(conn->dir->set->director_consistent_hashing); (void)mail_hosts_parse_and_add(orig_hosts_list, conn->dir->set->director_mail_servers); diff -r 389f084b3d2a -r b9df3d654710 src/director/mail-host.c --- a/src/director/mail-host.c Wed Nov 12 02:04:14 2014 +0200 +++ b/src/director/mail-host.c Wed Nov 12 03:29:04 2014 +0200 @@ -2,14 +2,22 @@ #include "lib.h" #include "array.h" +#include "bsearch-insert-pos.h" +#include "md5.h" #include "mail-host.h" #define VHOST_MULTIPLIER 100 +struct mail_vhost { + unsigned int hash; + struct mail_host *host; +}; + struct mail_host_list { ARRAY_TYPE(mail_host) hosts; - ARRAY(struct mail_host *) vhosts; + ARRAY(struct mail_vhost) vhosts; bool hosts_unsorted; + bool consistent_hashing; }; static int @@ -18,8 +26,71 @@ return net_ip_cmp(&(*h1)->ip, &(*h2)->ip); } -static void mail_hosts_sort(struct mail_host_list *list) +static int +mail_vhost_cmp(const struct mail_vhost *h1, const struct mail_vhost *h2) { + if (h1->hash < h2->hash) + return -1; + else if (h1->hash > h2->hash) + return 1; + /* hash collision. not ideal, but we'll need to keep the order + consistent across directors so compare the IPs next. */ + return net_ip_cmp(&h1->host->ip, &h2->host->ip); +} + +static int +mail_vhost_hash_cmp(const unsigned int *hash, const struct mail_vhost *vhost) +{ + if (vhost->hash < *hash) + return 1; + else if (vhost->hash > *hash) + return -1; + else + return 0; +} + +static void mail_vhost_add(struct mail_host_list *list, struct mail_host *host) +{ + struct mail_vhost *vhost; + struct md5_context md5_ctx, md5_ctx2; + unsigned char md5[MD5_RESULTLEN]; + const char *ip_str; + char num_str[MAX_INT_STRLEN]; + unsigned int i, j; + + ip_str = net_ip2addr(&host->ip); + + md5_init(&md5_ctx); + md5_update(&md5_ctx, ip_str, strlen(ip_str)); + + for (i = 0; i < host->vhost_count; i++) { + md5_ctx2 = md5_ctx; + i_snprintf(num_str, sizeof(num_str), "-%u", i); + md5_update(&md5_ctx2, num_str, strlen(num_str)); + md5_final(&md5_ctx2, md5); + + vhost = array_append_space(&list->vhosts); + vhost->host = host; + for (j = 0; j < sizeof(vhost->hash); j++) + vhost->hash = (vhost->hash << CHAR_BIT) | md5[j]; + } +} + +static void mail_hosts_sort_ring(struct mail_host_list *list) +{ + struct mail_host *const *hostp; + + /* rebuild vhosts */ + array_clear(&list->vhosts); + array_foreach(&list->hosts, hostp) + mail_vhost_add(list, *hostp); + array_sort(&list->vhosts, mail_vhost_cmp); + list->hosts_unsorted = FALSE; +} + +static void mail_hosts_sort_direct(struct mail_host_list *list) +{ + struct mail_vhost *vhost; struct mail_host *const *hostp; unsigned int i; @@ -28,12 +99,22 @@ /* rebuild vhosts */ array_clear(&list->vhosts); array_foreach(&list->hosts, hostp) { - for (i = 0; i < (*hostp)->vhost_count; i++) - array_append(&list->vhosts, hostp, 1); + for (i = 0; i < (*hostp)->vhost_count; i++) { + vhost = array_append_space(&list->vhosts); + vhost->host = *hostp; + } } list->hosts_unsorted = FALSE; } +static void mail_hosts_sort(struct mail_host_list *list) +{ + if (list->consistent_hashing) + mail_hosts_sort_ring(list); + else From dovecot at dovecot.org Wed Nov 12 04:59:53 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 12 Nov 2014 04:59:53 +0000 Subject: dovecot-2.2: director: Don't restart request timeout when direct... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/26679856fbd5 changeset: 18066:26679856fbd5 user: Timo Sirainen date: Wed Nov 12 06:46:45 2014 +0200 description: director: Don't restart request timeout when director state changes. If the state keeps changing all the time we'll never return timeouts for the requests. diffstat: src/director/main.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r b9df3d654710 -r 26679856fbd5 src/director/main.c --- a/src/director/main.c Wed Nov 12 03:29:04 2014 +0200 +++ b/src/director/main.c Wed Nov 12 06:46:45 2014 +0200 @@ -136,7 +136,7 @@ array_clear(&dir->pending_requests); array_append_array(&dir->pending_requests, &new_requests); - if (dir->to_request != NULL) + if (dir->to_request != NULL && array_count(&new_requests) == 0) timeout_remove(&dir->to_request); } From dovecot at dovecot.org Wed Nov 12 04:59:53 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 12 Nov 2014 04:59:53 +0000 Subject: dovecot-2.2: director: Added support for backend cluster "tags". Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a7e830b9b967 changeset: 18067:a7e830b9b967 user: Timo Sirainen date: Wed Nov 12 06:58:37 2014 +0200 description: director: Added support for backend cluster "tags". This allows using a single director ring for multiple backend clusters. By default everything has an empty tag. A passdb lookup can return "director_tag" field containing the wanted tag name. If there aren't any backend servers with the wanted tag, it's treated the same as if there aren't any backend servers available (= wait for 30 secs for a backend and then return temporary failure). Tags can be added to configuration by adding @tag suffix to IPs/hosts. For example: director_mail_servers = 10.0.0.100-10.0.0.110 at name1 10.0.0.120 at name2 "doveadm director add" can also add tags either with @tag suffix or with -t parameter. "doveadm director status user at domain" requires giving the user's correct tag with -t parameter or the results won't be correct (empty tag's results are shown). Tags can't currently be changed for an existing host without removing it first. diffstat: src/director/director-connection.c | 22 +++++++- src/director/director-request.c | 29 ++++++++-- src/director/director-request.h | 1 + src/director/director.c | 26 ++++++++- src/director/director.h | 2 + src/director/doveadm-connection.c | 38 ++++++++++++--- src/director/login-connection.c | 6 +- src/director/mail-host.c | 94 +++++++++++++++++++++++++++++-------- src/director/mail-host.h | 9 ++- src/director/main.c | 3 +- src/doveadm/doveadm-director.c | 47 +++++++++++++----- 11 files changed, 214 insertions(+), 63 deletions(-) diffs (truncated from 776 to 300 lines): diff -r 26679856fbd5 -r a7e830b9b967 src/director/director-connection.c --- a/src/director/director-connection.c Wed Nov 12 06:46:45 2014 +0200 +++ b/src/director/director-connection.c Wed Nov 12 06:58:37 2014 +0200 @@ -35,6 +35,7 @@ #include "istream.h" #include "ostream.h" #include "str.h" +#include "strescape.h" #include "master-service.h" #include "mail-host.h" #include "director.h" @@ -821,15 +822,18 @@ { struct mail_host *host; struct ip_addr ip; + const char *tag = ""; unsigned int vhost_count; bool update; - if (str_array_length(args) != 2 || + if (str_array_length(args) < 2 || net_addr2ip(args[0], &ip) < 0 || str_to_uint(args[1], &vhost_count) < 0) { director_cmd_error(conn, "Invalid parameters"); return FALSE; } + if (args[2] != NULL) + tag = args[2]; if (conn->ignore_host_events) { /* remote is sending hosts in a handshake, but it doesn't have a completed ring and we do. */ @@ -839,10 +843,17 @@ host = mail_host_lookup(conn->dir->mail_hosts, &ip); if (host == NULL) { - host = mail_host_add_ip(conn->dir->mail_hosts, &ip); + host = mail_host_add_ip(conn->dir->mail_hosts, &ip, tag); update = TRUE; } else { update = host->vhost_count != vhost_count; + if (strcmp(tag, host->tag) != 0) { + i_error("director(%s): Host %s changed tag from '%s' to '%s'", + conn->name, net_ip2addr(&host->ip), + host->tag, tag); + mail_host_set_tag(host, tag); + update = TRUE; + } } if (update) { @@ -1541,8 +1552,13 @@ str_printfa(str, "HOST-HAND-START\t%u\n", conn->dir->ring_handshaked); array_foreach(mail_hosts_get(conn->dir->mail_hosts), hostp) { - str_printfa(str, "HOST\t%s\t%u\n", + str_printfa(str, "HOST\t%s\t%u", net_ip2addr(&(*hostp)->ip), (*hostp)->vhost_count); + if ((*hostp)->tag[0] != '\0') { + str_append_c(str, '\t'); + str_append_tabescaped(str, (*hostp)->tag); + } + str_append_c(str, '\n'); } str_printfa(str, "HOST-HAND-END\t%u\n", conn->dir->ring_handshaked); } diff -r 26679856fbd5 -r a7e830b9b967 src/director/director-request.c --- a/src/director/director-request.c Wed Nov 12 06:46:45 2014 +0200 +++ b/src/director/director-request.c Wed Nov 12 06:58:37 2014 +0200 @@ -36,11 +36,18 @@ time_t create_time; unsigned int username_hash; enum director_request_delay_reason delay_reason; + char *username_tag; director_request_callback *callback; void *context; }; +static void director_request_free(struct director_request *request) +{ + i_free(request->username_tag); + i_free(request); +} + static const char * director_request_get_timeout_error(struct director_request *request, struct user *user, string_t *str) @@ -68,7 +75,10 @@ str_printfa(str, ", user refreshed %u secs ago", (unsigned int)(ioloop_time - user->timestamp)); } - str_printfa(str, "hash=%u)", request->username_hash); + str_printfa(str, ", hash=%u", request->username_hash); + if (request->username_tag != NULL) + str_printfa(str, ", tag=%s", request->username_tag); + str_append_c(str, ')'); return str_c(str); } @@ -103,7 +113,7 @@ T_BEGIN { request->callback(NULL, errormsg, request->context); } T_END; - i_free(request); + director_request_free(request); } if (array_count(&dir->pending_requests) == 0 && dir->to_request != NULL) @@ -111,6 +121,7 @@ } void director_request(struct director *dir, const char *username, + const char *tag, director_request_callback *callback, void *context) { struct director_request *request; @@ -121,6 +132,7 @@ request->dir = dir; request->create_time = ioloop_time; request->username_hash = username_hash; + request->username_tag = tag[0] == '\0' ? NULL : i_strdup(tag); request->callback = callback; request->context = context; @@ -159,7 +171,8 @@ } static bool -director_request_existing(struct director_request *request, struct user *user) +director_request_existing(struct director_request *request, struct user *user, + const char *tag) { struct director *dir = request->dir; struct mail_host *host; @@ -193,7 +206,7 @@ /* user is close to being expired. another director may have already expired it. */ - host = mail_host_get_by_hash(dir->mail_hosts, user->username_hash); + host = mail_host_get_by_hash(dir->mail_hosts, user->username_hash, tag); if (!dir->ring_synced) { /* try again later once ring is synced */ request->delay_reason = REQUEST_DELAY_RINGNOTSYNCED; @@ -253,6 +266,7 @@ struct director *dir = request->dir; struct mail_host *host; struct user *user; + const char *tag; if (!dir->ring_handshaked) { /* delay requests until ring handshaking is complete */ @@ -264,8 +278,9 @@ } user = user_directory_lookup(dir->users, request->username_hash); + tag = request->username_tag == NULL ? "" : request->username_tag; if (user != NULL) { - if (!director_request_existing(request, user)) + if (!director_request_existing(request, user, tag)) return FALSE; user_directory_refresh(dir->users, user); dir_debug("request: %u refreshed timeout to %u", @@ -280,7 +295,7 @@ return FALSE; } host = mail_host_get_by_hash(dir->mail_hosts, - request->username_hash); + request->username_hash, tag); if (host == NULL) { /* all hosts have been removed */ request->delay_reason = REQUEST_DELAY_NOHOSTS; @@ -299,6 +314,6 @@ T_BEGIN { request->callback(&user->host->ip, NULL, request->context); } T_END; - i_free(request); + director_request_free(request); return TRUE; } diff -r 26679856fbd5 -r a7e830b9b967 src/director/director-request.h --- a/src/director/director-request.h Wed Nov 12 06:46:45 2014 +0200 +++ b/src/director/director-request.h Wed Nov 12 06:58:37 2014 +0200 @@ -9,6 +9,7 @@ void *context); void director_request(struct director *dir, const char *username, + const char *tag, director_request_callback *callback, void *context); bool director_request_continue(struct director_request *request); diff -r 26679856fbd5 -r a7e830b9b967 src/director/director.c --- a/src/director/director.c Wed Nov 12 06:46:45 2014 +0200 +++ b/src/director/director.c Wed Nov 12 06:58:37 2014 +0200 @@ -4,6 +4,7 @@ #include "ioloop.h" #include "array.h" #include "str.h" +#include "strescape.h" #include "ipc-client.h" #include "user-directory.h" #include "mail-host.h" @@ -503,6 +504,8 @@ struct director_host *orig_src, struct mail_host *host) { + string_t *str; + /* update state in case this is the first mail host being added */ director_set_state_changed(dir); @@ -511,10 +514,25 @@ orig_src->last_seq++; } - director_update_send(dir, src, t_strdup_printf( - "HOST\t%s\t%u\t%u\t%s\t%u\n", - net_ip2addr(&orig_src->ip), orig_src->port, orig_src->last_seq, - net_ip2addr(&host->ip), host->vhost_count)); + str = t_str_new(128); + str_printfa(str, "HOST\t%s\t%u\t%u\t%s\t%u", + net_ip2addr(&orig_src->ip), orig_src->port, + orig_src->last_seq, + net_ip2addr(&host->ip), host->vhost_count); + if (host->tag[0] == '\0') + ; + else if (dir->ring_handshaked && + dir->ring_min_version < DIRECTOR_VERSION_TAGS) { + i_error("Ring has directors that don't support tags - removing host %s with tag '%s'", + net_ip2addr(&host->ip), host->tag); + director_remove_host(dir, NULL, NULL, host); + return; + } else { + str_append_c(str, '\t'); + str_append_tabescaped(str, host->tag); + } + str_append_c(str, '\n'); + director_update_send(dir, src, str_c(str)); director_sync(dir); } diff -r 26679856fbd5 -r a7e830b9b967 src/director/director.h --- a/src/director/director.h Wed Nov 12 06:46:45 2014 +0200 +++ b/src/director/director.h Wed Nov 12 06:58:37 2014 +0200 @@ -18,6 +18,8 @@ #define DIRECTOR_VERSION_USER_KICK 4 /* options supported in handshake */ #define DIRECTOR_VERSION_OPTIONS 5 +/* user tags supported */ +#define DIRECTOR_VERSION_TAGS 5 /* Minimum time between even attempting to communicate with a director that failed due to a protocol error. */ diff -r 26679856fbd5 -r a7e830b9b967 src/director/doveadm-connection.c --- a/src/director/doveadm-connection.c Wed Nov 12 06:46:45 2014 +0200 +++ b/src/director/doveadm-connection.c Wed Nov 12 06:58:37 2014 +0200 @@ -7,6 +7,7 @@ #include "ostream.h" #include "array.h" #include "str.h" +#include "strescape.h" #include "llist.h" #include "master-service.h" #include "user-directory.h" @@ -46,9 +47,11 @@ string_t *str = t_str_new(1024); array_foreach(mail_hosts_get(conn->dir->mail_hosts), hostp) { - str_printfa(str, "%s\t%u\t%u\n", + str_printfa(str, "%s\t%u\t%u\t", net_ip2addr(&(*hostp)->ip), (*hostp)->vhost_count, (*hostp)->user_count); + str_append_tabescaped(str, (*hostp)->tag); + str_append_c(str, '\n'); } str_append_c(str, '\n'); o_stream_nsend(conn->output, str_data(str), str_len(str)); @@ -244,14 +247,21 @@ doveadm_cmd_host_set(struct doveadm_connection *conn, const char *line) { struct director *dir = conn->dir; - const char *const *args; + const char *const *args, *ip_str, *tag = ""; struct mail_host *host; struct ip_addr ip; unsigned int vhost_count = UINT_MAX; args = t_strsplit_tab(line); - if (args[0] == NULL || - net_addr2ip(args[0], &ip) < 0 || + ip_str = args[0]; + if (ip_str != NULL) { + tag = strchr(ip_str, '@'); + if (tag == NULL) + tag = ""; + else + ip_str = t_strdup_until(ip_str, tag++); + } + if (ip_str == NULL || net_addr2ip(ip_str, &ip) < 0 || (args[1] != NULL && str_to_uint(args[1], &vhost_count) < 0)) { i_error("doveadm sent invalid HOST-SET parameters: %s", line); From pigeonhole at rename-it.nl Wed Nov 12 21:11:21 2014 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 12 Nov 2014 22:11:21 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: Made validator resolve all ta... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/c8e8321f2891 changeset: 1959:c8e8321f2891 user: Stephan Bosch date: Wed Nov 12 22:09:44 2014 +0100 description: lib-sieve: Made validator resolve all tagged command arguments before validating them. This is needed so that argument validation functions can reliably verify other arguments at arbitrary positions. diffstat: src/lib-sieve/sieve-validator.c | 34 +++++++++++++++++++++++----------- 1 files changed, 23 insertions(+), 11 deletions(-) diffs (57 lines): diff -r be2f902b41fb -r c8e8321f2891 src/lib-sieve/sieve-validator.c --- a/src/lib-sieve/sieve-validator.c Sun Nov 02 18:35:34 2014 +0100 +++ b/src/lib-sieve/sieve-validator.c Wed Nov 12 22:09:44 2014 +0100 @@ -890,16 +890,19 @@ struct sieve_ast_argument *arg; struct sieve_command_registration *cmd_reg = cmd->reg; - /* Validate any tags that might be present */ + /* Resolve tagged arguments */ arg = sieve_ast_argument_first(cmd->ast_node); + while ( arg != NULL ) { + void *arg_data = NULL; + struct sieve_tag_registration *tag_reg; + const struct sieve_argument_def *tag_def; - /* Visit tagged and optional arguments */ - while ( arg != NULL && sieve_ast_argument_type(arg) == SAAT_TAG ) { - struct sieve_ast_argument *parg; - void *arg_data = NULL; - struct sieve_tag_registration *tag_reg = - sieve_validator_command_tag_get(valdtr, cmd, arg, &arg_data); - const struct sieve_argument_def *tag_def; + if (sieve_ast_argument_type(arg) != SAAT_TAG) { + arg = sieve_ast_argument_next(arg); + continue; + } + + tag_reg = sieve_validator_command_tag_get(valdtr, cmd, arg, &arg_data); if ( tag_reg == NULL ) { sieve_argument_validate_error(valdtr, arg, @@ -923,13 +926,22 @@ (arg->ast, tag_def, tag_reg->ext, tag_reg->id_code); arg->argument->data = arg_data; + arg = sieve_ast_argument_next(arg); + } + + /* Validate tagged arguments */ + arg = sieve_ast_argument_first(cmd->ast_node); + while ( arg != NULL && sieve_ast_argument_type(arg) == SAAT_TAG) { + const struct sieve_argument_def *tag_def = arg->argument->def; + struct sieve_ast_argument *parg; + /* Scan backwards for any duplicates */ parg = sieve_ast_argument_prev(arg); while ( parg != NULL ) { if ( (sieve_ast_argument_type(parg) == SAAT_TAG && - parg->argument->def == tag_reg->tag_def) - || (tag_reg->id_code > 0 && parg->argument != NULL && - parg->argument->id_code == tag_reg->id_code) ) + parg->argument->def == tag_def) + || (arg->argument->id_code > 0 && parg->argument != NULL && + parg->argument->id_code == arg->argument->id_code) ) { const char *tag_id = sieve_ast_argument_tag(arg); const char *tag_desc = From pigeonhole at rename-it.nl Wed Nov 12 21:11:21 2014 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 12 Nov 2014 22:11:21 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: Implemented the index stringl... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/519972b17360 changeset: 1960:519972b17360 user: Stephan Bosch date: Wed Nov 12 22:10:23 2014 +0100 description: lib-sieve: Implemented the index stringlist class which selects an item of the parent at a particular index. diffstat: src/lib-sieve/sieve-stringlist.c | 132 +++++++++++++++++++++++++++++++++++++++ src/lib-sieve/sieve-stringlist.h | 8 ++ 2 files changed, 140 insertions(+), 0 deletions(-) diffs (165 lines): diff -r c8e8321f2891 -r 519972b17360 src/lib-sieve/sieve-stringlist.c --- a/src/lib-sieve/sieve-stringlist.c Wed Nov 12 22:09:44 2014 +0100 +++ b/src/lib-sieve/sieve-stringlist.c Wed Nov 12 22:10:23 2014 +0100 @@ -49,9 +49,11 @@ int count = 0; int ret; + sieve_stringlist_reset(strlist); while ( (ret=sieve_stringlist_next_item(strlist, &item)) > 0 ) { count++; } + sieve_stringlist_reset(strlist); return ( ret < 0 ? -1 : count ); } @@ -141,3 +143,133 @@ return ( strlist->count_empty || str_len(strlist->value) > 0 ); } + +/* + * Index Stringlist + */ + +/* Object */ + +static int sieve_index_stringlist_next_item + (struct sieve_stringlist *_strlist, string_t **str_r); +static void sieve_index_stringlist_reset + (struct sieve_stringlist *_strlist); +static int sieve_index_stringlist_get_length + (struct sieve_stringlist *_strlist); +static void sieve_index_stringlist_set_trace + (struct sieve_stringlist *strlist, bool trace); + +struct sieve_index_stringlist { + struct sieve_stringlist strlist; + + struct sieve_stringlist *source; + + int index; + unsigned int end:1; +}; + +struct sieve_stringlist *sieve_index_stringlist_create +(const struct sieve_runtime_env *renv, struct sieve_stringlist *source, + int index) +{ + struct sieve_index_stringlist *strlist; + + strlist = t_new(struct sieve_index_stringlist, 1); + strlist->strlist.runenv = renv; + strlist->strlist.exec_status = SIEVE_EXEC_OK; + strlist->strlist.next_item = sieve_index_stringlist_next_item; + strlist->strlist.reset = sieve_index_stringlist_reset; + strlist->strlist.get_length = sieve_index_stringlist_get_length; + strlist->strlist.set_trace = sieve_index_stringlist_set_trace; + strlist->source = source; + strlist->index = index; + + return &strlist->strlist; +} + +/* Implementation */ + +static int sieve_index_stringlist_next_item +(struct sieve_stringlist *_strlist, string_t **str_r) +{ + struct sieve_index_stringlist *strlist = + (struct sieve_index_stringlist *)_strlist; + int index, ret; + + if ( strlist->end ) { + *str_r = NULL; + return 0; + } + + if ( strlist->index < 0 ) { + int len = sieve_stringlist_get_length(strlist->source); + if (len < 0) { + _strlist->exec_status = strlist->source->exec_status; + return -1; + } + + if (len < -strlist->index) { + *str_r = NULL; + strlist->end = TRUE; + return 0; + } + index = len + 1 + strlist->index; + } else { + index = strlist->index; + } + + while ( index > 0 ) { + if ( (ret=sieve_stringlist_next_item(strlist->source, str_r)) <= 0 ) { + if (ret < 0) + _strlist->exec_status = strlist->source->exec_status; + return ret; + } + + index--; + } + + strlist->end = TRUE; + return 1; +} + +static void sieve_index_stringlist_reset +(struct sieve_stringlist *_strlist) +{ + struct sieve_index_stringlist *strlist = + (struct sieve_index_stringlist *)_strlist; + + sieve_stringlist_reset(strlist->source); + strlist->end = FALSE; +} + +static int sieve_index_stringlist_get_length +(struct sieve_stringlist *_strlist) +{ + struct sieve_index_stringlist *strlist = + (struct sieve_index_stringlist *)_strlist; + int len; + + len = sieve_stringlist_get_length(strlist->source); + if (len < 0) { + _strlist->exec_status = strlist->source->exec_status; + return -1; + } + + if ( strlist->index < 0 ) { + if ( -strlist->index >= len ) + return 0; + } else if ( strlist->index >= len ) { + return 0; + } + + return 1; +} + +static void sieve_index_stringlist_set_trace +(struct sieve_stringlist *_strlist, bool trace) +{ + struct sieve_index_stringlist *strlist = + (struct sieve_index_stringlist *)_strlist; + + sieve_stringlist_set_trace(strlist->source, trace); +} diff -r c8e8321f2891 -r 519972b17360 src/lib-sieve/sieve-stringlist.h --- a/src/lib-sieve/sieve-stringlist.h Wed Nov 12 22:09:44 2014 +0100 +++ b/src/lib-sieve/sieve-stringlist.h Wed Nov 12 22:10:23 2014 +0100 @@ -66,4 +66,12 @@ struct sieve_stringlist *sieve_single_stringlist_create_cstr (const struct sieve_runtime_env *renv, const char *cstr, bool count_empty); +/* + * Index Stringlist + */ + +struct sieve_stringlist *sieve_index_stringlist_create + (const struct sieve_runtime_env *renv, struct sieve_stringlist *source, + int index); + #endif /* __SIEVE_STRINGLIST_H */ From pigeonhole at rename-it.nl Wed Nov 12 21:11:21 2014 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 12 Nov 2014 22:11:21 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: Created message override infr... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/578aca9c0ba1 changeset: 1961:578aca9c0ba1 user: Stephan Bosch date: Wed Nov 12 22:10:24 2014 +0100 description: lib-sieve: Created message override infrastructure. This allows extension of test commands that evaluate the message somehow. Currently this is implemented only for the message header. Extensions can thereby influence what headers are evaluated and how. diffstat: src/lib-sieve/ext-envelope.c | 5 +- src/lib-sieve/plugins/date/tst-date.c | 19 ++- src/lib-sieve/sieve-address-parts.c | 26 +++- src/lib-sieve/sieve-address-parts.h | 7 +- src/lib-sieve/sieve-common.h | 2 + src/lib-sieve/sieve-match.c | 12 ++ src/lib-sieve/sieve-message.c | 195 ++++++++++++++++++++++++++++++++++ src/lib-sieve/sieve-message.h | 94 ++++++++++++++++ src/lib-sieve/tst-address.c | 26 +++- src/lib-sieve/tst-exists.c | 33 ++++- src/lib-sieve/tst-header.c | 18 ++- tests/compile/errors.svtest | 2 +- 12 files changed, 398 insertions(+), 41 deletions(-) diffs (truncated from 698 to 300 lines): diff -r 519972b17360 -r 578aca9c0ba1 src/lib-sieve/ext-envelope.c --- a/src/lib-sieve/ext-envelope.c Wed Nov 12 22:10:23 2014 +0100 +++ b/src/lib-sieve/ext-envelope.c Wed Nov 12 22:10:24 2014 +0100 @@ -436,10 +436,9 @@ struct sieve_command_registration *cmd_reg) { /* The order of these is not significant */ - sieve_comparators_link_tag(valdtr, cmd_reg, SIEVE_AM_OPT_COMPARATOR); + sieve_comparators_link_tag(valdtr, cmd_reg, SIEVE_MATCH_OPT_COMPARATOR); + sieve_match_types_link_tags(valdtr, cmd_reg, SIEVE_MATCH_OPT_MATCH_TYPE); sieve_address_parts_link_tags(valdtr, cmd_reg, SIEVE_AM_OPT_ADDRESS_PART); - sieve_match_types_link_tags(valdtr, cmd_reg, SIEVE_AM_OPT_MATCH_TYPE); - return TRUE; } diff -r 519972b17360 -r 578aca9c0ba1 src/lib-sieve/plugins/date/tst-date.c --- a/src/lib-sieve/plugins/date/tst-date.c Wed Nov 12 22:10:23 2014 +0100 +++ b/src/lib-sieve/plugins/date/tst-date.c Wed Nov 12 22:10:24 2014 +0100 @@ -139,7 +139,7 @@ */ enum tst_date_optional { - OPT_DATE_ZONE = SIEVE_MATCH_OPT_LAST, + OPT_DATE_ZONE = SIEVE_AM_OPT_LAST, OPT_DATE_LAST }; @@ -349,7 +349,7 @@ for (;;) { int opt; - if ( (opt=sieve_match_opr_optional_dump(denv, address, &opt_code)) < 0 ) + if ( (opt=sieve_message_opr_optional_dump(denv, address, &opt_code)) < 0 ) return FALSE; if ( opt == 0 ) break; @@ -387,6 +387,7 @@ SIEVE_MATCH_TYPE_DEFAULT(is_match_type); struct sieve_comparator cmp = SIEVE_COMPARATOR_DEFAULT(i_ascii_casemap_comparator); + ARRAY_TYPE(sieve_message_override) svmos; string_t *date_part = NULL, *zone = NULL; struct sieve_stringlist *hdr_list = NULL, *hdr_value_list; struct sieve_stringlist *value_list, *key_list; @@ -399,8 +400,10 @@ for (;;) { int opt; - if ( (opt=sieve_match_opr_optional_read - (renv, address, &opt_code, &ret, &cmp, &mcht)) < 0 ) + /* Optional operands */ + memset(&svmos, 0, sizeof(svmos)); + if ( (opt=sieve_message_opr_optional_read + (renv, address, &opt_code, &ret, NULL, &mcht, &cmp, &svmos)) < 0 ) return ret; if ( opt == 0 ) break; @@ -465,8 +468,14 @@ sieve_runtime_trace(renv, SIEVE_TRLVL_TESTS, "date test"); + /* Get header */ + sieve_runtime_trace_descend(renv); + if ( (ret=sieve_message_get_header_fields + (renv, hdr_list, &svmos, &hdr_value_list)) <= 0 ) + return ret; + sieve_runtime_trace_ascend(renv); + /* Create value stringlist */ - hdr_value_list = sieve_message_header_stringlist_create(renv, hdr_list, FALSE); value_list = ext_date_stringlist_create (renv, hdr_value_list, time_zone, dpart); diff -r 519972b17360 -r 578aca9c0ba1 src/lib-sieve/sieve-address-parts.c --- a/src/lib-sieve/sieve-address-parts.c Wed Nov 12 22:10:23 2014 +0100 +++ b/src/lib-sieve/sieve-address-parts.c Wed Nov 12 22:10:24 2014 +0100 @@ -362,10 +362,10 @@ return opt; switch ( *opt_code ) { - case SIEVE_AM_OPT_COMPARATOR: + case SIEVE_MATCH_OPT_COMPARATOR: opok = sieve_opr_comparator_dump(denv, address); break; - case SIEVE_AM_OPT_MATCH_TYPE: + case SIEVE_MATCH_OPT_MATCH_TYPE: opok = sieve_opr_match_type_dump(denv, address); break; case SIEVE_AM_OPT_ADDRESS_PART: @@ -406,13 +406,31 @@ } switch ( *opt_code ) { - case SIEVE_AM_OPT_COMPARATOR: + case SIEVE_MATCH_OPT_COMPARATOR: + if (cmp == NULL) { + sieve_runtime_trace_error(renv, "unexpected comparator operand"); + if ( exec_status != NULL ) + *exec_status = SIEVE_EXEC_BIN_CORRUPT; + return -1; + } status = sieve_opr_comparator_read(renv, address, cmp); break; - case SIEVE_AM_OPT_MATCH_TYPE: + case SIEVE_MATCH_OPT_MATCH_TYPE: + if (mtch == NULL) { + sieve_runtime_trace_error(renv, "unexpected match-type operand"); + if ( exec_status != NULL ) + *exec_status = SIEVE_EXEC_BIN_CORRUPT; + return -1; + } status = sieve_opr_match_type_read(renv, address, mtch); break; case SIEVE_AM_OPT_ADDRESS_PART: + if (addrp == NULL) { + sieve_runtime_trace_error(renv, "unexpected address-part operand"); + if ( exec_status != NULL ) + *exec_status = SIEVE_EXEC_BIN_CORRUPT; + return -1; + } status = sieve_opr_address_part_read(renv, address, addrp); break; default: diff -r 519972b17360 -r 578aca9c0ba1 src/lib-sieve/sieve-address-parts.h --- a/src/lib-sieve/sieve-address-parts.h Wed Nov 12 22:10:23 2014 +0100 +++ b/src/lib-sieve/sieve-address-parts.h Wed Nov 12 22:10:24 2014 +0100 @@ -7,6 +7,7 @@ #include "message-address.h" #include "sieve-common.h" +#include "sieve-match.h" #include "sieve-extensions.h" #include "sieve-objects.h" @@ -121,10 +122,8 @@ */ enum sieve_addrmatch_opt_operand { - SIEVE_AM_OPT_END, - SIEVE_AM_OPT_COMPARATOR, - SIEVE_AM_OPT_ADDRESS_PART, - SIEVE_AM_OPT_MATCH_TYPE + SIEVE_AM_OPT_ADDRESS_PART = SIEVE_MATCH_OPT_LAST, + SIEVE_AM_OPT_LAST }; int sieve_addrmatch_opr_optional_dump diff -r 519972b17360 -r 578aca9c0ba1 src/lib-sieve/sieve-common.h --- a/src/lib-sieve/sieve-common.h Wed Nov 12 22:10:23 2014 +0100 +++ b/src/lib-sieve/sieve-common.h Wed Nov 12 22:10:24 2014 +0100 @@ -147,6 +147,8 @@ /* sieve-message.h */ struct sieve_message_context; +struct sieve_message_override; +struct sieve_message_override_def; /* sieve-plugins.h */ struct sieve_plugin; diff -r 519972b17360 -r 578aca9c0ba1 src/lib-sieve/sieve-match.c --- a/src/lib-sieve/sieve-match.c Wed Nov 12 22:10:23 2014 +0100 +++ b/src/lib-sieve/sieve-match.c Wed Nov 12 22:10:24 2014 +0100 @@ -258,9 +258,21 @@ switch ( *opt_code ) { case SIEVE_MATCH_OPT_COMPARATOR: + if (cmp == NULL) { + sieve_runtime_trace_error(renv, "unexpected comparator operand"); + if ( exec_status != NULL ) + *exec_status = SIEVE_EXEC_BIN_CORRUPT; + return -1; + } status = sieve_opr_comparator_read(renv, address, cmp); break; case SIEVE_MATCH_OPT_MATCH_TYPE: + if (mcht == NULL) { + sieve_runtime_trace_error(renv, "unexpected match-type operand"); + if ( exec_status != NULL ) + *exec_status = SIEVE_EXEC_BIN_CORRUPT; + return -1; + } status = sieve_opr_match_type_read(renv, address, mcht); break; default: diff -r 519972b17360 -r 578aca9c0ba1 src/lib-sieve/sieve-message.c --- a/src/lib-sieve/sieve-message.c Wed Nov 12 22:10:23 2014 +0100 +++ b/src/lib-sieve/sieve-message.c Wed Nov 12 22:10:24 2014 +0100 @@ -20,8 +20,11 @@ #include "sieve-stringlist.h" #include "sieve-error.h" #include "sieve-extensions.h" +#include "sieve-code.h" +#include "sieve-address-parts.h" #include "sieve-runtime.h" #include "sieve-runtime-trace.h" +#include "sieve-match.h" #include "sieve-interpreter.h" #include "sieve-address.h" @@ -569,3 +572,195 @@ strlist->headers_index = 0; sieve_stringlist_reset(strlist->field_names); } + +/* + * Header override operand + */ + +const struct sieve_operand_class sieve_message_override_operand_class = + { "header-override" }; + +bool sieve_opr_message_override_dump +(const struct sieve_dumptime_env *denv, sieve_size_t *address) +{ + struct sieve_message_override svmo; + const struct sieve_message_override_def *hodef; + + if ( !sieve_opr_object_dump + (denv, &sieve_message_override_operand_class, address, &svmo.object) ) + return FALSE; + + hodef = svmo.def = + (const struct sieve_message_override_def *) svmo.object.def; + + if ( hodef->dump_context != NULL ) { + sieve_code_descend(denv); + if ( !hodef->dump_context(&svmo, denv, address) ) { + return FALSE; + } + sieve_code_ascend(denv); + } + + return TRUE; +} + +bool sieve_opr_message_override_read +(const struct sieve_runtime_env *renv, sieve_size_t *address, + struct sieve_message_override *svmo) +{ + const struct sieve_message_override_def *hodef; + + svmo->context = NULL; + + if ( !sieve_opr_object_read + (renv, &sieve_message_override_operand_class, address, &svmo->object) ) + return FALSE; + + hodef = svmo->def = + (const struct sieve_message_override_def *) svmo->object.def; + + if ( hodef->read_context != NULL && + !hodef->read_context(svmo, renv, address, &svmo->context) ) { + return FALSE; + } + + return TRUE; +} + +/* + * Optional operands + */ + +int sieve_message_opr_optional_dump +(const struct sieve_dumptime_env *denv, sieve_size_t *address, + signed int *opt_code) +{ + signed int _opt_code = 0; + bool final = FALSE, opok = TRUE; + + if ( opt_code == NULL ) { + opt_code = &_opt_code; + final = TRUE; + } + + while ( opok ) { + int opt; + + if ( (opt=sieve_addrmatch_opr_optional_dump + (denv, address, opt_code)) <= 0 ) + return opt; + + if ( *opt_code == SIEVE_OPT_MESSAGE_OVERRIDE ) { + opok = sieve_opr_message_override_dump(denv, address); + } else { + return ( final ? -1 : 1 ); + } + } + + return -1; +} + +int sieve_message_opr_optional_read +(const struct sieve_runtime_env *renv, sieve_size_t *address, + signed int *opt_code, int *exec_status, + struct sieve_address_part *addrp, struct sieve_match_type *mcht, + struct sieve_comparator *cmp, + ARRAY_TYPE(sieve_message_override) *svmos) +{ + signed int _opt_code = 0; From pigeonhole at rename-it.nl Wed Nov 12 21:11:21 2014 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 12 Nov 2014 22:11:21 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: Implemented the Sieve index e... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/1d06b61cbaf3 changeset: 1962:1d06b61cbaf3 user: Stephan Bosch date: Wed Nov 12 22:10:25 2014 +0100 description: lib-sieve: Implemented the Sieve index extension (RFC 5260). diffstat: Makefile.am | 2 + configure.ac | 1 + src/lib-sieve/Makefile.am | 1 + src/lib-sieve/plugins/Makefile.am | 1 + src/lib-sieve/plugins/index/Makefile.am | 13 + src/lib-sieve/plugins/index/ext-index-common.c | 15 + src/lib-sieve/plugins/index/ext-index-common.h | 32 ++ src/lib-sieve/plugins/index/ext-index.c | 69 ++++++ src/lib-sieve/plugins/index/tag-index.c | 272 +++++++++++++++++++++++++ src/lib-sieve/sieve-extensions.c | 4 +- tests/extensions/index/basic.svtest | 93 ++++++++ tests/extensions/index/errors.svtest | 20 + tests/extensions/index/errors/syntax.sieve | 20 + 13 files changed, 542 insertions(+), 1 deletions(-) diffs (truncated from 632 to 300 lines): diff -r 578aca9c0ba1 -r 1d06b61cbaf3 Makefile.am --- a/Makefile.am Wed Nov 12 22:10:24 2014 +0100 +++ b/Makefile.am Wed Nov 12 22:10:25 2014 +0100 @@ -140,6 +140,8 @@ tests/extensions/date/basic.svtest \ tests/extensions/date/date-parts.svtest \ tests/extensions/date/zones.svtest \ + tests/extensions/index/basic.svtest \ + tests/extensions/index/errors.svtest \ tests/extensions/spamvirustest/spamtest.svtest \ tests/extensions/spamvirustest/virustest.svtest \ tests/extensions/spamvirustest/spamtestplus.svtest \ diff -r 578aca9c0ba1 -r 1d06b61cbaf3 configure.ac --- a/configure.ac Wed Nov 12 22:10:24 2014 +0100 +++ b/configure.ac Wed Nov 12 22:10:25 2014 +0100 @@ -203,6 +203,7 @@ src/lib-sieve/plugins/editheader/Makefile src/lib-sieve/plugins/metadata/Makefile src/lib-sieve/plugins/duplicate/Makefile +src/lib-sieve/plugins/index/Makefile src/lib-sieve/plugins/vnd.dovecot/Makefile src/lib-sieve/plugins/vnd.dovecot/debug/Makefile src/lib-sieve-tool/Makefile diff -r 578aca9c0ba1 -r 1d06b61cbaf3 src/lib-sieve/Makefile.am --- a/src/lib-sieve/Makefile.am Wed Nov 12 22:10:24 2014 +0100 +++ b/src/lib-sieve/Makefile.am Wed Nov 12 22:10:25 2014 +0100 @@ -76,6 +76,7 @@ $(extdir)/ihave/libsieve_ext_ihave.la \ $(extdir)/editheader/libsieve_ext_editheader.la \ $(extdir)/duplicate/libsieve_ext_duplicate.la \ + $(extdir)/index/libsieve_ext_index.la \ $(extdir)/vnd.dovecot/debug/libsieve_ext_debug.la \ $(unfinished_plugins) diff -r 578aca9c0ba1 -r 1d06b61cbaf3 src/lib-sieve/plugins/Makefile.am --- a/src/lib-sieve/plugins/Makefile.am Wed Nov 12 22:10:24 2014 +0100 +++ b/src/lib-sieve/plugins/Makefile.am Wed Nov 12 22:10:25 2014 +0100 @@ -22,6 +22,7 @@ ihave \ editheader \ duplicate \ + index \ vnd.dovecot \ $(UNFINISHED) diff -r 578aca9c0ba1 -r 1d06b61cbaf3 src/lib-sieve/plugins/index/Makefile.am --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-sieve/plugins/index/Makefile.am Wed Nov 12 22:10:25 2014 +0100 @@ -0,0 +1,13 @@ +noinst_LTLIBRARIES = libsieve_ext_index.la + +AM_CPPFLAGS = \ + -I$(srcdir)/../.. \ + $(LIBDOVECOT_INCLUDE) + +libsieve_ext_index_la_SOURCES = \ + ext-index-common.c \ + ext-index.c \ + tag-index.c + +noinst_HEADERS = \ + ext-index-common.h diff -r 578aca9c0ba1 -r 1d06b61cbaf3 src/lib-sieve/plugins/index/ext-index-common.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-sieve/plugins/index/ext-index-common.c Wed Nov 12 22:10:25 2014 +0100 @@ -0,0 +1,15 @@ +/* Copyright (c) 2002-2014 Pigeonhole authors, see the included COPYING file + */ + +#include "lib.h" +#include "utc-offset.h" +#include "str.h" + +#include "sieve-common.h" +#include "sieve-stringlist.h" +#include "sieve-code.h" +#include "sieve-interpreter.h" +#include "sieve-message.h" + +#include "ext-index-common.h" + diff -r 578aca9c0ba1 -r 1d06b61cbaf3 src/lib-sieve/plugins/index/ext-index-common.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-sieve/plugins/index/ext-index-common.h Wed Nov 12 22:10:25 2014 +0100 @@ -0,0 +1,32 @@ +/* Copyright (c) 2002-2014 Pigeonhole authors, see the included COPYING file + */ + +#ifndef __EXT_INDEX_COMMON_H +#define __EXT_INDEX_COMMON_H + +#include "sieve-common.h" + +#include + +#define SIEVE_EXT_INDEX_HDR_OVERRIDE_SEQUENCE 100 + +/* + * Tagged arguments + */ + +extern const struct sieve_argument_def index_tag; +extern const struct sieve_argument_def last_tag; + +/* + * Operands + */ + +extern const struct sieve_operand_def index_operand; + +/* + * Extension + */ + +extern const struct sieve_extension_def index_extension; + +#endif /* __EXT_INDEX_COMMON_H */ diff -r 578aca9c0ba1 -r 1d06b61cbaf3 src/lib-sieve/plugins/index/ext-index.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-sieve/plugins/index/ext-index.c Wed Nov 12 22:10:25 2014 +0100 @@ -0,0 +1,69 @@ +/* Copyright (c) 2002-2014 Pigeonhole authors, see the included COPYING file + */ + +/* Extension index + * ------------------ + * + * Authors: Stephan Bosch + * Specification: RFC 5260 + * Implementation: full + * Status: testing + * + */ + +#include "lib.h" +#include "array.h" + +#include "sieve-common.h" +#include "sieve-message.h" +#include "sieve-extensions.h" +#include "sieve-commands.h" + +#include "sieve-validator.h" +#include "sieve-generator.h" +#include "sieve-binary.h" +#include "sieve-interpreter.h" +#include "sieve-dump.h" + +#include "ext-index-common.h" + +/* + * Extension + */ + +static bool ext_index_validator_load +(const struct sieve_extension *ext, struct sieve_validator *validator); + +const struct sieve_extension_def index_extension = { + .name = "index", + .validator_load = ext_index_validator_load, + SIEVE_EXT_DEFINE_OPERAND(index_operand) +}; + +static bool ext_index_validator_load +(const struct sieve_extension *ext, struct sieve_validator *valdtr) +{ + /* Register :index and :last tags with header, address and date test commands + * and we don't care whether these command are registered or even whether + * these will be registered at all. The validator handles either situation + * gracefully. + */ + sieve_validator_register_external_tag + (valdtr, "header", ext, &index_tag, SIEVE_OPT_MESSAGE_OVERRIDE); + sieve_validator_register_external_tag + (valdtr, "header", ext, &last_tag, 0); + + sieve_validator_register_external_tag + (valdtr, "address", ext, &index_tag, SIEVE_OPT_MESSAGE_OVERRIDE); + sieve_validator_register_external_tag + (valdtr, "address", ext, &last_tag, 0); + + sieve_validator_register_external_tag + (valdtr, "date", ext, &index_tag, SIEVE_OPT_MESSAGE_OVERRIDE); + sieve_validator_register_external_tag + (valdtr, "date", ext, &last_tag, 0); + + return TRUE; +} + + diff -r 578aca9c0ba1 -r 1d06b61cbaf3 src/lib-sieve/plugins/index/tag-index.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-sieve/plugins/index/tag-index.c Wed Nov 12 22:10:25 2014 +0100 @@ -0,0 +1,272 @@ +/* Copyright (c) 2002-2014 Pigeonhole authors, see the included COPYING file + */ + +#include "lib.h" +#include "mail-storage.h" +#include "mail-namespace.h" + +#include "sieve-common.h" +#include "sieve-stringlist.h" +#include "sieve-extensions.h" +#include "sieve-commands.h" +#include "sieve-binary.h" +#include "sieve-code.h" +#include "sieve-message.h" +#include "sieve-result.h" +#include "sieve-validator.h" +#include "sieve-generator.h" + +#include "ext-index-common.h" + +/* + * Tagged argument + */ + +static bool tag_index_validate + (struct sieve_validator *valdtr, struct sieve_ast_argument **arg, + struct sieve_command *cmd); +static bool tag_index_generate + (const struct sieve_codegen_env *cgenv, struct sieve_ast_argument *arg, + struct sieve_command *context); + +const struct sieve_argument_def index_tag = { + "index", + NULL, + tag_index_validate, + NULL, NULL, + tag_index_generate +}; + +static bool tag_last_validate + (struct sieve_validator *valdtr, struct sieve_ast_argument **arg, + struct sieve_command *cmd); + +const struct sieve_argument_def last_tag = { + "last", + NULL, + tag_last_validate, + NULL, NULL, NULL +}; + + +/* + * Header override + */ + +static bool svmo_index_dump_context + (const struct sieve_message_override *svmo, + const struct sieve_dumptime_env *denv, sieve_size_t *address); +static int svmo_index_read_context + (const struct sieve_message_override *svmo, + const struct sieve_runtime_env *renv, sieve_size_t *address, + void **ho_context); +static int svmo_index_header_override + (const struct sieve_message_override *svmo, + const struct sieve_runtime_env *renv, + struct sieve_stringlist **headers); + +const struct sieve_message_override_def index_header_override = { + SIEVE_OBJECT("index", &index_operand, 0), + .sequence = SIEVE_EXT_INDEX_HDR_OVERRIDE_SEQUENCE, + .dump_context = svmo_index_dump_context, + .read_context = svmo_index_read_context, + .header_override = svmo_index_header_override +}; + +/* + * Operand + */ + +static const struct sieve_extension_objects ext_header_overrides = + SIEVE_EXT_DEFINE_MESSAGE_OVERRIDE(index_header_override); + +const struct sieve_operand_def index_operand = { + "index operand", + &index_extension, + 0, + &sieve_message_override_operand_class, + &ext_header_overrides +}; + +/* + * Tag data + */ + +struct tag_index_data { + sieve_number_t fieldno; + unsigned int last:1; +}; + +/* + * Tag validation + */ + +static bool tag_index_validate +(struct sieve_validator *valdtr ATTR_UNUSED, + struct sieve_ast_argument **arg, struct sieve_command *cmd) From pigeonhole at rename-it.nl Wed Nov 12 21:21:20 2014 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 12 Nov 2014 22:21:20 +0100 Subject: dovecot-2.2-pigeonhole: Updated README with with implementation ... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/daa10b2102d2 changeset: 1963:daa10b2102d2 user: Stephan Bosch date: Wed Nov 12 22:21:14 2014 +0100 description: Updated README with with implementation status. diffstat: README | 22 ++++++++++++++++------ 1 files changed, 16 insertions(+), 6 deletions(-) diffs (65 lines): diff -r 1d06b61cbaf3 -r daa10b2102d2 README --- a/README Wed Nov 12 22:10:25 2014 +0100 +++ b/README Wed Nov 12 22:21:14 2014 +0100 @@ -110,6 +110,7 @@ subaddress (RFC 5233): fully supported, but with limited configurability. spamtest and virustest (RFC 5235): fully supported (v0.1.16+). date (RFC 5260; Section 4): fully supported (v0.1.12+). + index (RFC 5260; Section 6): fully supported (v0.4.7+). editheader (RFC 5293): fully supported (v0.3.0+). reject (RFC 5429; Section 2.2): fully supported. enotify (RFC 5435): fully supported (v0.1.3+). @@ -120,7 +121,7 @@ mailbox (RFC 5490; Section 3): fully supported (v0.1.10+), but ACL permissions are not verified for mailboxexists. include (RFC 6609): fully supported (v0.4.0+) - duplicate (draft v03): fully supported (v0.4.3+). + duplicate (RFC 7352): fully supported (v0.4.3+). regex (draft v08; not latest version): almost fully supported, but UTF-8 is not supported. @@ -135,30 +136,39 @@ The following Dovecot-specific Sieve extensions are available: vnd.dovecot.debug (v0.3.0+): - Allows logging debug messages + Allows logging debug messages. vnd.dovecot.pipe (v0.4.0+; sieve_extprograms plugin): - Implements piping messages to a pre-defined set of external programs + Implements piping messages to a pre-defined set of external programs. vnd.dovecot.filter (v0.4.0+; sieve_extprograms plugin): Implements filtering messages through a pre-defined set of external - programs + programs. vnd.dovecot.execute (v0.4.0+; sieve_extprograms plugin): Implements executing a pre-defined set of external programs with the - option to process string data through the external program + option to process string data through the external program. The following extensions are under development: + mboxmetadata and servermetadata (RFC 5490): partially implemented; + Dovecot adjustments needed. ereject (RFC 5429; page 4): implemented, but currently equal to reject Many more extensions to the language exist. Not all of these extensions are useful for Dovecot in particular, but many of them are. Currently, the author has taken notice of the following extensions: - index (RFC 5260; page 7): planned. foreverypart, mime, replace, enclose, and extracttext (RFC 5703): planned. + imapsieve (RFC 6785): planned. + envelope-dsn, envelope-deliverby, redirect-dsn and + redirect-deliverby (RFC 6009): planned; depends on lib-smtp changes in + Dovecot. + extlists (RFC 6134): under consideration. These extensions will be added as soon as the necessary infrastructure is available. +Check the TODO file for an up-to-date list of open issues and current +development. + Compiling and Configuring ========================= From dovecot at dovecot.org Wed Nov 12 23:42:54 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 12 Nov 2014 23:42:54 +0000 Subject: dovecot-2.2: stats: Added support for "global" stats type. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9c2bcafcf121 changeset: 18068:9c2bcafcf121 user: Timo Sirainen date: Thu Nov 13 01:41:07 2014 +0200 description: stats: Added support for "global" stats type. diffstat: src/stats/client-export.c | 34 ++++++++++++++++++++++++++++++++-- src/stats/mail-domain.c | 3 +++ src/stats/mail-stats.c | 27 +++++++++++++++++++++++++++ src/stats/mail-stats.h | 17 +++++++++++++++++ src/stats/main.c | 2 ++ 5 files changed, 81 insertions(+), 2 deletions(-) diffs (186 lines): diff -r a7e830b9b967 -r 9c2bcafcf121 src/stats/client-export.c --- a/src/stats/client-export.c Wed Nov 12 06:58:37 2014 +0200 +++ b/src/stats/client-export.c Thu Nov 13 01:41:07 2014 +0200 @@ -20,10 +20,11 @@ MAIL_EXPORT_LEVEL_SESSION, MAIL_EXPORT_LEVEL_USER, MAIL_EXPORT_LEVEL_DOMAIN, - MAIL_EXPORT_LEVEL_IP + MAIL_EXPORT_LEVEL_IP, + MAIL_EXPORT_LEVEL_GLOBAL }; static const char *mail_export_level_names[] = { - "command", "session", "user", "domain", "ip" + "command", "session", "user", "domain", "ip", "global" }; struct mail_export_filter { @@ -475,6 +476,32 @@ return 1; } +static int client_export_iter_global(struct client *client) +{ + struct client_export_cmd *cmd = client->cmd_export; + struct mail_global *g = &mail_global_stats; + + i_assert(cmd->level == MAIL_EXPORT_LEVEL_GLOBAL); + + if (!cmd->header_sent) { + o_stream_nsend_str(client->output, + "reset_timestamp\tlast_update" + "\tnum_logins\tnum_cmds\tnum_connected_sessions"MAIL_STATS_HEADER); + cmd->header_sent = TRUE; + } + + str_truncate(cmd->str, 0); + str_printfa(cmd->str, "\t%ld", (long)g->reset_timestamp); + client_export_timeval(cmd->str, &g->last_update); + str_printfa(cmd->str, "\t%u\t%u\t%u", + g->num_logins, g->num_cmds, g->num_connected_sessions); + client_export_mail_stats(cmd->str, &g->stats); + str_append_c(cmd->str, '\n'); + o_stream_nsend(client->output, str_data(cmd->str), + str_len(cmd->str)); + return 1; +} + static int client_export_more(struct client *client) { if (client->cmd_export->export_iter(client) == 0) @@ -584,6 +611,9 @@ mail_ip_ref(client->mail_ip_iter); cmd->export_iter = client_export_iter_ip; break; + case MAIL_EXPORT_LEVEL_GLOBAL: + cmd->export_iter = client_export_iter_global; + break; } i_assert(cmd->export_iter != NULL); return TRUE; diff -r a7e830b9b967 -r 9c2bcafcf121 src/stats/mail-domain.c --- a/src/stats/mail-domain.c Wed Nov 12 06:58:37 2014 +0200 +++ b/src/stats/mail-domain.c Thu Nov 13 01:41:07 2014 +0200 @@ -45,6 +45,7 @@ { domain->num_logins++; domain->num_connected_sessions++; + mail_global_login(); mail_domain_refresh(domain, NULL); } @@ -52,6 +53,7 @@ { i_assert(domain->num_connected_sessions > 0); domain->num_connected_sessions--; + mail_global_disconnected(); } struct mail_domain *mail_domain_lookup(const char *name) @@ -100,6 +102,7 @@ sorted_prev, sorted_next); DLLIST2_APPEND_FULL(&mail_domains_head, &mail_domains_tail, domain, sorted_prev, sorted_next); + mail_global_refresh(diff_stats); } void mail_domains_free_memory(void) diff -r a7e830b9b967 -r 9c2bcafcf121 src/stats/mail-stats.c --- a/src/stats/mail-stats.c Wed Nov 12 06:58:37 2014 +0200 +++ b/src/stats/mail-stats.c Thu Nov 13 01:41:07 2014 +0200 @@ -1,6 +1,7 @@ /* Copyright (c) 2011-2014 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "ioloop.h" #include "time-util.h" #include "mail-stats.h" @@ -39,6 +40,8 @@ EN("mcache", mail_cache_hits) }; +struct mail_global mail_global_stats; + static int mail_stats_parse_timeval(const char *value, struct timeval *tv) { const char *p, *secs_str; @@ -270,3 +273,27 @@ } } } + +void mail_global_init(void) +{ + mail_global_stats.reset_timestamp = ioloop_time; +} + +void mail_global_login(void) +{ + mail_global_stats.num_logins++; + mail_global_stats.num_connected_sessions++; +} + +void mail_global_disconnected(void) +{ + i_assert(mail_global_stats.num_connected_sessions > 0); + mail_global_stats.num_connected_sessions--; +} + +void mail_global_refresh(const struct mail_stats *diff_stats) +{ + if (diff_stats != NULL) + mail_stats_add(&mail_global_stats.stats, diff_stats); + mail_global_stats.last_update = ioloop_timeval; +} diff -r a7e830b9b967 -r 9c2bcafcf121 src/stats/mail-stats.h --- a/src/stats/mail-stats.h Wed Nov 12 06:58:37 2014 +0200 +++ b/src/stats/mail-stats.h Thu Nov 13 01:41:07 2014 +0200 @@ -107,6 +107,18 @@ struct mail_session *sessions; }; +struct mail_global { + time_t reset_timestamp; + + struct timeval last_update; + struct mail_stats stats; + unsigned int num_logins; + unsigned int num_cmds; + unsigned int num_connected_sessions; +}; + +extern struct mail_global mail_global_stats; + int mail_stats_parse(const char *const *args, struct mail_stats *stats_r, const char **error_r); /* diff1 is supposed to have smaller values than diff2. Returns TRUE if this @@ -116,4 +128,9 @@ struct mail_stats *diff_stats_r, const char **error_r); void mail_stats_add(struct mail_stats *dest, const struct mail_stats *src); +void mail_global_init(void); +void mail_global_login(void); +void mail_global_disconnected(void); +void mail_global_refresh(const struct mail_stats *diff_stats); + #endif diff -r a7e830b9b967 -r 9c2bcafcf121 src/stats/main.c --- a/src/stats/main.c Wed Nov 12 06:58:37 2014 +0200 +++ b/src/stats/main.c Thu Nov 13 01:41:07 2014 +0200 @@ -12,6 +12,7 @@ #include "mail-user.h" #include "mail-domain.h" #include "mail-ip.h" +#include "mail-stats.h" #include "client.h" static struct mail_server_connection *mail_server_conn = NULL; @@ -62,6 +63,7 @@ mail_users_init(); mail_domains_init(); mail_ips_init(); + mail_global_init(); master_service_init_finish(master_service); master_service_run(master_service, client_connected); From dovecot at dovecot.org Thu Nov 13 00:26:05 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 13 Nov 2014 00:26:05 +0000 Subject: dovecot-2.2: lib: Added str-table API for keeping reference coun... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4c956747c36f changeset: 18069:4c956747c36f user: Timo Sirainen date: Thu Nov 13 02:24:46 2014 +0200 description: lib: Added str-table API for keeping reference counted strings in a hash table. diffstat: src/lib/Makefile.am | 3 + src/lib/str-table.c | 77 ++++++++++++++++++++++++++++++++++++++++++++++++ src/lib/str-table.h | 19 +++++++++++ src/lib/test-lib.c | 1 + src/lib/test-lib.h | 1 + src/lib/test-str-table.c | 33 ++++++++++++++++++++ 6 files changed, 134 insertions(+), 0 deletions(-) diffs (190 lines): diff -r 9c2bcafcf121 -r 4c956747c36f src/lib/Makefile.am --- a/src/lib/Makefile.am Thu Nov 13 01:41:07 2014 +0200 +++ b/src/lib/Makefile.am Thu Nov 13 02:24:46 2014 +0200 @@ -126,6 +126,7 @@ str.c \ str-find.c \ str-sanitize.c \ + str-table.c \ strescape.c \ strfuncs.c \ strnum.c \ @@ -247,6 +248,7 @@ str.h \ str-find.h \ str-sanitize.h \ + str-table.h \ strescape.h \ strfuncs.h \ strnum.h \ @@ -311,6 +313,7 @@ test-strnum.c \ test-str-find.c \ test-str-sanitize.c \ + test-str-table.c \ test-time-util.c \ test-unichar.c \ test-utc-mktime.c \ diff -r 9c2bcafcf121 -r 4c956747c36f src/lib/str-table.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/str-table.c Thu Nov 13 02:24:46 2014 +0200 @@ -0,0 +1,77 @@ +/* Copyright (c) 2014 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "hash.h" +#include "str-table.h" + +struct str_table { + HASH_TABLE(char *, void *) hash; +}; + +struct str_table *str_table_init(void) +{ + struct str_table *table; + + table = i_new(struct str_table, 1); + hash_table_create(&table->hash, default_pool, 0, str_hash, strcmp); + return table; +} + +void str_table_deinit(struct str_table **_table) +{ + struct str_table *table = *_table; + struct hash_iterate_context *iter; + char *key; + void *value; + + *_table = NULL; + + iter = hash_table_iterate_init(table->hash); + while (hash_table_iterate(iter, table->hash, &key, &value)) + i_free(key); + hash_table_destroy(&table->hash); + i_free(table); +} + +bool str_table_is_empty(struct str_table *table) +{ + return hash_table_count(table->hash) == 0; +} + +const char *str_table_ref(struct str_table *table, const char *str) +{ + char *key; + void *value; + unsigned int ref; + + if (!hash_table_lookup_full(table->hash, str, &key, &value)) { + key = i_strdup(str); + ref = 1; + } else { + ref = POINTER_CAST_TO(value, unsigned int); + i_assert(ref > 0); + ref++; + } + hash_table_update(table->hash, key, POINTER_CAST(ref)); + return key; +} + +void str_table_unref(struct str_table *table, const char **str) +{ + char *key; + void *value; + unsigned int ref; + + if (!hash_table_lookup_full(table->hash, *str, &key, &value)) + i_unreached(); + + ref = POINTER_CAST_TO(value, unsigned int); + i_assert(ref > 0); + if (--ref > 0) + hash_table_update(table->hash, key, POINTER_CAST(ref)); + else { + hash_table_remove(table->hash, key); + i_free(key); + } + *str = NULL; +} diff -r 9c2bcafcf121 -r 4c956747c36f src/lib/str-table.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/str-table.h Thu Nov 13 02:24:46 2014 +0200 @@ -0,0 +1,19 @@ +#ifndef STR_TABLE_H +#define STR_TABLE_H + +/* Hash table containing string -> refcount. */ + +struct str_table *str_table_init(void); +void str_table_deinit(struct str_table **table); + +/* Returns TRUE if there are no referenced strings in the table. */ +bool str_table_is_empty(struct str_table *table); + +/* Return string allocated from the strtable and increase its reference + count. */ +const char *str_table_ref(struct str_table *table, const char *str); +/* Decrease string's reference count, freeing it if it reaches zero. + The str pointer must have been returned by the str_table_ref(). */ +void str_table_unref(struct str_table *table, const char **str); + +#endif diff -r 9c2bcafcf121 -r 4c956747c36f src/lib/test-lib.c --- a/src/lib/test-lib.c Thu Nov 13 01:41:07 2014 +0200 +++ b/src/lib/test-lib.c Thu Nov 13 02:24:46 2014 +0200 @@ -42,6 +42,7 @@ test_strnum, test_str_find, test_str_sanitize, + test_str_table, test_time_util, test_unichar, test_utc_mktime, diff -r 9c2bcafcf121 -r 4c956747c36f src/lib/test-lib.h --- a/src/lib/test-lib.h Thu Nov 13 01:41:07 2014 +0200 +++ b/src/lib/test-lib.h Thu Nov 13 02:24:46 2014 +0200 @@ -44,6 +44,7 @@ void test_strnum(void); void test_str_find(void); void test_str_sanitize(void); +void test_str_table(void); void test_time_util(void); void test_unichar(void); void test_utc_mktime(void); diff -r 9c2bcafcf121 -r 4c956747c36f src/lib/test-str-table.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/test-str-table.c Thu Nov 13 02:24:46 2014 +0200 @@ -0,0 +1,33 @@ +/* Copyright (c) 2014 Dovecot authors, see the included COPYING file */ + +#include "test-lib.h" +#include "str-table.h" + +void test_str_table(void) +{ + struct str_table *table; + const char *key1, *key2, *key1_copy, *key2_copy; + + test_begin("str_table"); + table = str_table_init(); + + key1 = str_table_ref(table, "str1"); + key2 = str_table_ref(table, "str2"); + test_assert(key1 != key2); + key1_copy = str_table_ref(table, "str1"); + test_assert(key1_copy == key1); + key2_copy = str_table_ref(table, "str2"); + test_assert(key2_copy == key2); + + str_table_unref(table, &key1); + test_assert(key1 == NULL); + str_table_unref(table, &key1_copy); + + str_table_unref(table, &key2); + str_table_unref(table, &key2_copy); + test_assert(str_table_is_empty(table)); + + str_table_deinit(&table); + test_assert(table == NULL); + test_end(); +} From dovecot at dovecot.org Thu Nov 13 00:30:07 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 13 Nov 2014 00:30:07 +0000 Subject: dovecot-2.2: stats: Use the new str-table API to reduce memory u... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/688da3675901 changeset: 18070:688da3675901 user: Timo Sirainen date: Thu Nov 13 02:28:46 2014 +0200 description: stats: Use the new str-table API to reduce memory usage for service strings. There are only a couple of services, while there can be a lot of sessions. diffstat: src/stats/mail-session.c | 10 +++++++--- src/stats/mail-stats.h | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diffs (70 lines): diff -r 4c956747c36f -r 688da3675901 src/stats/mail-session.c --- a/src/stats/mail-session.c Thu Nov 13 02:24:46 2014 +0200 +++ b/src/stats/mail-session.c Thu Nov 13 02:28:46 2014 +0200 @@ -4,6 +4,7 @@ #include "ioloop.h" #include "hash.h" #include "llist.h" +#include "str-table.h" #include "global-memory.h" #include "stats-settings.h" #include "mail-stats.h" @@ -28,12 +29,13 @@ static struct mail_session *mail_sessions_head, *mail_sessions_tail; static time_t session_guid_warn_hide_until; static bool session_guid_hide_warned = FALSE; +static struct str_table *services; struct mail_session *stable_mail_sessions; static size_t mail_session_memsize(const struct mail_session *session) { - return sizeof(*session) + strlen(session->service) + 1; + return sizeof(*session) + 1; } static void mail_session_disconnect(struct mail_session *session) @@ -99,7 +101,7 @@ } session = i_new(struct mail_session, 1); session->refcount = 1; /* unrefed at disconnect */ - session->service = i_strdup(args[2]); + session->service = str_table_ref(services, args[2]); memcpy(session->guid, session_guid, sizeof(session->guid)); session->pid = pid; session->last_update = ioloop_timeval; @@ -171,7 +173,7 @@ mail_ip_unref(&session->ip); } - i_free(session->service); + str_table_unref(services, &session->service); i_free(session); } @@ -319,6 +321,7 @@ ioloop_time + SESSION_GUID_WARN_HIDE_SECS; hash_table_create(&mail_sessions_hash, default_pool, 0, guid_128_hash, guid_128_cmp); + services = str_table_init(); } void mail_sessions_deinit(void) @@ -331,4 +334,5 @@ mail_session_free(mail_sessions_head); } hash_table_destroy(&mail_sessions_hash); + str_table_deinit(&services); } diff -r 4c956747c36f -r 688da3675901 src/stats/mail-stats.h --- a/src/stats/mail-stats.h Thu Nov 13 02:24:46 2014 +0200 +++ b/src/stats/mail-stats.h Thu Nov 13 02:28:46 2014 +0200 @@ -42,7 +42,7 @@ /* if guid is empty, the session no longer exists */ guid_128_t guid; struct mail_user *user; - char *service; + const char *service; pid_t pid; /* ip address may be NULL if there's none */ struct mail_ip *ip; From dovecot at dovecot.org Thu Nov 13 00:41:08 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 13 Nov 2014 00:41:08 +0000 Subject: dovecot-2.2: ssl-params: Removed duplicate lib-ssl-iostream.la l... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f348e669546c changeset: 18071:f348e669546c user: Timo Sirainen date: Thu Nov 13 02:37:39 2014 +0200 description: ssl-params: Removed duplicate lib-ssl-iostream.la linking diffstat: src/ssl-params/Makefile.am | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (14 lines): diff -r 688da3675901 -r f348e669546c src/ssl-params/Makefile.am --- a/src/ssl-params/Makefile.am Thu Nov 13 02:28:46 2014 +0200 +++ b/src/ssl-params/Makefile.am Thu Nov 13 02:37:39 2014 +0200 @@ -9,8 +9,8 @@ -I$(top_srcdir)/src/lib-ssl-iostream \ -DPKG_STATEDIR=\""$(statedir)"\" -ssl_params_LDADD = $(LIBDOVECOT) ../lib-ssl-iostream/libssl_iostream.la -ssl_params_DEPENDENCIES = $(LIBDOVECOT_DEPS) ../lib-ssl-iostream/libssl_iostream.la +ssl_params_LDADD = $(LIBDOVECOT) +ssl_params_DEPENDENCIES = $(LIBDOVECOT_DEPS) ssl_params_SOURCES = \ main.c \ ssl-params.c \ From dovecot at dovecot.org Thu Nov 13 00:41:08 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 13 Nov 2014 00:41:08 +0000 Subject: dovecot-2.2: stats: Use the same session ID string for stats tra... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0694bfec140a changeset: 18072:0694bfec140a user: Timo Sirainen date: Thu Nov 13 02:39:51 2014 +0200 description: stats: Use the same session ID string for stats tracking as everywhere else. Generate a new ID only if there is no session ID assigned. diffstat: src/plugins/imap-stats/imap-stats-plugin.c | 2 +- src/plugins/stats/stats-connection.c | 6 +- src/plugins/stats/stats-plugin.c | 10 ++- src/plugins/stats/stats-plugin.h | 3 +- src/stats/client-export.c | 5 +- src/stats/mail-session.c | 95 ++++++++++++------------------ src/stats/mail-stats.h | 4 +- 7 files changed, 56 insertions(+), 69 deletions(-) diffs (truncated from 362 to 300 lines): diff -r f348e669546c -r 0694bfec140a src/plugins/imap-stats/imap-stats-plugin.c --- a/src/plugins/imap-stats/imap-stats-plugin.c Thu Nov 13 02:37:39 2014 +0200 +++ b/src/plugins/imap-stats/imap-stats-plugin.c Thu Nov 13 02:39:51 2014 +0200 @@ -73,7 +73,7 @@ str = t_str_new(128); str_append(str, "UPDATE-CMD\t"); - str_append(str, guid_128_to_string(suser->session_guid)); + str_append(str, suser->stats_session_id); str_printfa(str, "\t%u\t", scmd->id); if (cmd->state == CLIENT_COMMAND_STATE_DONE) diff -r f348e669546c -r 0694bfec140a src/plugins/stats/stats-connection.c --- a/src/plugins/stats/stats-connection.c Thu Nov 13 02:37:39 2014 +0200 +++ b/src/plugins/stats/stats-connection.c Thu Nov 13 02:39:51 2014 +0200 @@ -113,7 +113,7 @@ str_append(str, "CONNECT\t"); /* required fields */ - str_append(str, guid_128_to_string(suser->session_guid)); + str_append(str, suser->stats_session_id); str_append_c(str, '\t'); str_append_tabescaped(str, user->username); str_append_c(str, '\t'); @@ -140,7 +140,7 @@ string_t *str = t_str_new(128); str_append(str, "DISCONNECT\t"); - str_append(str, guid_128_to_string(suser->session_guid)); + str_append(str, suser->stats_session_id); str_append_c(str, '\n'); stats_connection_send(conn, str); } @@ -153,7 +153,7 @@ string_t *str = t_str_new(128); str_append(str, "UPDATE-SESSION\t"); - str_append(str, guid_128_to_string(suser->session_guid)); + str_append(str, suser->stats_session_id); mail_stats_export(str, stats); diff -r f348e669546c -r 0694bfec140a src/plugins/stats/stats-plugin.c --- a/src/plugins/stats/stats-plugin.c Thu Nov 13 02:37:39 2014 +0200 +++ b/src/plugins/stats/stats-plugin.c Thu Nov 13 02:39:51 2014 +0200 @@ -631,7 +631,15 @@ suser->track_commands = TRUE; suser->stats_conn = global_stats_conn; - guid_128_generate(suser->session_guid); + if (user->session_id != NULL && user->session_id[0] != '\0') + suser->stats_session_id = user->session_id; + else { + guid_128_t guid; + + guid_128_generate(guid); + suser->stats_session_id = + p_strdup(user->pool, guid_128_to_string(guid)); + } suser->last_session_update = time(NULL); suser->ioloop_ctx = ioloop_ctx; diff -r f348e669546c -r 0694bfec140a src/plugins/stats/stats-plugin.h --- a/src/plugins/stats/stats-plugin.h Thu Nov 13 02:37:39 2014 +0200 +++ b/src/plugins/stats/stats-plugin.h Thu Nov 13 02:39:51 2014 +0200 @@ -2,7 +2,6 @@ #define STATS_PLUGIN_H #include "module-context.h" -#include "guid.h" #include "mail-user.h" #include "mail-storage-private.h" @@ -33,7 +32,7 @@ struct ioloop_context *ioloop_ctx; struct stats_connection *stats_conn; - guid_128_t session_guid; + const char *stats_session_id; unsigned int refresh_secs; bool track_commands; diff -r f348e669546c -r 0694bfec140a src/stats/client-export.c --- a/src/stats/client-export.c Thu Nov 13 02:37:39 2014 +0200 +++ b/src/stats/client-export.c Thu Nov 13 02:39:51 2014 +0200 @@ -276,8 +276,7 @@ str_append_tabescaped(cmd->str, command->args); str_append_c(cmd->str, '\t'); T_BEGIN { - str_append(cmd->str, - guid_128_to_string(command->session->guid)); + str_append(cmd->str, command->session->id); str_append_c(cmd->str, '\t'); str_append_tabescaped(cmd->str, command->session->user->name); @@ -321,7 +320,7 @@ str_truncate(cmd->str, 0); T_BEGIN { - str_append(cmd->str, guid_128_to_string(session->guid)); + str_append(cmd->str, session->id); str_append_c(cmd->str, '\t'); str_append_tabescaped(cmd->str, session->user->name); str_append_c(cmd->str, '\t'); diff -r f348e669546c -r 0694bfec140a src/stats/mail-session.c --- a/src/stats/mail-session.c Thu Nov 13 02:37:39 2014 +0200 +++ b/src/stats/mail-session.c Thu Nov 13 02:39:51 2014 +0200 @@ -18,37 +18,35 @@ stats plugin */ #define MAIL_SESSION_IDLE_TIMEOUT_MSECS (1000*60*15) /* If stats process crashes/restarts, existing processes keep sending status - updates to it, but this process doesn't know their GUIDs. If these missing - GUIDs are found within this many seconds of starting the stats process, + updates to it, but this process doesn't know their session IDs. If these + missing IDs are found within this many seconds of starting the stats process, don't log a warning about them. (On a larger installation this avoids flooding the error log with hundreds of warnings.) */ -#define SESSION_GUID_WARN_HIDE_SECS (60*5) +#define SESSION_ID_WARN_HIDE_SECS (60*5) -static HASH_TABLE(uint8_t *, struct mail_session *) mail_sessions_hash; +static HASH_TABLE(char *, struct mail_session *) mail_sessions_hash; /* sessions are sorted by their last_update timestamp, oldest first */ static struct mail_session *mail_sessions_head, *mail_sessions_tail; -static time_t session_guid_warn_hide_until; -static bool session_guid_hide_warned = FALSE; +static time_t session_id_warn_hide_until; +static bool session_id_hide_warned = FALSE; static struct str_table *services; struct mail_session *stable_mail_sessions; static size_t mail_session_memsize(const struct mail_session *session) { - return sizeof(*session) + 1; + return sizeof(*session) + strlen(session->id) + 1; } static void mail_session_disconnect(struct mail_session *session) { - uint8_t *guid_p = session->guid; - i_assert(!session->disconnected); mail_user_disconnected(session->user); if (session->ip != NULL) mail_ip_disconnected(session->ip); - hash_table_remove(mail_sessions_hash, guid_p); + hash_table_remove(mail_sessions_hash, session->id); session->disconnected = TRUE; timeout_remove(&session->to_idle); mail_session_unref(&session); @@ -64,8 +62,7 @@ session->pid == 0) { i_warning("Session %s (user %s, service %s) " "appears to have crashed, disconnecting it", - guid_128_to_string(session->guid), - session->user->name, session->service); + session->id, session->user->name, session->service); } mail_session_disconnect(session); } @@ -73,36 +70,31 @@ int mail_session_connect_parse(const char *const *args, const char **error_r) { struct mail_session *session; - guid_128_t session_guid; - uint8_t *guid_p; + const char *session_id; pid_t pid; struct ip_addr ip; unsigned int i; - /* [key=value ..] */ + /* [key=value ..] */ if (str_array_length(args) < 4) { *error_r = "CONNECT: Too few parameters"; return -1; } - if (guid_128_from_string(args[0], session_guid) < 0) { - *error_r = "CONNECT: Invalid GUID"; - return -1; - } + session_id = args[0]; if (str_to_pid(args[3], &pid) < 0) { *error_r = "CONNECT: Invalid pid"; return -1; } - guid_p = session_guid; - session = hash_table_lookup(mail_sessions_hash, guid_p); + session = hash_table_lookup(mail_sessions_hash, session_id); if (session != NULL) { - *error_r = "CONNECT: Duplicate session GUID"; + *error_r = "CONNECT: Duplicate session ID"; return -1; } session = i_new(struct mail_session, 1); session->refcount = 1; /* unrefed at disconnect */ + session->id = i_strdup(session_id); session->service = str_table_ref(services, args[2]); - memcpy(session->guid, session_guid, sizeof(session->guid)); session->pid = pid; session->last_update = ioloop_timeval; session->to_idle = timeout_add(MAIL_SESSION_IDLE_TIMEOUT_MSECS, @@ -115,8 +107,7 @@ session->ip = mail_ip_login(&ip); } - guid_p = session->guid; - hash_table_insert(mail_sessions_hash, guid_p, session); + hash_table_insert(mail_sessions_hash, session->id, session); DLLIST_PREPEND_FULL(&stable_mail_sessions, session, stable_prev, stable_next); DLLIST2_APPEND_FULL(&mail_sessions_head, &mail_sessions_tail, session, @@ -150,8 +141,6 @@ static void mail_session_free(struct mail_session *session) { - uint8_t *guid_p = session->guid; - i_assert(session->refcount == 0); global_memory_free(mail_session_memsize(session)); @@ -159,7 +148,7 @@ if (session->to_idle != NULL) timeout_remove(&session->to_idle); if (!session->disconnected) - hash_table_remove(mail_sessions_hash, guid_p); + hash_table_remove(mail_sessions_hash, session->id); DLLIST_REMOVE_FULL(&stable_mail_sessions, session, stable_prev, stable_next); DLLIST2_REMOVE_FULL(&mail_sessions_head, &mail_sessions_tail, session, @@ -174,65 +163,57 @@ } str_table_unref(services, &session->service); + i_free(session->id); i_free(session); } -static void mail_session_guid_lost(guid_128_t session_guid) +static void mail_session_id_lost(const char *session_id) { - if (ioloop_time < session_guid_warn_hide_until) { - if (session_guid_hide_warned) + if (ioloop_time < session_id_warn_hide_until) { + if (session_id_hide_warned) return; - session_guid_hide_warned = TRUE; + session_id_hide_warned = TRUE; i_warning("stats process appears to have crashed/restarted, " - "hiding missing session GUID warnings for %d seconds", - (int)(session_guid_warn_hide_until - ioloop_time)); + "hiding missing session ID warnings for %d seconds", + (int)(session_id_warn_hide_until - ioloop_time)); return; } - i_warning("Couldn't find session GUID: %s", - guid_128_to_string(session_guid)); + i_warning("Couldn't find session ID: %s", session_id); } -int mail_session_lookup(const char *guid, struct mail_session **session_r, +int mail_session_lookup(const char *id, struct mail_session **session_r, const char **error_r) { - guid_128_t session_guid; - uint8_t *guid_p; - - if (guid == NULL) { + if (id == NULL) { *error_r = "Too few parameters"; return -1; } - if (guid_128_from_string(guid, session_guid) < 0) { - *error_r = "Invalid GUID"; - return -1; - } - guid_p = session_guid; - *session_r = hash_table_lookup(mail_sessions_hash, guid_p); + *session_r = hash_table_lookup(mail_sessions_hash, id); if (*session_r == NULL) { - mail_session_guid_lost(session_guid); + mail_session_id_lost(id); return 0; } return 1; } -int mail_session_get(const char *guid, struct mail_session **session_r, +int mail_session_get(const char *id, struct mail_session **session_r, const char **error_r) { const char *new_args[5]; int ret; - if ((ret = mail_session_lookup(guid, session_r, error_r)) != 0) + if ((ret = mail_session_lookup(id, session_r, error_r)) != 0) From dovecot at dovecot.org Thu Nov 13 00:49:37 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 13 Nov 2014 00:49:37 +0000 Subject: dovecot-2.2: imap-stats: Fixed truncation of command args. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6c69fe41d553 changeset: 18073:6c69fe41d553 user: Timo Sirainen date: Thu Nov 13 02:48:22 2014 +0200 description: imap-stats: Fixed truncation of command args. diffstat: src/plugins/imap-stats/imap-stats-plugin.c | 11 ++++++++--- 1 files changed, 8 insertions(+), 3 deletions(-) diffs (36 lines): diff -r 0694bfec140a -r 6c69fe41d553 src/plugins/imap-stats/imap-stats-plugin.c --- a/src/plugins/imap-stats/imap-stats-plugin.c Thu Nov 13 02:39:51 2014 +0200 +++ b/src/plugins/imap-stats/imap-stats-plugin.c Thu Nov 13 02:48:22 2014 +0200 @@ -54,7 +54,7 @@ struct stats_user *suser = STATS_USER_CONTEXT(cmd->client->user); struct stats_client_command *scmd = IMAP_STATS_IMAP_CONTEXT(cmd); struct mail_stats stats, pre_trans_stats, trans_stats; - unsigned int args_pos = 0; + unsigned int args_pos = 0, args_len = 0; string_t *str; if (scmd == NULL) @@ -87,6 +87,7 @@ args_pos = str_len(str); if (cmd->args != NULL) str_append(str, cmd->args); + args_len = str_len(str) - args_pos; scmd->continued = TRUE; } @@ -95,9 +96,13 @@ if (str_len(str) > PIPE_BUF) { /* truncate the args so it fits */ + unsigned int delete_count = str_len(str) - PIPE_BUF; + i_assert(args_pos != 0); - str_delete(str, args_pos, str_len(str) - PIPE_BUF); - i_assert(str_len(str) == PIPE_BUF); + if (delete_count > args_len) + delete_count = args_len; + str_delete(str, args_pos + args_len - delete_count, + delete_count); } stats_connection_send(suser->stats_conn, str); From dovecot at dovecot.org Thu Nov 13 02:00:10 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 13 Nov 2014 02:00:10 +0000 Subject: dovecot-2.2: lmtp: Added support for DSN ORCPT parameter. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3db5fbb216d3 changeset: 18074:3db5fbb216d3 user: Timo Sirainen date: Thu Nov 13 03:58:51 2014 +0200 description: lmtp: Added support for DSN ORCPT parameter. If the parameter is specified, it's used as the original recipient address in Sieve. The ORCPT parameter is also proxied to destination LMTP server if used (without checking if it supports it). We don't currently advertise DSN extension, so this feature won't be enabled automatically. diffstat: src/lib-lda/lmtp-client.c | 39 +++++++++++++++++++++++++++- src/lib-lda/lmtp-client.h | 9 ++++++ src/lmtp/client.h | 3 ++ src/lmtp/commands.c | 64 +++++++++++++++++++++++++++++++++++++++++----- src/lmtp/lmtp-proxy.c | 5 ++- src/lmtp/lmtp-proxy.h | 1 + 6 files changed, 109 insertions(+), 12 deletions(-) diffs (284 lines): diff -r 6c69fe41d553 -r 3db5fbb216d3 src/lib-lda/lmtp-client.c --- a/src/lib-lda/lmtp-client.c Thu Nov 13 02:48:22 2014 +0200 +++ b/src/lib-lda/lmtp-client.c Thu Nov 13 03:58:51 2014 +0200 @@ -31,6 +31,8 @@ lmtp_callback_t *data_callback; void *context; + struct lmtp_recipient_params params; + unsigned int data_called:1; unsigned int failed:1; }; @@ -732,15 +734,35 @@ client->data_header = p_strdup(client->pool, str); } +static void lmtp_append_xtext(string_t *dest, const char *str) +{ + unsigned int i; + + for (i = 0; str[i] != '\0'; i++) { + if (str[i] >= 33 && str[i] <= 126 && + str[i] != '+' && str[i] != '=') + str_append_c(dest, str[i]); + else + str_printfa(dest, "+%02X", str[i]); + } +} + static void lmtp_client_send_rcpts(struct lmtp_client *client) { const struct lmtp_rcpt *rcpt; unsigned int i, count; + string_t *str = t_str_new(128); rcpt = array_get(&client->recipients, &count); for (i = client->rcpt_next_send_idx; i < count; i++) { - o_stream_nsend_str(client->output, - t_strdup_printf("RCPT TO:<%s>\r\n", rcpt[i].address)); + str_truncate(str, 0); + str_printfa(str, "RCPT TO:<%s>", rcpt[i].address); + if (rcpt->params.dsn_orcpt != NULL) { + str_append(str, " ORCPT="); + lmtp_append_xtext(str, rcpt->params.dsn_orcpt); + } + str_append(str, "\r\n"); + o_stream_nsend(client->output, str_data(str), str_len(str)); } client->rcpt_next_send_idx = i; } @@ -749,11 +771,24 @@ lmtp_callback_t *rcpt_to_callback, lmtp_callback_t *data_callback, void *context) { + struct lmtp_recipient_params params; + + memset(¶ms, 0, sizeof(params)); + lmtp_client_add_rcpt_params(client, address, ¶ms, rcpt_to_callback, + data_callback, context); +} + +void lmtp_client_add_rcpt_params(struct lmtp_client *client, const char *address, + const struct lmtp_recipient_params *params, + lmtp_callback_t *rcpt_to_callback, + lmtp_callback_t *data_callback, void *context) +{ struct lmtp_rcpt *rcpt; enum lmtp_client_result result; rcpt = array_append_space(&client->recipients); rcpt->address = p_strdup(client->pool, address); + rcpt->params.dsn_orcpt = p_strdup(client->pool, params->dsn_orcpt); rcpt->rcpt_to_callback = rcpt_to_callback; rcpt->data_callback = data_callback; rcpt->context = context; diff -r 6c69fe41d553 -r 3db5fbb216d3 src/lib-lda/lmtp-client.h --- a/src/lib-lda/lmtp-client.h Thu Nov 13 02:48:22 2014 +0200 +++ b/src/lib-lda/lmtp-client.h Thu Nov 13 03:58:51 2014 +0200 @@ -22,8 +22,13 @@ LMTP_CLIENT_RESULT_INTERNAL_ERROR = -1 }; +struct lmtp_recipient_params { + const char *dsn_orcpt; +}; + struct lmtp_client_settings { const char *my_hostname; + /* The whole MAIL FROM line, including parameters */ const char *mail_from; const char *dns_client_socket_path; @@ -66,6 +71,10 @@ void lmtp_client_add_rcpt(struct lmtp_client *client, const char *address, lmtp_callback_t *rcpt_to_callback, lmtp_callback_t *data_callback, void *context); +void lmtp_client_add_rcpt_params(struct lmtp_client *client, const char *address, + const struct lmtp_recipient_params *params, + lmtp_callback_t *rcpt_to_callback, + lmtp_callback_t *data_callback, void *context); /* Start sending input stream as DATA. */ void lmtp_client_send(struct lmtp_client *client, struct istream *data_input); /* Call this function whenever input stream can potentially be read forward. diff -r 6c69fe41d553 -r 3db5fbb216d3 src/lmtp/client.h --- a/src/lmtp/client.h Thu Nov 13 02:48:22 2014 +0200 +++ b/src/lmtp/client.h Thu Nov 13 03:58:51 2014 +0200 @@ -2,12 +2,15 @@ #define CLIENT_H #include "net.h" +#include "lmtp-client.h" #define CLIENT_MAIL_DATA_MAX_INMEMORY_SIZE (1024*128) struct mail_recipient { const char *address; const char *detail; /* +detail part is also in address */ + struct lmtp_recipient_params params; + struct mail_storage_service_user *service_user; }; diff -r 6c69fe41d553 -r 3db5fbb216d3 src/lmtp/commands.c --- a/src/lmtp/commands.c Thu Nov 13 02:48:22 2014 +0200 +++ b/src/lmtp/commands.c Thu Nov 13 03:58:51 2014 +0200 @@ -11,6 +11,7 @@ #include "ostream.h" #include "istream-dot.h" #include "safe-mkstemp.h" +#include "hex-dec.h" #include "var-expand.h" #include "restrict-access.h" #include "settings-parser.h" @@ -149,6 +150,33 @@ return 0; } +static const char * +parse_xtext(struct client *client, const char *value) +{ + const char *p; + string_t *str; + unsigned int i; + + p = strchr(value, '+'); + if (p == NULL) + return p_strdup(client->state_pool, value); + + /* + hexchar = ASCII "+" immediately followed by two upper case + hexadecimal digits + */ + str = t_str_new(128); + for (i = 0; value[i] != '\0'; i++) { + if (value[i] == '+' && value[i+1] != '\0' && value[i+2] != '\0') { + str_append_c(str, hex2dec((const void *)(value+i+1), 2)); + i += 2; + } else { + str_append_c(str, value[i]); + } + } + return p_strdup(client->state_pool, str_c(str)); +} + int cmd_mail(struct client *client, const char *args) { const char *addr, *const *argv; @@ -269,7 +297,8 @@ } static bool client_proxy_rcpt(struct client *client, const char *address, - const char *username, const char *detail) + const char *username, const char *detail, + const struct lmtp_recipient_params *params) { struct auth_master_connection *auth_conn; struct lmtp_proxy_rcpt_settings set; @@ -311,6 +340,7 @@ set.port = client->local_port; set.protocol = LMTP_CLIENT_PROTOCOL_LMTP; set.timeout_msecs = LMTP_PROXY_DEFAULT_TIMEOUT_MSECS; + set.params = *params; if (!client_proxy_rcpt_parse_fields(&set, fields, &username)) { /* not proxying this user */ @@ -537,7 +567,8 @@ { struct mail_recipient rcpt; struct mail_storage_service_input input; - const char *address, *username, *detail, *prefix; + const char *params, *address, *username, *detail, *prefix; + const char *const *argv; const char *error = NULL; int ret = 0; @@ -547,7 +578,7 @@ } if (strncasecmp(args, "TO:", 3) != 0 || - parse_address(args + 3, &address, &args) < 0) { + parse_address(args + 3, &address, ¶ms) < 0) { client_send_line(client, "501 5.5.4 Invalid parameters"); return 0; } @@ -555,16 +586,22 @@ memset(&rcpt, 0, sizeof(rcpt)); address = lmtp_unescape_address(address); - if (*args != '\0') { - client_send_line(client, "501 5.5.4 Unsupported options"); - return 0; + argv = t_strsplit(params, " "); + for (; *argv != NULL; argv++) { + if (strncasecmp(*argv, "ORCPT=", 6) == 0) { + rcpt.params.dsn_orcpt = parse_xtext(client, *argv + 6); + } else { + client_send_line(client, "501 5.5.4 Unsupported options"); + return 0; + } } rcpt_address_parse(client, address, &username, &detail); client_state_set(client, "RCPT TO", address); if (client->lmtp_set->lmtp_proxy) { - if (client_proxy_rcpt(client, address, username, detail)) + if (client_proxy_rcpt(client, address, username, detail, + &rcpt.params)) return 0; } @@ -647,6 +684,15 @@ return 0; } +static bool orcpt_get_valid_rfc822(const char *orcpt, const char **addr_r) +{ + if (orcpt == NULL || strncasecmp(orcpt, "rfc822;", 7) != 0) + return FALSE; + /* FIXME: we should verify the address further */ + *addr_r = orcpt + 7; + return TRUE; +} + static int client_deliver(struct client *client, const struct mail_recipient *rcpt, struct mail *src_mail, struct mail_deliver_session *session) @@ -709,7 +755,9 @@ dctx.src_mail = src_mail; dctx.src_envelope_sender = client->state.mail_from; dctx.dest_user = client->state.dest_user; - if (*dctx.set->lda_original_recipient_header != '\0') { + if (orcpt_get_valid_rfc822(rcpt->params.dsn_orcpt, &dctx.dest_addr)) { + /* used ORCPT */ + } else if (*dctx.set->lda_original_recipient_header != '\0') { dctx.dest_addr = mail_deliver_get_address(src_mail, dctx.set->lda_original_recipient_header); } diff -r 6c69fe41d553 -r 3db5fbb216d3 src/lmtp/lmtp-proxy.c --- a/src/lmtp/lmtp-proxy.c Thu Nov 13 02:48:22 2014 +0200 +++ b/src/lmtp/lmtp-proxy.c Thu Nov 13 03:58:51 2014 +0200 @@ -288,8 +288,9 @@ rcpt->address = p_strdup(proxy->pool, address); array_append(&proxy->rcpt_to, &rcpt, 1); - lmtp_client_add_rcpt(conn->client, address, lmtp_proxy_conn_rcpt_to, - lmtp_proxy_conn_data, rcpt); + lmtp_client_add_rcpt_params(conn->client, address, &set->params, + lmtp_proxy_conn_rcpt_to, + lmtp_proxy_conn_data, rcpt); return 0; } diff -r 6c69fe41d553 -r 3db5fbb216d3 src/lmtp/lmtp-proxy.h --- a/src/lmtp/lmtp-proxy.h Thu Nov 13 02:48:22 2014 +0200 +++ b/src/lmtp/lmtp-proxy.h Thu Nov 13 03:58:51 2014 +0200 @@ -22,6 +22,7 @@ unsigned int port; unsigned int timeout_msecs; enum lmtp_client_protocol protocol; + struct lmtp_recipient_params params; }; typedef void lmtp_proxy_finish_callback_t(void *context); From dovecot at dovecot.org Thu Nov 13 07:13:54 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 13 Nov 2014 07:13:54 +0000 Subject: dovecot-2.2: dsync: Set MAILBOX_TRANSACTION_FLAG_SYNC also for e... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8456ebd4fa80 changeset: 18075:8456ebd4fa80 user: Timo Sirainen date: Thu Nov 13 09:12:38 2014 +0200 description: dsync: Set MAILBOX_TRANSACTION_FLAG_SYNC also for exporter. This doesn't really do much because exporter shouldn't be writing anything, but it allows plugins to find out if transactions is coming from dsync. diffstat: src/doveadm/dsync/dsync-mailbox-export.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r 3db5fbb216d3 -r 8456ebd4fa80 src/doveadm/dsync/dsync-mailbox-export.c --- a/src/doveadm/dsync/dsync-mailbox-export.c Thu Nov 13 03:58:51 2014 +0200 +++ b/src/doveadm/dsync/dsync-mailbox-export.c Thu Nov 13 09:12:38 2014 +0200 @@ -363,7 +363,8 @@ wanted_headers = exporter->wanted_headers; } - exporter->trans = mailbox_transaction_begin(exporter->box, 0); + exporter->trans = mailbox_transaction_begin(exporter->box, + MAILBOX_TRANSACTION_FLAG_SYNC); search_ctx = mailbox_search_init(exporter->trans, search_args, NULL, wanted_fields, wanted_headers); mail_search_args_unref(&search_args); From dovecot at dovecot.org Thu Nov 13 07:55:08 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 13 Nov 2014 07:55:08 +0000 Subject: dovecot-2.2: lib-mail: message_search_msg() shouldn't clear istr... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/dc593d4dbeb3 changeset: 18076:dc593d4dbeb3 user: Timo Sirainen date: Thu Nov 13 09:35:59 2014 +0200 description: lib-mail: message_search_msg() shouldn't clear istream's stream_errno. If the input stream failed, retrying the operation without cached message_parts is unlikely to help. This also loses the original error message. diffstat: src/lib-mail/message-search.c | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diffs (11 lines): diff -r 8456ebd4fa80 -r dc593d4dbeb3 src/lib-mail/message-search.c --- a/src/lib-mail/message-search.c Thu Nov 13 09:12:38 2014 +0200 +++ b/src/lib-mail/message-search.c Thu Nov 13 09:35:59 2014 +0200 @@ -212,7 +212,6 @@ } if (message_parser_deinit(&parser_ctx, &new_parts) < 0) { /* broken parts */ - input->stream_errno = 0; ret = -1; } return ret; From dovecot at dovecot.org Thu Nov 13 07:55:08 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 13 Nov 2014 07:55:08 +0000 Subject: dovecot-2.2: lib: Never implicitly clear istream's stream_errno. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e3b52d11d038 changeset: 18077:e3b52d11d038 user: Timo Sirainen date: Thu Nov 13 09:39:24 2014 +0200 description: lib: Never implicitly clear istream's stream_errno. I'm not really sure why these were added in the first place. At some point the stream_errno was cleared in more places, such as just before reading, but those were also removed already. I don't see any benefit for these, just potential problems. diffstat: src/lib/istream.c | 6 +----- 1 files changed, 1 insertions(+), 5 deletions(-) diffs (24 lines): diff -r dc593d4dbeb3 -r e3b52d11d038 src/lib/istream.c --- a/src/lib/istream.c Thu Nov 13 09:35:59 2014 +0200 +++ b/src/lib/istream.c Thu Nov 13 09:39:24 2014 +0200 @@ -250,7 +250,6 @@ if (unlikely(stream->closed)) return; - stream->stream_errno = 0; _stream->seek(_stream, stream->v_offset + count, FALSE); } @@ -398,11 +397,8 @@ struct istream_private *_stream = stream->real_stream; const unsigned char *pos; - if (_stream->skip >= _stream->pos) { - if (!unlikely(stream->closed)) - stream->stream_errno = 0; + if (_stream->skip >= _stream->pos) return NULL; - } pos = memchr(_stream->buffer + _stream->skip, '\n', _stream->pos - _stream->skip); From dovecot at dovecot.org Thu Nov 13 07:55:08 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 13 Nov 2014 07:55:08 +0000 Subject: dovecot-2.2: lib-storage: Added "creator" for mail_user. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ba74ec089370 changeset: 18078:ba74ec089370 user: Timo Sirainen date: Thu Nov 13 09:53:35 2014 +0200 description: lib-storage: Added "creator" for mail_user. This allows fixing stats plugin to update stats also for shared mailboxes. That's a bit complicated change though, so I'm not sure how soon it'll happen. But at least this change makes it possible more easily later on. Might have other uses too. diffstat: src/lib-storage/index/shared/shared-storage.c | 1 + src/lib-storage/mail-user.h | 4 ++++ 2 files changed, 5 insertions(+), 0 deletions(-) diffs (25 lines): diff -r e3b52d11d038 -r ba74ec089370 src/lib-storage/index/shared/shared-storage.c --- a/src/lib-storage/index/shared/shared-storage.c Thu Nov 13 09:39:24 2014 +0200 +++ b/src/lib-storage/index/shared/shared-storage.c Thu Nov 13 09:53:35 2014 +0200 @@ -255,6 +255,7 @@ owner = mail_user_alloc(userdomain, user->set_info, user->unexpanded_set); + owner->creator = user; owner->autocreated = TRUE; owner->session_id = p_strdup(owner->pool, user->session_id); if (mail_user_init(owner, &error) < 0) { diff -r e3b52d11d038 -r ba74ec089370 src/lib-storage/mail-user.h --- a/src/lib-storage/mail-user.h Thu Nov 13 09:39:24 2014 +0200 +++ b/src/lib-storage/mail-user.h Thu Nov 13 09:53:35 2014 +0200 @@ -16,6 +16,10 @@ struct mail_user_vfuncs v, *vlast; int refcount; + /* User's creator if such exists. For example for autocreated shared + mailbox users their creator is the logged in user. */ + struct mail_user *creator; + const char *username; /* don't access the home directly. It may be set lazily. */ const char *_home; From dovecot at dovecot.org Thu Nov 13 08:02:53 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 13 Nov 2014 08:02:53 +0000 Subject: dovecot-2.2: lib: ostream-file checks "sendfile() not supported"... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/24d2d52d5355 changeset: 18079:24d2d52d5355 user: Timo Sirainen date: Thu Nov 13 10:01:37 2014 +0200 description: lib: ostream-file checks "sendfile() not supported" a bit better. There are other reasons why stream_errno=EINVAL could have been set, at least in theory. diffstat: src/lib/ostream-file.c | 22 ++++++++++++++-------- 1 files changed, 14 insertions(+), 8 deletions(-) diffs (67 lines): diff -r ba74ec089370 -r 24d2d52d5355 src/lib/ostream-file.c --- a/src/lib/ostream-file.c Thu Nov 13 09:53:35 2014 +0200 +++ b/src/lib/ostream-file.c Thu Nov 13 10:01:37 2014 +0200 @@ -697,13 +697,16 @@ } static off_t io_stream_sendfile(struct ostream_private *outstream, - struct istream *instream, int in_fd) + struct istream *instream, int in_fd, + bool *sendfile_not_supported_r) { struct file_ostream *foutstream = (struct file_ostream *)outstream; uoff_t start_offset; uoff_t in_size, offset, send_size, v_offset; ssize_t ret; + *sendfile_not_supported_r = FALSE; + if ((ret = i_stream_get_size(instream, TRUE, &in_size)) <= 0) { outstream->ostream.stream_errno = ret == 0 ? ESPIPE : instream->stream_errno; @@ -740,9 +743,10 @@ break; } } - - outstream->ostream.stream_errno = errno; - if (errno != EINVAL) { + if (errno == EINVAL) + *sendfile_not_supported_r = TRUE; + else { + outstream->ostream.stream_errno = errno; /* close only if error wasn't because sendfile() isn't supported */ stream_closed(foutstream); @@ -762,7 +766,8 @@ i_assert(!foutstream->file || instream->v_offset - start_offset == in_size); if (i_stream_read(instream) > 0) { - if (io_stream_sendfile(outstream, instream, in_fd) < 0) + if (io_stream_sendfile(outstream, instream, in_fd, + sendfile_not_supported_r) < 0) return -1; } } @@ -894,17 +899,18 @@ bool same_stream; int in_fd; off_t ret; + bool sendfile_not_supported; in_fd = !instream->readable_fd ? -1 : i_stream_get_fd(instream); if (!foutstream->no_sendfile && in_fd != -1 && in_fd != foutstream->fd && instream->seekable) { - ret = io_stream_sendfile(outstream, instream, in_fd); - if (ret >= 0 || outstream->ostream.stream_errno != EINVAL) + ret = io_stream_sendfile(outstream, instream, in_fd, + &sendfile_not_supported); + if (ret >= 0 || !sendfile_not_supported) return ret; /* sendfile() not supported (with this fd), fallback to regular sending. */ - outstream->ostream.stream_errno = 0; foutstream->no_sendfile = TRUE; } From dovecot at dovecot.org Thu Nov 13 08:10:53 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 13 Nov 2014 08:10:53 +0000 Subject: dovecot-2.2: lib: ostream.stream_errno is now exactly the same a... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4d8a52f5839a changeset: 18080:4d8a52f5839a user: Timo Sirainen date: Thu Nov 13 10:09:28 2014 +0200 description: lib: ostream.stream_errno is now exactly the same as last_failed_errno. There's a lot of code that checks only for stream_errno instead of last_failed_errno at the flush time where the stream_errno might have already been cleared. I'm not sure why these were separated in the first place. I don't think there are any stream errors which just correctly go away by retrying the operation. Perhaps some code could have attempted seeking and try to handle the ESPIPE error, but that kind of code paths are rare and they could clear the errors themselves if needed. diffstat: src/lib/ostream.c | 33 +++++++++++---------------------- 1 files changed, 11 insertions(+), 22 deletions(-) diffs (133 lines): diff -r 24d2d52d5355 -r 4d8a52f5839a src/lib/ostream.c --- a/src/lib/ostream.c Thu Nov 13 10:01:37 2014 +0200 +++ b/src/lib/ostream.c Thu Nov 13 10:09:28 2014 +0200 @@ -125,7 +125,7 @@ { struct ostream_private *_stream = stream->real_stream; - if (unlikely(stream->closed)) + if (unlikely(stream->closed || stream->stream_errno != 0)) return; _stream->cork(_stream, TRUE); @@ -135,10 +135,9 @@ { struct ostream_private *_stream = stream->real_stream; - if (unlikely(stream->closed)) + if (unlikely(stream->closed || stream->stream_errno != 0)) return; - stream->stream_errno = 0; _stream->cork(_stream, FALSE); if (stream->stream_errno != 0) errno = stream->last_failed_errno = stream->stream_errno; @@ -151,23 +150,16 @@ return _stream->corked; } -static void o_stream_clear_error(struct ostream *stream) -{ - stream->stream_errno = 0; - i_free_and_null(stream->real_stream->iostream.error); -} - int o_stream_flush(struct ostream *stream) { struct ostream_private *_stream = stream->real_stream; int ret = 1; - if (unlikely(stream->closed)) { + if (unlikely(stream->closed || stream->stream_errno != 0)) { errno = stream->stream_errno; return -1; } - o_stream_clear_error(stream); if (unlikely((ret = _stream->flush(_stream)) < 0)) { i_assert(stream->stream_errno != 0); stream->last_failed_errno = stream->stream_errno; @@ -180,7 +172,7 @@ { struct ostream_private *_stream = stream->real_stream; - if (unlikely(stream->closed)) + if (unlikely(stream->closed || stream->stream_errno != 0)) return; _stream->flush_pending(_stream, set); @@ -205,12 +197,11 @@ { struct ostream_private *_stream = stream->real_stream; - if (unlikely(stream->closed)) { + if (unlikely(stream->closed || stream->stream_errno != 0)) { errno = stream->stream_errno; return -1; } - o_stream_clear_error(stream); if (unlikely(_stream->seek(_stream, offset) < 0)) { i_assert(stream->stream_errno != 0); stream->last_failed_errno = stream->stream_errno; @@ -239,12 +230,11 @@ size_t total_size; ssize_t ret; - if (unlikely(stream->closed)) { + if (unlikely(stream->closed || stream->stream_errno != 0)) { errno = stream->stream_errno; return -1; } - o_stream_clear_error(stream); for (i = 0, total_size = 0; i < iov_count; i++) total_size += iov[i].iov_len; if (total_size == 0) @@ -282,7 +272,7 @@ void o_stream_nsendv(struct ostream *stream, const struct const_iovec *iov, unsigned int iov_count) { - if (unlikely(stream->closed)) + if (unlikely(stream->closed || stream->stream_errno != 0)) return; (void)o_stream_sendv(stream, iov, iov_count); stream->real_stream->last_errors_not_checked = TRUE; @@ -295,7 +285,7 @@ void o_stream_nflush(struct ostream *stream) { - if (unlikely(stream->closed)) + if (unlikely(stream->closed || stream->stream_errno != 0)) return; (void)o_stream_flush(stream); stream->real_stream->last_errors_not_checked = TRUE; @@ -328,12 +318,12 @@ struct ostream_private *_outstream = outstream->real_stream; off_t ret; - if (unlikely(outstream->closed || instream->closed)) { + if (unlikely(outstream->closed || instream->closed || + outstream->stream_errno != 0)) { errno = outstream->stream_errno; return -1; } - o_stream_clear_error(outstream); ret = _outstream->send_istream(_outstream, instream); if (unlikely(ret < 0)) { if (outstream->stream_errno != 0) { @@ -351,12 +341,11 @@ { int ret; - if (unlikely(stream->closed)) { + if (unlikely(stream->closed || stream->stream_errno != 0)) { errno = stream->stream_errno; return -1; } - o_stream_clear_error(stream); ret = stream->real_stream->write_at(stream->real_stream, data, size, offset); if (unlikely(ret < 0)) { From pigeonhole at rename-it.nl Thu Nov 13 08:36:22 2014 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 13 Nov 2014 09:36:22 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: Fixed bug in handling of bina... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/0ecc1c82065c changeset: 1964:0ecc1c82065c user: Stephan Bosch date: Thu Nov 13 09:36:13 2014 +0100 description: lib-sieve: Fixed bug in handling of binary errors for action side-effects and message overrides. diffstat: src/lib-sieve/sieve-actions.c | 47 +++++++++++++++++++++--------------------- src/lib-sieve/sieve-actions.h | 7 ++--- src/lib-sieve/sieve-message.c | 12 +++++----- src/lib-sieve/sieve-message.h | 7 ++--- 4 files changed, 36 insertions(+), 37 deletions(-) diffs (141 lines): diff -r daa10b2102d2 -r 0ecc1c82065c src/lib-sieve/sieve-actions.c --- a/src/lib-sieve/sieve-actions.c Wed Nov 12 22:21:14 2014 +0100 +++ b/src/lib-sieve/sieve-actions.c Thu Nov 13 09:36:13 2014 +0100 @@ -37,29 +37,6 @@ const struct sieve_operand_class sieve_side_effect_operand_class = { "SIDE-EFFECT" }; -bool sieve_opr_side_effect_read -(const struct sieve_runtime_env *renv, sieve_size_t *address, - struct sieve_side_effect *seffect) -{ - const struct sieve_side_effect_def *sdef; - - seffect->context = NULL; - - if ( !sieve_opr_object_read - (renv, &sieve_side_effect_operand_class, address, &seffect->object) ) - return FALSE; - - sdef = seffect->def = - (const struct sieve_side_effect_def *) seffect->object.def; - - if ( sdef->read_context != NULL && - !sdef->read_context(seffect, renv, address, &seffect->context) ) { - return FALSE; - } - - return TRUE; -} - bool sieve_opr_side_effect_dump (const struct sieve_dumptime_env *denv, sieve_size_t *address) { @@ -84,6 +61,30 @@ return TRUE; } +int sieve_opr_side_effect_read +(const struct sieve_runtime_env *renv, sieve_size_t *address, + struct sieve_side_effect *seffect) +{ + const struct sieve_side_effect_def *sdef; + int ret; + + seffect->context = NULL; + + if ( !sieve_opr_object_read + (renv, &sieve_side_effect_operand_class, address, &seffect->object) ) + return SIEVE_EXEC_BIN_CORRUPT; + + sdef = seffect->def = + (const struct sieve_side_effect_def *) seffect->object.def; + + if ( sdef->read_context != NULL && (ret=sdef->read_context + (seffect, renv, address, &seffect->context)) <= 0 ) { + return ret; + } + + return SIEVE_EXEC_OK; +} + /* * Optional operands */ diff -r daa10b2102d2 -r 0ecc1c82065c src/lib-sieve/sieve-actions.h --- a/src/lib-sieve/sieve-actions.h Wed Nov 12 22:21:14 2014 +0100 +++ b/src/lib-sieve/sieve-actions.h Thu Nov 13 09:36:13 2014 +0100 @@ -187,13 +187,12 @@ sieve_opr_object_emit(sblock, ext, &seff->obj_def); } -bool sieve_opr_side_effect_read +bool sieve_opr_side_effect_dump + (const struct sieve_dumptime_env *denv, sieve_size_t *address); +int sieve_opr_side_effect_read (const struct sieve_runtime_env *renv, sieve_size_t *address, struct sieve_side_effect *seffect); -bool sieve_opr_side_effect_dump - (const struct sieve_dumptime_env *denv, sieve_size_t *address); - /* * Optional operands */ diff -r daa10b2102d2 -r 0ecc1c82065c src/lib-sieve/sieve-message.c --- a/src/lib-sieve/sieve-message.c Wed Nov 12 22:21:14 2014 +0100 +++ b/src/lib-sieve/sieve-message.c Thu Nov 13 09:36:13 2014 +0100 @@ -604,27 +604,27 @@ return TRUE; } -bool sieve_opr_message_override_read +int sieve_opr_message_override_read (const struct sieve_runtime_env *renv, sieve_size_t *address, struct sieve_message_override *svmo) { const struct sieve_message_override_def *hodef; + int ret; svmo->context = NULL; if ( !sieve_opr_object_read (renv, &sieve_message_override_operand_class, address, &svmo->object) ) - return FALSE; + return SIEVE_EXEC_BIN_CORRUPT; hodef = svmo->def = (const struct sieve_message_override_def *) svmo->object.def; if ( hodef->read_context != NULL && - !hodef->read_context(svmo, renv, address, &svmo->context) ) { - return FALSE; - } + (ret=hodef->read_context(svmo, renv, address, &svmo->context)) <= 0 ) + return ret; - return TRUE; + return SIEVE_EXEC_OK; } /* diff -r daa10b2102d2 -r 0ecc1c82065c src/lib-sieve/sieve-message.h --- a/src/lib-sieve/sieve-message.h Wed Nov 12 22:21:14 2014 +0100 +++ b/src/lib-sieve/sieve-message.h Thu Nov 13 09:36:13 2014 +0100 @@ -135,13 +135,12 @@ sieve_opr_object_emit(sblock, ext, &seff->obj_def); } -bool sieve_opr_message_override_read +bool sieve_opr_message_override_dump + (const struct sieve_dumptime_env *denv, sieve_size_t *address); +int sieve_opr_message_override_read (const struct sieve_runtime_env *renv, sieve_size_t *address, struct sieve_message_override *svmo); -bool sieve_opr_message_override_dump - (const struct sieve_dumptime_env *denv, sieve_size_t *address); - /* * Optional operands */ From pigeonhole at rename-it.nl Thu Nov 13 08:37:30 2014 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 13 Nov 2014 09:37:30 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: Increased binary minor versio... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/637c68cc79c5 changeset: 1965:637c68cc79c5 user: Stephan Bosch date: Thu Nov 13 09:37:21 2014 +0100 description: lib-sieve: Increased binary minor version due to changes in optional operand codes. diffstat: src/lib-sieve/sieve-binary.h | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 0ecc1c82065c -r 637c68cc79c5 src/lib-sieve/sieve-binary.h --- a/src/lib-sieve/sieve-binary.h Thu Nov 13 09:36:13 2014 +0100 +++ b/src/lib-sieve/sieve-binary.h Thu Nov 13 09:37:21 2014 +0100 @@ -13,7 +13,7 @@ */ #define SIEVE_BINARY_VERSION_MAJOR 1 -#define SIEVE_BINARY_VERSION_MINOR 2 +#define SIEVE_BINARY_VERSION_MINOR 3 /* * Binary object From dovecot at dovecot.org Thu Nov 13 08:59:02 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 13 Nov 2014 08:59:02 +0000 Subject: dovecot-2.2: lib-storage: Fixed crash if mail_namespaces_init_ad... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/02f705783387 changeset: 18081:02f705783387 user: Timo Sirainen date: Thu Nov 13 10:57:07 2014 +0200 description: lib-storage: Fixed crash if mail_namespaces_init_add() failed due to invalid settings. Normally the invalid settings were caught by config process already. diffstat: src/lib-storage/mail-namespace.c | 8 +++++--- 1 files changed, 5 insertions(+), 3 deletions(-) diffs (18 lines): diff -r 4d8a52f5839a -r 02f705783387 src/lib-storage/mail-namespace.c --- a/src/lib-storage/mail-namespace.c Thu Nov 13 10:09:28 2014 +0200 +++ b/src/lib-storage/mail-namespace.c Thu Nov 13 10:57:07 2014 +0200 @@ -54,9 +54,11 @@ { struct mail_storage **storagep; - array_foreach_modifiable(&ns->all_storages, storagep) - mail_storage_unref(storagep); - array_free(&ns->all_storages); + if (array_is_created(&ns->all_storages)) { + array_foreach_modifiable(&ns->all_storages, storagep) + mail_storage_unref(storagep); + array_free(&ns->all_storages); + } if (ns->list != NULL) mailbox_list_destroy(&ns->list); From dovecot at dovecot.org Thu Nov 13 09:15:26 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 13 Nov 2014 09:15:26 +0000 Subject: dovecot-2.2: lib-storage: If storage creation fails, destroy all... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fab74a2b7468 changeset: 18082:fab74a2b7468 user: Timo Sirainen date: Thu Nov 13 11:14:08 2014 +0200 description: lib-storage: If storage creation fails, destroy all the already created namespaces/storages. diffstat: src/lib-storage/mail-namespace.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (15 lines): diff -r 02f705783387 -r fab74a2b7468 src/lib-storage/mail-namespace.c --- a/src/lib-storage/mail-namespace.c Thu Nov 13 10:57:07 2014 +0200 +++ b/src/lib-storage/mail-namespace.c Thu Nov 13 11:14:08 2014 +0200 @@ -387,8 +387,10 @@ if (mail_namespaces_init_add(user, ns_set[i], unexpanded_ns_set[i], ns_p, error_r) < 0) { - if (!ns_set[i]->ignore_on_failure) + if (!ns_set[i]->ignore_on_failure) { + mail_namespaces_deinit(&namespaces); return -1; + } if (mail_set->mail_debug) { i_debug("Skipping namespace %s: %s", ns_set[i]->prefix, *error_r); From dovecot at dovecot.org Thu Nov 13 09:30:58 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 13 Nov 2014 09:30:58 +0000 Subject: dovecot-2.2: lib-index: Make sure dovecot.index.cache parsing do... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8565c72f73ec changeset: 18083:8565c72f73ec user: Timo Sirainen date: Thu Nov 13 11:29:41 2014 +0200 description: lib-index: Make sure dovecot.index.cache parsing doesn't go to infinite loop. Although we guarantee this only for files generated with Dovecot v2.2+ because it's too much trouble to check for older versions. diffstat: src/lib-index/mail-cache-fields.c | 9 +++++++++ 1 files changed, 9 insertions(+), 0 deletions(-) diffs (19 lines): diff -r fab74a2b7468 -r 8565c72f73ec src/lib-index/mail-cache-fields.c --- a/src/lib-index/mail-cache-fields.c Thu Nov 13 11:14:08 2014 +0200 +++ b/src/lib-index/mail-cache-fields.c Thu Nov 13 11:29:41 2014 +0200 @@ -234,6 +234,15 @@ "next_offset in field header loops"); return -1; } + /* In Dovecot v2.2+ we don't try to use any holes, + so next_offset must always be larger than current offset. + also makes it easier to guarantee there aren't any loops + (which we don't bother doing for old files) */ + if (next_offset < offset && cache->hdr->minor_version != 0) { + mail_cache_set_corrupted(cache, + "next_offset in field header decreases"); + return -1; + } offset = next_offset; if (cache->mmap_base != NULL || cache->map_with_read) { From dovecot at dovecot.org Fri Nov 14 00:16:53 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 14 Nov 2014 00:16:53 +0000 Subject: dovecot-2.2: lib: Added iostream_temp_create_named() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4e10ca59e907 changeset: 18084:4e10ca59e907 user: Timo Sirainen date: Fri Nov 14 01:19:37 2014 +0200 description: lib: Added iostream_temp_create_named() diffstat: src/lib/iostream-temp.c | 36 +++++++++++++++++++++++++++++------- src/lib/iostream-temp.h | 3 +++ 2 files changed, 32 insertions(+), 7 deletions(-) diffs (107 lines): diff -r 8565c72f73ec -r 4e10ca59e907 src/lib/iostream-temp.c --- a/src/lib/iostream-temp.c Thu Nov 13 11:29:41 2014 +0200 +++ b/src/lib/iostream-temp.c Fri Nov 14 01:19:37 2014 +0200 @@ -21,6 +21,7 @@ struct istream *dupstream; uoff_t dupstream_offset, dupstream_start_offset; + char *name; buffer_t *buf; int fd; @@ -39,6 +40,7 @@ if (tstream->buf != NULL) buffer_free(&tstream->buf); i_free(tstream->temp_path_prefix); + i_free(tstream->name); } static int o_stream_temp_move_to_fd(struct temp_ostream *tstream) @@ -214,6 +216,13 @@ struct ostream *iostream_temp_create(const char *temp_path_prefix, enum iostream_temp_flags flags) { + return iostream_temp_create_named(temp_path_prefix, flags, ""); +} + +struct ostream *iostream_temp_create_named(const char *temp_path_prefix, + enum iostream_temp_flags flags, + const char *name) +{ struct temp_ostream *tstream; struct ostream *output; @@ -228,7 +237,14 @@ tstream->fd = -1; output = o_stream_create(&tstream->ostream, NULL, -1); - o_stream_set_name(output, "(temp iostream)"); + tstream->name = i_strdup(name); + if (name[0] == '\0') { + o_stream_set_name(output, t_strdup_printf( + "(temp iostream in %s)", temp_path_prefix)); + } else { + o_stream_set_name(output, t_strdup_printf( + "(temp iostream in %s for %s)", temp_path_prefix, name)); + } return output; } @@ -244,8 +260,14 @@ (struct temp_ostream *)(*output)->real_stream; struct istream *input, *input2; uoff_t abs_offset, size; + const char *for_path; int fd; + if (tstream->name[0] == '\0') + for_path = ""; + else + for_path = t_strdup_printf(" for %s", tstream->name); + if (tstream->dupstream != NULL && !tstream->dupstream->closed) { abs_offset = tstream->dupstream->real_stream->abs_start_offset + tstream->dupstream_start_offset; @@ -261,8 +283,8 @@ i_stream_unref(&input2); } i_stream_set_name(input, t_strdup_printf( - "(Temp file in %s, from %s)", tstream->temp_path_prefix, - i_stream_get_name(tstream->dupstream))); + "(Temp file in %s%s, from %s)", tstream->temp_path_prefix, + for_path, i_stream_get_name(tstream->dupstream))); i_stream_unref(&tstream->dupstream); } else if (tstream->dupstream != NULL) { /* return the original failed stream. */ @@ -271,14 +293,14 @@ int fd = tstream->fd; input = i_stream_create_fd_autoclose(&tstream->fd, max_buffer_size); i_stream_set_name(input, t_strdup_printf( - "(Temp file fd %d in %s, %"PRIuUOFF_T" bytes)", - fd, tstream->temp_path_prefix, tstream->fd_size)); + "(Temp file fd %d in %s%s, %"PRIuUOFF_T" bytes)", + fd, tstream->temp_path_prefix, for_path, tstream->fd_size)); } else { input = i_stream_create_from_data(tstream->buf->data, tstream->buf->used); i_stream_set_name(input, t_strdup_printf( - "(Temp buffer in %s, %"PRIuSIZE_T" bytes)", - tstream->temp_path_prefix, tstream->buf->used)); + "(Temp buffer in %s%s, %"PRIuSIZE_T" bytes)", + tstream->temp_path_prefix, for_path, tstream->buf->used)); i_stream_add_destroy_callback(input, iostream_temp_buf_destroyed, tstream->buf); tstream->buf = NULL; diff -r 8565c72f73ec -r 4e10ca59e907 src/lib/iostream-temp.h --- a/src/lib/iostream-temp.h Thu Nov 13 11:29:41 2014 +0200 +++ b/src/lib/iostream-temp.h Fri Nov 14 01:19:37 2014 +0200 @@ -12,6 +12,9 @@ memory, and later to a temporary file that is immediately unlinked. */ struct ostream *iostream_temp_create(const char *temp_path_prefix, enum iostream_temp_flags flags); +struct ostream *iostream_temp_create_named(const char *temp_path_prefix, + enum iostream_temp_flags flags, + const char *name); /* Finished writing to stream. Return input stream for it and free the output stream. */ struct istream *iostream_temp_finish(struct ostream **output, From dovecot at dovecot.org Fri Nov 14 00:16:53 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 14 Nov 2014 00:16:53 +0000 Subject: dovecot-2.2: lib-fs: fs-metawrap now supports adding more metada... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3eba1442493d changeset: 18085:3eba1442493d user: Timo Sirainen date: Fri Nov 14 02:00:12 2014 +0200 description: lib-fs: fs-metawrap now supports adding more metadata fields while file is being saved. This potentially adds yet another temporary file write though. diffstat: src/lib-fs/fs-metawrap.c | 115 +++++++++++++++++++++++++++++++++++++++++----- 1 files changed, 101 insertions(+), 14 deletions(-) diffs (200 lines): diff -r 4e10ca59e907 -r 3eba1442493d src/lib-fs/fs-metawrap.c --- a/src/lib-fs/fs-metawrap.c Fri Nov 14 01:19:37 2014 +0200 +++ b/src/lib-fs/fs-metawrap.c Fri Nov 14 02:00:12 2014 +0200 @@ -6,9 +6,11 @@ #include "strescape.h" #include "istream.h" #include "istream-private.h" +#include "istream-concat.h" #include "istream-metawrap.h" #include "ostream.h" #include "ostream-metawrap.h" +#include "iostream-temp.h" #include "fs-api-private.h" #define MAX_METADATA_LINE_LEN 8192 @@ -24,8 +26,13 @@ struct fs_file *super, *super_read; enum fs_open_mode open_mode; struct istream *input; + bool metadata_read; + struct ostream *super_output; - bool metadata_read; + struct ostream *temp_output; + string_t *metadata_header; + uoff_t metadata_write_size; + bool metadata_changed_since_write; }; struct metawrap_fs_iter { @@ -132,6 +139,8 @@ if (file->super_read != file->super && file->super_read != NULL) fs_file_deinit(&file->super_read); + if (file->metadata_header != NULL) + str_free(&file->metadata_header); fs_file_deinit(&file->super); i_free(file->file.path); i_free(file); @@ -179,8 +188,10 @@ if (!file->fs->wrap_metadata) fs_set_metadata(file->super, key, value); - else + else { fs_default_set_metadata(_file, key, value); + file->metadata_changed_since_write = TRUE; + } } static int @@ -269,15 +280,11 @@ return fs_write_via_stream(_file, data, size); } -static void fs_metawrap_write_metadata(void *context) +static void +fs_metawrap_append_metadata(struct metawrap_fs_file *file, string_t *str) { - struct metawrap_fs_file *file = context; const struct fs_metadata *metadata; - string_t *str = t_str_new(256); - ssize_t ret; - /* FIXME: if fs_set_metadata() is called later the changes are - ignored. we'd need to write via temporary file then. */ array_foreach(&file->file.metadata, metadata) { str_append_tabescaped(str, metadata->key); str_append_c(str, ':'); @@ -285,11 +292,23 @@ str_append_c(str, '\n'); } str_append_c(str, '\n'); +} + +static void fs_metawrap_write_metadata(void *context) +{ + struct metawrap_fs_file *file = context; + string_t *str = t_str_new(256); + ssize_t ret; + + fs_metawrap_append_metadata(file, str); + file->metadata_write_size = str_len(str); + ret = o_stream_send(file->file.output, str_data(str), str_len(str)); if (ret < 0) o_stream_close(file->file.output); else i_assert((size_t)ret == str_len(str)); + file->metadata_changed_since_write = FALSE; } static void fs_metawrap_write_stream(struct fs_file *_file) @@ -298,18 +317,46 @@ i_assert(_file->output == NULL); - file->super_output = fs_write_stream(file->super); - if (!file->fs->wrap_metadata) + if (!file->fs->wrap_metadata) { + file->super_output = fs_write_stream(file->super); _file->output = file->super_output; - else { - _file->output = o_stream_create_metawrap(file->super_output, + } else { + file->temp_output = + iostream_temp_create_named(_file->fs->temp_path_prefix, + IOSTREAM_TEMP_FLAG_TRY_FD_DUP, + fs_file_path(_file)); + _file->output = o_stream_create_metawrap(file->temp_output, fs_metawrap_write_metadata, file); } } +static struct istream * +fs_metawrap_create_updated_istream(struct metawrap_fs_file *file, + struct istream *input) +{ + struct istream *input2, *inputs[3]; + + if (file->metadata_header != NULL) + str_truncate(file->metadata_header, 0); + else + file->metadata_header = str_new(default_pool, 1024); + fs_metawrap_append_metadata(file, file->metadata_header); + inputs[0] = i_stream_create_from_data(str_data(file->metadata_header), + str_len(file->metadata_header)); + + i_stream_seek(input, file->metadata_write_size); + inputs[1] = i_stream_create_limit(input, (uoff_t)-1); + inputs[2] = NULL; + input2 = i_stream_create_concat(inputs); + i_stream_unref(&inputs[0]); + i_stream_unref(&inputs[1]); + return input2; +} + static int fs_metawrap_write_stream_finish(struct fs_file *_file, bool success) { struct metawrap_fs_file *file = (struct metawrap_fs_file *)_file; + struct istream *input; int ret; if (_file->output != NULL) { @@ -320,13 +367,53 @@ else o_stream_unref(&_file->output); } + if (!success) { + if (file->temp_output != NULL) + o_stream_destroy(&file->temp_output); + if (file->super_output != NULL) + fs_write_stream_abort(file->super, &file->super_output); + return -1; + } - if (!success) { + if (file->super_output != NULL) { + /* no metawrap */ + i_assert(file->temp_output == NULL); + return fs_write_stream_finish(file->super, &file->super_output); + } + if (file->temp_output == NULL) { + /* finishing up */ + i_assert(file->super_output == NULL); + return fs_write_stream_finish(file->super, &file->super_output); + } + /* finish writing the temporary file */ + input = iostream_temp_finish(&file->temp_output, IO_BLOCK_SIZE); + if (file->metadata_changed_since_write) { + /* we'll need to recreate the metadata. do this by creating a + new istream combining the new metadata header and the + old body. */ + struct istream *input2 = input; + + input = fs_metawrap_create_updated_istream(file, input); + i_stream_unref(&input2); + } + file->super_output = fs_write_stream(file->super); + if (o_stream_send_istream(file->super_output, input) >= 0) + ret = fs_write_stream_finish(file->super, &file->super_output); + else if (input->stream_errno != 0) { + fs_set_error(_file->fs, "read(%s) failed: %s", + i_stream_get_name(input), + i_stream_get_error(input)); fs_write_stream_abort(file->super, &file->super_output); ret = -1; } else { - ret = fs_write_stream_finish(file->super, &file->super_output); + i_assert(file->super_output->stream_errno != 0); + fs_set_error(_file->fs, "write(%s) failed: %s", + o_stream_get_name(file->super_output), + o_stream_get_error(file->super_output)); + fs_write_stream_abort(file->super, &file->super_output); + ret = -1; } + i_stream_unref(&input); return ret; } From dovecot at dovecot.org Fri Nov 14 06:57:16 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 14 Nov 2014 06:57:16 +0000 Subject: dovecot-2.2: doveadm: If passdb lookup returns port, use it to o... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2fff557c9837 changeset: 18086:2fff557c9837 user: Timo Sirainen date: Thu Nov 13 22:55:59 2014 -0800 description: doveadm: If passdb lookup returns port, use it to override the doveadm_port setting. diffstat: src/doveadm/doveadm-mail-server.c | 10 +++++++--- 1 files changed, 7 insertions(+), 3 deletions(-) diffs (40 lines): diff -r 3eba1442493d -r 2fff557c9837 src/doveadm/doveadm-mail-server.c --- a/src/doveadm/doveadm-mail-server.c Fri Nov 14 02:00:12 2014 +0200 +++ b/src/doveadm/doveadm-mail-server.c Thu Nov 13 22:55:59 2014 -0800 @@ -179,7 +179,7 @@ struct auth_user_info info; pool_t pool; const char *auth_socket_path, *proxy_host, *const *fields; - unsigned int i; + unsigned int i, proxy_port; bool proxying; int ret; @@ -214,12 +214,17 @@ so just continue with the default host */ } else { proxy_host = NULL; proxying = FALSE; + proxy_port = ctx->set->doveadm_port; for (i = 0; fields[i] != NULL; i++) { if (strncmp(fields[i], "proxy", 5) == 0 && (fields[i][5] == '\0' || fields[i][5] == '=')) proxying = TRUE; else if (strncmp(fields[i], "host=", 5) == 0) proxy_host = fields[i]+5; + else if (strncmp(fields[i], "port=", 5) == 0) { + if (str_to_uint(fields[i]+5, &proxy_port) < 0) + proxy_port = 0; + } } if (!proxying) ret = 0; @@ -233,8 +238,7 @@ } ret = -1; } else { - *host_r = t_strdup_printf("%s:%u", proxy_host, - ctx->set->doveadm_port); + *host_r = t_strdup_printf("%s:%u", proxy_host, proxy_port); } } pool_unref(&pool); From dovecot at dovecot.org Fri Nov 14 08:24:35 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 14 Nov 2014 08:24:35 +0000 Subject: dovecot-2.2: imap: Added client_disconnect_if_inconsistent() to ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0f81d0ea3a48 changeset: 18087:0f81d0ea3a48 user: Timo Sirainen date: Fri Nov 14 02:29:27 2014 +0200 description: imap: Added client_disconnect_if_inconsistent() to remove code duplication. Based on patch by Stephan Bosch diffstat: src/imap/imap-commands-util.c | 24 ++++++++++++------------ src/imap/imap-commands-util.h | 2 ++ 2 files changed, 14 insertions(+), 12 deletions(-) diffs (60 lines): diff -r 2fff557c9837 -r 0f81d0ea3a48 src/imap/imap-commands-util.c --- a/src/imap/imap-commands-util.c Thu Nov 13 22:55:59 2014 -0800 +++ b/src/imap/imap-commands-util.c Fri Nov 14 02:29:27 2014 +0200 @@ -170,6 +170,16 @@ error)); } +void client_disconnect_if_inconsistent(struct client *client) +{ + if (client->mailbox != NULL && + mailbox_is_inconsistent(client->mailbox)) { + /* we can't do forced CLOSE, so have to disconnect */ + client_disconnect_with_error(client, + "IMAP session state is inconsistent, please relogin."); + } +} + void client_send_box_error(struct client_command_context *cmd, struct mailbox *box) { @@ -186,12 +196,7 @@ client_send_tagline(cmd, imap_get_error_string(cmd, error_string, error)); - if (cmd->client->mailbox != NULL && - mailbox_is_inconsistent(cmd->client->mailbox)) { - /* we can't do forced CLOSE, so have to disconnect */ - client_disconnect_with_error(cmd->client, - "IMAP session state is inconsistent, please relogin."); - } + client_disconnect_if_inconsistent(cmd->client); } void client_send_untagged_storage_error(struct client *client, @@ -203,12 +208,7 @@ error_string = mail_storage_get_last_error(storage, &error); client_send_line(client, t_strconcat("* NO ", error_string, NULL)); - if (client->mailbox != NULL && - mailbox_is_inconsistent(client->mailbox)) { - /* we can't do forced CLOSE, so have to disconnect */ - client_disconnect_with_error(client, - "IMAP session state is inconsistent, please relogin."); - } + client_disconnect_if_inconsistent(client); } bool client_parse_mail_flags(struct client_command_context *cmd, diff -r 2fff557c9837 -r 0f81d0ea3a48 src/imap/imap-commands-util.h --- a/src/imap/imap-commands-util.h Thu Nov 13 22:55:59 2014 -0800 +++ b/src/imap/imap-commands-util.h Fri Nov 14 02:29:27 2014 +0200 @@ -29,6 +29,8 @@ imap_get_error_string(struct client_command_context *cmd, const char *error_string, enum mail_error error); +void client_disconnect_if_inconsistent(struct client *client); + /* Send last mailbox list error message to client. */ void client_send_list_error(struct client_command_context *cmd, struct mailbox_list *list); From dovecot at dovecot.org Fri Nov 14 09:01:20 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 14 Nov 2014 09:01:20 +0000 Subject: dovecot-2.2: imap: namespace { order } parameter now specifies t... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/19b9ecfffb39 changeset: 18088:19b9ecfffb39 user: Timo Sirainen date: Fri Nov 14 00:59:57 2014 -0800 description: imap: namespace { order } parameter now specifies the order returned in NAMESPACE reply. This is relevant only if there are multiple visible namespaces with the same type. The default order depends primarily on the length of the prefix because of internal sorting (the order of equal length prefixes can't really be relied on). diffstat: src/imap/cmd-namespace.c | 79 ++++++++++++++++++++++---------- src/lib-storage/mail-storage-settings.c | 2 + src/lib-storage/mail-storage-settings.h | 1 + 3 files changed, 57 insertions(+), 25 deletions(-) diffs (132 lines): diff -r 0f81d0ea3a48 -r 19b9ecfffb39 src/imap/cmd-namespace.c --- a/src/imap/cmd-namespace.c Fri Nov 14 02:29:27 2014 +0200 +++ b/src/imap/cmd-namespace.c Fri Nov 14 00:59:57 2014 -0800 @@ -7,44 +7,73 @@ #include "imap-commands.h" #include "mail-namespace.h" +struct namespace_order { + int secondary_order; + struct mail_namespace *ns; +}; + +static int namespace_order_cmp(const struct namespace_order *no1, + const struct namespace_order *no2) +{ + if (no1->ns->set->order < no2->ns->set->order) + return -1; + if (no1->ns->set->order > no2->ns->set->order) + return 1; + + if (no1->secondary_order < no2->secondary_order) + return -1; + if (no1->secondary_order > no2->secondary_order) + return 1; + return 0; +} + static void list_namespaces(struct mail_namespace *ns, enum mail_namespace_type type, string_t *str) { - string_t *mutf7_prefix = t_str_new(64); + ARRAY(struct namespace_order) ns_order; + struct namespace_order *no; + unsigned int count = 0; + string_t *mutf7_prefix; char ns_sep; - bool found = FALSE; + + t_array_init(&ns_order, 4); while (ns != NULL) { if (ns->type == type && (ns->flags & NAMESPACE_FLAG_HIDDEN) == 0) { - if (!found) { - str_append_c(str, '('); - found = TRUE; - } - ns_sep = mail_namespace_get_sep(ns); - str_append_c(str, '('); - - str_truncate(mutf7_prefix, 0); - if (imap_utf8_to_utf7(ns->prefix, mutf7_prefix) < 0) { - i_panic("LIST: Namespace prefix not UTF-8: %s", - ns->prefix); - } - - imap_append_string(str, str_c(mutf7_prefix)); - str_append(str, " \""); - if (ns_sep == '\\') - str_append_c(str, '\\'); - str_append_c(str, ns_sep); - str_append(str, "\")"); + no = array_append_space(&ns_order); + no->ns = ns; + no->secondary_order = ++count; } - ns = ns->next; } - if (found) - str_append_c(str, ')'); - else + if (array_count(&ns_order) == 0) { str_append(str, "NIL"); + return; + } + array_sort(&ns_order, namespace_order_cmp); + + mutf7_prefix = t_str_new(64); + str_append_c(str, '('); + array_foreach_modifiable(&ns_order, no) { + ns_sep = mail_namespace_get_sep(no->ns); + str_append_c(str, '('); + + str_truncate(mutf7_prefix, 0); + if (imap_utf8_to_utf7(no->ns->prefix, mutf7_prefix) < 0) { + i_panic("LIST: Namespace prefix not UTF-8: %s", + no->ns->prefix); + } + + imap_append_string(str, str_c(mutf7_prefix)); + str_append(str, " \""); + if (ns_sep == '\\') + str_append_c(str, '\\'); + str_append_c(str, ns_sep); + str_append(str, "\")"); + } + str_append_c(str, ')'); } bool cmd_namespace(struct client_command_context *cmd) diff -r 0f81d0ea3a48 -r 19b9ecfffb39 src/lib-storage/mail-storage-settings.c --- a/src/lib-storage/mail-storage-settings.c Fri Nov 14 02:29:27 2014 +0200 +++ b/src/lib-storage/mail-storage-settings.c Fri Nov 14 00:59:57 2014 -0800 @@ -171,6 +171,7 @@ DEF(SET_BOOL, subscriptions), DEF(SET_BOOL, ignore_on_failure), DEF(SET_BOOL, disabled), + DEF(SET_UINT, order), DEFLIST_UNIQUE(mailboxes, "mailbox", &mailbox_setting_parser_info), @@ -191,6 +192,7 @@ .subscriptions = TRUE, .ignore_on_failure = FALSE, .disabled = FALSE, + .order = 0, .mailboxes = ARRAY_INIT }; diff -r 0f81d0ea3a48 -r 19b9ecfffb39 src/lib-storage/mail-storage-settings.h --- a/src/lib-storage/mail-storage-settings.h Fri Nov 14 02:29:27 2014 +0200 +++ b/src/lib-storage/mail-storage-settings.h Fri Nov 14 00:59:57 2014 -0800 @@ -62,6 +62,7 @@ bool subscriptions; bool ignore_on_failure; bool disabled; + unsigned int order; ARRAY(struct mailbox_settings *) mailboxes; struct mail_user_settings *user_set; From dovecot at dovecot.org Fri Nov 14 22:13:48 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 14 Nov 2014 22:13:48 +0000 Subject: dovecot-2.2: lib: array/hash - protect macro parameters Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/922ac3245e17 changeset: 18089:922ac3245e17 user: Phil Carmody date: Sat Nov 15 00:06:09 2014 +0200 description: lib: array/hash - protect macro parameters If a pointer parameter p is ``ptr + offset'', then sizeof(*p) becomes sizeof(*ptr + offset), which isn't what was wanted. sizeof(*(p)) is the safe expression to use instead. Ditto for just ``*data'' in array.h. The macros in hash.h which have been changed have been reindented for better readability and consistency. The bracketting of elem in (elem)++ in both of the array_foreach* macros isn't actually useful, as elem participates in token-pasting elsewhere. However, the two macros have been made more similar to each other for better parallelism. Signed-off-by: Phil Carmody diffstat: src/lib/array.h | 4 ++-- src/lib/hash.h | 20 ++++++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) diffs (58 lines): diff -r 19b9ecfffb39 -r 922ac3245e17 src/lib/array.h --- a/src/lib/array.h Fri Nov 14 00:59:57 2014 -0800 +++ b/src/lib/array.h Sat Nov 15 00:06:09 2014 +0200 @@ -46,7 +46,7 @@ (typeof(*(array)->v_modifiable)) # define ARRAY_TYPE_CHECK(array, data) \ COMPILE_ERROR_IF_TYPES_NOT_COMPATIBLE( \ - **(array)->v_modifiable, *data) + **(array)->v_modifiable, *(data)) #else # define ARRAY_TYPE_CAST_CONST(array) # define ARRAY_TYPE_CAST_MODIFIABLE(array) @@ -64,7 +64,7 @@ (const char *)(elem = ARRAY_TYPE_CAST_MODIFIABLE(array) \ buffer_get_modifiable_data((array)->arr.buffer, NULL)) + \ (array)->arr.buffer->used; \ - elem != elem ## _end; elem++) + elem != elem ## _end; (elem)++) #else # define array_foreach(array, elem) \ for (elem = *(array)->v; \ diff -r 19b9ecfffb39 -r 922ac3245e17 src/lib/hash.h --- a/src/lib/hash.h Fri Nov 14 00:59:57 2014 -0800 +++ b/src/lib/hash.h Sat Nov 15 00:06:09 2014 +0200 @@ -85,11 +85,14 @@ #ifndef __cplusplus # define hash_table_lookup_full(table, lookup_key, orig_key_r, value_r) \ hash_table_lookup_full((table)._table, \ - (void *)((const char *)(lookup_key) + COMPILE_ERROR_IF_TYPES2_NOT_COMPATIBLE((table)._const_key, (table)._key, lookup_key)), \ - (void *)((orig_key_r) + COMPILE_ERROR_IF_TYPES_NOT_COMPATIBLE((table)._keyp, orig_key_r) + \ - COMPILE_ERROR_IF_TRUE(sizeof(*orig_key_r) != sizeof(void *))), \ - (void *)((value_r) + COMPILE_ERROR_IF_TYPES_NOT_COMPATIBLE((table)._valuep, value_r) + \ - COMPILE_ERROR_IF_TRUE(sizeof(*value_r) != sizeof(void *)))) + (void *)((const char *)(lookup_key) + \ + COMPILE_ERROR_IF_TYPES2_NOT_COMPATIBLE((table)._const_key, (table)._key, lookup_key)), \ + (void *)((orig_key_r) + \ + COMPILE_ERROR_IF_TYPES_NOT_COMPATIBLE((table)._keyp, orig_key_r) + \ + COMPILE_ERROR_IF_TRUE(sizeof(*(orig_key_r)) != sizeof(void *))), \ + (void *)((value_r) + \ + COMPILE_ERROR_IF_TYPES_NOT_COMPATIBLE((table)._valuep, value_r) + \ + COMPILE_ERROR_IF_TRUE(sizeof(*(value_r)) != sizeof(void *)))) #else /* C++ requires (void **) casting, but that's not possible with strict aliasing, so .. we'll just disable the type checks */ @@ -134,9 +137,10 @@ #ifndef __cplusplus # define hash_table_iterate(ctx, table, key_r, value_r) \ hash_table_iterate(ctx, \ - (void *)((key_r) + COMPILE_ERROR_IF_TYPES_NOT_COMPATIBLE((table)._keyp, key_r) + \ - COMPILE_ERROR_IF_TRUE(sizeof(*key_r) != sizeof(void *)) + \ - COMPILE_ERROR_IF_TRUE(sizeof(*value_r) != sizeof(void *))), \ + (void *)((key_r) + \ + COMPILE_ERROR_IF_TYPES_NOT_COMPATIBLE((table)._keyp, key_r) + \ + COMPILE_ERROR_IF_TRUE(sizeof(*(key_r)) != sizeof(void *)) + \ + COMPILE_ERROR_IF_TRUE(sizeof(*(value_r)) != sizeof(void *))), \ (void *)((value_r) + COMPILE_ERROR_IF_TYPES_NOT_COMPATIBLE((table)._valuep, value_r))) #else /* C++ requires (void **) casting, but that's not possible with strict From dovecot at dovecot.org Fri Nov 14 22:15:38 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 14 Nov 2014 22:15:38 +0000 Subject: dovecot-2.2: lib: array/hash - enable typof checks for more comp... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/77638cc62ca8 changeset: 18090:77638cc62ca8 user: Phil Carmody date: Sat Nov 15 00:14:20 2014 +0200 description: lib: array/hash - enable typof checks for more compilers We have a HAVE_TYPEOF macro from autoconf, we may as well make use of it. Signed-off-by: Phil Carmody diffstat: src/lib/array.h | 4 ++-- src/lib/hash.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diffs (33 lines): diff -r 922ac3245e17 -r 77638cc62ca8 src/lib/array.h --- a/src/lib/array.h Sat Nov 15 00:06:09 2014 +0200 +++ b/src/lib/array.h Sat Nov 15 00:14:20 2014 +0200 @@ -2,7 +2,7 @@ #define ARRAY_H /* Array is a buffer accessible using fixed size elements. As long as the - compiler provides typeof() function, the array provides type safety. If + compiler provides a typeof() operator, the array provides type safety. If a wrong type is tried to be added to the array, or if the array's contents are tried to be used using a wrong type, the compiler will give a warning. @@ -39,7 +39,7 @@ #define t_array_init(array, init_count) \ p_array_init(array, pool_datastack_create(), init_count) -#ifdef __GNUC__ +#ifdef HAVE_TYPEOF # define ARRAY_TYPE_CAST_CONST(array) \ (typeof(*(array)->v)) # define ARRAY_TYPE_CAST_MODIFIABLE(array) \ diff -r 922ac3245e17 -r 77638cc62ca8 src/lib/hash.h --- a/src/lib/hash.h Sat Nov 15 00:06:09 2014 +0200 +++ b/src/lib/hash.h Sat Nov 15 00:14:20 2014 +0200 @@ -3,7 +3,7 @@ struct hash_table; -#ifdef __GNUC__ +#ifdef HAVE_TYPEOF # define HASH_VALUE_CAST(table) (typeof((table)._value)) #else # define HASH_VALUE_CAST(table) From dovecot at dovecot.org Fri Nov 14 22:31:24 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 14 Nov 2014 22:31:24 +0000 Subject: dovecot-2.2: lib-ssl-iostream: Close the plain iostreams when ss... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2d87ec863b76 changeset: 18091:2d87ec863b76 user: Timo Sirainen date: Sat Nov 15 00:27:27 2014 +0200 description: lib-ssl-iostream: Close the plain iostreams when ssl-iostream is destroyed. diffstat: src/lib-ssl-iostream/iostream-openssl.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (14 lines): diff -r 77638cc62ca8 -r 2d87ec863b76 src/lib-ssl-iostream/iostream-openssl.c --- a/src/lib-ssl-iostream/iostream-openssl.c Sat Nov 15 00:14:20 2014 +0200 +++ b/src/lib-ssl-iostream/iostream-openssl.c Sat Nov 15 00:27:27 2014 +0200 @@ -294,6 +294,10 @@ (void)SSL_shutdown(ssl_io->ssl); (void)openssl_iostream_more(ssl_io); (void)o_stream_flush(ssl_io->plain_output); + /* close the plain i/o streams, because their fd may be closed soon, + but we may still keep this ssl-iostream referenced until later. */ + i_stream_close(ssl_io->plain_input); + o_stream_close(ssl_io->plain_output); ssl_iostream_unref(&ssl_io); } From dovecot at dovecot.org Fri Nov 14 22:31:29 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 14 Nov 2014 22:31:29 +0000 Subject: dovecot-2.2: lib-master: Don't even attempt to initialize SSL co... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ef8b7e44e96c changeset: 18092:ef8b7e44e96c user: Timo Sirainen date: Sat Nov 15 00:30:06 2014 +0200 description: lib-master: Don't even attempt to initialize SSL context if ssl=no diffstat: src/lib-master/master-service-ssl.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (14 lines): diff -r 2d87ec863b76 -r ef8b7e44e96c src/lib-master/master-service-ssl.c --- a/src/lib-master/master-service-ssl.c Sat Nov 15 00:27:27 2014 +0200 +++ b/src/lib-master/master-service-ssl.c Sat Nov 15 00:30:06 2014 +0200 @@ -106,6 +106,10 @@ i_assert(service->listeners != NULL || service->socket_count == 0); set = master_service_ssl_settings_get(service); + if (strcmp(set->ssl, "no") == 0) { + /* SSL disabled, don't use it */ + return; + } memset(&ssl_set, 0, sizeof(ssl_set)); ssl_set.protocols = set->ssl_protocols; From dovecot at dovecot.org Sat Nov 15 00:24:17 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 15 Nov 2014 00:24:17 +0000 Subject: dovecot-2.2: lib-fs: Added fs_lookup_metadata() wrapper to fs_ge... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/372de41933c0 changeset: 18093:372de41933c0 user: Timo Sirainen date: Sat Nov 15 02:22:56 2014 +0200 description: lib-fs: Added fs_lookup_metadata() wrapper to fs_get_metadata() diffstat: src/lib-fs/fs-api.c | 18 ++++++++++++++++++ src/lib-fs/fs-api.h | 4 ++++ 2 files changed, 22 insertions(+), 0 deletions(-) diffs (42 lines): diff -r ef8b7e44e96c -r 372de41933c0 src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Sat Nov 15 00:30:06 2014 +0200 +++ b/src/lib-fs/fs-api.c Sat Nov 15 02:22:56 2014 +0200 @@ -265,6 +265,24 @@ return ret; } +int fs_lookup_metadata(struct fs_file *file, const char *key, + const char **value_r) +{ + const ARRAY_TYPE(fs_metadata) *metadata; + const struct fs_metadata *md; + + if (fs_get_metadata(file, &metadata) < 0) + return -1; + array_foreach(metadata, md) { + if (strcmp(md->key, key) == 0) { + *value_r = md->value; + return 1; + } + } + *value_r = NULL; + return 0; +} + const char *fs_file_path(struct fs_file *file) { return file->fs->v.get_path == NULL ? file->path : diff -r ef8b7e44e96c -r 372de41933c0 src/lib-fs/fs-api.h --- a/src/lib-fs/fs-api.h Sat Nov 15 00:30:06 2014 +0200 +++ b/src/lib-fs/fs-api.h Sat Nov 15 02:22:56 2014 +0200 @@ -132,6 +132,10 @@ /* Return file's all metadata. */ int fs_get_metadata(struct fs_file *file, const ARRAY_TYPE(fs_metadata) **metadata_r); +/* Wrapper to fs_get_metadata() to lookup a specific key. Returns 1 if value_r + is set, 0 if key wasn't found, -1 if error. */ +int fs_lookup_metadata(struct fs_file *file, const char *key, + const char **value_r); /* Returns the path given to fs_open(). If file was opened with FS_OPEN_MODE_CREATE_UNIQUE_128 and the write has already finished, From dovecot at dovecot.org Sat Nov 15 00:28:57 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 15 Nov 2014 00:28:57 +0000 Subject: dovecot-2.2: lib-imap-storage: Created new METADATA API. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a350812e07bf changeset: 18094:a350812e07bf user: Stephan Bosch date: Sat Nov 15 02:27:27 2014 +0200 description: lib-imap-storage: Created new METADATA API. diffstat: dovecot-config.in.in | 2 +- src/imap/Makefile.am | 2 - src/imap/cmd-getmetadata.c | 147 +++++++++------- src/imap/cmd-setmetadata.c | 140 +++++++++------ src/imap/imap-metadata.c | 85 --------- src/imap/imap-metadata.h | 13 - src/lib-imap-storage/Makefile.am | 6 +- src/lib-imap-storage/imap-metadata.c | 307 +++++++++++++++++++++++++++++++++++ src/lib-imap-storage/imap-metadata.h | 57 ++++++ 9 files changed, 531 insertions(+), 228 deletions(-) diffs (truncated from 1068 to 300 lines): diff -r 372de41933c0 -r a350812e07bf dovecot-config.in.in --- a/dovecot-config.in.in Sat Nov 15 02:22:56 2014 +0200 +++ b/dovecot-config.in.in Sat Nov 15 02:27:27 2014 +0200 @@ -23,7 +23,7 @@ LIBDOVECOT_INCLUDE="-I$(incdir) -I$(incdir)/src/lib -I$(incdir)/src/lib-dict -I$(incdir)/src/lib-dns -I$(incdir)/src/lib-http -I$(incdir)/src/lib-mail -I$(incdir)/src/lib-imap -I$(incdir)/src/lib-fs -I$(incdir)/src/lib-charset -I$(incdir)/src/lib-auth -I$(incdir)/src/lib-master -I$(incdir)/src/lib-ssl-iostream -I$(incdir)/src/lib-compression -I$(incdir)/src/lib-settings -I$(incdir)/src/lib-test" LIBDOVECOT_LDA_INCLUDE="-I$(incdir)/src/lib-lda -I$(incdir)/src/lda" LIBDOVECOT_DOVEADM_INCLUDE="-I$(incdir)/src/doveadm" -LIBDOVECOT_STORAGE_INCLUDE="-I$(incdir)/src/lib-index -I$(incdir)/src/lib-storage -I$(incdir)/src/lib-storage/list -I$(incdir)/src/lib-storage/index -I$(incdir)/src/lib-storage/index/raw -I$(incdir)/src/plugins/quota" +LIBDOVECOT_STORAGE_INCLUDE="-I$(incdir)/src/lib-index -I$(incdir)/src/lib-storage -I$(incdir)/src/lib-storage/list -I$(incdir)/src/lib-storage/index -I$(incdir)/src/lib-storage/index/raw -I$(incdir)/src/lib-imap-storage -I$(incdir)/src/plugins/quota" LIBDOVECOT_DSYNC_INCLUDE="-I$(incdir)/src/doveadm/dsync" LIBDOVECOT_LOGIN_INCLUDE="-I$(incdir)/src/login-common" LIBDOVECOT_IMAP_INCLUDE="-I$(incdir)/src/imap" diff -r 372de41933c0 -r a350812e07bf src/imap/Makefile.am --- a/src/imap/Makefile.am Sat Nov 15 02:22:56 2014 +0200 +++ b/src/imap/Makefile.am Sat Nov 15 02:27:27 2014 +0200 @@ -72,7 +72,6 @@ imap-fetch.c \ imap-fetch-body.c \ imap-list.c \ - imap-metadata.c \ imap-notify.c \ imap-search.c \ imap-search-args.c \ @@ -90,7 +89,6 @@ imap-expunge.h \ imap-fetch.h \ imap-list.h \ - imap-metadata.h \ imap-notify.h \ imap-search.h \ imap-search-args.h \ diff -r 372de41933c0 -r a350812e07bf src/imap/cmd-getmetadata.c --- a/src/imap/cmd-getmetadata.c Sat Nov 15 02:22:56 2014 +0200 +++ b/src/imap/cmd-getmetadata.c Sat Nov 15 02:27:27 2014 +0200 @@ -13,7 +13,7 @@ struct client_command_context *cmd; struct mailbox *box; - struct mailbox_transaction_context *trans; + struct imap_metadata_transaction *trans; struct mailbox_list_iterate_context *list_iter; ARRAY_TYPE(const_string) entries; @@ -24,16 +24,16 @@ struct istream *cur_stream; uoff_t cur_stream_offset, cur_stream_size; - struct mailbox_attribute_iter *iter; + struct imap_metadata_iter *iter; string_t *iter_entry_prefix; - const char *key_prefix; unsigned int entry_idx; bool first_entry_sent; bool failed; }; -static bool cmd_getmetadata_iter_next(struct imap_getmetadata_context *ctx); +static bool +cmd_getmetadata_mailbox_iter_next(struct imap_getmetadata_context *ctx); static bool cmd_getmetadata_parse_options(struct imap_getmetadata_context *ctx, @@ -81,7 +81,7 @@ imap_metadata_parse_entry_names(struct imap_getmetadata_context *ctx, const struct imap_arg *entries) { - const char *value; + const char *value, *error; p_array_init(&ctx->entries, ctx->cmd->pool, 4); for (; !IMAP_ARG_IS_EOL(entries); entries++) { @@ -89,8 +89,10 @@ client_send_command_error(ctx->cmd, "Entry isn't astring"); return FALSE; } - if (!imap_metadata_verify_entry_name(ctx->cmd, value)) + if (!imap_metadata_verify_entry_name(value, &error)) { + client_send_command_error(ctx->cmd, error); return FALSE; + } /* names are case-insensitive so we'll always lowercase them */ value = p_strdup(ctx->cmd->pool, t_str_lcase(value)); @@ -110,9 +112,14 @@ ctx->first_entry_sent = TRUE; str_append(str, "* METADATA "); - if (imap_utf8_to_utf7(mailbox_get_vname(ctx->box), mailbox_mutf7) < 0) - i_unreached(); - imap_append_astring(str, str_c(mailbox_mutf7)); + if (ctx->box == NULL) { + /* server metadata reply */ + str_append(str, "\"\""); + } else { + if (imap_utf8_to_utf7(mailbox_get_vname(ctx->box), mailbox_mutf7) < 0) + i_unreached(); + imap_append_astring(str, str_c(mailbox_mutf7)); + } str_append(str, " ("); /* nothing can be sent until untagged METADATA is finished */ @@ -140,33 +147,22 @@ static void cmd_getmetadata_send_entry(struct imap_getmetadata_context *ctx, const char *entry, bool require_reply) { - enum mail_attribute_type type; + struct client *client = ctx->cmd->client; struct mail_attribute_value value; + const char *error_string; enum mail_error error; uoff_t value_len; - const char *key; string_t *str; - imap_metadata_entry2key(entry, ctx->key_prefix, &type, &key); - if (ctx->key_prefix == NULL && - strncmp(key, MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT, - strlen(MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT)) == 0) { - /* skip over dovecot's internal attributes. (if key_prefix - isn't NULL, we're getting server metadata, which is handled - inside the private metadata.) */ - if (require_reply) - cmd_getmetadata_send_nil_reply(ctx, entry); - return; - } - - if (mailbox_attribute_get_stream(ctx->trans, type, key, &value) < 0) { - (void)mailbox_get_last_error(ctx->box, &error); + if (imap_metadata_get_stream(ctx->trans, entry, &value) < 0) { + error_string = imap_metadata_transaction_get_last_error( + ctx->trans, &error); if (error != MAIL_ERROR_NOTFOUND && error != MAIL_ERROR_PERM) { - client_send_untagged_storage_error(ctx->cmd->client, - mailbox_get_storage(ctx->box)); + client_send_line(client, t_strconcat("* NO ", error_string, NULL)); ctx->failed = TRUE; } } + if (value.value != NULL) value_len = strlen(value.value); else if (value.value_stream != NULL) { @@ -198,10 +194,10 @@ str = metadata_add_entry(ctx, entry); if (value.value != NULL) { str_printfa(str, " {%"PRIuUOFF_T"}\r\n%s", value_len, value.value); - o_stream_send(ctx->cmd->client->output, str_data(str), str_len(str)); + o_stream_nsend(client->output, str_data(str), str_len(str)); } else { str_printfa(str, " ~{%"PRIuUOFF_T"}\r\n", value_len); - o_stream_send(ctx->cmd->client->output, str_data(str), str_len(str)); + o_stream_nsend(client->output, str_data(str), str_len(str)); ctx->cur_stream_offset = 0; ctx->cur_stream_size = value_len; @@ -245,35 +241,39 @@ return FALSE; } -static int cmd_getmetadata_send_entry_tree(struct imap_getmetadata_context *ctx, +static int +cmd_getmetadata_send_entry_tree(struct imap_getmetadata_context *ctx, const char *entry) { - const char *key; - enum mail_attribute_type type; + struct client *client = ctx->cmd->client; - if (o_stream_get_buffer_used_size(ctx->cmd->client->output) >= + if (o_stream_get_buffer_used_size(client->output) >= CLIENT_OUTPUT_OPTIMAL_SIZE) { - if (o_stream_flush(ctx->cmd->client->output) <= 0) { - o_stream_set_flush_pending(ctx->cmd->client->output, TRUE); + if (o_stream_flush(client->output) <= 0) { + o_stream_set_flush_pending(client->output, TRUE); return 0; } } if (ctx->iter != NULL) { + const char *subentry; + /* DEPTH iteration */ do { - key = mailbox_attribute_iter_next(ctx->iter); - if (key == NULL) { + subentry = imap_metadata_iter_next(ctx->iter); + if (subentry == NULL) { /* iteration finished, get to the next entry */ - if (mailbox_attribute_iter_deinit(&ctx->iter) < 0) { - client_send_untagged_storage_error(ctx->cmd->client, - mailbox_get_storage(ctx->box)); + if (imap_metadata_iter_deinit(&ctx->iter) < 0) { + enum mail_error error; + client_send_line(client, t_strconcat("* NO ", + imap_metadata_transaction_get_last_error(ctx->trans, &error), + NULL)); ctx->failed = TRUE; } return -1; } - } while (ctx->depth == 1 && strchr(key, '/') != NULL); - entry = t_strconcat(str_c(ctx->iter_entry_prefix), key, NULL); + } while (ctx->depth == 1 && strchr(subentry, '/') != NULL); + entry = t_strconcat(str_c(ctx->iter_entry_prefix), subentry, NULL); } cmd_getmetadata_send_entry(ctx, entry, ctx->iter == NULL); @@ -295,21 +295,19 @@ str_append(ctx->iter_entry_prefix, entry); str_append_c(ctx->iter_entry_prefix, '/'); - imap_metadata_entry2key(entry, ctx->key_prefix, &type, &key); - ctx->iter = mailbox_attribute_iter_init(ctx->box, type, - key[0] == '\0' ? "" : t_strconcat(key, "/", NULL)); + ctx->iter = imap_metadata_iter_init(ctx->trans, entry); return 1; } } -static void cmd_getmetadata_mailbox_deinit(struct imap_getmetadata_context *ctx) +static void cmd_getmetadata_iter_deinit(struct imap_getmetadata_context *ctx) { if (ctx->iter != NULL) - (void)mailbox_attribute_iter_deinit(&ctx->iter); - if (ctx->box != NULL) { - (void)mailbox_transaction_commit(&ctx->trans); + (void)imap_metadata_iter_deinit(&ctx->iter); + if (ctx->trans != NULL) + (void)imap_metadata_transaction_commit(&ctx->trans, NULL, NULL); + if (ctx->box != NULL) mailbox_free(&ctx->box); - } ctx->first_entry_sent = FALSE; ctx->entry_idx = 0; } @@ -318,7 +316,7 @@ { struct client_command_context *cmd = ctx->cmd; - cmd_getmetadata_mailbox_deinit(ctx); + cmd_getmetadata_iter_deinit(ctx); cmd->client->output_cmd_lock = NULL; if (ctx->list_iter != NULL && @@ -366,33 +364,46 @@ if (ctx->first_entry_sent) o_stream_nsend_str(cmd->client->output, ")\r\n"); - cmd_getmetadata_mailbox_deinit(ctx); + cmd_getmetadata_iter_deinit(ctx); if (ctx->list_iter != NULL) - return cmd_getmetadata_iter_next(ctx); + return cmd_getmetadata_mailbox_iter_next(ctx); cmd_getmetadata_deinit(ctx); return TRUE; } -static int -cmd_getmetadata_try_mailbox(struct imap_getmetadata_context *ctx, - struct mail_namespace *ns, const char *mailbox) +static bool +cmd_getmetadata_start(struct imap_getmetadata_context *ctx) { struct client_command_context *cmd = ctx->cmd; - ctx->box = mailbox_alloc(ns->list, mailbox, MAILBOX_FLAG_READONLY); - if (mailbox_open(ctx->box) < 0) - return -1; - ctx->trans = mailbox_transaction_begin(ctx->box, 0); - if (ctx->depth > 0) ctx->iter_entry_prefix = str_new(cmd->pool, 128); if (!cmd_getmetadata_continue(cmd)) { cmd->state = CLIENT_COMMAND_STATE_WAIT_OUTPUT; cmd->func = cmd_getmetadata_continue; - return 0; + return FALSE; } - return 1; + return TRUE; +} + +static bool +cmd_getmetadata_server(struct imap_getmetadata_context *ctx) +{ + ctx->trans = imap_metadata_transaction_begin_server(ctx->cmd->client->user); + return cmd_getmetadata_start(ctx); +} + +static int +cmd_getmetadata_try_mailbox(struct imap_getmetadata_context *ctx, + struct mail_namespace *ns, const char *mailbox) +{ + ctx->box = mailbox_alloc(ns->list, mailbox, MAILBOX_FLAG_READONLY); + if (mailbox_open(ctx->box) < 0) + return -1; + From dovecot at dovecot.org Sat Nov 15 01:27:35 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 15 Nov 2014 01:27:35 +0000 Subject: dovecot-2.2: lib: connection: Now uses io_add_istream() instead ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7ef360d3fb09 changeset: 18095:7ef360d3fb09 user: Stephan Bosch date: Sat Nov 15 02:36:22 2014 +0200 description: lib: connection: Now uses io_add_istream() instead of io_add(). diffstat: src/lib/connection.c | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (21 lines): diff -r a350812e07bf -r 7ef360d3fb09 src/lib/connection.c --- a/src/lib/connection.c Sat Nov 15 02:27:27 2014 +0200 +++ b/src/lib/connection.c Sat Nov 15 02:36:22 2014 +0200 @@ -124,7 +124,7 @@ o_stream_set_no_error_handling(conn->output, TRUE); o_stream_set_name(conn->output, conn->name); } - conn->io = io_add(conn->fd_in, IO_READ, *conn->list->v.input, conn); + conn->io = io_add_istream(conn->input, *conn->list->v.input, conn); if (set->input_idle_timeout_secs != 0) { conn->to = timeout_add(set->input_idle_timeout_secs*1000, connection_idle_timeout, conn); @@ -225,7 +225,7 @@ o_stream_set_no_error_handling(conn->output, TRUE); o_stream_set_name(conn->output, conn->name); - conn->io = io_add(conn->fd_in, IO_READ, *list->v.input, conn); + conn->io = io_add_istream(conn->input, *list->v.input, conn); DLLIST_PREPEND(&list->connections, conn); list->connections_count++; From dovecot at dovecot.org Sat Nov 15 01:27:36 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 15 Nov 2014 01:27:36 +0000 Subject: dovecot-2.2: lib-http: client: Removed connection code that is a... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8eaaff8e2c63 changeset: 18096:8eaaff8e2c63 user: Stephan Bosch date: Sat Nov 15 02:37:03 2014 +0200 description: lib-http: client: Removed connection code that is already in src/lib/connection.c. diffstat: src/lib-http/http-client-connection.c | 38 +---------------------------------- 1 files changed, 1 insertions(+), 37 deletions(-) diffs (55 lines): diff -r 7ef360d3fb09 -r 8eaaff8e2c63 src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Sat Nov 15 02:36:22 2014 +0200 +++ b/src/lib-http/http-client-connection.c Sat Nov 15 02:37:03 2014 +0200 @@ -1142,42 +1142,6 @@ http_client_connection_close(&conn); } -// FIXME: put something like this in lib/connection.c -static void -_connection_init_from_streams(struct connection_list *list, - struct connection *conn, const char *name, - struct istream *input, struct ostream *output) -{ - i_assert(name != NULL); - - conn->list = list; - conn->name = i_strdup(name); - conn->fd_in = i_stream_get_fd(input); - conn->fd_out = o_stream_get_fd(output); - - i_assert(conn->fd_in >= 0); - i_assert(conn->fd_out >= 0); - i_assert(conn->io == NULL); - i_assert(conn->input == NULL); - i_assert(conn->output == NULL); - i_assert(conn->to == NULL); - - conn->input = input; - i_stream_set_name(conn->input, conn->name); - - conn->output = output; - o_stream_set_no_error_handling(conn->output, TRUE); - o_stream_set_name(conn->output, conn->name); - - conn->io = io_add_istream(conn->input, *list->v.input, conn); - - DLLIST_PREPEND(&list->connections, conn); - list->connections_count++; - - if (list->v.client_connected != NULL) - list->v.client_connected(conn, TRUE); -} - static void http_client_connection_tunnel_response(const struct http_response *response, struct http_client_connection *conn) @@ -1196,7 +1160,7 @@ http_client_request_start_tunnel(conn->connect_request, &tunnel); conn->connect_request = NULL; - _connection_init_from_streams + connection_init_from_streams (conn->client->conn_list, &conn->conn, name, tunnel.input, tunnel.output); conn->connect_initialized = TRUE; } From dovecot at dovecot.org Sat Nov 15 01:27:56 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 15 Nov 2014 01:27:56 +0000 Subject: dovecot-2.2: Implemented dot output stream. This outputs data in... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2866a692ec47 changeset: 18097:2866a692ec47 user: Stephan Bosch date: Sat Nov 15 02:37:45 2014 +0200 description: Implemented dot output stream. This outputs data in the dot-escaped format needed for the SMTP data command. The sequence CRLF.CRLF is sent upon closing the stream. diffstat: src/lib-mail/Makefile.am | 7 + src/lib-mail/ostream-dot.c | 180 ++++++++++++++++++++++++++++++++++++++++ src/lib-mail/ostream-dot.h | 9 ++ src/lib-mail/test-ostream-dot.c | 77 +++++++++++++++++ 4 files changed, 273 insertions(+), 0 deletions(-) diffs (truncated from 316 to 300 lines): diff -r 8eaaff8e2c63 -r 2866a692ec47 src/lib-mail/Makefile.am --- a/src/lib-mail/Makefile.am Sat Nov 15 02:37:03 2014 +0200 +++ b/src/lib-mail/Makefile.am Sat Nov 15 02:37:45 2014 +0200 @@ -28,6 +28,7 @@ message-part-serialize.c \ message-search.c \ message-size.c \ + ostream-dot.c \ quoted-printable.c \ rfc2231-parser.c \ rfc822-parser.c @@ -56,6 +57,7 @@ message-part-serialize.h \ message-search.h \ message-size.h \ + ostream-dot.h \ quoted-printable.h \ rfc2231-parser.h \ rfc822-parser.h @@ -79,6 +81,7 @@ test-message-id \ test-message-parser \ test-message-part \ + test-ostream-dot \ test-quoted-printable \ test-rfc2231-parser @@ -157,6 +160,10 @@ test_message_part_LDADD = message-part.lo message-parser.lo message-header-parser.lo message-size.lo rfc822-parser.lo rfc2231-parser.lo $(test_libs) test_message_part_DEPENDENCIES = $(test_deps) +test_ostream_dot_SOURCES = test-ostream-dot.c +test_ostream_dot_LDADD = ostream-dot.lo $(test_libs) +test_ostream_dot_DEPENDENCIES = $(test_deps) + test_quoted_printable_SOURCES = test-quoted-printable.c test_quoted_printable_LDADD = quoted-printable.lo $(test_libs) test_quoted_printable_DEPENDENCIES = $(test_deps) diff -r 8eaaff8e2c63 -r 2866a692ec47 src/lib-mail/ostream-dot.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-mail/ostream-dot.c Sat Nov 15 02:37:45 2014 +0200 @@ -0,0 +1,180 @@ +/* Copyright (c) 2013-2014 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ostream-private.h" +#include "ostream-dot.h" + +enum dot_ostream_state { + STREAM_STATE_INIT = 0, + STREAM_STATE_NONE, + STREAM_STATE_CR, + STREAM_STATE_CRLF, +}; + +struct dot_ostream { + struct ostream_private ostream; + + enum dot_ostream_state state; +}; + +static void +o_stream_dot_close(struct iostream_private *stream, + bool close_parent) +{ + struct dot_ostream *dstream = (struct dot_ostream *)stream; + + if (dstream->state == STREAM_STATE_CRLF) + (void)o_stream_send(dstream->ostream.parent, ".\r\n", 3); + else + (void)o_stream_send(dstream->ostream.parent, "\r\n.\r\n", 5); + (void)o_stream_flush(&dstream->ostream.ostream); + + if (close_parent) + o_stream_close(dstream->ostream.parent); +} + +static ssize_t +o_stream_dot_sendv(struct ostream_private *stream, + const struct const_iovec *iov, unsigned int iov_count) +{ + struct dot_ostream *dstream = (struct dot_ostream *)stream; + ARRAY(struct const_iovec) iov_arr; + const struct const_iovec *iov_new; + size_t max_bytes, sent, added; + unsigned int count, i; + ssize_t ret; + + if ((ret=o_stream_flush(stream->parent)) <= 0) { + /* error / we still couldn't flush existing data to + parent stream. */ + o_stream_copy_error_from_parent(stream); + return ret; + } + + /* check for dots */ + t_array_init(&iov_arr, iov_count + 32); + max_bytes = o_stream_get_buffer_avail_size(stream->parent); // FIXME: what if max_buffer_size is 0? + sent = added = 0; + for (i = 0; i < iov_count && max_bytes > 0; i++) { + size_t size = iov[i].iov_len, chunk; + const char *data = iov[i].iov_base, *p, *pend; + struct const_iovec iovn; + + p = data; + pend = CONST_PTR_OFFSET(data, size); + for (; p < pend && (size_t)(p-data) < (max_bytes-2); p++) { + char add = 0; + + switch (dstream->state) { + /* none */ + case STREAM_STATE_NONE: + switch (*p) { + case '\n': + dstream->state = STREAM_STATE_CRLF; + /* add missing CR */ + add = '\r'; + break; + case '\r': + dstream->state = STREAM_STATE_CR; + break; + } + break; + /* got CR */ + case STREAM_STATE_CR: + switch (*p) { + case '\r': + break; + case '\n': + dstream->state = STREAM_STATE_CRLF; + break; + default: + dstream->state = STREAM_STATE_NONE; + break; + } + break; + /* got CRLF, or the first line */ + case STREAM_STATE_INIT: + case STREAM_STATE_CRLF: + switch (*p) { + case '\r': + dstream->state = STREAM_STATE_CR; + break; + case '\n': + dstream->state = STREAM_STATE_CRLF; + /* add missing CR */ + add = '\r'; + break; + case '.': + /* add dot */ + add = '.'; + default: + dstream->state = STREAM_STATE_NONE; + break; + } + break; + } + + if (add != 0) { + chunk = (size_t)(p - data); + if (chunk > 0) { + /* forward chunk to new iovec */ + iovn.iov_base = data; + iovn.iov_len = chunk; + array_append(&iov_arr, &iovn, 1); + data = p; + max_bytes -= chunk; + sent += chunk; + } + /* insert byte (substitute one with pair) */ + data++; + iovn.iov_base = (add == '\r' ? "\r\n" : ".."); + iovn.iov_len = 2; + array_append(&iov_arr, &iovn, 1); + max_bytes -= 2; + added++; + sent++; + } + } + + if (max_bytes == 0) + break; + chunk = ((size_t)(p-data) >= (max_bytes-2) ? + max_bytes - 2 : (size_t)(p - data)); + if (chunk > 0) { + iovn.iov_base = data; + iovn.iov_len = chunk; + array_append(&iov_arr, &iovn, 1); + max_bytes -= chunk; + sent += chunk; + } + } + + /* send */ + iov_new = array_get(&iov_arr, &count); + if (count == 0) { + ret = 0; + } else if ((ret=o_stream_sendv(stream->parent, iov_new, count)) <= 0) { + i_assert(ret < 0); + o_stream_copy_error_from_parent(stream); + return -1; + } + + /* all must be sent */ + i_assert((size_t)ret == sent + added); + + stream->ostream.offset += sent; + return sent; +} + +struct ostream * +o_stream_create_dot(struct ostream *output) +{ + struct dot_ostream *dstream; + + dstream = i_new(struct dot_ostream, 1); + dstream->ostream.sendv = o_stream_dot_sendv; + dstream->ostream.iostream.close = o_stream_dot_close; + dstream->ostream.max_buffer_size = output->real_stream->max_buffer_size; + return o_stream_create(&dstream->ostream, output, o_stream_get_fd(output)); +} diff -r 8eaaff8e2c63 -r 2866a692ec47 src/lib-mail/ostream-dot.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-mail/ostream-dot.h Sat Nov 15 02:37:45 2014 +0200 @@ -0,0 +1,9 @@ +#ifndef OSTREAM_DOT_H +#define OSTREAM_DOT_H + +/* Create output stream for writing SMTP DATA style message: Add additional "." + to lines beginning with it. Write line that contains only "." upon close(). + */ +struct ostream *o_stream_create_dot(struct ostream *output); + +#endif diff -r 8eaaff8e2c63 -r 2866a692ec47 src/lib-mail/test-ostream-dot.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-mail/test-ostream-dot.c Sat Nov 15 02:37:45 2014 +0200 @@ -0,0 +1,77 @@ +/* Copyright (c) 2013-2014 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "buffer.h" +#include "str.h" +#include "istream.h" +#include "ostream.h" +#include "ostream-dot.h" +#include "test-common.h" + +struct dot_test { + const char *input; + const char *output; +}; + +static void test_ostream_dot_one(const struct dot_test *test) +{ + struct istream *test_input; + struct ostream *output, *test_output; + buffer_t *output_data; + const unsigned char *data; + size_t size; + ssize_t ret; + + test_input = test_istream_create(test->input); + output_data = buffer_create_dynamic(pool_datastack_create(), 1024); + test_output = o_stream_create_buffer(output_data); + + output = o_stream_create_dot(test_output); + + while ((ret = i_stream_read(test_input)) > 0 || ret == -2) { + data = i_stream_get_data(test_input, &size); + ret = o_stream_send(output, data, size); + test_assert(ret >= 0); + if (ret <= 0) + break; + i_stream_skip(test_input, ret); + } + + test_assert(test_input->eof); + + o_stream_unref(&output); + o_stream_unref(&test_output); + + test_assert(strcmp(str_c(output_data), test->output) == 0); + + i_stream_unref(&test_input); +} + +static void test_ostream_dot(void) +{ + static struct dot_test tests[] = { + { "foo\r\n.\r\n", "foo\r\n..\r\n.\r\n" }, + { "foo\n.\n", "foo\r\n..\r\n.\r\n" }, + { ".foo\r\n.\r\nfoo\r\n", "..foo\r\n..\r\nfoo\r\n.\r\n" }, + { ".foo\n.\nfoo\n", "..foo\r\n..\r\nfoo\r\n.\r\n" }, + { "\r\n", "\r\n.\r\n" }, + { "\n", "\r\n.\r\n" }, + { "", "\r\n.\r\n" }, + }; + unsigned int i; From dovecot at dovecot.org Sat Nov 15 01:28:28 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 15 Nov 2014 01:28:28 +0000 Subject: dovecot-2.2: Added support for base32 encode/decode. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1663b76430e1 changeset: 18098:1663b76430e1 user: Stephan Bosch date: Sat Nov 15 02:39:04 2014 +0200 description: Added support for base32 encode/decode. diffstat: src/lib/Makefile.am | 3 + src/lib/base32.c | 326 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/lib/base32.h | 47 +++++++ src/lib/test-base32.c | 190 +++++++++++++++++++++++++++++ src/lib/test-lib.c | 1 + src/lib/test-lib.h | 1 + 6 files changed, 568 insertions(+), 0 deletions(-) diffs (truncated from 624 to 300 lines): diff -r 2866a692ec47 -r 1663b76430e1 src/lib/Makefile.am --- a/src/lib/Makefile.am Sat Nov 15 02:37:45 2014 +0200 +++ b/src/lib/Makefile.am Sat Nov 15 02:39:04 2014 +0200 @@ -16,6 +16,7 @@ aqueue.c \ askpass.c \ backtrace-string.c \ + base32.c \ base64.c \ bits.c \ bsearch-insert-pos.c \ @@ -149,6 +150,7 @@ array-decl.h \ askpass.h \ backtrace-string.h \ + base32.h \ base64.h \ bits.h \ bsearch-insert-pos.h \ @@ -278,6 +280,7 @@ test-lib.c \ test-array.c \ test-aqueue.c \ + test-base32.c \ test-base64.c \ test-bits.c \ test-bsearch-insert-pos.c \ diff -r 2866a692ec47 -r 1663b76430e1 src/lib/base32.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/base32.c Sat Nov 15 02:39:04 2014 +0200 @@ -0,0 +1,326 @@ +/* Copyright (c) 2007-2014 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "base32.h" +#include "buffer.h" + +static const char b32enc[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; + +static const char b32hexenc[] = + "0123456789ABCDEFGHIJKLMNOPQRSTUV"; + +static const unsigned char b32dec[256] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0-7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 8-15 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 16-23 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 24-31 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 32-39 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 40-47 */ + 0xff, 0xff, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 48-55 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 56-63 */ + 0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, /* 64-71 */ + 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, /* 72-79 */ + 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, /* 80-87 */ + 0x17, 0x18, 0x19, 0xff, 0xff, 0xff, 0xff, 0xff, /* 88-95 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 96-103 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 104-111 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 112-119 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 120-127 */ + + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 128-255 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +static const unsigned char b32hexdec[256] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 0-7 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 8-15 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 16-23 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 24-31 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 32-39 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 40-47 */ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 48-55 */ + 0x08, 0x09, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 56-63 */ + 0xff, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, /* 64-71 */ + 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, /* 72-79 */ + 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0xff, /* 80-87 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 88-95 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 96-103 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 104-111 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 112-119 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 120-127 */ + + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, /* 128-255 */ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +static void +base32_encode_with_alphabet(const char *alph, + bool pad, const void *src, size_t src_size, buffer_t *dest) +{ + const unsigned char *src_c = src; + unsigned char tmp[8], endb; + size_t src_pos; + + /* [5 3][2 5 1][4 4][1 5 2][3 5] + (5)(3 2)(5)(1 4)(4 1)(5)(2 3)(5) + */ + + /* encode main part */ + for (src_pos = 0; src_pos + 4 < src_size; src_pos += 5) { + tmp[0] = alph[src_c[src_pos] >> 3]; + tmp[1] = alph[((src_c[src_pos] & 0x07) << 2) | + ((src_c[src_pos+1] >> 6) & 0x03)]; + tmp[2] = alph[((src_c[src_pos+1] >> 1) & 0x1f)]; + tmp[3] = alph[((src_c[src_pos+1] & 0x01) << 4) | + (src_c[src_pos+2] >> 4)]; + tmp[4] = alph[((src_c[src_pos+2] & 0x0f) << 1) | + (src_c[src_pos+3] >> 7)]; + tmp[5] = alph[((src_c[src_pos+3] >> 2) & 0x1f)]; + tmp[6] = alph[((src_c[src_pos+3] & 0x03) << 3) | + (src_c[src_pos+4] >> 5)]; + tmp[7] = alph[src_c[src_pos+4] & 0x1f]; + buffer_append(dest, tmp, 8); + } + + /* encode last < 5 bytes if any */ + if (src_pos < src_size) { + tmp[0] = alph[src_c[src_pos] >> 3]; + switch (src_size - src_pos) { + case 1: + tmp[1] = alph[((src_c[src_pos] & 0x07) << 2)]; + endb = 2; + break; + case 2: + tmp[1] = alph[((src_c[src_pos] & 0x07) << 2) | + ((src_c[src_pos+1] >> 6) & 0x03)]; + tmp[2] = alph[((src_c[src_pos+1] >> 1) & 0x1f)]; + tmp[3] = alph[((src_c[src_pos+1] & 0x01) << 4)]; + endb = 4; + break; + case 3: + tmp[1] = alph[((src_c[src_pos] & 0x07) << 2) | + ((src_c[src_pos+1] >> 6) & 0x03)]; + tmp[2] = alph[((src_c[src_pos+1] >> 1) & 0x1f)]; + tmp[3] = alph[((src_c[src_pos+1] & 0x01) << 4) | + (src_c[src_pos+2] >> 4)]; + tmp[4] = alph[((src_c[src_pos+2] & 0x0f) << 1)]; + endb = 5; + break; + case 4: + tmp[1] = alph[((src_c[src_pos] & 0x07) << 2) | + ((src_c[src_pos+1] >> 6) & 0x03)]; + tmp[2] = alph[((src_c[src_pos+1] >> 1) & 0x1f)]; + tmp[3] = alph[((src_c[src_pos+1] & 0x01) << 4) | + (src_c[src_pos+2] >> 4)]; + tmp[4] = alph[((src_c[src_pos+2] & 0x0f) << 1) | + (src_c[src_pos+3] >> 7)]; + tmp[5] = alph[((src_c[src_pos+3] >> 2) & 0x1f)]; + tmp[6] = alph[((src_c[src_pos+3] & 0x03) << 3)]; + endb = 7; + break; + default: + i_unreached(); + } + + /* add padding if required */ + if (pad) { + memset(&tmp[endb], '=', sizeof(tmp)-endb); + buffer_append(dest, tmp, 8); + } else { + buffer_append(dest, tmp, endb); + } + } +} + +void base32_encode(bool pad, const void *src, size_t src_size, + buffer_t *dest) +{ + base32_encode_with_alphabet(b32enc, pad, src, src_size, dest); +} + +void base32hex_encode(bool pad, const void *src, size_t src_size, + buffer_t *dest) +{ + base32_encode_with_alphabet(b32hexenc, pad, src, src_size, dest); +} + +#define IS_EMPTY(c) \ + ((c) == '\n' || (c) == '\r' || (c) == ' ' || (c) == '\t') + +static int +base32_decode_with_alphabet(const unsigned char *alph, + const void *src, size_t src_size, size_t *src_pos_r, + buffer_t *dest) +{ + const unsigned char *src_c = src; + size_t block_pos, src_pos; + unsigned char output[5], ipos, opos; + int ret = 1; + + /* (5)(3 2)(5)(1 4)(4 1)(5)(2 3)(5) + [5 3][2 5 1][4 4][1 5 2][3 5] + */ + ipos = opos = 0; + block_pos = 0; + for (src_pos = 0; src_pos < src_size; src_pos++) { + unsigned char input = alph[src_c[src_pos]]; + + if (input == 0xff) { + if (unlikely(!IS_EMPTY(src_c[src_pos]))) + break; + continue; + } + + ipos++; + switch (ipos) { + case 1: + output[0] = input << 3; + opos = 0; + break; + case 2: + output[0] |= input >> 2; + output[1] = (input & 0x03) << 6; + opos = 1; + break; + case 3: + output[1] |= input << 1; + opos = 1; + break; + case 4: + output[1] |= input >> 4; + output[2] = (input & 0x0f) << 4; + opos = 2; + break; + case 5: + output[2] |= input >> 1; + output[3] = (input & 0x01) << 7; + opos = 3; + break; + case 6: + output[3] |= input << 2; + opos = 3; + break; + case 7: + output[3] |= input >> 3; + output[4] = ((input & 0x07) << 5); + opos = 4; + break; + case 8: + output[4] |= input; + buffer_append(dest, output, 5); + ipos = 0; + opos = 0; + block_pos = src_pos; + break; + default: + i_unreached(); + } + } + + if (ipos > 0) { + for (; src_pos < src_size; src_pos++) { + if (src_c[src_pos] != '=') { + if (unlikely(!IS_EMPTY(src_c[src_pos]))) { + ret = -1; + break; + } + continue; + } + if (++ipos >= 8) { + buffer_append(dest, output, opos); + ipos = 0; + ret = 0; + src_pos++; + break; + } + } + } + + if (src_pos_r != NULL) { + if (ipos == 0) { From pigeonhole at rename-it.nl Sun Nov 16 22:16:50 2014 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 16 Nov 2014 23:16:50 +0100 Subject: dovecot-2.2-pigeonhole: Merged concurrent changes. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/0677417707c0 changeset: 1967:0677417707c0 user: Stephan Bosch date: Sun Nov 16 23:16:38 2014 +0100 description: Merged concurrent changes. diffstat: README | 22 ++++++++++++++----- src/lib-sieve/sieve-actions.c | 47 +++++++++++++++++++++--------------------- src/lib-sieve/sieve-actions.h | 7 ++--- src/lib-sieve/sieve-binary.h | 2 +- src/lib-sieve/sieve-message.c | 12 +++++----- src/lib-sieve/sieve-message.h | 7 ++--- 6 files changed, 53 insertions(+), 44 deletions(-) diffs (218 lines): diff -r 4da1e84f32e4 -r 0677417707c0 README --- a/README Sun Nov 16 23:13:58 2014 +0100 +++ b/README Sun Nov 16 23:16:38 2014 +0100 @@ -110,6 +110,7 @@ subaddress (RFC 5233): fully supported, but with limited configurability. spamtest and virustest (RFC 5235): fully supported (v0.1.16+). date (RFC 5260; Section 4): fully supported (v0.1.12+). + index (RFC 5260; Section 6): fully supported (v0.4.7+). editheader (RFC 5293): fully supported (v0.3.0+). reject (RFC 5429; Section 2.2): fully supported. enotify (RFC 5435): fully supported (v0.1.3+). @@ -120,7 +121,7 @@ mailbox (RFC 5490; Section 3): fully supported (v0.1.10+), but ACL permissions are not verified for mailboxexists. include (RFC 6609): fully supported (v0.4.0+) - duplicate (draft v03): fully supported (v0.4.3+). + duplicate (RFC 7352): fully supported (v0.4.3+). regex (draft v08; not latest version): almost fully supported, but UTF-8 is not supported. @@ -135,30 +136,39 @@ The following Dovecot-specific Sieve extensions are available: vnd.dovecot.debug (v0.3.0+): - Allows logging debug messages + Allows logging debug messages. vnd.dovecot.pipe (v0.4.0+; sieve_extprograms plugin): - Implements piping messages to a pre-defined set of external programs + Implements piping messages to a pre-defined set of external programs. vnd.dovecot.filter (v0.4.0+; sieve_extprograms plugin): Implements filtering messages through a pre-defined set of external - programs + programs. vnd.dovecot.execute (v0.4.0+; sieve_extprograms plugin): Implements executing a pre-defined set of external programs with the - option to process string data through the external program + option to process string data through the external program. The following extensions are under development: + mboxmetadata and servermetadata (RFC 5490): partially implemented; + Dovecot adjustments needed. ereject (RFC 5429; page 4): implemented, but currently equal to reject Many more extensions to the language exist. Not all of these extensions are useful for Dovecot in particular, but many of them are. Currently, the author has taken notice of the following extensions: - index (RFC 5260; page 7): planned. foreverypart, mime, replace, enclose, and extracttext (RFC 5703): planned. + imapsieve (RFC 6785): planned. + envelope-dsn, envelope-deliverby, redirect-dsn and + redirect-deliverby (RFC 6009): planned; depends on lib-smtp changes in + Dovecot. + extlists (RFC 6134): under consideration. These extensions will be added as soon as the necessary infrastructure is available. +Check the TODO file for an up-to-date list of open issues and current +development. + Compiling and Configuring ========================= diff -r 4da1e84f32e4 -r 0677417707c0 src/lib-sieve/sieve-actions.c --- a/src/lib-sieve/sieve-actions.c Sun Nov 16 23:13:58 2014 +0100 +++ b/src/lib-sieve/sieve-actions.c Sun Nov 16 23:16:38 2014 +0100 @@ -37,29 +37,6 @@ const struct sieve_operand_class sieve_side_effect_operand_class = { "SIDE-EFFECT" }; -bool sieve_opr_side_effect_read -(const struct sieve_runtime_env *renv, sieve_size_t *address, - struct sieve_side_effect *seffect) -{ - const struct sieve_side_effect_def *sdef; - - seffect->context = NULL; - - if ( !sieve_opr_object_read - (renv, &sieve_side_effect_operand_class, address, &seffect->object) ) - return FALSE; - - sdef = seffect->def = - (const struct sieve_side_effect_def *) seffect->object.def; - - if ( sdef->read_context != NULL && - !sdef->read_context(seffect, renv, address, &seffect->context) ) { - return FALSE; - } - - return TRUE; -} - bool sieve_opr_side_effect_dump (const struct sieve_dumptime_env *denv, sieve_size_t *address) { @@ -84,6 +61,30 @@ return TRUE; } +int sieve_opr_side_effect_read +(const struct sieve_runtime_env *renv, sieve_size_t *address, + struct sieve_side_effect *seffect) +{ + const struct sieve_side_effect_def *sdef; + int ret; + + seffect->context = NULL; + + if ( !sieve_opr_object_read + (renv, &sieve_side_effect_operand_class, address, &seffect->object) ) + return SIEVE_EXEC_BIN_CORRUPT; + + sdef = seffect->def = + (const struct sieve_side_effect_def *) seffect->object.def; + + if ( sdef->read_context != NULL && (ret=sdef->read_context + (seffect, renv, address, &seffect->context)) <= 0 ) { + return ret; + } + + return SIEVE_EXEC_OK; +} + /* * Optional operands */ diff -r 4da1e84f32e4 -r 0677417707c0 src/lib-sieve/sieve-actions.h --- a/src/lib-sieve/sieve-actions.h Sun Nov 16 23:13:58 2014 +0100 +++ b/src/lib-sieve/sieve-actions.h Sun Nov 16 23:16:38 2014 +0100 @@ -187,13 +187,12 @@ sieve_opr_object_emit(sblock, ext, &seff->obj_def); } -bool sieve_opr_side_effect_read +bool sieve_opr_side_effect_dump + (const struct sieve_dumptime_env *denv, sieve_size_t *address); +int sieve_opr_side_effect_read (const struct sieve_runtime_env *renv, sieve_size_t *address, struct sieve_side_effect *seffect); -bool sieve_opr_side_effect_dump - (const struct sieve_dumptime_env *denv, sieve_size_t *address); - /* * Optional operands */ diff -r 4da1e84f32e4 -r 0677417707c0 src/lib-sieve/sieve-binary.h --- a/src/lib-sieve/sieve-binary.h Sun Nov 16 23:13:58 2014 +0100 +++ b/src/lib-sieve/sieve-binary.h Sun Nov 16 23:16:38 2014 +0100 @@ -13,7 +13,7 @@ */ #define SIEVE_BINARY_VERSION_MAJOR 1 -#define SIEVE_BINARY_VERSION_MINOR 2 +#define SIEVE_BINARY_VERSION_MINOR 3 /* * Binary object diff -r 4da1e84f32e4 -r 0677417707c0 src/lib-sieve/sieve-message.c --- a/src/lib-sieve/sieve-message.c Sun Nov 16 23:13:58 2014 +0100 +++ b/src/lib-sieve/sieve-message.c Sun Nov 16 23:16:38 2014 +0100 @@ -604,27 +604,27 @@ return TRUE; } -bool sieve_opr_message_override_read +int sieve_opr_message_override_read (const struct sieve_runtime_env *renv, sieve_size_t *address, struct sieve_message_override *svmo) { const struct sieve_message_override_def *hodef; + int ret; svmo->context = NULL; if ( !sieve_opr_object_read (renv, &sieve_message_override_operand_class, address, &svmo->object) ) - return FALSE; + return SIEVE_EXEC_BIN_CORRUPT; hodef = svmo->def = (const struct sieve_message_override_def *) svmo->object.def; if ( hodef->read_context != NULL && - !hodef->read_context(svmo, renv, address, &svmo->context) ) { - return FALSE; - } + (ret=hodef->read_context(svmo, renv, address, &svmo->context)) <= 0 ) + return ret; - return TRUE; + return SIEVE_EXEC_OK; } /* diff -r 4da1e84f32e4 -r 0677417707c0 src/lib-sieve/sieve-message.h --- a/src/lib-sieve/sieve-message.h Sun Nov 16 23:13:58 2014 +0100 +++ b/src/lib-sieve/sieve-message.h Sun Nov 16 23:16:38 2014 +0100 @@ -135,13 +135,12 @@ sieve_opr_object_emit(sblock, ext, &seff->obj_def); } -bool sieve_opr_message_override_read +bool sieve_opr_message_override_dump + (const struct sieve_dumptime_env *denv, sieve_size_t *address); +int sieve_opr_message_override_read (const struct sieve_runtime_env *renv, sieve_size_t *address, struct sieve_message_override *svmo); -bool sieve_opr_message_override_dump - (const struct sieve_dumptime_env *denv, sieve_size_t *address); - /* * Optional operands */ From pigeonhole at rename-it.nl Sun Nov 16 22:16:49 2014 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sun, 16 Nov 2014 23:16:49 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: Finished support for mboxmeta... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/4da1e84f32e4 changeset: 1966:4da1e84f32e4 user: Stephan Bosch date: Sun Nov 16 23:13:58 2014 +0100 description: lib-sieve: Finished support for mboxmetadata and servermetadata extensions. diffstat: src/lib-sieve/Makefile.am | 4 +- src/lib-sieve/plugins/Makefile.am | 3 +- src/lib-sieve/plugins/metadata/Makefile.am | 3 +- src/lib-sieve/plugins/metadata/ext-metadata-common.h | 4 + src/lib-sieve/plugins/metadata/tst-metadata.c | 147 ++++++++++++++++-- src/lib-sieve/plugins/metadata/tst-metadataexists.c | 137 ++++++++++++++--- src/lib-sieve/sieve-extensions.c | 7 +- src/sieve-tools/sieve-test.c | 2 +- 8 files changed, 256 insertions(+), 51 deletions(-) diffs (truncated from 518 to 300 lines): diff -r 1d06b61cbaf3 -r 4da1e84f32e4 src/lib-sieve/Makefile.am --- a/src/lib-sieve/Makefile.am Wed Nov 12 22:10:25 2014 +0100 +++ b/src/lib-sieve/Makefile.am Sun Nov 16 23:13:58 2014 +0100 @@ -44,8 +44,7 @@ if BUILD_UNFINISHED unfinished_storages = -unfinished_plugins = \ - $(extdir)/metadata/libsieve_ext_metadata.la +unfinished_plugins = endif strgdir = $(top_builddir)/src/lib-sieve/storage @@ -77,6 +76,7 @@ $(extdir)/editheader/libsieve_ext_editheader.la \ $(extdir)/duplicate/libsieve_ext_duplicate.la \ $(extdir)/index/libsieve_ext_index.la \ + $(extdir)/metadata/libsieve_ext_metadata.la \ $(extdir)/vnd.dovecot/debug/libsieve_ext_debug.la \ $(unfinished_plugins) diff -r 1d06b61cbaf3 -r 4da1e84f32e4 src/lib-sieve/plugins/Makefile.am --- a/src/lib-sieve/plugins/Makefile.am Wed Nov 12 22:10:25 2014 +0100 +++ b/src/lib-sieve/plugins/Makefile.am Sun Nov 16 23:13:58 2014 +0100 @@ -1,5 +1,5 @@ if BUILD_UNFINISHED -UNFINISHED = metadata +UNFINISHED = endif SUBDIRS = \ @@ -23,6 +23,7 @@ editheader \ duplicate \ index \ + metadata \ vnd.dovecot \ $(UNFINISHED) diff -r 1d06b61cbaf3 -r 4da1e84f32e4 src/lib-sieve/plugins/metadata/Makefile.am --- a/src/lib-sieve/plugins/metadata/Makefile.am Wed Nov 12 22:10:25 2014 +0100 +++ b/src/lib-sieve/plugins/metadata/Makefile.am Sun Nov 16 23:13:58 2014 +0100 @@ -5,7 +5,8 @@ AM_CPPFLAGS = \ -I$(srcdir)/../.. \ -I$(srcdir)/../variables \ - $(LIBDOVECOT_INCLUDE) + $(LIBDOVECOT_INCLUDE) \ + $(LIBDOVECOT_STORAGE_INCLUDE) tests = \ tst-metadata.c \ diff -r 1d06b61cbaf3 -r 4da1e84f32e4 src/lib-sieve/plugins/metadata/ext-metadata-common.h --- a/src/lib-sieve/plugins/metadata/ext-metadata-common.h Wed Nov 12 22:10:25 2014 +0100 +++ b/src/lib-sieve/plugins/metadata/ext-metadata-common.h Sun Nov 16 23:13:58 2014 +0100 @@ -4,6 +4,10 @@ #ifndef __EXT_METADATA_COMMON_H #define __EXT_METADATA_COMMON_H +#include "lib.h" +#include "mail-storage.h" +#include "imap-metadata.h" + #include "sieve-common.h" /* diff -r 1d06b61cbaf3 -r 4da1e84f32e4 src/lib-sieve/plugins/metadata/tst-metadata.c --- a/src/lib-sieve/plugins/metadata/tst-metadata.c Wed Nov 12 22:10:25 2014 +0100 +++ b/src/lib-sieve/plugins/metadata/tst-metadata.c Sun Nov 16 23:13:58 2014 +0100 @@ -1,7 +1,12 @@ /* Copyright (c) 2002-2014 Pigeonhole authors, see the included COPYING file */ +#include "lib.h" +#include "str-sanitize.h" +#include "istream.h" + #include "sieve-common.h" +#include "sieve-limits.h" #include "sieve-commands.h" #include "sieve-stringlist.h" #include "sieve-code.h" @@ -15,6 +20,10 @@ #include "ext-metadata-common.h" +#include + +#define TST_METADATA_MAX_MATCH_SIZE SIEVE_MAX_STRING_LEN + /* * Test definitions */ @@ -102,7 +111,8 @@ */ static bool tst_metadata_registered -(struct sieve_validator *valdtr, const struct sieve_extension *ext ATTR_UNUSED, +(struct sieve_validator *valdtr, + const struct sieve_extension *ext ATTR_UNUSED, struct sieve_command_registration *cmd_reg) { /* The order of these is not significant */ @@ -125,7 +135,9 @@ const struct sieve_comparator cmp_default = SIEVE_COMPARATOR_DEFAULT(i_ascii_casemap_comparator); unsigned int arg_index = 1; + const char *error; + /* mailbox */ if ( sieve_command_is(tst, metadata_test) ) { if ( !sieve_validate_positional_argument (valdtr, tst, arg, "mailbox", arg_index++, SAAT_STRING) ) { @@ -138,6 +150,7 @@ arg = sieve_ast_argument_next(arg); } + /* annotation-name */ if ( !sieve_validate_positional_argument (valdtr, tst, arg, "annotation-name", arg_index++, SAAT_STRING) ) { return FALSE; @@ -146,10 +159,25 @@ if ( !sieve_validator_argument_activate(valdtr, tst, arg, FALSE) ) return FALSE; + if ( sieve_argument_is_string_literal(arg) ) { + string_t *aname = sieve_ast_argument_str(arg); + + if ( !imap_metadata_verify_entry_name(str_c(aname), &error) ) { + char *lcerror = t_strdup_noconst(error); + lcerror[0] = i_tolower(lcerror[0]); + sieve_argument_validate_warning + (valdtr, arg, "%s test: " + "specified annotation name `%s' is invalid: %s", + sieve_command_identifier(tst), + str_sanitize(str_c(aname), 256), lcerror); + } + } + arg = sieve_ast_argument_next(arg); + /* key-list */ if ( !sieve_validate_positional_argument - (valdtr, tst, arg, "key list", arg_index++, SAAT_STRING_LIST) ) { + (valdtr, tst, arg, "key-list", arg_index++, SAAT_STRING_LIST) ) { return FALSE; } @@ -217,6 +245,63 @@ * Code execution */ +static inline const char *_lc_error(const char *error) +{ + char *lcerror = t_strdup_noconst(error); + lcerror[0] = i_tolower(lcerror[0]); + + return lcerror; +} + +static int tst_metadata_get_annotation +(const struct sieve_runtime_env *renv, const char *mailbox, + const char *aname, const char **annotation_r) +{ + struct mail_user *user = renv->scriptenv->user; + struct mailbox *box; + struct imap_metadata_transaction *imtrans; + struct mail_attribute_value avalue; + int status, ret; + + *annotation_r = NULL; + + if ( user == NULL ) + return SIEVE_EXEC_OK; + + if ( mailbox != NULL ) { + struct mail_namespace *ns; + ns = mail_namespace_find(user->namespaces, mailbox); + box = mailbox_alloc(ns->list, mailbox, 0); + imtrans = imap_metadata_transaction_begin(box); + } else { + imtrans = imap_metadata_transaction_begin_server(user); + } + + status = SIEVE_EXEC_OK; + ret = imap_metadata_get(imtrans, aname, &avalue); + if (ret < 0) { + enum mail_error error_code; + const char *error; + + error = imap_metadata_transaction_get_last_error + (imtrans, &error_code); + + sieve_runtime_error(renv, NULL, "%s test: " + "failed to retrieve annotation `%s': %s%s", + (mailbox != NULL ? "metadata" : "servermetadata"), + str_sanitize(aname, 256), _lc_error(error), + (error_code == MAIL_ERROR_TEMP ? " (temporary failure)" : "")); + + status = ( error_code == MAIL_ERROR_TEMP ? + SIEVE_EXEC_TEMP_FAILURE : SIEVE_EXEC_FAILURE ); + + } else if (avalue.value != NULL) { + *annotation_r = avalue.value; + } + (void)imap_metadata_transaction_commit(&imtrans, NULL, NULL); + return status; +} + static int tst_metadata_operation_execute (const struct sieve_runtime_env *renv, sieve_size_t *address) { @@ -225,9 +310,9 @@ SIEVE_MATCH_TYPE_DEFAULT(is_match_type); struct sieve_comparator cmp = SIEVE_COMPARATOR_DEFAULT(i_ascii_casemap_comparator); - string_t *mailbox, *annotation_name; + string_t *mailbox, *aname; struct sieve_stringlist *value_list, *key_list; - const char *annotation = NULL; + const char *annotation = NULL, *error; int match, ret; /* @@ -247,7 +332,7 @@ /* Read annotation-name */ if ( (ret=sieve_opr_string_read - (renv, address, "annotation-name", &annotation_name)) <= 0 ) + (renv, address, "annotation-name", &aname)) <= 0 ) return ret; /* Read key-list */ @@ -263,24 +348,48 @@ sieve_runtime_trace(renv, SIEVE_TRLVL_TESTS, "metadata test"); else sieve_runtime_trace(renv, SIEVE_TRLVL_TESTS, "servermetadata test"); + sieve_runtime_trace_descend(renv); + + if ( !imap_metadata_verify_entry_name(str_c(aname), &error) ) { + sieve_runtime_warning(renv, NULL, "%s test: " + "specified annotation name `%s' is invalid: %s", + (metadata ? "metadata" : "servermetadata"), + str_sanitize(str_c(aname), 256), _lc_error(error)); + sieve_interpreter_set_test_result(renv->interp, FALSE); + return SIEVE_EXEC_OK; + } + + if ( metadata ) { + sieve_runtime_trace(renv, SIEVE_TRLVL_TESTS, + "retrieving annotation `%s' from mailbox `%s'", + str_sanitize(str_c(aname), 256), + str_sanitize(str_c(mailbox), 80)); + } else { + sieve_runtime_trace(renv, SIEVE_TRLVL_TESTS, + "retrieving server annotation `%s'", + str_sanitize(str_c(aname), 256)); + } /* Get annotation */ - annotation = "FIXME"; + if ( (ret=tst_metadata_get_annotation + (renv, (metadata ? str_c(mailbox) : NULL), str_c(aname), &annotation)) + == SIEVE_EXEC_OK ) { + /* Perform match */ + if ( annotation != NULL ) { + /* Create value stringlist */ + value_list = sieve_single_stringlist_create_cstr(renv, annotation, FALSE); - /* Perform match */ - if ( annotation != NULL ) { - /* Create value stringlist */ - value_list = sieve_single_stringlist_create_cstr(renv, annotation, FALSE); - - /* Perform match */ - if ( (match=sieve_match(renv, &mcht, &cmp, value_list, key_list, &ret)) - < 0 ) - return ret; - } else { - match = 0; + /* Perform match */ + if ( (match=sieve_match + (renv, &mcht, &cmp, value_list, key_list, &ret)) < 0 ) + return ret; + } else { + match = 0; + } } /* Set test result for subsequent conditional jump */ - sieve_interpreter_set_test_result(renv->interp, match > 0); - return SIEVE_EXEC_OK; + if (ret == SIEVE_EXEC_OK) + sieve_interpreter_set_test_result(renv->interp, match > 0); + return ret; } diff -r 1d06b61cbaf3 -r 4da1e84f32e4 src/lib-sieve/plugins/metadata/tst-metadataexists.c --- a/src/lib-sieve/plugins/metadata/tst-metadataexists.c Wed Nov 12 22:10:25 2014 +0100 +++ b/src/lib-sieve/plugins/metadata/tst-metadataexists.c Sun Nov 16 23:13:58 2014 +0100 @@ -17,6 +17,9 @@ From pigeonhole at rename-it.nl Mon Nov 17 01:20:48 2014 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 17 Nov 2014 02:20:48 +0100 Subject: dovecot-2.2-pigeonhole: Added basic testsuite items for new Siev... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/a4f2d618bae2 changeset: 1968:a4f2d618bae2 user: Stephan Bosch date: Mon Nov 17 02:20:40 2014 +0100 description: Added basic testsuite items for new Sieve metadata support. diffstat: Makefile.am | 1 + src/lib-sieve/plugins/metadata/tst-metadata.c | 2 + src/testsuite/Makefile.am | 3 +- src/testsuite/cmd-test-imap-metadata.c | 176 ++++++++++++++++++++++++++ src/testsuite/ext-testsuite.c | 4 +- src/testsuite/testsuite-common.h | 3 + src/testsuite/testsuite-mailstore.c | 133 +++++++++++++++++- src/testsuite/testsuite-mailstore.h | 12 + src/testsuite/testsuite.c | 2 +- tests/extensions/metadata/execute.svtest | 58 ++++++++ 10 files changed, 379 insertions(+), 15 deletions(-) diffs (truncated from 563 to 300 lines): diff -r 0677417707c0 -r a4f2d618bae2 Makefile.am --- a/Makefile.am Sun Nov 16 23:16:38 2014 +0100 +++ b/Makefile.am Mon Nov 17 02:20:40 2014 +0100 @@ -158,6 +158,7 @@ tests/extensions/duplicate/errors.svtest \ tests/extensions/duplicate/execute.svtest \ tests/extensions/duplicate/execute-vnd.svtest \ + tests/extensions/metadata/execute.svtest \ tests/extensions/vnd.dovecot/debug/execute.svtest \ tests/deprecated/notify/basic.svtest \ tests/deprecated/notify/mailto.svtest \ diff -r 0677417707c0 -r a4f2d618bae2 src/lib-sieve/plugins/metadata/tst-metadata.c --- a/src/lib-sieve/plugins/metadata/tst-metadata.c Sun Nov 16 23:16:38 2014 +0100 +++ b/src/lib-sieve/plugins/metadata/tst-metadata.c Mon Nov 17 02:20:40 2014 +0100 @@ -299,6 +299,8 @@ *annotation_r = avalue.value; } (void)imap_metadata_transaction_commit(&imtrans, NULL, NULL); + if ( box != NULL ) + mailbox_free(&box); return status; } diff -r 0677417707c0 -r a4f2d618bae2 src/testsuite/Makefile.am --- a/src/testsuite/Makefile.am Sun Nov 16 23:16:38 2014 +0100 +++ b/src/testsuite/Makefile.am Mon Nov 17 02:20:40 2014 +0100 @@ -24,7 +24,8 @@ cmd-test-result.c \ cmd-test-message.c \ cmd-test-mailbox.c \ - cmd-test-binary.c + cmd-test-binary.c \ + cmd-test-imap-metadata.c tests = \ tst-test-script-compile.c \ diff -r 0677417707c0 -r a4f2d618bae2 src/testsuite/cmd-test-imap-metadata.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/testsuite/cmd-test-imap-metadata.c Mon Nov 17 02:20:40 2014 +0100 @@ -0,0 +1,176 @@ +/* Copyright (c) 2002-2014 Pigeonhole authors, see the included COPYING file + */ + +#include "sieve-common.h" +#include "sieve-commands.h" +#include "sieve-validator.h" +#include "sieve-generator.h" +#include "sieve-interpreter.h" +#include "sieve-code.h" +#include "sieve-binary.h" +#include "sieve-dump.h" + +#include "testsuite-common.h" +#include "testsuite-mailstore.h" + +/* + * Commands + */ + +static bool cmd_test_imap_metadata_validate + (struct sieve_validator *valdtr, struct sieve_command *cmd); +static bool cmd_test_imap_metadata_generate + (const struct sieve_codegen_env *cgenv, struct sieve_command *ctx); + +/* Test_mailbox_create command + * + * Syntax: + * test_imap_metadata_set + * + */ + +const struct sieve_command_def cmd_test_imap_metadata_set = { + "test_imap_metadata_set", + SCT_COMMAND, + 3, 0, FALSE, FALSE, + NULL, NULL, + cmd_test_imap_metadata_validate, + NULL, + cmd_test_imap_metadata_generate, + NULL +}; + +/* + * Operations + */ + +static bool cmd_test_imap_metadata_operation_dump + (const struct sieve_dumptime_env *denv, sieve_size_t *address); +static int cmd_test_imap_metadata_operation_execute + (const struct sieve_runtime_env *renv, sieve_size_t *address); + +/* Test_mailbox_create operation */ + +const struct sieve_operation_def test_imap_metadata_set_operation = { + "TEST_IMAP_METADATA_SET", + &testsuite_extension, + TESTSUITE_OPERATION_TEST_IMAP_METADATA_SET, + cmd_test_imap_metadata_operation_dump, + cmd_test_imap_metadata_operation_execute +}; + +/* + * Validation + */ + +static bool cmd_test_imap_metadata_validate +(struct sieve_validator *valdtr, struct sieve_command *cmd) +{ + struct sieve_ast_argument *arg = cmd->first_positional; + + if ( !sieve_validate_positional_argument + (valdtr, cmd, arg, "mailbox", 1, SAAT_STRING) ) + return FALSE; + + if (!sieve_validator_argument_activate(valdtr, cmd, arg, FALSE)) + return FALSE; + + arg = sieve_ast_argument_next(arg); + + if ( !sieve_validate_positional_argument + (valdtr, cmd, arg, "annotation", 2, SAAT_STRING) ) + return FALSE; + + if (!sieve_validator_argument_activate(valdtr, cmd, arg, FALSE)) + return FALSE; + + arg = sieve_ast_argument_next(arg); + + if ( !sieve_validate_positional_argument + (valdtr, cmd, arg, "value", 3, SAAT_STRING) ) + return FALSE; + + if (!sieve_validator_argument_activate(valdtr, cmd, arg, FALSE)) + return FALSE; + return TRUE; +} + +/* + * Code generation + */ + +static bool cmd_test_imap_metadata_generate +(const struct sieve_codegen_env *cgenv, struct sieve_command *cmd) +{ + /* Emit operation */ + if ( sieve_command_is(cmd, cmd_test_imap_metadata_set) ) + sieve_operation_emit + (cgenv->sblock, cmd->ext, &test_imap_metadata_set_operation); + else + i_unreached(); + + /* Generate arguments */ + if ( !sieve_generate_arguments(cgenv, cmd, NULL) ) + return FALSE; + return TRUE; +} + +/* + * Code dump + */ + +static bool cmd_test_imap_metadata_operation_dump +(const struct sieve_dumptime_env *denv, sieve_size_t *address) +{ + sieve_code_dumpf(denv, "%s:", sieve_operation_mnemonic(denv->oprtn)); + + sieve_code_descend(denv); + + return (sieve_opr_string_dump(denv, address, "mailbox") && + sieve_opr_string_dump(denv, address, "annotation") && + sieve_opr_string_dump(denv, address, "value")); +} + +/* + * Intepretation + */ + +static int cmd_test_imap_metadata_operation_execute +(const struct sieve_runtime_env *renv, sieve_size_t *address) +{ + const struct sieve_operation *oprtn = renv->oprtn; + string_t *mailbox = NULL, *annotation = NULL, *value = NULL; + int ret; + + /* + * Read operands + */ + + if ( (ret=sieve_opr_string_read + (renv, address, "mailbox", &mailbox)) <= 0 ) + return ret; + if ( (ret=sieve_opr_string_read + (renv, address, "annotation", &annotation)) <= 0 ) + return ret; + if ( (ret=sieve_opr_string_read + (renv, address, "value", &value)) <= 0 ) + return ret; + + /* + * Perform operation + */ + + if ( sieve_operation_is(oprtn, test_imap_metadata_set_operation) ) { + if ( sieve_runtime_trace_active(renv, SIEVE_TRLVL_COMMANDS) ) { + sieve_runtime_trace(renv, 0, "testsuite/test_imap_metadata_set command"); + sieve_runtime_trace_descend(renv); + sieve_runtime_trace(renv, 0, "set annotation `%s'", str_c(mailbox)); + } + + if (testsuite_mailstore_set_imap_metadata + (str_c(mailbox), str_c(annotation), str_c(value)) < 0) + return SIEVE_EXEC_FAILURE; + } + + return SIEVE_EXEC_OK; +} diff -r 0677417707c0 -r a4f2d618bae2 src/testsuite/ext-testsuite.c --- a/src/testsuite/ext-testsuite.c Sun Nov 16 23:16:38 2014 +0100 +++ b/src/testsuite/ext-testsuite.c Mon Nov 17 02:20:40 2014 +0100 @@ -74,7 +74,8 @@ &test_mailbox_create_operation, &test_mailbox_delete_operation, &test_binary_load_operation, - &test_binary_save_operation + &test_binary_save_operation, + &test_imap_metadata_set_operation }; /* @@ -134,6 +135,7 @@ sieve_validator_register_command(valdtr, ext, &cmd_test_mailbox_delete); sieve_validator_register_command(valdtr, ext, &cmd_test_binary_load); sieve_validator_register_command(valdtr, ext, &cmd_test_binary_save); + sieve_validator_register_command(valdtr, ext, &cmd_test_imap_metadata_set); sieve_validator_register_command(valdtr, ext, &tst_test_script_compile); sieve_validator_register_command(valdtr, ext, &tst_test_script_run); diff -r 0677417707c0 -r a4f2d618bae2 src/testsuite/testsuite-common.h --- a/src/testsuite/testsuite-common.h Sun Nov 16 23:16:38 2014 +0100 +++ b/src/testsuite/testsuite-common.h Mon Nov 17 02:20:40 2014 +0100 @@ -78,6 +78,7 @@ extern const struct sieve_command_def cmd_test_mailbox_delete; extern const struct sieve_command_def cmd_test_binary_load; extern const struct sieve_command_def cmd_test_binary_save; +extern const struct sieve_command_def cmd_test_imap_metadata_set; /* * Tests @@ -117,6 +118,7 @@ TESTSUITE_OPERATION_TEST_MAILBOX_DELETE, TESTSUITE_OPERATION_TEST_BINARY_LOAD, TESTSUITE_OPERATION_TEST_BINARY_SAVE, + TESTSUITE_OPERATION_TEST_IMAP_METADATA_SET }; extern const struct sieve_operation_def test_operation; @@ -141,6 +143,7 @@ extern const struct sieve_operation_def test_mailbox_delete_operation; extern const struct sieve_operation_def test_binary_load_operation; extern const struct sieve_operation_def test_binary_save_operation; +extern const struct sieve_operation_def test_imap_metadata_set_operation; /* * Operands diff -r 0677417707c0 -r a4f2d618bae2 src/testsuite/testsuite-mailstore.c --- a/src/testsuite/testsuite-mailstore.c Sun Nov 16 23:16:38 2014 +0100 +++ b/src/testsuite/testsuite-mailstore.c Mon Nov 17 02:20:40 2014 +0100 @@ -6,10 +6,13 @@ #include "imem.h" #include "array.h" #include "strfuncs.h" +#include "str-sanitize.h" +#include "abspath.h" #include "unlink-directory.h" #include "env-util.h" #include "mail-namespace.h" #include "mail-storage.h" +#include "imap-metadata.h" #include "sieve-common.h" #include "sieve-error.h" @@ -34,7 +37,10 @@ * State */ -static char *testsuite_mailstore_tmp = NULL; +static struct mail_user *testsuite_mailstore_user = NULL; + +static char *testsuite_mailstore_location = NULL; +static char *testsuite_mailstore_attrs = NULL; static char *testsuite_mailstore_folder = NULL; static struct mailbox *testsuite_mailstore_box = NULL; @@ -47,28 +53,66 @@ void testsuite_mailstore_init(void) { - testsuite_mailstore_tmp = i_strconcat - (testsuite_tmp_dir_get(), "/mailstore", NULL); + struct mail_user *mail_user_dovecot, *mail_user; From dovecot at dovecot.org Wed Nov 19 08:38:12 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 19 Nov 2014 08:38:12 +0000 Subject: dovecot-2.2: mbox: istream_raw_mbox_seek() needs to reset all co... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/31262a892ba7 changeset: 18100:31262a892ba7 user: Timo Sirainen date: Wed Nov 19 17:37:18 2014 +0900 description: mbox: istream_raw_mbox_seek() needs to reset all corruption errors. We now reset all errors, but it doesn't really matter. diffstat: src/lib-storage/index/mbox/istream-raw-mbox.c | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diffs (13 lines): diff -r 8973329d1ceb -r 31262a892ba7 src/lib-storage/index/mbox/istream-raw-mbox.c --- a/src/lib-storage/index/mbox/istream-raw-mbox.c Sat Nov 15 03:26:16 2014 +0200 +++ b/src/lib-storage/index/mbox/istream-raw-mbox.c Wed Nov 19 17:37:18 2014 +0900 @@ -688,6 +688,9 @@ i_assert(rstream->locked); + /* reset any (corruption) errors */ + stream->stream_errno = 0; + i_free_and_null(stream->real_stream->iostream.error); rstream->corrupted = FALSE; rstream->eof = FALSE; rstream->istream.istream.eof = FALSE; From dovecot at dovecot.org Wed Nov 19 08:38:12 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 19 Nov 2014 08:38:12 +0000 Subject: dovecot-2.2: mbox: Added more (and earlier) detection for errors... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/80ed82a93c1a changeset: 18101:80ed82a93c1a user: Timo Sirainen date: Wed Nov 19 17:37:57 2014 +0900 description: mbox: Added more (and earlier) detection for errors in mbox istreams. diffstat: src/lib-storage/index/mbox/istream-raw-mbox.c | 67 ++++++++++++++++--------- src/lib-storage/index/mbox/istream-raw-mbox.h | 13 +++- src/lib-storage/index/mbox/mbox-mail.c | 22 ++++++-- src/lib-storage/index/mbox/mbox-sync-parse.c | 10 +++- src/lib-storage/index/mbox/mbox-sync-private.h | 4 +- src/lib-storage/index/mbox/mbox-sync-rewrite.c | 27 ++++++--- src/lib-storage/index/mbox/mbox-sync.c | 41 ++++++++++----- 7 files changed, 122 insertions(+), 62 deletions(-) diffs (truncated from 441 to 300 lines): diff -r 31262a892ba7 -r 80ed82a93c1a src/lib-storage/index/mbox/istream-raw-mbox.c --- a/src/lib-storage/index/mbox/istream-raw-mbox.c Wed Nov 19 17:37:18 2014 +0900 +++ b/src/lib-storage/index/mbox/istream-raw-mbox.c Wed Nov 19 17:37:57 2014 +0900 @@ -506,7 +506,8 @@ return rstream->from_offset; } -uoff_t istream_raw_mbox_get_header_offset(struct istream *stream) +int istream_raw_mbox_get_header_offset(struct istream *stream, + uoff_t *hdr_offset_r) { struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream->real_stream; @@ -520,13 +521,17 @@ i_error("Unexpectedly lost From-line from mbox file %s at " "%"PRIuUOFF_T, i_stream_get_name(stream), rstream->from_offset); - return (uoff_t)-1; + return -1; } + if (stream->stream_errno != 0) + return -1; - return rstream->hdr_offset; + *hdr_offset_r = rstream->hdr_offset; + return 0; } -uoff_t istream_raw_mbox_get_body_offset(struct istream *stream) +int istream_raw_mbox_get_body_offset(struct istream *stream, + uoff_t *body_offset_r) { struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream->real_stream; @@ -534,8 +539,10 @@ i_assert(rstream->seeked); - if (rstream->body_offset != (uoff_t)-1) - return rstream->body_offset; + if (rstream->body_offset != (uoff_t)-1) { + *body_offset_r = rstream->body_offset; + return 0; + } offset = stream->v_offset; i_stream_seek(stream, rstream->hdr_offset); @@ -551,27 +558,30 @@ } else { i_assert(rstream->body_offset != (uoff_t)-1); } - break; + return -1; } } i_stream_seek(stream, offset); - return rstream->body_offset; + *body_offset_r = rstream->body_offset; + return 0; } -uoff_t istream_raw_mbox_get_body_size(struct istream *stream, - uoff_t expected_body_size) +int istream_raw_mbox_get_body_size(struct istream *stream, + uoff_t expected_body_size, + uoff_t *body_size_r) { struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream->real_stream; const unsigned char *data; size_t size; - uoff_t old_offset, body_size, next_body_offset; + uoff_t old_offset, body_offset, body_size, next_body_offset; i_assert(rstream->seeked); i_assert(rstream->hdr_offset != (uoff_t)-1); - (void)istream_raw_mbox_get_body_offset(stream); + if (istream_raw_mbox_get_body_offset(stream, &body_offset) < 0) + return -1; body_size = rstream->mail_size == (uoff_t)-1 ? (uoff_t)-1 : rstream->mail_size - (rstream->body_offset - rstream->hdr_offset); @@ -580,8 +590,10 @@ /* if we already have the existing body size, use it as long as it's >= expected body_size. otherwise the previous parsing may have stopped at a From_-line that belongs to the body. */ - if (body_size != (uoff_t)-1 && body_size >= expected_body_size) - return body_size; + if (body_size != (uoff_t)-1 && body_size >= expected_body_size) { + *body_size_r = body_size; + return 0; + } next_body_offset = rstream->body_offset + expected_body_size; /* If header_missing_eoh is set, the message body begins with @@ -600,21 +612,27 @@ rstream->mail_size = next_body_offset - rstream->hdr_offset; i_stream_seek(stream, old_offset); - return expected_body_size; + *body_size_r = expected_body_size; + return 0; } /* invalid expected_body_size */ } - if (body_size != (uoff_t)-1) - return body_size; + if (body_size != (uoff_t)-1) { + *body_size_r = body_size; + return 0; + } /* have to read through the message body */ while (i_stream_read_data(stream, &data, &size, 0) > 0) i_stream_skip(stream, size); i_stream_seek(stream, old_offset); + if (stream->stream_errno != 0) + return -1; i_assert(rstream->mail_size != (uoff_t)-1); - return rstream->mail_size - + *body_size_r = rstream->mail_size - (rstream->body_offset - rstream->hdr_offset); + return 0; } time_t istream_raw_mbox_get_received_time(struct istream *stream) @@ -651,13 +669,15 @@ return rstream->crlf_ending; } -void istream_raw_mbox_next(struct istream *stream, uoff_t expected_body_size) +int istream_raw_mbox_next(struct istream *stream, uoff_t expected_body_size) { struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream->real_stream; uoff_t body_size; - body_size = istream_raw_mbox_get_body_size(stream, expected_body_size); + if (istream_raw_mbox_get_body_size(stream, expected_body_size, + &body_size) < 0) + return -1; rstream->mail_size = (uoff_t)-1; rstream->received_time = rstream->next_received_time; @@ -678,6 +698,7 @@ rstream->eof = FALSE; rstream->istream.istream.eof = FALSE; + return 0; } int istream_raw_mbox_seek(struct istream *stream, uoff_t offset) @@ -698,10 +719,8 @@ /* if seeked is FALSE, we unlocked in the middle. don't try to use any cached state then. */ if (rstream->mail_size != (uoff_t)-1 && rstream->seeked && - rstream->hdr_offset + rstream->mail_size == offset) { - istream_raw_mbox_next(stream, (uoff_t)-1); - return 0; - } + rstream->hdr_offset + rstream->mail_size == offset) + return istream_raw_mbox_next(stream, (uoff_t)-1); if (offset == rstream->from_offset && rstream->seeked) { /* back to beginning of current message */ diff -r 31262a892ba7 -r 80ed82a93c1a src/lib-storage/index/mbox/istream-raw-mbox.h --- a/src/lib-storage/index/mbox/istream-raw-mbox.h Wed Nov 19 17:37:18 2014 +0900 +++ b/src/lib-storage/index/mbox/istream-raw-mbox.h Wed Nov 19 17:37:57 2014 +0900 @@ -9,15 +9,18 @@ /* Return offset to beginning of the "\nFrom"-line. */ uoff_t istream_raw_mbox_get_start_offset(struct istream *stream); /* Return offset to beginning of the headers. */ -uoff_t istream_raw_mbox_get_header_offset(struct istream *stream); +int istream_raw_mbox_get_header_offset(struct istream *stream, + uoff_t *hdr_offset_r); /* Return offset to beginning of the body. */ -uoff_t istream_raw_mbox_get_body_offset(struct istream *stream); +int istream_raw_mbox_get_body_offset(struct istream *stream, + uoff_t *body_offset_r); /* Return the number of bytes in the body of this message. If expected_body_size isn't (uoff_t)-1, we'll use it as potentially valid body size to avoid actually reading through the whole message. */ -uoff_t istream_raw_mbox_get_body_size(struct istream *stream, - uoff_t expected_body_size); +int istream_raw_mbox_get_body_size(struct istream *stream, + uoff_t expected_body_size, + uoff_t *body_size_r); /* Return received time of current message, or (time_t)-1 if the timestamp is broken. */ @@ -30,7 +33,7 @@ /* Jump to next message. If expected_body_size isn't (uoff_t)-1, we'll use it as potentially valid body size. */ -void istream_raw_mbox_next(struct istream *stream, uoff_t expected_body_size); +int istream_raw_mbox_next(struct istream *stream, uoff_t expected_body_size); /* Seek to message at given offset. offset must point to beginning of "\nFrom ", or 0 for beginning of file. Returns -1 if it offset doesn't diff -r 31262a892ba7 -r 80ed82a93c1a src/lib-storage/index/mbox/mbox-mail.c --- a/src/lib-storage/index/mbox/mbox-mail.c Wed Nov 19 17:37:18 2014 +0900 +++ b/src/lib-storage/index/mbox/mbox-mail.c Wed Nov 19 17:37:57 2014 +0900 @@ -304,10 +304,10 @@ return -1; /* our header size varies, so don't do any caching */ - body_offset = istream_raw_mbox_get_body_offset(mbox->mbox_stream); - if (body_offset == (uoff_t)-1) { + if (istream_raw_mbox_get_body_offset(mbox->mbox_stream, &body_offset) < 0) { mail_storage_set_critical(_mail->box->storage, - "Couldn't get mbox size"); + "mbox %s: Couldn't get body offset for uid=%u", + mailbox_get_path(&mbox->box), mail->mail.mail.uid); return -1; } @@ -319,8 +319,13 @@ body_size = (uoff_t)-1; /* verify that the calculated body size is correct */ - body_size = istream_raw_mbox_get_body_size(mbox->mbox_stream, - body_size); + if (istream_raw_mbox_get_body_size(mbox->mbox_stream, + body_size, &body_size) < 0) { + mail_storage_set_critical(_mail->box->storage, + "mbox %s: Couldn't get body size for uid=%u", + mailbox_get_path(&mbox->box), mail->mail.mail.uid); + return -1; + } data->physical_size = hdr_size.physical_size + body_size; *size_r = data->physical_size; @@ -352,7 +357,12 @@ } raw_stream = mbox->mbox_stream; - hdr_offset = istream_raw_mbox_get_header_offset(raw_stream); + if (istream_raw_mbox_get_header_offset(raw_stream, &hdr_offset) < 0) { + mail_storage_set_critical(mbox->box.storage, + "mbox %s: Couldn't get header offset for uid=%u", + mailbox_get_path(&mbox->box), mail->mail.mail.uid); + return -1; + } i_stream_seek(raw_stream, hdr_offset); if (next_offset != (uoff_t)-1) diff -r 31262a892ba7 -r 80ed82a93c1a src/lib-storage/index/mbox/mbox-sync-parse.c --- a/src/lib-storage/index/mbox/mbox-sync-parse.c Wed Nov 19 17:37:18 2014 +0900 +++ b/src/lib-storage/index/mbox/mbox-sync-parse.c Wed Nov 19 17:37:57 2014 +0900 @@ -453,8 +453,8 @@ return strcasecmp(key, func->header); } -void mbox_sync_parse_next_mail(struct istream *input, - struct mbox_sync_mail_context *ctx) +int mbox_sync_parse_next_mail(struct istream *input, + struct mbox_sync_mail_context *ctx) { struct mbox_sync_context *sync_ctx = ctx->sync_ctx; struct message_header_parser_ctx *hdr_ctx; @@ -545,6 +545,12 @@ } ctx->body_offset = input->v_offset; + if (input->stream_errno != 0) { + mbox_sync_set_critical(ctx->sync_ctx, "read(%s) failed: %s", + i_stream_get_name(input), i_stream_get_error(input)); + return -1; + } + return 0; } bool mbox_sync_parse_match_mail(struct mbox_mailbox *mbox, diff -r 31262a892ba7 -r 80ed82a93c1a src/lib-storage/index/mbox/mbox-sync-private.h --- a/src/lib-storage/index/mbox/mbox-sync-private.h Wed Nov 19 17:37:18 2014 +0900 +++ b/src/lib-storage/index/mbox/mbox-sync-private.h Wed Nov 19 17:37:57 2014 +0900 @@ -158,8 +158,8 @@ void mbox_sync_set_critical(struct mbox_sync_context *sync_ctx, const char *fmt, ...) ATTR_FORMAT(2, 3); -void mbox_sync_parse_next_mail(struct istream *input, - struct mbox_sync_mail_context *ctx); +int mbox_sync_parse_next_mail(struct istream *input, + struct mbox_sync_mail_context *ctx); bool mbox_sync_parse_match_mail(struct mbox_mailbox *mbox, struct mail_index_view *view, uint32_t seq); diff -r 31262a892ba7 -r 80ed82a93c1a src/lib-storage/index/mbox/mbox-sync-rewrite.c --- a/src/lib-storage/index/mbox/mbox-sync-rewrite.c Wed Nov 19 17:37:18 2014 +0900 +++ b/src/lib-storage/index/mbox/mbox-sync-rewrite.c Wed Nov 19 17:37:57 2014 +0900 @@ -318,11 +318,11 @@ return 1; } -static void mbox_sync_read_next(struct mbox_sync_context *sync_ctx, - struct mbox_sync_mail_context *mail_ctx, From dovecot at dovecot.org Wed Nov 19 20:41:07 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 19 Nov 2014 20:41:07 +0000 Subject: dovecot-2.2: auth: Increase timeout for multi-line auth-worker r... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c3736d065d54 changeset: 18102:c3736d065d54 user: Timo Sirainen date: Thu Nov 20 05:40:55 2014 +0900 description: auth: Increase timeout for multi-line auth-worker replies to 30 minutes. This should help with running long-running doveadm -A commands. diffstat: src/auth/auth-worker-server.c | 19 +++++++++++++++++-- 1 files changed, 17 insertions(+), 2 deletions(-) diffs (52 lines): diff -r 80ed82a93c1a -r c3736d065d54 src/auth/auth-worker-server.c --- a/src/auth/auth-worker-server.c Wed Nov 19 17:37:57 2014 +0900 +++ b/src/auth/auth-worker-server.c Thu Nov 20 05:40:55 2014 +0900 @@ -17,7 +17,13 @@ #include #include +/* Initial lookup timeout */ #define AUTH_WORKER_LOOKUP_TIMEOUT_SECS 60 +/* Timeout for multi-line replies, e.g. listing users. This should be a much + higher value, because e.g. doveadm could be doing some long-running commands + for the users. And because of buffering this timeout is for handling + multiple users, not just one. */ +#define AUTH_WORKER_RESUME_TIMEOUT_SECS (30*60) #define AUTH_WORKER_MAX_IDLE_SECS (60*5) #define AUTH_WORKER_ABORT_SECS 60 #define AUTH_WORKER_DELAY_WARN_SECS 3 @@ -47,6 +53,7 @@ unsigned int restart:1; unsigned int shutdown:1; unsigned int timeout_pending_resume:1; + unsigned int resuming:1; }; static ARRAY(struct auth_worker_connection *) connections = ARRAY_INIT; @@ -276,8 +283,16 @@ { if (strncmp(line, "*\t", 2) == 0) { /* multi-line reply, not finished yet */ - timeout_reset(conn->to); + if (conn->resuming) + timeout_reset(conn->to); + else { + conn->resuming = TRUE; + timeout_remove(&conn->to); + conn->to = timeout_add(AUTH_WORKER_RESUME_TIMEOUT_SECS * 1000, + auth_worker_call_timeout, conn); + } } else { + conn->resuming = FALSE; conn->request = NULL; conn->timeout_pending_resume = FALSE; timeout_remove(&conn->to); @@ -416,7 +431,7 @@ { conn->timeout_pending_resume = FALSE; timeout_remove(&conn->to); - conn->to = timeout_add(AUTH_WORKER_LOOKUP_TIMEOUT_SECS * 1000, + conn->to = timeout_add(AUTH_WORKER_RESUME_TIMEOUT_SECS * 1000, auth_worker_call_timeout, conn); worker_input(conn); } From pigeonhole at rename-it.nl Thu Nov 20 00:04:08 2014 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 20 Nov 2014 01:04:08 +0100 Subject: dovecot-2.2-pigeonhole: testsuite: Created tests for the Sieve s... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/d0ca187be049 changeset: 1970:d0ca187be049 user: Stephan Bosch date: Thu Nov 20 00:53:41 2014 +0100 description: testsuite: Created tests for the Sieve servermetadata extension. diffstat: src/testsuite/cmd-test-imap-metadata.c | 150 ++++++++++++++++++++++++++---- src/testsuite/testsuite-mailstore.c | 1 + tests/extensions/metadata/execute.svtest | 64 ++++++++++++- 3 files changed, 191 insertions(+), 24 deletions(-) diffs (truncated from 324 to 300 lines): diff -r b7fad6e195a8 -r d0ca187be049 src/testsuite/cmd-test-imap-metadata.c --- a/src/testsuite/cmd-test-imap-metadata.c Thu Nov 20 00:51:09 2014 +0100 +++ b/src/testsuite/cmd-test-imap-metadata.c Thu Nov 20 00:53:41 2014 +0100 @@ -17,6 +17,9 @@ * Commands */ +static bool cmd_test_imap_metadata_registered + (struct sieve_validator *valdtr, const struct sieve_extension *ext, + struct sieve_command_registration *cmd_reg); static bool cmd_test_imap_metadata_validate (struct sieve_validator *valdtr, struct sieve_command *cmd); static bool cmd_test_imap_metadata_generate @@ -32,8 +35,9 @@ const struct sieve_command_def cmd_test_imap_metadata_set = { "test_imap_metadata_set", SCT_COMMAND, - 3, 0, FALSE, FALSE, - NULL, NULL, + 2, 0, FALSE, FALSE, + cmd_test_imap_metadata_registered, + NULL, cmd_test_imap_metadata_validate, NULL, cmd_test_imap_metadata_generate, @@ -41,6 +45,21 @@ }; /* + * Command tags + */ + +static bool cmd_test_imap_metadata_validate_mailbox_tag + (struct sieve_validator *valdtr, struct sieve_ast_argument **arg, + struct sieve_command *cmd); + +static const struct sieve_argument_def test_imap_metadata_mailbox_tag = { + "mailbox", + NULL, + cmd_test_imap_metadata_validate_mailbox_tag, + NULL, NULL, NULL +}; + +/* * Operations */ @@ -59,6 +78,53 @@ cmd_test_imap_metadata_operation_execute }; +/* Codes for optional arguments */ + +enum cmd_vacation_optional { + OPT_END, + OPT_MAILBOX +}; + +/* + * Tag validation + */ + +static bool cmd_test_imap_metadata_validate_mailbox_tag +(struct sieve_validator *valdtr, struct sieve_ast_argument **arg, + struct sieve_command *cmd) +{ + struct sieve_ast_argument *tag = *arg; + + /* Delete this tag */ + *arg = sieve_ast_arguments_detach(*arg, 1); + + /* Check syntax: + * :mailbox string + */ + if ( !sieve_validate_tag_parameter + (valdtr, cmd, tag, *arg, NULL, 0, SAAT_STRING, FALSE) ) { + return FALSE; + } + + /* Skip parameter */ + *arg = sieve_ast_argument_next(*arg); + return TRUE; +} + +/* + * Command registration + */ + +static bool cmd_test_imap_metadata_registered +(struct sieve_validator *valdtr, const struct sieve_extension *ext, + struct sieve_command_registration *cmd_reg) +{ + sieve_validator_register_tag + (valdtr, cmd_reg, ext, &test_imap_metadata_mailbox_tag, OPT_MAILBOX); + return TRUE; +} + + /* * Validation */ @@ -69,15 +135,6 @@ struct sieve_ast_argument *arg = cmd->first_positional; if ( !sieve_validate_positional_argument - (valdtr, cmd, arg, "mailbox", 1, SAAT_STRING) ) - return FALSE; - - if (!sieve_validator_argument_activate(valdtr, cmd, arg, FALSE)) - return FALSE; - - arg = sieve_ast_argument_next(arg); - - if ( !sieve_validate_positional_argument (valdtr, cmd, arg, "annotation", 2, SAAT_STRING) ) return FALSE; @@ -122,13 +179,36 @@ static bool cmd_test_imap_metadata_operation_dump (const struct sieve_dumptime_env *denv, sieve_size_t *address) { + int opt_code = 0; + sieve_code_dumpf(denv, "%s:", sieve_operation_mnemonic(denv->oprtn)); - sieve_code_descend(denv); - return (sieve_opr_string_dump(denv, address, "mailbox") && - sieve_opr_string_dump(denv, address, "annotation") && - sieve_opr_string_dump(denv, address, "value")); + /* Dump optional operands */ + + for (;;) { + int opt; + bool opok = TRUE; + + if ( (opt=sieve_opr_optional_dump(denv, address, &opt_code)) < 0 ) + return FALSE; + + if ( opt == 0 ) break; + + switch ( opt_code ) { + case OPT_MAILBOX: + opok = sieve_opr_string_dump(denv, address, "mailbox"); + break; + default: + return FALSE; + } + + if ( !opok ) return FALSE; + } + + return + ( sieve_opr_string_dump(denv, address, "annotation") && + sieve_opr_string_dump(denv, address, "value") ); } /* @@ -139,6 +219,7 @@ (const struct sieve_runtime_env *renv, sieve_size_t *address) { const struct sieve_operation *oprtn = renv->oprtn; + int opt_code = 0; string_t *mailbox = NULL, *annotation = NULL, *value = NULL; int ret; @@ -146,9 +227,30 @@ * Read operands */ - if ( (ret=sieve_opr_string_read - (renv, address, "mailbox", &mailbox)) <= 0 ) - return ret; + /* Optional operands */ + + for (;;) { + int opt; + + if ( (opt=sieve_opr_optional_read(renv, address, &opt_code)) < 0 ) + return SIEVE_EXEC_BIN_CORRUPT; + + if ( opt == 0 ) break; + + switch ( opt_code ) { + case OPT_MAILBOX: + ret = sieve_opr_string_read(renv, address, "mailbox", &mailbox); + break; + default: + sieve_runtime_trace_error(renv, "unknown optional operand"); + ret = SIEVE_EXEC_BIN_CORRUPT; + } + + if ( ret <= 0 ) return ret; + } + + /* Fixed operands */ + if ( (ret=sieve_opr_string_read (renv, address, "annotation", &annotation)) <= 0 ) return ret; @@ -164,11 +266,19 @@ if ( sieve_runtime_trace_active(renv, SIEVE_TRLVL_COMMANDS) ) { sieve_runtime_trace(renv, 0, "testsuite/test_imap_metadata_set command"); sieve_runtime_trace_descend(renv); - sieve_runtime_trace(renv, 0, "set annotation `%s'", str_c(mailbox)); + if (mailbox == NULL) { + sieve_runtime_trace(renv, 0, + "set server annotation `%s'", str_c(annotation)); + } else { + sieve_runtime_trace(renv, 0, + "set annotation `%s' for mailbox `%s'", + str_c(annotation), str_c(mailbox)); + } } if (testsuite_mailstore_set_imap_metadata - (str_c(mailbox), str_c(annotation), str_c(value)) < 0) + (( mailbox == NULL ? NULL : str_c(mailbox) ), + str_c(annotation), str_c(value)) < 0) return SIEVE_EXEC_FAILURE; } diff -r b7fad6e195a8 -r d0ca187be049 src/testsuite/testsuite-mailstore.c --- a/src/testsuite/testsuite-mailstore.c Thu Nov 20 00:51:09 2014 +0100 +++ b/src/testsuite/testsuite-mailstore.c Thu Nov 20 00:53:41 2014 +0100 @@ -256,6 +256,7 @@ box = mailbox_alloc(ns->list, mailbox, 0); imtrans = imap_metadata_transaction_begin(box); } else { + box = NULL; imtrans = imap_metadata_transaction_begin_server (testsuite_mailstore_user); } diff -r b7fad6e195a8 -r d0ca187be049 tests/extensions/metadata/execute.svtest --- a/tests/extensions/metadata/execute.svtest Thu Nov 20 00:51:09 2014 +0100 +++ b/tests/extensions/metadata/execute.svtest Thu Nov 20 00:53:41 2014 +0100 @@ -1,5 +1,6 @@ require "vnd.dovecot.testsuite"; require "mboxmetadata"; +require "servermetadata"; require "fileinto"; test "MetadataExists - None exist" { @@ -8,8 +9,8 @@ } } -test_imap_metadata_set "INBOX" "/private/frop" "FROP!"; -test_imap_metadata_set "INBOX" "/private/friep" "FRIEP!"; +test_imap_metadata_set :mailbox "INBOX" "/private/frop" "FROP!"; +test_imap_metadata_set :mailbox "INBOX" "/private/friep" "FRIEP!"; test "MetadataExists - Not all exist" { if metadataexists "INBOX" @@ -18,8 +19,8 @@ } } -test_imap_metadata_set "INBOX" "/private/friep" "FRIEP!"; -test_imap_metadata_set "INBOX" "/private/frml" "FRML!"; +test_imap_metadata_set :mailbox "INBOX" "/private/friep" "FRIEP!"; +test_imap_metadata_set :mailbox "INBOX" "/private/frml" "FRML!"; test "MetadataExists - One exists" { if not metadataexists "INBOX" ["/private/frop"] { @@ -56,3 +57,58 @@ test_fail "unexpected match for /private/frml"; } } + +test "ServermetadataExists - None exist" { + if servermetadataexists "/private/frop" { + test_fail "servermetadataexists confirms existence of unknown annotation"; + } +} + +test_imap_metadata_set "/private/frop" "FROP!"; +test_imap_metadata_set "/private/friep" "FRIEP!"; + +test "ServermetadataExists - Not all exist" { + if servermetadataexists + ["/private/frop", "/private/friep", "/private/frml"] { + test_fail "metadataexists confirms existence of unknown annotation"; + } +} + +test_imap_metadata_set "/private/friep" "FRIEP!"; +test_imap_metadata_set "/private/frml" "FRML!"; + +test "ServermetadataExists - One exists" { + if not servermetadataexists ["/private/frop"] { + test_fail "servermetadataexists fails to recognize annotation"; + } +} + +test "ServermetadataExists - All exist" { + if not servermetadataexists + ["/private/frop", "/private/friep", "/private/frml"] { + test_fail "metadataexists fails to recognize annotations"; + } From pigeonhole at rename-it.nl Thu Nov 20 00:04:08 2014 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 20 Nov 2014 01:04:08 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: Flush duplicate database duri... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/57b48c0580cd changeset: 1971:57b48c0580cd user: Stephan Bosch date: Thu Nov 20 01:04:00 2014 +0100 description: lib-sieve: Flush duplicate database during start phase of result execution rather than commit phase. This will release the duplicate database lock earlier, even before a mail storage transation is started. diffstat: src/lib-sieve/sieve-result.c | 15 ++++++++------- 1 files changed, 8 insertions(+), 7 deletions(-) diffs (50 lines): diff -r d0ca187be049 -r 57b48c0580cd src/lib-sieve/sieve-result.c --- a/src/lib-sieve/sieve-result.c Thu Nov 20 00:53:41 2014 +0100 +++ b/src/lib-sieve/sieve-result.c Thu Nov 20 01:04:00 2014 +0100 @@ -1106,8 +1106,10 @@ (struct sieve_result *result, struct sieve_result_action *first, struct sieve_result_action **last_r) { + const struct sieve_script_env *senv = result->action_env.scriptenv; struct sieve_result_action *rac = first; int status = SIEVE_EXEC_OK; + bool dup_flushed = FALSE; while ( status == SIEVE_EXEC_OK && rac != NULL ) { struct sieve_action *act = &rac->action; @@ -1118,6 +1120,12 @@ continue; } + if ((act->def->flags & SIEVE_ACTFLAG_MAIL_STORAGE) != 0 && + !dup_flushed) { + sieve_action_duplicate_flush(senv); + dup_flushed = TRUE; + } + if ( act->def->start != NULL ) { status = act->def->start (act, &result->action_env, &rac->tr_context); @@ -1283,10 +1291,8 @@ struct sieve_result_action *last, bool *implicit_keep, bool *keep) { - const struct sieve_script_env *senv = result->action_env.scriptenv; struct sieve_result_action *rac; int commit_status = status; - bool dup_flushed = FALSE; /* First commit/rollback all storage actions */ rac = first; @@ -1299,11 +1305,6 @@ continue; } - if (!dup_flushed) { - sieve_action_duplicate_flush(senv); - dup_flushed = TRUE; - } - status = sieve_result_action_commit_or_rollback (result, rac, status, implicit_keep, keep, &commit_status); From pigeonhole at rename-it.nl Thu Nov 20 00:04:08 2014 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 20 Nov 2014 01:04:08 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: servermetadata extension: Fix... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/b7fad6e195a8 changeset: 1969:b7fad6e195a8 user: Stephan Bosch date: Thu Nov 20 00:51:09 2014 +0100 description: lib-sieve: servermetadata extension: Fixed segfault. diffstat: src/lib-sieve/plugins/metadata/tst-metadata.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r a4f2d618bae2 -r b7fad6e195a8 src/lib-sieve/plugins/metadata/tst-metadata.c --- a/src/lib-sieve/plugins/metadata/tst-metadata.c Mon Nov 17 02:20:40 2014 +0100 +++ b/src/lib-sieve/plugins/metadata/tst-metadata.c Thu Nov 20 00:51:09 2014 +0100 @@ -274,6 +274,7 @@ box = mailbox_alloc(ns->list, mailbox, 0); imtrans = imap_metadata_transaction_begin(box); } else { + box = NULL; imtrans = imap_metadata_transaction_begin_server(user); } From dovecot at dovecot.org Thu Nov 20 06:15:36 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 20 Nov 2014 06:15:36 +0000 Subject: dovecot-2.2: lib-index: If transaction log is unexpectedly recre... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/df1d0c0e7ae2 changeset: 18103:df1d0c0e7ae2 user: Timo Sirainen date: Thu Nov 20 15:15:20 2014 +0900 description: lib-index: If transaction log is unexpectedly recreated while it was locked, log a nice error. Instead of just assert-crashing. diffstat: src/lib-index/mail-transaction-log-file.c | 9 +++++---- src/lib-index/mail-transaction-log.c | 13 +++++++++++-- 2 files changed, 16 insertions(+), 6 deletions(-) diffs (76 lines): diff -r c3736d065d54 -r df1d0c0e7ae2 src/lib-index/mail-transaction-log-file.c --- a/src/lib-index/mail-transaction-log-file.c Thu Nov 20 05:40:55 2014 +0900 +++ b/src/lib-index/mail-transaction-log-file.c Thu Nov 20 15:15:20 2014 +0900 @@ -817,7 +817,7 @@ /* success */ file->fd = new_fd; mail_transaction_log_file_add_to_list(file); - return 0; + return 1; } int mail_transaction_log_file_create(struct mail_transaction_log_file *file, @@ -827,7 +827,7 @@ struct dotlock_settings new_dotlock_set; struct dotlock *dotlock; mode_t old_mask; - int fd; + int fd, ret; i_assert(!MAIL_INDEX_IS_IN_MEMORY(index)); @@ -855,12 +855,13 @@ /* either fd gets used or the dotlock gets deleted and returned fd is for the existing file */ - if (mail_transaction_log_file_create2(file, fd, reset, &dotlock) < 0) { + ret = mail_transaction_log_file_create2(file, fd, reset, &dotlock); + if (ret < 0) { if (dotlock != NULL) file_dotlock_delete(&dotlock); return -1; } - return 0; + return ret; } int mail_transaction_log_file_open(struct mail_transaction_log_file *file) diff -r c3736d065d54 -r df1d0c0e7ae2 src/lib-index/mail-transaction-log.c --- a/src/lib-index/mail-transaction-log.c Thu Nov 20 05:40:55 2014 +0900 +++ b/src/lib-index/mail-transaction-log.c Thu Nov 20 15:15:20 2014 +0900 @@ -233,6 +233,7 @@ struct mail_transaction_log_file *file; const char *path = log->head->filepath; struct stat st; + int ret; i_assert(log->head->locked); @@ -258,17 +259,25 @@ file->last_mtime = st.st_mtime; file->last_size = st.st_size; - if (mail_transaction_log_file_create(file, reset) < 0) { + if ((ret = mail_transaction_log_file_create(file, reset)) < 0) { mail_transaction_log_file_free(&file); return -1; } + if (ret == 0) { + mail_index_set_error(log->index, + "Transaction log %s was recreated while we had it locked - " + "locking is broken (lock_method=%s)", path, + file_lock_method_to_str(log->index->lock_method)); + mail_transaction_log_file_free(&file); + return -1; + } + i_assert(file->locked); } if (--log->head->refcount == 0) mail_transaction_logs_clean(log); else { /* the newly created log file is already locked */ - i_assert(file->locked); mail_transaction_log_file_unlock(log->head, !log->index->log_sync_locked ? "rotating" : "rotating while syncing"); From dovecot at dovecot.org Thu Nov 20 13:03:43 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 20 Nov 2014 13:03:43 +0000 Subject: dovecot-2.2: stats: Removed extra TAB from global stats exporting. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5de5be00b369 changeset: 18104:5de5be00b369 user: Timo Sirainen date: Thu Nov 20 22:03:25 2014 +0900 description: stats: Removed extra TAB from global stats exporting. diffstat: src/stats/client-export.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r df1d0c0e7ae2 -r 5de5be00b369 src/stats/client-export.c --- a/src/stats/client-export.c Thu Nov 20 15:15:20 2014 +0900 +++ b/src/stats/client-export.c Thu Nov 20 22:03:25 2014 +0900 @@ -490,7 +490,7 @@ } str_truncate(cmd->str, 0); - str_printfa(cmd->str, "\t%ld", (long)g->reset_timestamp); + str_printfa(cmd->str, "%ld", (long)g->reset_timestamp); client_export_timeval(cmd->str, &g->last_update); str_printfa(cmd->str, "\t%u\t%u\t%u", g->num_logins, g->num_cmds, g->num_connected_sessions); From pigeonhole at rename-it.nl Fri Nov 21 00:24:06 2014 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Fri, 21 Nov 2014 01:24:06 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: metadata extensions: Implemen... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/4f53a9230f44 changeset: 1972:4f53a9230f44 user: Stephan Bosch date: Fri Nov 21 01:23:15 2014 +0100 description: lib-sieve: metadata extensions: Implemented proper checking of annotation names everywhere. Also added testsuite items to test syntax. diffstat: Makefile.am | 1 + src/lib-sieve/plugins/metadata/tst-metadataexists.c | 45 ++++++++++++++++++- tests/extensions/metadata/errors.svtest | 18 +++++++ tests/extensions/metadata/errors/syntax.sieve | 48 +++++++++++++++++++++ tests/extensions/metadata/execute.svtest | 29 ++++++++++++- 5 files changed, 138 insertions(+), 3 deletions(-) diffs (215 lines): diff -r 57b48c0580cd -r 4f53a9230f44 Makefile.am --- a/Makefile.am Thu Nov 20 01:04:00 2014 +0100 +++ b/Makefile.am Fri Nov 21 01:23:15 2014 +0100 @@ -159,6 +159,7 @@ tests/extensions/duplicate/execute.svtest \ tests/extensions/duplicate/execute-vnd.svtest \ tests/extensions/metadata/execute.svtest \ + tests/extensions/metadata/errors.svtest \ tests/extensions/vnd.dovecot/debug/execute.svtest \ tests/deprecated/notify/basic.svtest \ tests/deprecated/notify/mailto.svtest \ diff -r 57b48c0580cd -r 4f53a9230f44 src/lib-sieve/plugins/metadata/tst-metadataexists.c --- a/src/lib-sieve/plugins/metadata/tst-metadataexists.c Thu Nov 20 01:04:00 2014 +0100 +++ b/src/lib-sieve/plugins/metadata/tst-metadataexists.c Fri Nov 21 01:23:15 2014 +0100 @@ -104,10 +104,41 @@ * Test validation */ +struct _validate_context { + struct sieve_validator *valdtr; + struct sieve_command *tst; +}; + +static int tst_metadataexists_annotation_validate +(void *context, struct sieve_ast_argument *arg) +{ + struct _validate_context *valctx = + (struct _validate_context *)context; + + if ( sieve_argument_is_string_literal(arg) ) { + const char *aname = sieve_ast_strlist_strc(arg); + const char *error; + + if ( !imap_metadata_verify_entry_name(aname, &error) ) { + char *lcerror = t_strdup_noconst(error); + lcerror[0] = i_tolower(lcerror[0]); + sieve_argument_validate_warning + (valctx->valdtr, arg, "%s test: " + "specified annotation name `%s' is invalid: %s", + sieve_command_identifier(valctx->tst), + str_sanitize(aname, 256), lcerror); + } + } + + return TRUE; /* Can't check at compile time */ +} + static bool tst_metadataexists_validate (struct sieve_validator *valdtr, struct sieve_command *tst) { struct sieve_ast_argument *arg = tst->first_positional; + struct sieve_ast_argument *aarg; + struct _validate_context valctx; unsigned int arg_index = 1; if ( sieve_command_is(tst, metadataexists_test) ) { @@ -127,7 +158,16 @@ return FALSE; } - return sieve_validator_argument_activate(valdtr, tst, arg, FALSE); + if ( !sieve_validator_argument_activate(valdtr, tst, arg, FALSE) ) + return FALSE; + + aarg = arg; + memset(&valctx, 0, sizeof(valctx)); + valctx.valdtr = valdtr; + valctx.tst = tst; + + return sieve_ast_stringlist_map + (&aarg, (void*)&valctx, tst_metadataexists_annotation_validate); } /* @@ -232,7 +272,8 @@ "specified annotation name `%s' is invalid: %s", (mailbox != NULL ? "metadataexists" : "servermetadataexists"), str_sanitize(str_c(aname), 256), _lc_error(error)); - continue; + all_exist = FALSE; + break;; } ret = imap_metadata_get(imtrans, str_c(aname), &avalue); diff -r 57b48c0580cd -r 4f53a9230f44 tests/extensions/metadata/errors.svtest --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/extensions/metadata/errors.svtest Fri Nov 21 01:23:15 2014 +0100 @@ -0,0 +1,18 @@ +require "vnd.dovecot.testsuite"; + +require "relational"; +require "comparator-i;ascii-numeric"; + +/* + * Invalid syntax + */ + +test "Invalid Syntax" { + if test_script_compile "errors/syntax.sieve" { + test_fail "compile should have failed"; + } + + if not test_error :count "eq" :comparator "i;ascii-numeric" "27" { + test_fail "wrong number of errors reported"; + } +} diff -r 57b48c0580cd -r 4f53a9230f44 tests/extensions/metadata/errors/syntax.sieve --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/extensions/metadata/errors/syntax.sieve Fri Nov 21 01:23:15 2014 +0100 @@ -0,0 +1,48 @@ +require "mboxmetadata"; +require "servermetadata"; + +# 1-4: Used as a command +metadata; +metadataexists; +servermetadata; +servermetadataexists; + +# 5-8: Used with no argument +if metadata {} +if metadataexists {} +if servermetadata {} +if servermetadataexists {} + +# 9-10: Used with one string argument +if metadata "frop" { } +if servermetadata "frop" { } +if metadataexists "frop" { } + +# Used with one number argument +if metadata 13123123 { } +if servermetadata 123123 { } +if metadataexists 123123 { } +if servermetadataexists 123123 {} + +# Used with one string list argument +if metadata ["frop"] { } +if servermetadata ["frop"] { } +if metadataexists ["frop"] { } + +# Used with unknown tag +if metadata :frop "frop" { } +if servermetadata :frop "frop" { } +if metadataexists :frop "frop" { } +if servermetadataexists :frop "frop" {} + +# Invalid arguments +if metadata "/private/frop" "friep" {} +if servermetadata "INBOX" "/private/frop" "friep" {} +if metadataexists 23 "/private/frop" {} +if servermetadataexists "INBOX" "/private/frop" {} + +# Invalid annotations +if metadata "INBOX" "frop" "friep" {} +if servermetadata "frop" "friep" {} +if metadataexists "INBOX" ["/private/frop", "/friep"] { } +if servermetadataexists ["/private/frop", "/friep", "/private/friep"] { } diff -r 57b48c0580cd -r 4f53a9230f44 tests/extensions/metadata/execute.svtest --- a/tests/extensions/metadata/execute.svtest Thu Nov 20 01:04:00 2014 +0100 +++ b/tests/extensions/metadata/execute.svtest Fri Nov 21 01:23:15 2014 +0100 @@ -35,6 +35,13 @@ } } +test "MetadataExists - Invalid" { + if metadataexists "INBOX" + ["/shared/frop", "/friep", "/private/frml"] { + test_fail "metadataexists accepted invalid annotation name"; + } +} + test "Metadata" { if not metadata :is "INBOX" "/private/frop" "FROP!" { test_fail "invalid metadata value for /private/frop"; @@ -58,6 +65,12 @@ } } +test "Metadata - Invalid" { + if metadata :contains "INBOX" "/frop" "" { + test_fail "erroneously found a value for \"/frop\""; + } +} + test "ServermetadataExists - None exist" { if servermetadataexists "/private/frop" { test_fail "servermetadataexists confirms existence of unknown annotation"; @@ -86,7 +99,14 @@ test "ServermetadataExists - All exist" { if not servermetadataexists ["/private/frop", "/private/friep", "/private/frml"] { - test_fail "metadataexists fails to recognize annotations"; + test_fail "servermetadataexists fails to recognize annotations"; + } +} + +test "ServermetadataExists - Invalid" { + if servermetadataexists + ["frop", "/private/friep", "/private/frml"] { + test_fail "servermetadataexists accepted invalid annotation name"; } } @@ -112,3 +132,10 @@ test_fail "unexpected match for /private/frml"; } } + +test "Servermetadata - Invalid" { + if servermetadata :contains "/frop" "" { + test_fail "erroneously found a value for \"/frop\""; + } +} + From dovecot at dovecot.org Fri Nov 21 09:17:13 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 21 Nov 2014 09:17:13 +0000 Subject: dovecot-2.2: lmtp: Added TLS security information to Received: h... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1d811ffd1832 changeset: 18105:1d811ffd1832 user: Timo Sirainen date: Fri Nov 21 18:17:00 2014 +0900 description: lmtp: Added TLS security information to Received: header when STARTTLS was used. diffstat: src/lmtp/commands.c | 7 ++++++- 1 files changed, 6 insertions(+), 1 deletions(-) diffs (17 lines): diff -r 5de5be00b369 -r 1d811ffd1832 src/lmtp/commands.c --- a/src/lmtp/commands.c Thu Nov 20 22:03:25 2014 +0900 +++ b/src/lmtp/commands.c Fri Nov 21 18:17:00 2014 +0900 @@ -1004,7 +1004,12 @@ host = net_ip2addr(&client->remote_ip); if (host[0] != '\0') str_printfa(str, " ([%s])", host); - str_printfa(str, "\r\n\tby %s ("PACKAGE_NAME") with LMTP id %s", + str_append(str, "\r\n"); + if (client->ssl_iostream != NULL) { + str_printfa(str, "\t(using %s)\r\n", + ssl_iostream_get_security_string(client->ssl_iostream)); + } + str_printfa(str, "\tby %s ("PACKAGE_NAME") with LMTP id %s", client->my_domain, client->state.session_id); str_append(str, "\r\n\t"); From pigeonhole at rename-it.nl Fri Nov 21 23:17:29 2014 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Sat, 22 Nov 2014 00:17:29 +0100 Subject: dovecot-2.2-pigeonhole: lib-sieve: Consolidated checking mailbox... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/e7979b6a9d4b changeset: 1973:e7979b6a9d4b user: Stephan Bosch date: Sat Nov 22 00:17:13 2014 +0100 description: lib-sieve: Consolidated checking mailbox name. diffstat: src/lib-sieve/ext-fileinto.c | 7 ++- src/lib-sieve/plugins/mailbox/tst-mailboxexists.c | 40 ++++++++++++++++++++- src/lib-sieve/plugins/metadata/tst-metadata.c | 23 ++++++++++++ src/lib-sieve/plugins/metadata/tst-metadataexists.c | 24 ++++++++++++ src/lib-sieve/sieve-actions.c | 11 +++++ src/lib-sieve/sieve-actions.h | 7 +++ 6 files changed, 108 insertions(+), 4 deletions(-) diffs (224 lines): diff -r 4f53a9230f44 -r e7979b6a9d4b src/lib-sieve/ext-fileinto.c --- a/src/lib-sieve/ext-fileinto.c Fri Nov 21 01:23:15 2014 +0100 +++ b/src/lib-sieve/ext-fileinto.c Sat Nov 22 00:17:13 2014 +0100 @@ -117,11 +117,12 @@ /* Check name validity when folder argument is not a variable */ if ( sieve_argument_is_string_literal(arg) ) { - const char *folder = sieve_ast_argument_strc(arg); + const char *folder = sieve_ast_argument_strc(arg), *error; - if ( !uni_utf8_str_is_valid(folder) ) { + if ( !sieve_mailbox_check_name(folder, &error) ) { sieve_command_validate_error(valdtr, cmd, - "folder name specified for fileinto command is not utf-8: %s", folder); + "invalid folder name `%s' specified for fileinto command: %s", + str_sanitize(folder, 256), error); return FALSE; } } diff -r 4f53a9230f44 -r e7979b6a9d4b src/lib-sieve/plugins/mailbox/tst-mailboxexists.c --- a/src/lib-sieve/plugins/mailbox/tst-mailboxexists.c Fri Nov 21 01:23:15 2014 +0100 +++ b/src/lib-sieve/plugins/mailbox/tst-mailboxexists.c Sat Nov 22 00:17:13 2014 +0100 @@ -7,6 +7,7 @@ #include "mail-namespace.h" #include "sieve-common.h" +#include "sieve-actions.h" #include "sieve-extensions.h" #include "sieve-commands.h" #include "sieve-stringlist.h" @@ -62,17 +63,54 @@ * Test validation */ +struct _validate_context { + struct sieve_validator *valdtr; + struct sieve_command *tst; +}; + +static int tst_mailboxexists_mailbox_validate +(void *context, struct sieve_ast_argument *arg) +{ + struct _validate_context *valctx = + (struct _validate_context *)context; + + if ( sieve_argument_is_string_literal(arg) ) { + const char *mailbox = sieve_ast_argument_strc(arg), *error; + + if ( !sieve_mailbox_check_name(mailbox, &error) ) { + sieve_argument_validate_warning + (valctx->valdtr, arg, "%s test: " + "invalid mailbox name `%s' specified: %s", + sieve_command_identifier(valctx->tst), + str_sanitize(mailbox, 256), error); + } + } + + return TRUE; +} + static bool tst_mailboxexists_validate (struct sieve_validator *valdtr, struct sieve_command *tst) { struct sieve_ast_argument *arg = tst->first_positional; + struct sieve_ast_argument *aarg; + struct _validate_context valctx; if ( !sieve_validate_positional_argument (valdtr, tst, arg, "mailbox-names", 1, SAAT_STRING_LIST) ) { return FALSE; } - return sieve_validator_argument_activate(valdtr, tst, arg, FALSE); + if ( !sieve_validator_argument_activate(valdtr, tst, arg, FALSE) ) + return FALSE; + + aarg = arg; + memset(&valctx, 0, sizeof(valctx)); + valctx.valdtr = valdtr; + valctx.tst = tst; + + return sieve_ast_stringlist_map + (&aarg, (void*)&valctx, tst_mailboxexists_mailbox_validate); } /* diff -r 4f53a9230f44 -r e7979b6a9d4b src/lib-sieve/plugins/metadata/tst-metadata.c --- a/src/lib-sieve/plugins/metadata/tst-metadata.c Fri Nov 21 01:23:15 2014 +0100 +++ b/src/lib-sieve/plugins/metadata/tst-metadata.c Sat Nov 22 00:17:13 2014 +0100 @@ -8,6 +8,7 @@ #include "sieve-common.h" #include "sieve-limits.h" #include "sieve-commands.h" +#include "sieve-actions.h" #include "sieve-stringlist.h" #include "sieve-code.h" #include "sieve-comparators.h" @@ -147,6 +148,19 @@ if ( !sieve_validator_argument_activate(valdtr, tst, arg, FALSE) ) return FALSE; + /* Check name validity when mailbox argument is not a variable */ + if ( sieve_argument_is_string_literal(arg) ) { + const char *mailbox = sieve_ast_argument_strc(arg), *error; + + if ( !sieve_mailbox_check_name(mailbox, &error) ) { + sieve_argument_validate_warning + (valdtr, arg, "%s test: " + "invalid mailbox name `%s' specified: %s", + sieve_command_identifier(tst), + str_sanitize(mailbox, 256), error); + } + } + arg = sieve_ast_argument_next(arg); } @@ -363,6 +377,15 @@ } if ( metadata ) { + if ( !sieve_mailbox_check_name(str_c(mailbox), &error) ) { + sieve_runtime_warning(renv, NULL, "%s test: " + "invalid mailbox name `%s' specified: %s", + (metadata ? "metadata" : "servermetadata"), + str_sanitize(str_c(mailbox), 256), error); + sieve_interpreter_set_test_result(renv->interp, FALSE); + return SIEVE_EXEC_OK; + } + sieve_runtime_trace(renv, SIEVE_TRLVL_TESTS, "retrieving annotation `%s' from mailbox `%s'", str_sanitize(str_c(aname), 256), diff -r 4f53a9230f44 -r e7979b6a9d4b src/lib-sieve/plugins/metadata/tst-metadataexists.c --- a/src/lib-sieve/plugins/metadata/tst-metadataexists.c Fri Nov 21 01:23:15 2014 +0100 +++ b/src/lib-sieve/plugins/metadata/tst-metadataexists.c Sat Nov 22 00:17:13 2014 +0100 @@ -8,6 +8,7 @@ #include "sieve-common.h" #include "sieve-commands.h" +#include "sieve-actions.h" #include "sieve-stringlist.h" #include "sieve-code.h" #include "sieve-validator.h" @@ -150,6 +151,19 @@ if ( !sieve_validator_argument_activate(valdtr, tst, arg, FALSE) ) return FALSE; + /* Check name validity when mailbox argument is not a variable */ + if ( sieve_argument_is_string_literal(arg) ) { + const char *mailbox = sieve_ast_argument_strc(arg), *error; + + if ( !sieve_mailbox_check_name(mailbox, &error) ) { + sieve_argument_validate_warning + (valdtr, arg, "%s test: " + "invalid mailbox name `%s' specified: %s", + sieve_command_identifier(tst), + str_sanitize(mailbox, 256), error); + } + } + arg = sieve_ast_argument_next(arg); } @@ -327,6 +341,7 @@ struct sieve_stringlist *anames; string_t *mailbox; bool trace = FALSE, all_exist = TRUE; + const char *error; int ret; /* @@ -348,6 +363,15 @@ * Perform operation */ + if ( metadata && !sieve_mailbox_check_name(str_c(mailbox), &error) ) { + sieve_runtime_warning(renv, NULL, "%s test: " + "invalid mailbox name `%s' specified: %s", + (metadata ? "metadataexists" : "servermetadataexists"), + str_sanitize(str_c(mailbox), 256), error); + sieve_interpreter_set_test_result(renv->interp, FALSE); + return SIEVE_EXEC_OK; + } + if ( sieve_runtime_trace_active(renv, SIEVE_TRLVL_TESTS) ) { if ( metadata ) sieve_runtime_trace(renv, SIEVE_TRLVL_TESTS, "metadataexists test"); diff -r 4f53a9230f44 -r e7979b6a9d4b src/lib-sieve/sieve-actions.c --- a/src/lib-sieve/sieve-actions.c Fri Nov 21 01:23:15 2014 +0100 +++ b/src/lib-sieve/sieve-actions.c Sat Nov 22 00:17:13 2014 +0100 @@ -932,6 +932,17 @@ return result; } +/* + * Mailbox + */ +bool sieve_mailbox_check_name(const char *mailbox, const char **error_r) +{ + if ( !uni_utf8_str_is_valid(mailbox) ) { + *error_r = "mailbox is utf-8"; + return FALSE; + } + return TRUE; +} diff -r 4f53a9230f44 -r e7979b6a9d4b src/lib-sieve/sieve-actions.h --- a/src/lib-sieve/sieve-actions.h Fri Nov 21 01:23:15 2014 +0100 +++ b/src/lib-sieve/sieve-actions.h Sat Nov 22 00:17:13 2014 +0100 @@ -273,4 +273,11 @@ (const struct sieve_action_exec_env *aenv, const char *sender, const char *recipient, const char *reason); +/* + * Mailbox + */ + +// FIXME: move this to a more appropriate location +bool sieve_mailbox_check_name(const char *mailbox, const char **error_r); + #endif /* __SIEVE_ACTIONS_H */ From dovecot at dovecot.org Tue Nov 25 00:56:57 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 25 Nov 2014 00:56:57 +0000 Subject: dovecot-2.2: lib-storage: Don't fetch wanted_fields for messages... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2553f3aae9bb changeset: 18106:2553f3aae9bb user: Timo Sirainen date: Tue Nov 25 02:56:37 2014 +0200 description: lib-storage: Don't fetch wanted_fields for messages that don't match the search query. For example "doveadm fetch text subject foo" was opening all the mails, even though only a few matched the subject (that was hopefully already in cache file). The behavior still isn't perfect though. The wanted_fields should probably be split into search_wanted_fields and fetch_wanted_fields, but the current behavior is likely good enough for now. diffstat: src/lib-storage/index/index-mail.c | 13 +++++++++---- src/lib-storage/index/index-mail.h | 1 + src/lib-storage/index/index-search.c | 8 ++++++++ 3 files changed, 18 insertions(+), 4 deletions(-) diffs (74 lines): diff -r 1d811ffd1832 -r 2553f3aae9bb src/lib-storage/index/index-mail.c --- a/src/lib-storage/index/index-mail.c Fri Nov 21 18:17:00 2014 +0900 +++ b/src/lib-storage/index/index-mail.c Tue Nov 25 02:56:37 2014 +0200 @@ -1435,9 +1435,9 @@ mail->data.save_envelope = TRUE; } -static void index_mail_update_access_parts(struct index_mail *mail) +void index_mail_update_access_parts(struct mail *_mail) { - struct mail *_mail = &mail->mail.mail; + struct index_mail *mail = (struct index_mail *)_mail; struct index_mail_data *data = &mail->data; const struct mail_cache_field *cache_fields = mail->ibox->cache_fields; struct mail_cache_view *cache_view = _mail->transaction->cache_view; @@ -1569,7 +1569,12 @@ return; } - index_mail_update_access_parts(mail); + if (!mail->search_mail) + index_mail_update_access_parts(_mail); + else { + /* searching code will call the index_mail_update_access_parts() + after we know the mail is actually wanted to be fetched. */ + } mail->data.initialized = TRUE; } @@ -1660,7 +1665,7 @@ mailbox_header_lookup_init(_mail->box, array_idx(&names, 0)); } - index_mail_update_access_parts(mail); + index_mail_update_access_parts(_mail); } void index_mail_set_uid_cache_updates(struct mail *_mail, bool set) diff -r 1d811ffd1832 -r 2553f3aae9bb src/lib-storage/index/index-mail.h --- a/src/lib-storage/index/index-mail.h Fri Nov 21 18:17:00 2014 +0900 +++ b/src/lib-storage/index/index-mail.h Tue Nov 25 02:56:37 2014 +0200 @@ -171,6 +171,7 @@ void index_mail_add_temp_wanted_fields(struct mail *mail, enum mail_fetch_field fields, struct mailbox_header_lookup_ctx *headers); +void index_mail_update_access_parts(struct mail *mail); void index_mail_close(struct mail *mail); void index_mail_close_streams(struct index_mail *mail); void index_mail_free(struct mail *mail); diff -r 1d811ffd1832 -r 2553f3aae9bb src/lib-storage/index/index-search.c --- a/src/lib-storage/index/index-search.c Fri Nov 21 18:17:00 2014 +0900 +++ b/src/lib-storage/index/index-search.c Tue Nov 25 02:56:37 2014 +0200 @@ -1550,6 +1550,13 @@ mail_search_args_reset(_ctx->args->args, FALSE); if (match != 0) { + /* either matched or result is still unknown. + anyway we're far enough now that we probably want + to update the access_parts. the only problem here is + if searching would want fewer access_parts than the + fetching part, but that's probably not a big problem + usually. */ + index_mail_update_access_parts(mail); ret = 1; break; } @@ -1730,6 +1737,7 @@ mailp = array_idx(&ctx->mails, 0); mail_set_seq(*mailp, seq); + index_mail_update_access_parts(*mailp); *mail_r = *mailp; return TRUE; } From dovecot at dovecot.org Tue Nov 25 01:10:57 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 25 Nov 2014 01:10:57 +0000 Subject: dovecot-2.2: dict-redis: Added support for expire_secs=n parameter. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5ee74bc88f18 changeset: 18107:5ee74bc88f18 user: Timo Sirainen date: Tue Nov 25 03:10:41 2014 +0200 description: dict-redis: Added support for expire_secs=n parameter. It performs EXPIRE command for the added keys immediately after the SET. diffstat: src/lib-dict/dict-redis.c | 34 ++++++++++++++++++++++++++-------- 1 files changed, 26 insertions(+), 8 deletions(-) diffs (79 lines): diff -r 2553f3aae9bb -r 5ee74bc88f18 src/lib-dict/dict-redis.c --- a/src/lib-dict/dict-redis.c Tue Nov 25 02:56:37 2014 +0200 +++ b/src/lib-dict/dict-redis.c Tue Nov 25 03:10:41 2014 +0200 @@ -43,7 +43,7 @@ struct redis_dict { struct dict dict; - char *username, *key_prefix; + char *username, *key_prefix, *expire_value; unsigned int timeout_msecs; struct ioloop *ioloop, *prev_ioloop; @@ -316,7 +316,7 @@ { struct redis_dict *dict; struct ip_addr ip; - unsigned int port = REDIS_DEFAULT_PORT; + unsigned int secs, port = REDIS_DEFAULT_PORT; const char *const *args, *unix_path = NULL; int ret = 0; @@ -351,6 +351,14 @@ } else if (strncmp(*args, "prefix=", 7) == 0) { i_free(dict->key_prefix); dict->key_prefix = i_strdup(*args + 7); + } else if (strncmp(*args, "expire_secs=", 12) == 0) { + if (str_to_uint(*args + 12, &secs) < 0 || secs == 0) { + *error_r = t_strdup_printf( + "Invalid expire_secs: %s", *args+14); + ret = -1; + } + i_free(dict->expire_value); + dict->expire_value = i_strdup(*args + 12); } else if (strncmp(*args, "timeout_msecs=", 14) == 0) { if (str_to_uint(*args+14, &dict->timeout_msecs) < 0) { *error_r = t_strdup_printf( @@ -404,6 +412,7 @@ str_free(&dict->conn.last_reply); array_free(&dict->replies); array_free(&dict->input_states); + i_free(dict->expire_value); i_free(dict->key_prefix); i_free(dict->username); i_free(dict); @@ -630,19 +639,28 @@ struct redis_dict_transaction_context *ctx = (struct redis_dict_transaction_context *)_ctx; struct redis_dict *dict = (struct redis_dict *)_ctx->dict; - const char *cmd; + string_t *cmd; if (redis_check_transaction(ctx) < 0) return; key = redis_dict_get_full_key(dict, key); - cmd = t_strdup_printf("*3\r\n$3\r\nSET\r\n$%u\r\n%s\r\n$%u\r\n%s\r\n", - (unsigned int)strlen(key), key, - (unsigned int)strlen(value), value); - if (o_stream_send_str(dict->conn.conn.output, cmd) < 0) - ctx->failed = TRUE; + cmd = t_str_new(128); + str_printfa(cmd, "*3\r\n$3\r\nSET\r\n$%u\r\n%s\r\n$%u\r\n%s\r\n", + (unsigned int)strlen(key), key, + (unsigned int)strlen(value), value); redis_input_state_add(dict, REDIS_INPUT_STATE_MULTI); ctx->cmd_count++; + if (dict->expire_value != NULL) { + str_printfa(cmd, "*3\r\n$6\r\nEXPIRE\r\n$%u\r\n%s\r\n$%u\r\n%s\r\n", + (unsigned int)strlen(key), key, + (unsigned int)strlen(dict->expire_value), + dict->expire_value); + redis_input_state_add(dict, REDIS_INPUT_STATE_MULTI); + ctx->cmd_count++; + } + if (o_stream_send(dict->conn.conn.output, str_data(cmd), str_len(cmd)) < 0) + ctx->failed = TRUE; } static void redis_unset(struct dict_transaction_context *_ctx, From dovecot at dovecot.org Tue Nov 25 01:12:42 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 25 Nov 2014 01:12:42 +0000 Subject: dovecot-2.2: dict-redis: Small error handling fix to previous ch... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fd43098e23d0 changeset: 18108:fd43098e23d0 user: Timo Sirainen date: Tue Nov 25 03:12:25 2014 +0200 description: dict-redis: Small error handling fix to previous change. diffstat: src/lib-dict/dict-redis.c | 8 +++++--- 1 files changed, 5 insertions(+), 3 deletions(-) diffs (22 lines): diff -r 5ee74bc88f18 -r fd43098e23d0 src/lib-dict/dict-redis.c --- a/src/lib-dict/dict-redis.c Tue Nov 25 03:10:41 2014 +0200 +++ b/src/lib-dict/dict-redis.c Tue Nov 25 03:12:25 2014 +0200 @@ -352,13 +352,15 @@ i_free(dict->key_prefix); dict->key_prefix = i_strdup(*args + 7); } else if (strncmp(*args, "expire_secs=", 12) == 0) { - if (str_to_uint(*args + 12, &secs) < 0 || secs == 0) { + const char *value = *args + 12; + + if (str_to_uint(value, &secs) < 0 || secs == 0) { *error_r = t_strdup_printf( - "Invalid expire_secs: %s", *args+14); + "Invalid expire_secs: %s", value); ret = -1; } i_free(dict->expire_value); - dict->expire_value = i_strdup(*args + 12); + dict->expire_value = i_strdup(value); } else if (strncmp(*args, "timeout_msecs=", 14) == 0) { if (str_to_uint(*args+14, &dict->timeout_msecs) < 0) { *error_r = t_strdup_printf( From dovecot at dovecot.org Tue Nov 25 01:36:12 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 25 Nov 2014 01:36:12 +0000 Subject: dovecot-2.2: lib: guid - turn uint8_t * parameters into guid_128_t Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/88e985e7f2b6 changeset: 18109:88e985e7f2b6 user: Phil Carmody date: Tue Nov 25 03:31:34 2014 +0200 description: lib: guid - turn uint8_t * parameters into guid_128_t As we expose the type's internals, there's no concrete difference between the two, but it might help static code checkers to detect misuse of the library, or sloppy typing. Signed-off-by: Phil Carmody diffstat: src/lib/guid.c | 4 ++-- src/lib/guid.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diffs (32 lines): diff -r fd43098e23d0 -r 88e985e7f2b6 src/lib/guid.c --- a/src/lib/guid.c Tue Nov 25 03:12:25 2014 +0200 +++ b/src/lib/guid.c Tue Nov 25 03:31:34 2014 +0200 @@ -114,12 +114,12 @@ return binary_to_hex(guid, GUID_128_SIZE); } -unsigned int guid_128_hash(const uint8_t *guid) +unsigned int guid_128_hash(const guid_128_t guid) { return mem_hash(guid, GUID_128_SIZE); } -int guid_128_cmp(const uint8_t *guid1, const uint8_t *guid2) +int guid_128_cmp(const guid_128_t guid1, const guid_128_t guid2) { return memcmp(guid1, guid2, GUID_128_SIZE); } diff -r fd43098e23d0 -r 88e985e7f2b6 src/lib/guid.h --- a/src/lib/guid.h Tue Nov 25 03:12:25 2014 +0200 +++ b/src/lib/guid.h Tue Nov 25 03:31:34 2014 +0200 @@ -26,8 +26,8 @@ int guid_128_from_string(const char *str, guid_128_t guid_r); /* guid_128 hash/cmp functions for hash.h */ -unsigned int guid_128_hash(const uint8_t *guid); -int guid_128_cmp(const uint8_t *guid1, const uint8_t *guid2); +unsigned int guid_128_hash(const guid_128_t guid); +int guid_128_cmp(const guid_128_t guid1, const guid_128_t guid2); /* Return the hash of host used by guid_128_generate(). */ void guid_128_host_hash_get(const char *host, From dovecot at dovecot.org Tue Nov 25 01:53:53 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 25 Nov 2014 01:53:53 +0000 Subject: dovecot-2.2: lib: test-array - test array_cmp() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a45ee7e221b0 changeset: 18110:a45ee7e221b0 user: Phil Carmody date: Tue Nov 25 03:43:02 2014 +0200 description: lib: test-array - test array_cmp() Signed-off-by: Phil Carmody diffstat: src/lib/test-array.c | 37 +++++++++++++++++++++++++++++++++++++ 1 files changed, 37 insertions(+), 0 deletions(-) diffs (56 lines): diff -r 88e985e7f2b6 -r a45ee7e221b0 src/lib/test-array.c --- a/src/lib/test-array.c Tue Nov 25 03:31:34 2014 +0200 +++ b/src/lib/test-array.c Tue Nov 25 03:43:02 2014 +0200 @@ -3,6 +3,8 @@ #include "test-lib.h" #include "array.h" +#include /* rand() */ + struct foo { unsigned int a, b, c; }; @@ -51,8 +53,43 @@ test_end(); } +static void test_array_cmp(void) +{ + static const unsigned short deltas[] = { + -32768, -16384, -512, -256, -128, -64, -2, -1, + 0, 1, 2, 64, 128, 256, 512, 16384, 32768 + }; + +#define NELEMS 5u + ARRAY(unsigned short) arr1, arr2; + unsigned short elems[NELEMS]; + unsigned int i; + + test_begin("array compare (ushort)"); + t_array_init(&arr1, NELEMS); + t_array_init(&arr2, NELEMS); + for (i = 0; i < NELEMS; i++) { + elems[i] = rand(); + array_append(&arr2, &elems[i], 1); + } + array_append(&arr1, elems, NELEMS); + test_assert(array_cmp(&arr1, &arr2) == 1); + for (i = 0; i < 256; i++) { + unsigned int j = rand() % NELEMS; + unsigned short tmp = *array_idx(&arr2, j); + unsigned short repl = tmp + deltas[rand() % N_ELEMENTS(deltas)]; + + array_idx_set(&arr2, j, &repl); + test_assert_idx(array_cmp(&arr1, &arr2) == (tmp == repl), i); + array_idx_set(&arr2, j, &tmp); + test_assert_idx(array_cmp(&arr1, &arr2) == TRUE, i); + } + test_end(); +} + void test_array(void) { test_array_foreach(); test_array_reverse(); + test_array_cmp(); } From dovecot at dovecot.org Tue Nov 25 01:53:53 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 25 Nov 2014 01:53:53 +0000 Subject: dovecot-2.2: lib: array - element-wise comparison via callback Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b8ac4e5a1002 changeset: 18111:b8ac4e5a1002 user: Phil Carmody date: Tue Nov 25 03:43:02 2014 +0200 description: lib: array - element-wise comparison via callback 2 are provided - with and without a context pointer. Signed-off-by: Phil Carmody diffstat: src/lib/array.c | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ src/lib/array.h | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 0 deletions(-) diffs (118 lines): diff -r a45ee7e221b0 -r b8ac4e5a1002 src/lib/array.c --- a/src/lib/array.c Tue Nov 25 03:43:02 2014 +0200 +++ b/src/lib/array.c Tue Nov 25 03:43:02 2014 +0200 @@ -69,6 +69,59 @@ return buffer_cmp(array1->buffer, array2->buffer); } +bool array_equal_fn_i(const struct array *array1, const struct array *array2, + int (*cmp)(const void *, const void*)) +{ + unsigned int count1, count2, i, size; + + if (!array_is_created_i(array1) || array1->buffer->used == 0) + return !array_is_created_i(array2) || array2->buffer->used == 0; + + if (!array_is_created_i(array2)) + return FALSE; + + count1 = array_count_i(array1); count2 = array_count_i(array2); + if (count1 != count2) + return FALSE; + + size = array1->element_size; + i_assert(size == array2->element_size); + + for (i = 0; i < count1; i++) { + if (cmp(CONST_PTR_OFFSET(array1->buffer->data, i * size), + CONST_PTR_OFFSET(array2->buffer->data, i * size)) != 0) + return FALSE; + } + return TRUE; +} + +bool array_equal_fn_ctx_i(const struct array *array1, const struct array *array2, + int (*cmp)(const void *, const void *, const void *), + const void *context) +{ + unsigned int count1, count2, i, size; + + if (!array_is_created_i(array1) || array1->buffer->used == 0) + return !array_is_created_i(array2) || array2->buffer->used == 0; + + if (!array_is_created_i(array2)) + return FALSE; + + count1 = array_count_i(array1); count2 = array_count_i(array2); + if (count1 != count2) + return FALSE; + + size = array1->element_size; + i_assert(size == array2->element_size); + + for (i = 0; i < count1; i++) { + if (cmp(CONST_PTR_OFFSET(array1->buffer->data, i * size), + CONST_PTR_OFFSET(array2->buffer->data, i * size), context) != 0) + return FALSE; + } + return TRUE; +} + void array_reverse_i(struct array *array) { const size_t element_size = array->element_size; diff -r a45ee7e221b0 -r b8ac4e5a1002 src/lib/array.h --- a/src/lib/array.h Tue Nov 25 03:43:02 2014 +0200 +++ b/src/lib/array.h Tue Nov 25 03:43:02 2014 +0200 @@ -47,10 +47,15 @@ # define ARRAY_TYPE_CHECK(array, data) \ COMPILE_ERROR_IF_TYPES_NOT_COMPATIBLE( \ **(array)->v_modifiable, *(data)) +# define ARRAY_TYPES_CHECK(array1, array2) \ + COMPILE_ERROR_IF_TYPES_NOT_COMPATIBLE( \ + **(array1)->v_modifiable, **(array2)->v_modifiable) + #else # define ARRAY_TYPE_CAST_CONST(array) # define ARRAY_TYPE_CAST_MODIFIABLE(array) # define ARRAY_TYPE_CHECK(array, data) 0 +# define ARRAY_TYPES_CHECK(array1, array2) 0 #endif /* usage: struct foo *foo; array_foreach(foo_arr, foo) { .. } */ @@ -259,6 +264,35 @@ #define array_cmp(array1, array2) \ array_cmp_i(&(array1)->arr, &(array2)->arr) +/* Test equality via a comparator */ +bool array_equal_fn_i(const struct array *array1, + const struct array *array2, + int (*cmp)(const void*, const void *)) ATTR_PURE; +#define array_equal_fn(array1, array2, cmp) \ + array_equal_fn_i(&(array1)->arr + \ + ARRAY_TYPES_CHECK(array1, array2), \ + &(array2)->arr + \ + CALLBACK_TYPECHECK(cmp, int (*)(typeof(*(array1)->v), \ + typeof(*(array2)->v))), \ + (int (*)(const void *, const void *))cmp) +bool array_equal_fn_ctx_i(const struct array *array1, + const struct array *array2, + int (*cmp)(const void*, const void *, const void *), + const void *context) ATTR_PURE; +/* Same, but with a context pointer. + context can't be void* as ``const typeof(context)'' won't compile, + so ``const typeof(*context)*'' is required instead, and that requires a + complete type. */ +#define array_equal_fn_ctx(array1, array2, cmp, ctx) \ + array_equal_fn_ctx_i(&(array1)->arr + \ + ARRAY_TYPES_CHECK(array1, array2), \ + &(array2)->arr + \ + CALLBACK_TYPECHECK(cmp, int (*)(typeof(*(array1)->v), \ + typeof(*(array2)->v), \ + const typeof(*ctx)*)), \ + (int (*)(const void *, const void *, const void *))cmp, \ + ctx) + void array_reverse_i(struct array *array); #define array_reverse(array) \ array_reverse_i(&(array)->arr) From dovecot at dovecot.org Tue Nov 25 01:54:01 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 25 Nov 2014 01:54:01 +0000 Subject: dovecot-2.2: lib: test-array - new equality testers Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5522d5ce87a5 changeset: 18112:5522d5ce87a5 user: Phil Carmody date: Tue Nov 25 03:44:31 2014 +0200 description: lib: test-array - new equality testers One using strings that get dereferenced, the other using the context pointer to pass in external data. Signed-off-by: Phil Carmody diffstat: src/lib/test-array.c | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 files changed, 85 insertions(+), 2 deletions(-) diffs (127 lines): diff -r b8ac4e5a1002 -r 5522d5ce87a5 src/lib/test-array.c --- a/src/lib/test-array.c Tue Nov 25 03:43:02 2014 +0200 +++ b/src/lib/test-array.c Tue Nov 25 03:44:31 2014 +0200 @@ -52,7 +52,19 @@ } test_end(); } - +static int test_compare_ushort(const unsigned short *c1, const unsigned short *c2) +{ + return *c1 > *c2 ? 1 + : *c1 < *c2 ? -1 + : 0; +} +static int test_compare_ushort_fuzz(const unsigned short *c1, const unsigned short *c2, const int *pfuzz) +{ + int d = (int)*c1 - (int)*c2; + if (d <= *pfuzz && -d <= *pfuzz) + return 0; + return d; +} static void test_array_cmp(void) { static const unsigned short deltas[] = { @@ -62,8 +74,9 @@ #define NELEMS 5u ARRAY(unsigned short) arr1, arr2; - unsigned short elems[NELEMS]; + unsigned short elems[NELEMS+1]; unsigned int i; + int fuzz; test_begin("array compare (ushort)"); t_array_init(&arr1, NELEMS); @@ -74,6 +87,10 @@ } array_append(&arr1, elems, NELEMS); test_assert(array_cmp(&arr1, &arr2) == 1); + test_assert(array_equal_fn(&arr1, &arr2, test_compare_ushort) == 1); + fuzz = 0; + test_assert(array_equal_fn_ctx(&arr1, &arr2, test_compare_ushort_fuzz, &fuzz) == 1); + for (i = 0; i < 256; i++) { unsigned int j = rand() % NELEMS; unsigned short tmp = *array_idx(&arr2, j); @@ -81,9 +98,74 @@ array_idx_set(&arr2, j, &repl); test_assert_idx(array_cmp(&arr1, &arr2) == (tmp == repl), i); + test_assert_idx(array_equal_fn(&arr1, &arr2, test_compare_ushort) == (tmp == repl), i); + fuzz = (int)tmp - (int)repl; + if (fuzz < 0) + fuzz = -fuzz; + test_assert_idx(array_equal_fn_ctx(&arr1, &arr2, test_compare_ushort_fuzz, &fuzz) == 1, i); + if (fuzz > 0) { + fuzz--; + test_assert_idx(array_equal_fn_ctx(&arr1, &arr2, test_compare_ushort_fuzz, &fuzz) == 0, i); + } array_idx_set(&arr2, j, &tmp); test_assert_idx(array_cmp(&arr1, &arr2) == TRUE, i); + test_assert_idx(array_equal_fn(&arr1, &arr2, test_compare_ushort) == 1, i); + fuzz = 0; + test_assert_idx(array_equal_fn_ctx(&arr1, &arr2, test_compare_ushort_fuzz, &fuzz) == 1, i); } + elems[NELEMS] = 0; + array_append(&arr2, &elems[NELEMS], 1); + test_assert(array_cmp(&arr1, &arr2) == 0); + test_assert(array_equal_fn(&arr1, &arr2, test_compare_ushort) == 0); + test_assert_idx(array_equal_fn_ctx(&arr1, &arr2, test_compare_ushort_fuzz, &fuzz) == 0, i); + + test_end(); +} + +static int test_compare_string(const char *const *c1, const char *const *c2) +{ + return strcmp(*c1, *c2); +} +static void test_array_cmp_str(void) +{ +#define NELEMS 5u + ARRAY(const char *) arr1, arr2; + const char *elemstrs[NELEMS+1]; + unsigned int i; + + test_begin("array compare (char*)"); + t_array_init(&arr1, NELEMS); + t_array_init(&arr2, NELEMS); + for (i = 0; i < NELEMS; i++) { + elemstrs[i] = t_strdup_printf("%x", rand()); /* never 0-length */ + array_append(&arr2, &elemstrs[i], 1); + } + array_append(&arr1, elemstrs, NELEMS); + test_assert(array_cmp(&arr1, &arr2) == 1); /* pointers shared, so identical */ + test_assert(array_equal_fn(&arr1, &arr2, test_compare_string) == 1); /* therefore value same */ + for (i = 0; i < 2560; i++) { + unsigned int j = rand() % NELEMS; + const char *ostr = *array_idx(&arr2, j); + unsigned int olen = strlen(ostr); + unsigned int rc = rand() % (olen + 1); + char ochar = ostr[rc]; + char buf[12]; + const char *bufp = buf; + memcpy(buf, ostr, olen+1); + buf[rc] = rand() % (CHAR_MAX + 1 - CHAR_MIN) + CHAR_MIN; + if(rc == olen) + buf[rc+1] = '\0'; + array_idx_set(&arr2, j, &bufp); + test_assert(array_cmp(&arr1, &arr2) == 0); /* pointers now differ */ + test_assert_idx(array_equal_fn(&arr1, &arr2, test_compare_string) + == (strcmp(ostr, buf) == 0), i); /* sometimes still the same */ + test_assert_idx(array_equal_fn(&arr1, &arr2, test_compare_string) + == (ochar == buf[rc]), i); /* ditto */ + array_idx_set(&arr2, j, &ostr); + test_assert(array_cmp(&arr1, &arr2) == 1); /* pointers now same again */ + test_assert_idx(array_equal_fn(&arr1, &arr2, test_compare_string) == 1, i); /* duh! */ + } + /* length differences being detected are tested in other tests */ test_end(); } @@ -92,4 +174,5 @@ test_array_foreach(); test_array_reverse(); test_array_cmp(); + test_array_cmp_str(); } From dovecot at dovecot.org Tue Nov 25 01:54:06 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 25 Nov 2014 01:54:06 +0000 Subject: dovecot-2.2: lib: test-array - fatal tests Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9141a16aff4e changeset: 18113:9141a16aff4e user: Phil Carmody date: Tue Nov 25 03:44:55 2014 +0200 description: lib: test-array - fatal tests Tests both of the i_asserts which aren't already protected by TYPEOF compile-time checks. Signed-off-by: Phil Carmody diffstat: src/lib/test-array.c | 25 +++++++++++++++++++++++++ src/lib/test-lib.c | 1 + src/lib/test-lib.h | 1 + 3 files changed, 27 insertions(+), 0 deletions(-) diffs (54 lines): diff -r 5522d5ce87a5 -r 9141a16aff4e src/lib/test-array.c --- a/src/lib/test-array.c Tue Nov 25 03:44:31 2014 +0200 +++ b/src/lib/test-array.c Tue Nov 25 03:44:55 2014 +0200 @@ -176,3 +176,28 @@ test_array_cmp(); test_array_cmp_str(); } + +enum fatal_test_state fatal_array(int stage) +{ + switch(stage) { + case 0: { + ARRAY(double) ad; + test_begin("fatal_array"); + t_array_init(&ad, 3); + (void)array_idx(&ad, 3); + return FATAL_TEST_FAILURE; + } break; + + case 1: { + ARRAY(double) ad; + ARRAY(short) as; + t_array_init(&ad, 2); + t_array_init(&as, 8); + array_copy(&ad.arr, 1, &as.arr, 0, 4); + return FATAL_TEST_FAILURE; + } break; + + } + test_end(); + return FATAL_TEST_FINISHED; +} diff -r 5522d5ce87a5 -r 9141a16aff4e src/lib/test-lib.c --- a/src/lib/test-lib.c Tue Nov 25 03:44:31 2014 +0200 +++ b/src/lib/test-lib.c Tue Nov 25 03:44:55 2014 +0200 @@ -52,6 +52,7 @@ NULL }; static enum fatal_test_state (*fatal_functions[])(int) = { + fatal_array, fatal_data_stack, fatal_mempool, fatal_printf_format_fix, diff -r 5522d5ce87a5 -r 9141a16aff4e src/lib/test-lib.h --- a/src/lib/test-lib.h Tue Nov 25 03:44:31 2014 +0200 +++ b/src/lib/test-lib.h Tue Nov 25 03:44:55 2014 +0200 @@ -6,6 +6,7 @@ void test_aqueue(void); void test_array(void); +enum fatal_test_state fatal_array(int); void test_base32(void); void test_base64(void); void test_bits(void); From dovecot at dovecot.org Tue Nov 25 01:54:06 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 25 Nov 2014 01:54:06 +0000 Subject: dovecot-2.2: lib: test-array - fix off-by-one in test_reverse Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e9ac32255ef1 changeset: 18114:e9ac32255ef1 user: Phil Carmody date: Tue Nov 25 03:45:35 2014 +0200 description: lib: test-array - fix off-by-one in test_reverse i is not used as an index, but as a count. Therefore, it can attain the array size in the loop without causing any buffer overflows. Signed-off-by: Phil Carmody diffstat: src/lib/test-array.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 9141a16aff4e -r e9ac32255ef1 src/lib/test-array.c --- a/src/lib/test-array.c Tue Nov 25 03:44:55 2014 +0200 +++ b/src/lib/test-array.c Tue Nov 25 03:45:35 2014 +0200 @@ -41,7 +41,7 @@ test_begin("array reverse"); t_array_init(&intarr, 5); - for (i = 0; i < N_ELEMENTS(input); i++) { + for (i = 0; i <= N_ELEMENTS(input); i++) { array_clear(&intarr); array_append(&intarr, input, i); array_reverse(&intarr); From dovecot at dovecot.org Tue Nov 25 01:54:20 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 25 Nov 2014 01:54:20 +0000 Subject: dovecot-2.2: lib: array - additional compile-time checks to arra... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b7074256cc65 changeset: 18115:b7074256cc65 user: Phil Carmody date: Tue Nov 25 03:48:42 2014 +0200 description: lib: array - additional compile-time checks to array_append The new two-array helper macro works in this context too. Note: array_copy doesn't have any protection currently, it isn't obvious how to wrap that with some safety without breaking interfaces. It only seems to have one internal user, lib/aqueue. Signed-off-by: Phil Carmody diffstat: src/lib/array.h | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (13 lines): diff -r e9ac32255ef1 -r b7074256cc65 src/lib/array.h --- a/src/lib/array.h Tue Nov 25 03:45:35 2014 +0200 +++ b/src/lib/array.h Tue Nov 25 03:48:42 2014 +0200 @@ -164,7 +164,8 @@ buffer_append_buf(dest_array->buffer, src_array->buffer, 0, (size_t)-1); } #define array_append_array(dest_array, src_array) \ - array_append_array_i(&(dest_array)->arr, &(src_array)->arr) + array_append_array_i(&(dest_array)->arr + ARRAY_TYPES_CHECK(dest_array, src_array), \ + &(src_array)->arr) static inline void array_insert_i(struct array *array, unsigned int idx, From dovecot at dovecot.org Fri Nov 28 08:58:05 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 28 Nov 2014 08:58:05 +0000 Subject: dovecot-2.2: lib-test: more delicate *assert() behaviour when do... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/247f3742a3c2 changeset: 18116:247f3742a3c2 user: Phil Carmody date: Fri Nov 28 10:57:43 2014 +0200 description: lib-test: more delicate *assert() behaviour when doing fatal tests We shouldn't be i_asserting ourselves when we're in the state of wanting a test to fail. Therefore instead test_assert to get the assert message, and ensure that the success flag is cleared to get the FAILED at the end. Signed-off-by: Phil Carmody diffstat: src/lib-test/test-common.c | 20 +++++++++++++------- 1 files changed, 13 insertions(+), 7 deletions(-) diffs (52 lines): diff -r b7074256cc65 -r 247f3742a3c2 src/lib-test/test-common.c --- a/src/lib-test/test-common.c Tue Nov 25 03:48:42 2014 +0200 +++ b/src/lib-test/test-common.c Fri Nov 28 10:57:43 2014 +0200 @@ -9,6 +9,10 @@ #include /* for fatal tests */ +/* To test the firing of i_assert, we need non-local jumps, i.e. setjmp */ +static volatile bool expecting_fatal = FALSE; +static jmp_buf fatal_jmpbuf; + #define OUT_NAME_ALIGN 70 static char *test_prefix; @@ -145,9 +149,12 @@ void test_begin(const char *name) { - i_assert(test_prefix == NULL); + test_success = TRUE; + if (!expecting_fatal) + i_assert(test_prefix == NULL); + else + test_assert((test_success = (test_prefix == NULL))); test_prefix = i_strdup(name); - test_success = TRUE; } bool test_has_failed(void) @@ -185,7 +192,10 @@ void test_end(void) { - i_assert(test_prefix != NULL); + if (!expecting_fatal) + i_assert(test_prefix != NULL); + else + test_assert(test_prefix != NULL); test_out("", test_success); if (!test_success) @@ -260,10 +270,6 @@ test_success = FALSE; } -/* To test the firing of i_assert, we need non-local jumps, i.e. setjmp */ -static volatile bool expecting_fatal = FALSE; -static jmp_buf fatal_jmpbuf; - static void ATTR_FORMAT(2, 0) ATTR_NORETURN test_fatal_handler(const struct failure_context *ctx, const char *format, va_list args) From dovecot at dovecot.org Fri Nov 28 08:58:05 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 28 Nov 2014 08:58:05 +0000 Subject: dovecot-2.2: lib: test-array - remove possibility to optimise ou... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8a4b536705ad changeset: 18117:8a4b536705ad user: Phil Carmody date: Fri Nov 28 10:57:43 2014 +0200 description: lib: test-array - remove possibility to optimise out tests (void) on a call to a "pure" function was causing some compilers (gcc 4.8) to not reach the assert within the pure function. Technically it's not a pure function, as (in the assert fail case) it has effects other than its return value. However, simply removing the void cast (and then actually using the return value) also prevents this optimisation. An additional fatal test was added - there's a difference between an access outside the allocated range and outside the initialised range. Signed-off-by: Phil Carmody diffstat: src/lib/array.h | 1 + src/lib/test-array.c | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 1 deletions(-) diffs (52 lines): diff -r 247f3742a3c2 -r 8a4b536705ad src/lib/array.h --- a/src/lib/array.h Fri Nov 28 10:57:43 2014 +0200 +++ b/src/lib/array.h Fri Nov 28 10:57:43 2014 +0200 @@ -197,6 +197,7 @@ #define array_get(array, count) \ ARRAY_TYPE_CAST_CONST(array)array_get_i(&(array)->arr, count) +/* Re: i_assert() vs. pure: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51971#c1 */ static inline const void * ATTR_PURE array_idx_i(const struct array *array, unsigned int idx) { diff -r 247f3742a3c2 -r 8a4b536705ad src/lib/test-array.c --- a/src/lib/test-array.c Fri Nov 28 10:57:43 2014 +0200 +++ b/src/lib/test-array.c Fri Nov 28 10:57:43 2014 +0200 @@ -179,20 +179,36 @@ enum fatal_test_state fatal_array(int stage) { + double tmpd[2] = { 42., -42. }; + short tmps[8] = {1,2,3,4,5,6,7,8}; + switch(stage) { case 0: { ARRAY(double) ad; test_begin("fatal_array"); t_array_init(&ad, 3); - (void)array_idx(&ad, 3); + /* allocation big enough, but memory not initialised */ + if (array_idx(&ad, 0) == NULL) + return FATAL_TEST_FAILURE; return FATAL_TEST_FAILURE; } break; case 1: { ARRAY(double) ad; + t_array_init(&ad, 2); + array_append(&ad, tmpd, 2); + /* actual out of range address requested */ + if (array_idx(&ad, 2) == NULL) + return FATAL_TEST_FAILURE; + return FATAL_TEST_FAILURE; + } break; + + case 2: { + ARRAY(double) ad; ARRAY(short) as; t_array_init(&ad, 2); t_array_init(&as, 8); + array_append(&as, tmps, 2); array_copy(&ad.arr, 1, &as.arr, 0, 4); return FATAL_TEST_FAILURE; } break; From dovecot at dovecot.org Fri Nov 28 08:58:08 2014 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 28 Nov 2014 08:58:08 +0000 Subject: dovecot-2.2: lib: array - array_foreach_idx is just a generic pt... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/67b46a7f4ed2 changeset: 18118:67b46a7f4ed2 user: Phil Carmody date: Fri Nov 28 10:57:43 2014 +0200 description: lib: array - array_foreach_idx is just a generic ptr_to_idx helper Signed-off-by: Phil Carmody diffstat: src/lib/array.h | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (15 lines): diff -r 8a4b536705ad -r 67b46a7f4ed2 src/lib/array.h --- a/src/lib/array.h Fri Nov 28 10:57:43 2014 +0200 +++ b/src/lib/array.h Fri Nov 28 10:57:43 2014 +0200 @@ -82,8 +82,10 @@ (elem)++) #endif +#define array_ptr_to_idx(array, elem) \ + ((elem) - (array)->v[0]) #define array_foreach_idx(array, elem) \ - ((elem) - (array)->v[0]) + array_ptr_to_idx(array, elem) static inline void array_create_from_buffer_i(struct array *array, buffer_t *buffer,