From dovecot at dovecot.org Wed Apr 1 23:13:55 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 01 Apr 2015 23:13:55 +0000 Subject: dovecot-2.2: dsync: Improved error message when remote dsync-ser... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9e85aeb92fa8 changeset: 18395:9e85aeb92fa8 user: Timo Sirainen date: Thu Apr 02 08:12:41 2015 +0900 description: dsync: Improved error message when remote dsync-server couldn't be started. diffstat: src/doveadm/doveadm-dsync.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (14 lines): diff -r 89cbb9347de7 -r 9e85aeb92fa8 src/doveadm/doveadm-dsync.c --- a/src/doveadm/doveadm-dsync.c Wed Apr 01 01:16:57 2015 +0900 +++ b/src/doveadm/doveadm-dsync.c Thu Apr 02 08:12:41 2015 +0900 @@ -700,7 +700,9 @@ break; default: ctx->error = p_strdup_printf(ctx->ctx.pool, - "Failed to start dsync-server command: %u", exit_code); + "Failed to start remote dsync-server command: " + "Remote exit_code=%u %s", + exit_code, error == NULL ? "" : error); break; } io_loop_stop(current_ioloop); From dovecot at dovecot.org Thu Apr 2 23:55:08 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 02 Apr 2015 23:55:08 +0000 Subject: dovecot-2.2: configure: Added --with-textcat parameter Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/33685b996cf2 changeset: 18396:33685b996cf2 user: Timo Sirainen date: Fri Apr 03 08:53:45 2015 +0900 description: configure: Added --with-textcat parameter diffstat: configure.ac | 28 +++++++++++++++++++--------- 1 files changed, 19 insertions(+), 9 deletions(-) diffs (45 lines): diff -r 9e85aeb92fa8 -r 33685b996cf2 configure.ac --- a/configure.ac Thu Apr 02 08:12:41 2015 +0900 +++ b/configure.ac Fri Apr 03 08:53:45 2015 +0900 @@ -164,6 +164,11 @@ TEST_WITH(stemmer, $withval), want_stemmer=auto) +AC_ARG_WITH(textcat, +AS_HELP_STRING([--with-textcat], [Build with libtextcat support (for CLucene) (auto)]), + TEST_WITH(textcat, $withval), + want_textcat=auto) + AC_ARG_WITH(solr, AS_HELP_STRING([--with-solr], [Build with Solr full text search support]), TEST_WITH(solr, $withval), @@ -2742,15 +2747,20 @@ AC_CHECK_LIB(stemmer, sb_stemmer_new, [ have_lucene_stemmer=yes AC_DEFINE(HAVE_LUCENE_STEMMER,, Define if you want stemming support for CLucene) - AC_CHECK_LIB(textcat, special_textcat_Init, [ - have_lucene_textcat=yes - AC_DEFINE(HAVE_LUCENE_TEXTCAT,, Define if you want textcat support for CLucene) - ], [ - AC_CHECK_LIB(exttextcat, special_textcat_Init, [ - have_lucene_exttextcat=yes - AC_DEFINE(HAVE_LUCENE_EXTTEXTCAT,, Define if you want textcat (Debian version) support for CLucene) - ]) - ]) + if test $want_textcat != no; then + AC_CHECK_LIB(textcat, special_textcat_Init, [ + have_lucene_textcat=yes + AC_DEFINE(HAVE_LUCENE_TEXTCAT,, Define if you want textcat support for CLucene) + ], [ + AC_CHECK_LIB(exttextcat, special_textcat_Init, [ + have_lucene_exttextcat=yes + AC_DEFINE(HAVE_LUCENE_EXTTEXTCAT,, Define if you want textcat (Debian version) support for CLucene) + ]) + ]) + if test $want_textcat = yes && test "$have_lucene_exttextcat" != yes && test "$have_lucene_textcat" != yes; then + AC_ERROR([Can't build with textcat support: libtextcat or libexttextcat not found]) + fi + fi ], [ if test $want_stemmer = yes; then AC_ERROR([Can't build with stemmer support: libstemmer not found]) From dovecot at dovecot.org Mon Apr 6 02:40:53 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 06 Apr 2015 02:40:53 +0000 Subject: dovecot-2.2: rawlog: Flush output to disk every second instead o... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/ee001c131952 changeset: 18397:ee001c131952 user: Timo Sirainen date: Mon Apr 06 11:39:34 2015 +0900 description: rawlog: Flush output to disk every second instead of only when buffer is full. diffstat: src/util/rawlog.c | 21 +++++++++++++++++++++ 1 files changed, 21 insertions(+), 0 deletions(-) diffs (59 lines): diff -r 33685b996cf2 -r ee001c131952 src/util/rawlog.c --- a/src/util/rawlog.c Fri Apr 03 08:53:45 2015 +0900 +++ b/src/util/rawlog.c Mon Apr 06 11:39:34 2015 +0900 @@ -21,6 +21,7 @@ #include #define OUTBUF_THRESHOLD IO_BLOCK_SIZE +#define RAWLOG_TIMEOUT_FLUSH_MSECS 1000 static struct ioloop *ioloop; @@ -36,6 +37,7 @@ int client_in_fd, client_out_fd, server_fd; struct io *client_io, *server_io; struct ostream *client_output, *server_output; + struct timeout *to_flush; struct ostream *in_output, *out_output; enum rawlog_flags flags; @@ -64,6 +66,8 @@ io_remove(&proxy->client_io); if (proxy->server_io != NULL) io_remove(&proxy->server_io); + if (proxy->to_flush != NULL) + timeout_remove(&proxy->to_flush); o_stream_destroy(&proxy->client_output); o_stream_destroy(&proxy->server_output); @@ -103,6 +107,18 @@ } T_END; } +static void proxy_flush_timeout(struct rawlog_proxy *proxy) +{ + bool flushed = TRUE; + + if (o_stream_flush(proxy->in_output) == 0) + flushed = FALSE; + if (o_stream_flush(proxy->out_output) == 0) + flushed = FALSE; + if (flushed) + timeout_remove(&proxy->to_flush); +} + static void proxy_write_data(struct rawlog_proxy *proxy, struct ostream *output, bool *prev_lf, const void *data, size_t size) { @@ -119,6 +135,11 @@ if ((proxy->flags & RAWLOG_FLAG_LOG_BOUNDARIES) != 0) o_stream_nsend_str(output, ">>>\n"); + + if (proxy->to_flush == NULL) { + proxy->to_flush = timeout_add(RAWLOG_TIMEOUT_FLUSH_MSECS, + proxy_flush_timeout, proxy); + } } static void proxy_write_in(struct rawlog_proxy *proxy, From dovecot at dovecot.org Mon Apr 6 03:08:53 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 06 Apr 2015 03:08:53 +0000 Subject: dovecot-2.2: lib-index: Added reason_r parameter to mail_transac... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8146fdc0de34 changeset: 18398:8146fdc0de34 user: Timo Sirainen date: Mon Apr 06 12:07:32 2015 +0900 description: lib-index: Added reason_r parameter to mail_transaction_log_view_set() This is used to improve some of the error messages about index corruption. diffstat: src/doveadm/dsync/dsync-transaction-log-scan.c | 13 ++++-- src/lib-index/mail-index-modseq.c | 5 +- src/lib-index/mail-index-sync-update.c | 8 ++- src/lib-index/mail-index-sync.c | 11 +++-- src/lib-index/mail-index-view-sync.c | 15 ++++++- src/lib-index/mail-transaction-log-view.c | 46 +++++++++++++++++++------ src/lib-index/mail-transaction-log.h | 2 +- src/lib-index/test-mail-transaction-log-view.c | 13 +++--- src/lib-storage/mailbox-get.c | 5 +- src/lib-storage/test-mailbox-get.c | 2 +- 10 files changed, 81 insertions(+), 39 deletions(-) diffs (truncated from 455 to 300 lines): diff -r ee001c131952 -r 8146fdc0de34 src/doveadm/dsync/dsync-transaction-log-scan.c --- a/src/doveadm/dsync/dsync-transaction-log-scan.c Mon Apr 06 11:39:34 2015 +0900 +++ b/src/doveadm/dsync/dsync-transaction-log-scan.c Mon Apr 06 12:07:32 2015 +0900 @@ -350,6 +350,7 @@ { uint32_t log_seq, end_seq; uoff_t log_offset, end_offset; + const char *reason; bool reset; int ret; @@ -371,7 +372,7 @@ ret = mail_transaction_log_view_set(log_view, log_seq, log_offset, end_seq, end_offset, - &reset); + &reset, &reason); if (ret != 0) return ret; } @@ -390,13 +391,14 @@ } ret = mail_transaction_log_view_set(log_view, log_seq, log_offset, - end_seq, end_offset, &reset); + end_seq, end_offset, + &reset, &reason); if (ret == 0) { /* we shouldn't get here. _view_set_all() already reserved all the log files, the _view_set() only removed unwanted ones. */ - i_error("%s: Couldn't set transaction log view (seq %u..%u)", - view->index->filepath, log_seq, end_seq); + i_error("%s: Couldn't set transaction log view (seq %u..%u): %s", + view->index->filepath, log_seq, end_seq, reason); ret = -1; } if (ret < 0) @@ -566,6 +568,7 @@ struct mail_transaction_log_view *log_view; const struct mail_transaction_header *hdr; const void *data; + const char *reason; bool reset, found = FALSE; i_assert(uid > 0); @@ -578,7 +581,7 @@ scan->last_log_seq, scan->last_log_offset, (uint32_t)-1, (uoff_t)-1, - &reset) > 0) { + &reset, &reason) > 0) { while (!found && mail_transaction_log_view_next(log_view, &hdr, &data) > 0) { switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) { diff -r ee001c131952 -r 8146fdc0de34 src/lib-index/mail-index-modseq.c --- a/src/lib-index/mail-index-modseq.c Mon Apr 06 11:39:34 2015 +0900 +++ b/src/lib-index/mail-index-modseq.c Mon Apr 06 12:07:32 2015 +0900 @@ -403,6 +403,7 @@ const struct mail_index_modseq_header *hdr; const struct mail_transaction_header *thdr; const void *tdata; + const char *reason; uint32_t ext_map_idx; uint32_t end_seq; uoff_t end_offset; @@ -437,12 +438,12 @@ ret = mail_transaction_log_view_set(ctx->log_view, I_MAX(1, hdr->log_seq), hdr->log_offset, - end_seq, end_offset, &reset); + end_seq, end_offset, &reset, &reason); if (ret <= 0) { /* missing files / error - try with only the last file */ ret = mail_transaction_log_view_set(ctx->log_view, end_seq, 0, end_seq, end_offset, - &reset); + &reset, &reason); /* since we don't know if we skipped some changes, set all modseqs to beginning of the latest file. */ cur_modseq = mail_transaction_log_view_get_prev_modseq( diff -r ee001c131952 -r 8146fdc0de34 src/lib-index/mail-index-sync-update.c --- a/src/lib-index/mail-index-sync-update.c Mon Apr 06 11:39:34 2015 +0900 +++ b/src/lib-index/mail-index-sync-update.c Mon Apr 06 12:07:32 2015 +0900 @@ -911,6 +911,7 @@ const void *tdata; uint32_t prev_seq; uoff_t start_offset, prev_offset; + const char *reason; int ret; bool had_dirty, reset; @@ -952,14 +953,15 @@ view = mail_index_view_open_with_map(index, map); ret = mail_transaction_log_view_set(view->log_view, map->hdr.log_file_seq, start_offset, - (uint32_t)-1, (uoff_t)-1, &reset); + (uint32_t)-1, (uoff_t)-1, + &reset, &reason); if (ret <= 0) { mail_index_view_close(&view); if (force && ret == 0) { /* the seq/offset is probably broken */ mail_index_set_error(index, "Index %s: Lost log for " - "seq=%u offset=%"PRIuUOFF_T, index->filepath, - map->hdr.log_file_seq, start_offset); + "seq=%u offset=%"PRIuUOFF_T": %s", index->filepath, + map->hdr.log_file_seq, start_offset, reason); (void)mail_index_fsck(index); } /* can't use it. sync by re-reading index. */ diff -r ee001c131952 -r 8146fdc0de34 src/lib-index/mail-index-sync.c --- a/src/lib-index/mail-index-sync.c Mon Apr 06 11:39:34 2015 +0900 +++ b/src/lib-index/mail-index-sync.c Mon Apr 06 12:07:32 2015 +0900 @@ -284,6 +284,7 @@ { uint32_t log_seq; uoff_t log_offset; + const char *reason; bool reset; int ret; @@ -291,15 +292,15 @@ ret = mail_transaction_log_view_set(view->log_view, start_file_seq, start_file_offset, - log_seq, log_offset, &reset); + log_seq, log_offset, &reset, &reason); if (ret < 0) return -1; if (ret == 0) { /* either corrupted or the file was deleted for some reason. either way, we can't go forward */ mail_index_set_error(view->index, - "Unexpected transaction log desync with index %s", - view->index->filepath); + "Unexpected transaction log desync with index %s: %s", + view->index->filepath, reason); return 0; } return 1; @@ -529,6 +530,7 @@ const void *data; uint32_t log_seq; uoff_t log_offset; + const char *reason; bool reset; int ret; @@ -544,7 +546,8 @@ if (mail_transaction_log_view_set(view->log_view, view->map->hdr.log_file_seq, view->map->hdr.log_file_tail_offset, - log_seq, log_offset, &reset) <= 0) { + log_seq, log_offset, + &reset, &reason) <= 0) { /* let the actual syncing handle the error */ return TRUE; } diff -r ee001c131952 -r 8146fdc0de34 src/lib-index/mail-index-view-sync.c --- a/src/lib-index/mail-index-view-sync.c Mon Apr 06 11:39:34 2015 +0900 +++ b/src/lib-index/mail-index-view-sync.c Mon Apr 06 12:07:32 2015 +0900 @@ -48,6 +48,7 @@ const struct mail_index_header *hdr = &view->index->map->hdr; uint32_t start_seq, end_seq; uoff_t start_offset, end_offset; + const char *reason; int ret; start_seq = view->log_file_expunge_seq; @@ -71,7 +72,7 @@ ret = mail_transaction_log_view_set(view->log_view, start_seq, start_offset, end_seq, end_offset, - reset_r); + reset_r, &reason); if (ret <= 0) return ret; @@ -489,7 +490,9 @@ struct mail_index_view *view = ctx->view; uint32_t seq; uoff_t offset; + const char *reason; bool reset; + int ret; /* replace the view's map */ view->index->map->refcount++; @@ -501,9 +504,15 @@ view->log_file_head_offset = offset = view->map->hdr.log_file_head_offset; - if (mail_transaction_log_view_set(view->log_view, seq, offset, - seq, offset, &reset) <= 0) + ret = mail_transaction_log_view_set(view->log_view, seq, offset, + seq, offset, &reset, &reason); + if (ret < 0) return -1; + if (ret == 0) { + mail_index_set_error(view->index, "Failed to fix view for %s: %s", + view->index->filepath, reason); + return 0; + } view->inconsistent = FALSE; return 0; } diff -r ee001c131952 -r 8146fdc0de34 src/lib-index/mail-transaction-log-view.c --- a/src/lib-index/mail-transaction-log-view.c Mon Apr 06 11:39:34 2015 +0900 +++ b/src/lib-index/mail-transaction-log-view.c Mon Apr 06 12:07:32 2015 +0900 @@ -61,7 +61,7 @@ int mail_transaction_log_view_set(struct mail_transaction_log_view *view, uint32_t min_file_seq, uoff_t min_file_offset, uint32_t max_file_seq, uoff_t max_file_offset, - bool *reset_r) + bool *reset_r, const char **reason_r) { struct mail_transaction_log_file *file, *const *files; uoff_t start_offset, end_offset; @@ -70,10 +70,12 @@ int ret; *reset_r = FALSE; + *reason_r = NULL; if (view->log == NULL) { /* transaction log is closed already. this log view shouldn't be used anymore. */ + *reason_r = "Log already closed"; return -1; } @@ -82,6 +84,9 @@ start from the beginning */ if (view->log->files->hdr.prev_file_seq != 0) { /* but it doesn't */ + *reason_r = t_strdup_printf( + "Wanted log beginning, but found prev_file_seq=%u", + view->log->files->hdr.prev_file_seq); return 0; } @@ -124,10 +129,11 @@ if (min_file_seq == max_file_seq && min_file_offset > max_file_offset) { /* log file offset is probably corrupted in the index file. */ - mail_transaction_log_view_set_corrupted(view, - "file_seq=%u, min_file_offset (%"PRIuUOFF_T + *reason_r = t_strdup_printf( + "Invalid offset: file_seq=%u, min_file_offset (%"PRIuUOFF_T ") > max_file_offset (%"PRIuUOFF_T")", min_file_seq, min_file_offset, max_file_offset); + mail_transaction_log_view_set_corrupted(view, "%s", *reason_r); return -1; } @@ -142,8 +148,11 @@ ret = mail_transaction_log_find_file(view->log, seq, nfs_flush, &file); if (ret <= 0) { - if (ret < 0) + if (ret < 0) { + *reason_r = t_strdup_printf( + "Failed to find file seq=%u", seq); return -1; + } /* not found / corrupted */ file = NULL; @@ -166,6 +175,9 @@ if (file == NULL || file->hdr.file_seq > max_file_seq) { /* missing files in the middle */ + *reason_r = t_strdup_printf( + "Missing middle file seq=%u (between %u..%u)", + seq, min_file_seq, max_file_seq); return 0; } @@ -198,18 +210,20 @@ if (min_file_offset < view->tail->hdr.hdr_size) { /* log file offset is probably corrupted in the index file. */ - mail_transaction_log_view_set_corrupted(view, - "file_seq=%u, min_file_offset (%"PRIuUOFF_T + *reason_r = t_strdup_printf( + "Invalid min_file_offset: file_seq=%u, min_file_offset (%"PRIuUOFF_T ") < hdr_size (%u)", min_file_seq, min_file_offset, view->tail->hdr.hdr_size); + mail_transaction_log_view_set_corrupted(view, "%s", *reason_r); return -1; } if (max_file_offset < view->head->hdr.hdr_size) { /* log file offset is probably corrupted in the index file. */ - mail_transaction_log_view_set_corrupted(view, - "file_seq=%u, min_file_offset (%"PRIuUOFF_T + *reason_r = t_strdup_printf( + "Invalid max_file_offset: file_seq=%u, min_file_offset (%"PRIuUOFF_T ") < hdr_size (%u)", max_file_seq, max_file_offset, view->head->hdr.hdr_size); From pigeonhole at rename-it.nl Mon Apr 6 15:04:22 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Mon, 06 Apr 2015 17:04:22 +0200 Subject: dovecot-2.2-pigeonhole: Fixed some inconsistencies in the INSTAL... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/a8256d9a8f12 changeset: 2035:a8256d9a8f12 user: Stephan Bosch date: Mon Apr 06 17:04:13 2015 +0200 description: Fixed some inconsistencies in the INSTALL documentation. Forgot to adjust a few things when the location types were introduced and documented. diffstat: INSTALL | 60 ++++++++++++++++++++------------ doc/example-config/conf.d/90-sieve.conf | 2 +- 2 files changed, 38 insertions(+), 24 deletions(-) diffs (90 lines): diff -r 2ebd3bd76e1e -r a8256d9a8f12 INSTALL --- a/INSTALL Sat Mar 28 10:49:23 2015 +0100 +++ b/INSTALL Mon Apr 06 17:04:13 2015 +0200 @@ -565,28 +565,43 @@ plugin section of the config file. These settings are the same ones used by the LDA Sieve plugin. - sieve_dir = ~/sieve - This specifies the path to the directory where the uploaded scripts are - stored. Scripts are stored as separate files with extension '.sieve'. All - other files are ignored when scripts are listed by a ManageSieve client. The - Sieve interpreter also uses this setting to locate the user's personal - scripts for use with the Sieve include extension. A storage location - specified by sieve_dir is always generated automatically if it does not exist - (as far as the system permits the user to do so; no root privileges are - used). This is similar to the behavior of the mail daemons regarding the - mail_location configuration. + sieve = file:~/sieve;active=~/.dovecot.sieve + This specifies the location where the scripts that are uploaded through + ManageSieve are stored. During delivery, the LDA Sieve plugin uses this + location setting to find the active script for Sieve filtering. The Sieve + "include" extension uses this location for retrieving ":personal" scripts. + If the location type does not allow uploading scripts, the ManageSieve + service cannot be used. Currently, only the 'file' location type supports + ManageSieve. - sieve = ~/.dovecot.sieve - This specifies the location of the symbolic link pointing to the active - script in the Sieve storage directory. The Sieve interpreter uses this - setting to locate the main script file that needs to be executed upon - delivery. When using ManageSieve, this is a symbolic link managed by the - ManageSieve service. ManageSieve thereby determines which script (if any) in - the sieve_dir directory is executed for incoming messages. If a regular file - already exists at the location specified by the sieve setting, it is moved to - the sieve_dir location before the symbolic link is installed. It is renamed - to dovecot.orig.sieve and therefore listed as `dovecot.orig' by a ManageSieve - client. + For the 'file' location type: + + * The location is the path to the storage directory for all the user's + personal Sieve scripts. Scripts are stored as separate files with + extension .sieve. All other files are ignored when scripts are listed + by a ManageSieve client. The storage directory is always generated + automatically if it does not exist (as far as the system permits the + user to do so; no root privileges are used). This is similar to the + behavior of the mail daemons regarding the mail_location configuration. + + * ManageSieve maintains a symbolic link pointing to the currently active + script (the script executed at delivery). The location of this symbolic + link can be configured using the ;active= option. If a regular + file already exists at the location specified by in the ;active= + location option, it is moved to the storage directory before the + symbolic link is installed. It is renamed to dovecot.orig.sieve and + therefore listed as dovecot.orig by a ManageSieve client. + + NOTE: It is not wise to place this active symbolic link inside your + mail store, as it may be mistaken for a mail folder. Inside a + maildir for instance, the default .dovecot.sieve would show up + as phantom folder /dovecot/sieve in your IMAP tree. + + For Pigeonhole versions before v0.3.1, this setting can only be a + filesystem path pointing to a script file, or - when ManageSieve is used - + it is the location of the symbolic link pointing to the active script in + the storage directory. That storage directory is then configured using the + deprecated `sieve_dir' setting. The following provides an example configuration for Sieve and ManageSieve. Only sections and settings relevant to ManageSieve are shown. Refer to @@ -614,8 +629,7 @@ # *** conf.d/90-sieve.conf *** plugin { - sieve=~/.dovecot.sieve - sieve_dir=~/sieve + sieve=file:~/sieve;active=~/.dovecot.sieve } # *** dovecot.conf *** diff -r 2ebd3bd76e1e -r a8256d9a8f12 doc/example-config/conf.d/90-sieve.conf --- a/doc/example-config/conf.d/90-sieve.conf Sat Mar 28 10:49:23 2015 +0100 +++ b/doc/example-config/conf.d/90-sieve.conf Mon Apr 06 17:04:13 2015 +0200 @@ -9,7 +9,7 @@ # locations. The default `file' location type is a local filesystem path # pointing to a Sieve script file or a directory containing multiple Sieve # script files. More complex setups can use other location types such as -# `ldap'or `dict' to fetch Sieve scripts from remote databases. +# `ldap' or `dict' to fetch Sieve scripts from remote databases. # # All settings that specify the location of one ore more Sieve scripts accept # the following syntax: From dovecot at dovecot.org Wed Apr 8 04:56:59 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 08 Apr 2015 04:56:59 +0000 Subject: dovecot-2.2: lib: Added json-tree API for parsing JSON input int... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6bde7868cffd changeset: 18399:6bde7868cffd user: Timo Sirainen date: Wed Apr 08 13:55:35 2015 +0900 description: lib: Added json-tree API for parsing JSON input into a tree structure. This makes it easier to access complicated JSON structs that can fit into memory. diffstat: src/lib/Makefile.am | 3 + src/lib/json-tree.c | 167 +++++++++++++++++++++++++++++++++++++++++++++++ src/lib/json-tree.h | 40 +++++++++++ src/lib/test-json-tree.c | 114 ++++++++++++++++++++++++++++++++ src/lib/test-lib.c | 1 + src/lib/test-lib.h | 1 + 6 files changed, 326 insertions(+), 0 deletions(-) diffs (truncated from 382 to 300 lines): diff -r 8146fdc0de34 -r 6bde7868cffd src/lib/Makefile.am --- a/src/lib/Makefile.am Mon Apr 06 12:07:32 2015 +0900 +++ b/src/lib/Makefile.am Wed Apr 08 13:55:35 2015 +0900 @@ -86,6 +86,7 @@ ioloop-epoll.c \ ioloop-kqueue.c \ json-parser.c \ + json-tree.c \ lib.c \ lib-signals.c \ md4.c \ @@ -212,6 +213,7 @@ ioloop-private.h \ ioloop-notify-fd.h \ json-parser.h \ + json-tree.h \ lib.h \ lib-signals.h \ llist.h \ @@ -301,6 +303,7 @@ test-istream-seekable.c \ test-istream-tee.c \ test-json-parser.c \ + test-json-tree.c \ test-llist.c \ test-mempool-alloconly.c \ test-network.c \ diff -r 8146fdc0de34 -r 6bde7868cffd src/lib/json-tree.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/json-tree.c Wed Apr 08 13:55:35 2015 +0900 @@ -0,0 +1,167 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "json-tree.h" + +struct json_tree { + pool_t pool; + struct json_tree_node *root, *cur, *cur_child; +}; + +struct json_tree *json_tree_init(void) +{ + struct json_tree *tree; + pool_t pool; + + pool = pool_alloconly_create("json tree", 1024); + tree = p_new(pool, struct json_tree, 1); + tree->pool = pool; + tree->root = tree->cur = p_new(tree->pool, struct json_tree_node, 1); + tree->cur->value_type = JSON_TYPE_OBJECT; + return tree; +} + +void json_tree_deinit(struct json_tree **_tree) +{ + struct json_tree *tree = *_tree; + + *_tree = NULL; + pool_unref(&tree->pool); +} + +static void +json_tree_append_child(struct json_tree *tree, enum json_type type, + const char *value) +{ + struct json_tree_node *node; + + node = p_new(tree->pool, struct json_tree_node, 1); + node->parent = tree->cur; + node->value_type = type; + node->value.str = p_strdup(tree->pool, value); + + if (tree->cur_child == NULL) + tree->cur->value.child = node; + else + tree->cur_child->next = node; + tree->cur_child = node; +} + +static void +json_tree_set_cur(struct json_tree *tree, struct json_tree_node *node) +{ + tree->cur = node; + tree->cur_child = tree->cur->value.child; + if (tree->cur_child != NULL) { + while (tree->cur_child->next != NULL) + tree->cur_child = tree->cur_child->next; + } +} + +static int +json_tree_append_value(struct json_tree *tree, enum json_type type, + const char *value) +{ + switch (tree->cur->value_type) { + case JSON_TYPE_OBJECT_KEY: + /* "key": value - we already added the node and set its key, + so now just set the value */ + tree->cur->value_type = type; + tree->cur->value.str = p_strdup(tree->pool, value); + json_tree_set_cur(tree, tree->cur->parent); + break; + case JSON_TYPE_ARRAY: + /* element in array - add a new node */ + json_tree_append_child(tree, type, value); + break; + default: + return -1; + } + return 0; +} + +int json_tree_append(struct json_tree *tree, enum json_type type, + const char *value) +{ + switch (type) { + case JSON_TYPE_OBJECT_KEY: + if (tree->cur->value_type != JSON_TYPE_OBJECT) + return -1; + json_tree_append_child(tree, type, NULL); + json_tree_set_cur(tree, tree->cur_child); + tree->cur->key = p_strdup(tree->pool, value); + break; + case JSON_TYPE_ARRAY: + if (json_tree_append_value(tree, type, NULL) < 0) + return -1; + json_tree_set_cur(tree, tree->cur_child); + break; + case JSON_TYPE_OBJECT: + if (tree->cur->value_type == JSON_TYPE_OBJECT_KEY) + tree->cur->value_type = JSON_TYPE_OBJECT; + else if (tree->cur->value_type == JSON_TYPE_ARRAY) { + json_tree_append_child(tree, type, NULL); + json_tree_set_cur(tree, tree->cur_child); + } else { + return -1; + } + break; + case JSON_TYPE_OBJECT_END: + if (tree->cur->parent == NULL || + tree->cur->value_type != JSON_TYPE_OBJECT) + return -1; + json_tree_set_cur(tree, tree->cur->parent); + break; + case JSON_TYPE_ARRAY_END: + if (tree->cur->parent == NULL || + tree->cur->value_type != JSON_TYPE_ARRAY) + return -1; + json_tree_set_cur(tree, tree->cur->parent); + break; + case JSON_TYPE_STRING: + case JSON_TYPE_NUMBER: + case JSON_TYPE_TRUE: + case JSON_TYPE_FALSE: + case JSON_TYPE_NULL: + if (json_tree_append_value(tree, type, value) < 0) + return -1; + break; + } + return 0; +} + +struct json_tree_node *json_tree_root(struct json_tree *tree) +{ + return tree->root; +} + +struct json_tree_node * +json_tree_find_key(struct json_tree_node *node, const char *key) +{ + for (; node != NULL; node = node->next) { + if (node->key != NULL && strcmp(node->key, key) == 0) + return node; + } + return NULL; +} + +struct json_tree_node * +json_tree_find_child_with(struct json_tree_node *node, + const char *key, const char *value) +{ + struct json_tree_node *child; + + i_assert(node->value_type == JSON_TYPE_OBJECT || + node->value_type == JSON_TYPE_ARRAY); + + for (node = node->value.child; node != NULL; node = node->next) { + if (node->value_type != JSON_TYPE_OBJECT) + continue; + + child = json_tree_find_key(node->value.child, key); + if (child != NULL && child->value.str != NULL && + strcmp(child->value.str, value) == 0) + return node; + } + return NULL; +} diff -r 8146fdc0de34 -r 6bde7868cffd src/lib/json-tree.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/json-tree.h Wed Apr 08 13:55:35 2015 +0900 @@ -0,0 +1,40 @@ +#ifndef JSON_TREE_H +#define JSON_TREE_H + +#include "json-parser.h" + +struct json_tree_node { + /* object key, or NULL if we're in a list */ + const char *key; + struct json_tree_node *parent, *next; + + enum json_type value_type; + struct { + /* for JSON_TYPE_OBJECT and JSON_TYPE_ARRAY */ + struct json_tree_node *child; + /* for other types */ + const char *str; + } value; +}; + +struct json_tree *json_tree_init(void); +void json_tree_deinit(struct json_tree **tree); + +/* Append data to a tree. The type/value should normally come from json-parser. + Returns 0 on success, -1 if the input was invalid (which should never happen + if it's coming from json-parser). */ +int json_tree_append(struct json_tree *tree, enum json_type type, + const char *value); + +/* Return the root node. */ +struct json_tree_node *json_tree_root(struct json_tree *tree); +/* Find a node with the specified key (from node's siblings) */ +struct json_tree_node * +json_tree_find_key(struct json_tree_node *node, const char *key); +/* Find an object node (from an array), which contains the specified key=value + in its values. */ +struct json_tree_node * +json_tree_find_child_with(struct json_tree_node *node, + const char *key, const char *value); + +#endif diff -r 8146fdc0de34 -r 6bde7868cffd src/lib/test-json-tree.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/test-json-tree.c Wed Apr 08 13:55:35 2015 +0900 @@ -0,0 +1,114 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "test-lib.h" +#include "json-tree.h" + +struct { + enum json_type type; + const char *value; +} test_input[] = { + { JSON_TYPE_OBJECT_KEY, "key-str" }, + { JSON_TYPE_STRING, "string" }, + { JSON_TYPE_OBJECT_KEY, "key-num" }, + { JSON_TYPE_NUMBER, "1234" }, + { JSON_TYPE_OBJECT_KEY, "key-true" }, + { JSON_TYPE_TRUE, "true" }, + { JSON_TYPE_OBJECT_KEY, "key-false" }, + { JSON_TYPE_FALSE, "false" }, + { JSON_TYPE_OBJECT_KEY, "key-null" }, + { JSON_TYPE_NULL, NULL }, + + { JSON_TYPE_OBJECT_KEY, "key-obj-empty" }, + { JSON_TYPE_OBJECT, NULL }, + { JSON_TYPE_OBJECT_END, NULL }, + + { JSON_TYPE_OBJECT_KEY, "key-obj" }, + { JSON_TYPE_OBJECT, NULL }, + { JSON_TYPE_OBJECT_KEY, "sub" }, + { JSON_TYPE_STRING, "value" }, + { JSON_TYPE_OBJECT_END, NULL }, + + { JSON_TYPE_OBJECT_KEY, "key-arr-empty" }, + { JSON_TYPE_ARRAY, NULL }, + { JSON_TYPE_ARRAY_END, NULL }, + + { JSON_TYPE_OBJECT_KEY, "key-arr" }, + { JSON_TYPE_ARRAY, NULL }, + { JSON_TYPE_STRING, "foo" }, + { JSON_TYPE_ARRAY, NULL }, + { JSON_TYPE_TRUE, "true" }, + { JSON_TYPE_ARRAY_END, NULL }, + { JSON_TYPE_OBJECT, NULL }, + { JSON_TYPE_OBJECT_KEY, "aobj" }, + { JSON_TYPE_ARRAY, NULL }, + { JSON_TYPE_ARRAY_END, NULL }, + { JSON_TYPE_OBJECT_END, NULL }, + { JSON_TYPE_OBJECT, NULL }, + { JSON_TYPE_OBJECT_KEY, "aobj-key" }, + { JSON_TYPE_STRING, "value1" }, + { JSON_TYPE_OBJECT_END, NULL }, + { JSON_TYPE_OBJECT, NULL }, + { JSON_TYPE_OBJECT_KEY, "aobj-key" }, + { JSON_TYPE_STRING, "value2" }, + { JSON_TYPE_OBJECT_END, NULL }, + { JSON_TYPE_ARRAY_END, NULL } From pigeonhole at rename-it.nl Wed Apr 8 21:48:54 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 08 Apr 2015 23:48:54 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: editheader extension: The ADD... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/1a8e8af5376c changeset: 2036:1a8e8af5376c user: Stephan Bosch date: Wed Apr 08 23:48:30 2015 +0200 description: lib-sieve: editheader extension: The ADDHEADER operation mnemonic was in lower case. diffstat: src/lib-sieve/plugins/editheader/cmd-addheader.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r a8256d9a8f12 -r 1a8e8af5376c src/lib-sieve/plugins/editheader/cmd-addheader.c --- a/src/lib-sieve/plugins/editheader/cmd-addheader.c Mon Apr 06 17:04:13 2015 +0200 +++ b/src/lib-sieve/plugins/editheader/cmd-addheader.c Wed Apr 08 23:48:30 2015 +0200 @@ -76,7 +76,7 @@ (const struct sieve_runtime_env *renv, sieve_size_t *address); const struct sieve_operation_def addheader_operation = { - "addheader", + "ADDHEADER", &editheader_extension, EXT_EDITHEADER_OPERATION_ADDHEADER, cmd_addheader_operation_dump, From dovecot at dovecot.org Mon Apr 13 07:06:10 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 13 Apr 2015 07:06:10 +0000 Subject: dovecot-2.2: replication plugin: Don't crash for users who don't... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/631cd10e62b2 changeset: 18400:631cd10e62b2 user: Timo Sirainen date: Mon Apr 13 10:04:47 2015 +0300 description: replication plugin: Don't crash for users who don't have replication enabled. diffstat: src/plugins/replication/replication-plugin.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (21 lines): diff -r 6bde7868cffd -r 631cd10e62b2 src/plugins/replication/replication-plugin.c --- a/src/plugins/replication/replication-plugin.c Wed Apr 08 13:55:35 2015 +0900 +++ b/src/plugins/replication/replication-plugin.c Mon Apr 13 10:04:47 2015 +0300 @@ -191,6 +191,8 @@ } ruser = REPLICATION_USER_CONTEXT(ns->user); + if (ruser == NULL) + return; if (priority == REPLICATION_PRIORITY_SYNC) { if (replication_notify_sync(ns->user) == 0) { @@ -254,7 +256,7 @@ REPLICATION_USER_CONTEXT(ctx->ns->user); enum replication_priority priority; - if (ctx->new_messages || changes->changed) { + if (ruser != NULL && (ctx->new_messages || changes->changed)) { priority = !ctx->new_messages ? REPLICATION_PRIORITY_LOW : ruser->sync_secs == 0 ? REPLICATION_PRIORITY_HIGH : REPLICATION_PRIORITY_SYNC; From dovecot at dovecot.org Mon Apr 13 17:39:33 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 13 Apr 2015 17:39:33 +0000 Subject: dovecot-2.2: auth: Setting userdb fields from cache didn't set h... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/08b2f79e8212 changeset: 18401:08b2f79e8212 user: Timo Sirainen date: Mon Apr 13 20:37:48 2015 +0300 description: auth: Setting userdb fields from cache didn't set handle any special fields. The special fields were relatively rarely used though. diffstat: src/auth/auth-request.c | 17 +++++++---------- 1 files changed, 7 insertions(+), 10 deletions(-) diffs (64 lines): diff -r 631cd10e62b2 -r 08b2f79e8212 src/auth/auth-request.c --- a/src/auth/auth-request.c Mon Apr 13 10:04:47 2015 +0300 +++ b/src/auth/auth-request.c Mon Apr 13 20:37:48 2015 +0300 @@ -46,6 +46,8 @@ static void get_log_prefix(string_t *str, struct auth_request *auth_request, const char *subsystem); +static void +auth_request_userdb_import(struct auth_request *request, const char *args); struct auth_request * auth_request_new(const struct mech_module *mech) @@ -985,7 +987,6 @@ static bool auth_request_lookup_user_cache(struct auth_request *request, const char *key, - struct auth_fields **reply_r, enum userdb_result *result_r, bool use_expired) { @@ -1007,13 +1008,13 @@ if (*value == '\0') { /* negative cache entry */ *result_r = USERDB_RESULT_USER_UNKNOWN; - *reply_r = auth_fields_init(request->pool); + request->userdb_reply = auth_fields_init(request->pool); return TRUE; } + request->userdb_reply = auth_fields_init(request->pool); + auth_request_userdb_import(request, value); *result_r = USERDB_RESULT_OK; - *reply_r = auth_fields_init(request->pool); - auth_fields_import(*reply_r, value, 0); return TRUE; } @@ -1114,11 +1115,9 @@ request was expired in cache, fallback to using cached expired record. */ const char *cache_key = userdb->cache_key; - struct auth_fields *reply; - if (auth_request_lookup_user_cache(request, cache_key, &reply, + if (auth_request_lookup_user_cache(request, cache_key, &result, TRUE)) { - request->userdb_reply = reply; auth_request_log_info(request, AUTH_SUBSYS_DB, "Falling back to expired data from cache"); } @@ -1141,12 +1140,10 @@ /* (for now) auth_cache is shared between passdb and userdb */ cache_key = passdb_cache == NULL ? NULL : userdb->cache_key; if (cache_key != NULL) { - struct auth_fields *reply; enum userdb_result result; - if (auth_request_lookup_user_cache(request, cache_key, &reply, + if (auth_request_lookup_user_cache(request, cache_key, &result, FALSE)) { - request->userdb_reply = reply; request->private_callback.userdb(result, request); return; } From dovecot at dovecot.org Mon Apr 13 17:39:33 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 13 Apr 2015 17:39:33 +0000 Subject: dovecot-2.2: auth: If passdb/userdb changes the username, add th... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/635f9c7d5991 changeset: 18402:635f9c7d5991 user: Timo Sirainen date: Mon Apr 13 20:38:10 2015 +0300 description: auth: If passdb/userdb changes the username, add the changed username also to the cache. diffstat: src/auth/auth-request.c | 7 +++++++ 1 files changed, 7 insertions(+), 0 deletions(-) diffs (17 lines): diff -r 08b2f79e8212 -r 635f9c7d5991 src/auth/auth-request.c --- a/src/auth/auth-request.c Mon Apr 13 20:37:48 2015 +0300 +++ b/src/auth/auth-request.c Mon Apr 13 20:38:10 2015 +0300 @@ -973,6 +973,13 @@ auth_fields_append(request->userdb_reply, str, AUTH_FIELD_FLAG_CHANGED, AUTH_FIELD_FLAG_CHANGED); + if (strcmp(request->user, request->translated_username) != 0) { + /* username was changed by passdb or userdb */ + if (str_len(str) > 0) + str_append_c(str, '\t'); + str_append(str, "user="); + str_append_tabescaped(str, request->user); + } if (str_len(str) == 0) { /* no userdb fields. but we can't save an empty string, since that means "user unknown". */ From dovecot at dovecot.org Tue Apr 14 06:59:24 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 14 Apr 2015 06:59:24 +0000 Subject: dovecot-2.2: dict-file: Use tabescaping for keys and values to a... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/93dab55ae36e changeset: 18403:93dab55ae36e user: Timo Sirainen date: Tue Apr 14 09:58:01 2015 +0300 description: dict-file: Use tabescaping for keys and values to allow LFs in them. Although this makes the format slightly incompatible, it's doubtful anybody is using dict values containing \001 characters. diffstat: src/lib-dict/dict-file.c | 18 ++++++++++++------ 1 files changed, 12 insertions(+), 6 deletions(-) diffs (54 lines): diff -r 635f9c7d5991 -r 93dab55ae36e src/lib-dict/dict-file.c --- a/src/lib-dict/dict-file.c Mon Apr 13 20:38:10 2015 +0300 +++ b/src/lib-dict/dict-file.c Tue Apr 14 09:58:01 2015 +0300 @@ -3,6 +3,8 @@ #include "lib.h" #include "array.h" #include "hash.h" +#include "str.h" +#include "strescape.h" #include "home-expand.h" #include "mkdir-parents.h" #include "file-lock.h" @@ -177,12 +179,12 @@ while ((key = i_stream_read_next_line(input)) != NULL) { /* strdup() before the second read */ - key = p_strdup(dict->hash_pool, key); + key = str_tabunescape(p_strdup(dict->hash_pool, key)); if ((value = i_stream_read_next_line(input)) == NULL) break; - value = p_strdup(dict->hash_pool, value); + value = str_tabunescape(p_strdup(dict->hash_pool, value)); hash_table_insert(dict->hash, key, value); } i_stream_destroy(&input); @@ -501,6 +503,7 @@ struct hash_iterate_context *iter; struct ostream *output; char *key, *value; + string_t *str; int fd = -1; *atomic_inc_not_found_r = FALSE; @@ -558,11 +561,14 @@ output = o_stream_create_fd(fd, 0, FALSE); o_stream_cork(output); iter = hash_table_iterate_init(dict->hash); + str = t_str_new(256); while (hash_table_iterate(iter, dict->hash, &key, &value)) { - o_stream_nsend_str(output, key); - o_stream_nsend(output, "\n", 1); - o_stream_nsend_str(output, value); - o_stream_nsend(output, "\n", 1); + str_truncate(str, 0); + str_append_tabescaped(str, key); + str_append_c(str, '\n'); + str_append_tabescaped(str, value); + str_append_c(str, '\n'); + o_stream_nsend(output, str_data(str), str_len(str)); } hash_table_iterate_deinit(&iter); From dovecot at dovecot.org Wed Apr 15 18:02:44 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 15 Apr 2015 18:02:44 +0000 Subject: dovecot-2.2: imapc: Added imapc_features=zimbra-workarounds Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1c17f06d5e52 changeset: 18404:1c17f06d5e52 user: Timo Sirainen date: Wed Apr 15 21:01:11 2015 +0300 description: imapc: Added imapc_features=zimbra-workarounds Zimbra (at least v5.0.18_GA_3011.RHEL4_64) can return different headers depending on whether the whole message body was fetched or only (partial) headers. This probably happens only for invalid characters that are translated into '?'. With this workaround enabled we don't use FETCH BODY.PEEK[], but we do FETCH (BODY.PEEK[HEADER] BODY.PEEK[TEXT]) and merge the results together. This way the results are always consistent and headers don't suddenly change. diffstat: src/lib-storage/index/imapc/imapc-mail-fetch.c | 96 +++++++++++++++++++++---- src/lib-storage/index/imapc/imapc-mail.c | 2 + src/lib-storage/index/imapc/imapc-mail.h | 3 +- src/lib-storage/index/imapc/imapc-save.c | 5 +- src/lib-storage/index/imapc/imapc-settings.c | 1 + src/lib-storage/index/imapc/imapc-settings.h | 3 +- 6 files changed, 90 insertions(+), 20 deletions(-) diffs (268 lines): diff -r 93dab55ae36e -r 1c17f06d5e52 src/lib-storage/index/imapc/imapc-mail-fetch.c --- a/src/lib-storage/index/imapc/imapc-mail-fetch.c Tue Apr 14 09:58:01 2015 +0300 +++ b/src/lib-storage/index/imapc/imapc-mail-fetch.c Wed Apr 15 21:01:11 2015 +0300 @@ -4,6 +4,7 @@ #include "str.h" #include "ioloop.h" #include "istream.h" +#include "istream-concat.h" #include "istream-header-filter.h" #include "message-header-parser.h" #include "imap-arg.h" @@ -218,9 +219,18 @@ str_append_c(str, ' '); } - if ((fields & MAIL_FETCH_STREAM_BODY) != 0) - str_append(str, "BODY.PEEK[] "); - else if ((fields & MAIL_FETCH_STREAM_HEADER) != 0) + if ((fields & MAIL_FETCH_STREAM_BODY) != 0) { + if (!IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_ZIMBRA_WORKAROUNDS)) + str_append(str, "BODY.PEEK[] "); + else { + /* BODY.PEEK[] can return different headers than + BODY.PEEK[HEADER] (e.g. invalid 8bit chars replaced + with '?' in HEADER) - this violates IMAP protocol + and messes up dsync since it sometimes fetches the + full body and sometimes only the headers. */ + str_append(str, "BODY.PEEK[HEADER] BODY.PEEK[TEXT] "); + } + } else if ((fields & MAIL_FETCH_STREAM_HEADER) != 0) str_append(str, "BODY.PEEK[HEADER] "); else if (headers != NULL) { mail->fetching_headers = @@ -267,8 +277,9 @@ } else { return; } + mail->header_fetched = TRUE; mail->body_fetched = TRUE; - imapc_mail_init_stream(mail, TRUE); + imapc_mail_init_stream(mail); } static enum mail_fetch_field @@ -450,7 +461,7 @@ *input = filter_input; } -void imapc_mail_init_stream(struct imapc_mail *mail, bool have_body) +void imapc_mail_init_stream(struct imapc_mail *mail) { struct index_mail *imail = &mail->imail; struct mail *_mail = &imail->mail.mail; @@ -469,7 +480,7 @@ index_mail_close_streams(imail); return; } - } else if (have_body) { + } else if (mail->body_fetched) { ret = i_stream_get_size(imail->data.stream, TRUE, &size); if (ret < 0) { index_mail_close_streams(imail); @@ -482,7 +493,7 @@ imail->data.virtual_size = size; } - imail->data.stream_has_only_header = !have_body; + imail->data.stream_has_only_header = !mail->body_fetched; if (index_mail_init_stream(imail, NULL, NULL, &input) < 0) index_mail_close_streams(imail); } @@ -490,52 +501,100 @@ static void imapc_fetch_stream(struct imapc_mail *mail, const struct imapc_untagged_reply *reply, - const struct imap_arg *arg, bool body) + const struct imap_arg *arg, + bool have_header, bool have_body) { struct index_mail *imail = &mail->imail; + struct istream *hdr_stream = NULL; const char *value; int fd; if (imail->data.stream != NULL) { - if (!body) + i_assert(mail->header_fetched); + if (mail->body_fetched || !have_body) return; - /* maybe the existing stream has no body. replace it. */ + if (have_header) { + /* replace the existing stream */ + } else if (mail->fd == -1) { + /* append this body stream to the existing + header stream */ + hdr_stream = imail->data.stream; + i_stream_ref(hdr_stream); + } else { + /* append this body stream to the existing + header stream. we'll need to recreate the stream + with autoclosed fd. */ + if (lseek(mail->fd, 0, SEEK_SET) < 0) + i_error("lseek(imapc) failed: %m"); + hdr_stream = i_stream_create_fd_autoclose(&mail->fd, 0); + } index_mail_close_streams(imail); if (mail->fd != -1) { if (close(mail->fd) < 0) i_error("close(imapc mail) failed: %m"); mail->fd = -1; } + } else { + if (!have_header) { + /* BODY.PEEK[TEXT] received - we can't currently handle + this before receiving BODY.PEEK[HEADER] reply */ + return; + } } if (arg->type == IMAP_ARG_LITERAL_SIZE) { - if (!imapc_find_lfile_arg(reply, arg, &fd)) + if (!imapc_find_lfile_arg(reply, arg, &fd)) { + if (hdr_stream != NULL) + i_stream_unref(&hdr_stream); return; + } if ((fd = dup(fd)) == -1) { i_error("dup() failed: %m"); + if (hdr_stream != NULL) + i_stream_unref(&hdr_stream); return; } mail->fd = fd; imail->data.stream = i_stream_create_fd(fd, 0, FALSE); } else { if (!imap_arg_get_nstring(arg, &value)) - return; + value = NULL; if (value == NULL) { mail_set_expunged(&imail->mail.mail); + if (hdr_stream != NULL) + i_stream_unref(&hdr_stream); return; } if (mail->body == NULL) { mail->body = buffer_create_dynamic(default_pool, arg->str_len + 1); + } else if (!have_header && hdr_stream != NULL) { + /* header is already in the buffer - add body now + without destroying the existing header data */ + i_stream_unref(&hdr_stream); + } else { + buffer_set_used_size(mail->body, 0); } - buffer_set_used_size(mail->body, 0); buffer_append(mail->body, value, arg->str_len); imail->data.stream = i_stream_create_from_data(mail->body->data, mail->body->used); } - mail->body_fetched = body; + if (have_header) + mail->header_fetched = TRUE; + mail->body_fetched = have_body; - imapc_mail_init_stream(mail, body); + if (hdr_stream != NULL) { + struct istream *inputs[3]; + + inputs[0] = hdr_stream; + inputs[1] = imail->data.stream; + inputs[2] = NULL; + imail->data.stream = i_stream_create_concat(inputs); + i_stream_unref(&inputs[0]); + i_stream_unref(&inputs[1]); + } + + imapc_mail_init_stream(mail); } static void @@ -623,10 +682,13 @@ break; if (strcasecmp(key, "BODY[]") == 0) { - imapc_fetch_stream(mail, reply, &args[i+1], TRUE); + imapc_fetch_stream(mail, reply, &args[i+1], TRUE, TRUE); match = TRUE; } else if (strcasecmp(key, "BODY[HEADER]") == 0) { - imapc_fetch_stream(mail, reply, &args[i+1], FALSE); + imapc_fetch_stream(mail, reply, &args[i+1], TRUE, FALSE); + match = TRUE; + } else if (strcasecmp(key, "BODY[TEXT]") == 0) { + imapc_fetch_stream(mail, reply, &args[i+1], FALSE, TRUE); match = TRUE; } else if (strcasecmp(key, "BODY[HEADER.FIELDS") == 0) { imapc_fetch_header_stream(mail, reply, &args[i+1]); diff -r 93dab55ae36e -r 1c17f06d5e52 src/lib-storage/index/imapc/imapc-mail.c --- a/src/lib-storage/index/imapc/imapc-mail.c Tue Apr 14 09:58:01 2015 +0300 +++ b/src/lib-storage/index/imapc/imapc-mail.c Wed Apr 15 21:01:11 2015 +0300 @@ -386,6 +386,8 @@ } if (mail->body != NULL) buffer_free(&mail->body); + mail->header_fetched = FALSE; + mail->body_fetched = FALSE; } static int imapc_mail_get_hdr_hash(struct index_mail *imail) diff -r 93dab55ae36e -r 1c17f06d5e52 src/lib-storage/index/imapc/imapc-mail.h --- a/src/lib-storage/index/imapc/imapc-mail.h Tue Apr 14 09:58:01 2015 +0300 +++ b/src/lib-storage/index/imapc/imapc-mail.h Wed Apr 15 21:01:11 2015 +0300 @@ -17,6 +17,7 @@ int fd; buffer_t *body; + bool header_fetched; bool body_fetched; bool header_list_fetched; }; @@ -31,7 +32,7 @@ const char *const *headers); bool imapc_mail_prefetch(struct mail *mail); void imapc_mail_fetch_flush(struct imapc_mailbox *mbox); -void imapc_mail_init_stream(struct imapc_mail *mail, bool have_body); +void imapc_mail_init_stream(struct imapc_mail *mail); void imapc_mail_fetch_update(struct imapc_mail *mail, const struct imapc_untagged_reply *reply, diff -r 93dab55ae36e -r 1c17f06d5e52 src/lib-storage/index/imapc/imapc-save.c --- a/src/lib-storage/index/imapc/imapc-save.c Tue Apr 14 09:58:01 2015 +0300 +++ b/src/lib-storage/index/imapc/imapc-save.c Wed Apr 15 21:01:11 2015 +0300 @@ -152,8 +152,11 @@ imail->data.forced_no_caching = TRUE; if (ctx->fd != -1) { + struct imapc_mail *imapc_mail = (struct imapc_mail *)imail; imail->data.stream = i_stream_create_fd_autoclose(&ctx->fd, 0); - imapc_mail_init_stream((struct imapc_mail *)imail, TRUE); + imapc_mail->header_fetched = TRUE; + imapc_mail->body_fetched = TRUE; + imapc_mail_init_stream(imapc_mail); } ctx->save_count++; diff -r 93dab55ae36e -r 1c17f06d5e52 src/lib-storage/index/imapc/imapc-settings.c --- a/src/lib-storage/index/imapc/imapc-settings.c Tue Apr 14 09:58:01 2015 +0300 +++ b/src/lib-storage/index/imapc/imapc-settings.c Wed Apr 15 21:01:11 2015 +0300 @@ -86,6 +86,7 @@ { "fetch-headers", IMAPC_FEATURE_FETCH_HEADERS }, { "gmail-migration", IMAPC_FEATURE_GMAIL_MIGRATION }, { "search", IMAPC_FEATURE_SEARCH }, + { "zimbra-workarounds", IMAPC_FEATURE_ZIMBRA_WORKAROUNDS }, { NULL, 0 } }; diff -r 93dab55ae36e -r 1c17f06d5e52 src/lib-storage/index/imapc/imapc-settings.h --- a/src/lib-storage/index/imapc/imapc-settings.h Tue Apr 14 09:58:01 2015 +0300 +++ b/src/lib-storage/index/imapc/imapc-settings.h Wed Apr 15 21:01:11 2015 +0300 @@ -7,7 +7,8 @@ IMAPC_FEATURE_GUID_FORCED = 0x02, IMAPC_FEATURE_FETCH_HEADERS = 0x04, IMAPC_FEATURE_GMAIL_MIGRATION = 0x08, - IMAPC_FEATURE_SEARCH = 0x10 + IMAPC_FEATURE_SEARCH = 0x10, + IMAPC_FEATURE_ZIMBRA_WORKAROUNDS = 0x20 }; /* */ From dovecot at dovecot.org Thu Apr 16 08:48:05 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 16 Apr 2015 08:48:05 +0000 Subject: dovecot-2.2: lib: test-array build warnings on Solaris 10 Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7c90285a72a6 changeset: 18405:7c90285a72a6 user: Phil Carmody date: Thu Apr 16 11:46:22 2015 +0300 description: lib: test-array build warnings on Solaris 10 Solaris cc doesn't think I've read N1570 6.7.9p11, 6.5.16.1p2, and 6.3.1.3p2, and wants to save me from myself. Reported-by: Juergen Obermann Signed-off-by: Phil Carmody diffstat: src/lib/test-array.c | 8 +++++--- 1 files changed, 5 insertions(+), 3 deletions(-) diffs (32 lines): diff -r 1c17f06d5e52 -r 7c90285a72a6 src/lib/test-array.c --- a/src/lib/test-array.c Wed Apr 15 21:01:11 2015 +0300 +++ b/src/lib/test-array.c Thu Apr 16 11:46:22 2015 +0300 @@ -126,7 +126,7 @@ static void test_array_cmp(void) { static const unsigned short deltas[] = { - -32768, -16384, -512, -256, -128, -64, -2, -1, + 0x8000, 0xc000, 0xfe00, 0xff00, 0xff80, 0xffc0, 0xfffe, 0xffff, 0, 1, 2, 64, 128, 256, 512, 16384, 32768 }; @@ -151,7 +151,8 @@ for (i = 0; i < 256; i++) { unsigned int j = rand() % NELEMS; - unsigned short tmp = *array_idx(&arr2, j); + const unsigned short *ptmp = array_idx(&arr2, j); + unsigned short tmp = *ptmp; unsigned short repl = tmp + deltas[rand() % N_ELEMENTS(deltas)]; array_idx_set(&arr2, j, &repl); @@ -203,7 +204,8 @@ test_assert(array_equal_fn(&arr1, &arr2, test_compare_string) == 1); /* therefore value same */ for (i = 0; i < 2560; i++) { unsigned int j = rand() % NELEMS; - const char *ostr = *array_idx(&arr2, j); + const char *const *ostr_p = array_idx(&arr2, j); + const char *ostr = *ostr_p; unsigned int olen = strlen(ostr); unsigned int rc = rand() % (olen + 1); char ochar = ostr[rc]; From dovecot at dovecot.org Thu Apr 16 10:00:33 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 16 Apr 2015 10:00:33 +0000 Subject: dovecot-2.2: stats: Fixed compiling with some OSes. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/736730cf45f0 changeset: 18406:736730cf45f0 user: Timo Sirainen date: Thu Apr 16 12:59:05 2015 +0300 description: stats: Fixed compiling with some OSes. struct timeval wasn't necessarily already defined by existing includes. diffstat: src/stats/mail-stats.h | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 7c90285a72a6 -r 736730cf45f0 src/stats/mail-stats.h --- a/src/stats/mail-stats.h Thu Apr 16 11:46:22 2015 +0300 +++ b/src/stats/mail-stats.h Thu Apr 16 12:59:05 2015 +0300 @@ -1,6 +1,8 @@ #ifndef MAIL_STATS_H #define MAIL_STATS_H +#include + #include "net.h" #include "guid.h" #include "stats.h" From dovecot at dovecot.org Thu Apr 16 11:21:19 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 16 Apr 2015 11:21:19 +0000 Subject: dovecot-2.2: doveadm mailbox metadata: The keys must always be l... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0f4b010b78b6 changeset: 18407:0f4b010b78b6 user: Timo Sirainen date: Thu Apr 16 14:19:51 2015 +0300 description: doveadm mailbox metadata: The keys must always be lowercased. The keys need to be case-insensitive in IMAP, so the IMAP code always also lowercases them. diffstat: src/doveadm/doveadm-mail-mailbox-metadata.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 736730cf45f0 -r 0f4b010b78b6 src/doveadm/doveadm-mail-mailbox-metadata.c --- a/src/doveadm/doveadm-mail-mailbox-metadata.c Thu Apr 16 12:59:05 2015 +0300 +++ b/src/doveadm/doveadm-mail-mailbox-metadata.c Thu Apr 16 14:19:51 2015 +0300 @@ -78,6 +78,7 @@ i_fatal_status(EX_USAGE, "Invalid metadata key '%s': " "Must begin with /private or /shared", arg); } + *key_r = t_str_lcase(*key_r); } static void From dovecot at dovecot.org Thu Apr 16 15:21:42 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 16 Apr 2015 15:21:42 +0000 Subject: dovecot-2.2: lib-fs: fs_class_register() can now be called exter... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4ced4305d960 changeset: 18408:4ced4305d960 user: Timo Sirainen date: Thu Apr 16 18:20:15 2015 +0300 description: lib-fs: fs_class_register() can now be called externally. diffstat: src/lib-fs/fs-api-private.h | 2 ++ src/lib-fs/fs-api.c | 6 +++++- 2 files changed, 7 insertions(+), 1 deletions(-) diffs (36 lines): diff -r 0f4b010b78b6 -r 4ced4305d960 src/lib-fs/fs-api-private.h --- a/src/lib-fs/fs-api-private.h Thu Apr 16 14:19:51 2015 +0300 +++ b/src/lib-fs/fs-api-private.h Thu Apr 16 18:20:15 2015 +0300 @@ -138,6 +138,8 @@ extern const struct fs fs_class_sis; extern const struct fs fs_class_sis_queue; +void fs_class_register(const struct fs *fs_class); + void fs_set_error(struct fs *fs, const char *fmt, ...) ATTR_FORMAT(2, 3); void fs_set_critical(struct fs *fs, const char *fmt, ...) ATTR_FORMAT(2, 3); diff -r 0f4b010b78b6 -r 4ced4305d960 src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Thu Apr 16 14:19:51 2015 +0300 +++ b/src/lib-fs/fs-api.c Thu Apr 16 18:20:15 2015 +0300 @@ -16,6 +16,8 @@ static struct module *fs_modules = NULL; static ARRAY(const struct fs *) fs_classes; +static void fs_classes_init(void); + static int fs_alloc(const struct fs *fs_class, const char *args, const struct fs_settings *set, struct fs **fs_r, const char **error_r) @@ -45,8 +47,10 @@ return 0; } -static void fs_class_register(const struct fs *fs_class) +void fs_class_register(const struct fs *fs_class) { + if (!array_is_created(&fs_classes)) + fs_classes_init(); array_append(&fs_classes, &fs_class, 1); } From dovecot at dovecot.org Fri Apr 17 08:21:51 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 17 Apr 2015 08:21:51 +0000 Subject: dovecot-2.2: dovecot-config: Added lib-stats to LIBDOVECOT_INCLUDE Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/87ec8d53c9fa changeset: 18409:87ec8d53c9fa user: Timo Sirainen date: Fri Apr 17 11:20:23 2015 +0300 description: dovecot-config: Added lib-stats to LIBDOVECOT_INCLUDE diffstat: dovecot-config.in.in | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 4ced4305d960 -r 87ec8d53c9fa dovecot-config.in.in --- a/dovecot-config.in.in Thu Apr 16 18:20:15 2015 +0300 +++ b/dovecot-config.in.in Fri Apr 17 11:20:23 2015 +0300 @@ -20,7 +20,7 @@ LIBDOVECOT_STORAGE_DEPS="@LIBDOVECOT_STORAGE_DEPS@" LIBDOVECOT_DSYNC_DEPS="@LIBDOVECOT_DSYNC@" -LIBDOVECOT_INCLUDE="-I$(incdir) -I$(incdir)/src/lib -I$(incdir)/src/lib-dict -I$(incdir)/src/lib-dns -I$(incdir)/src/lib-http -I$(incdir)/src/lib-mail -I$(incdir)/src/lib-imap -I$(incdir)/src/lib-fs -I$(incdir)/src/lib-charset -I$(incdir)/src/lib-auth -I$(incdir)/src/lib-master -I$(incdir)/src/lib-ssl-iostream -I$(incdir)/src/lib-compression -I$(incdir)/src/lib-settings -I$(incdir)/src/lib-test -I$(incdir)/src/lib-sasl" +LIBDOVECOT_INCLUDE="-I$(incdir) -I$(incdir)/src/lib -I$(incdir)/src/lib-dict -I$(incdir)/src/lib-dns -I$(incdir)/src/lib-http -I$(incdir)/src/lib-mail -I$(incdir)/src/lib-imap -I$(incdir)/src/lib-fs -I$(incdir)/src/lib-charset -I$(incdir)/src/lib-auth -I$(incdir)/src/lib-master -I$(incdir)/src/lib-ssl-iostream -I$(incdir)/src/lib-compression -I$(incdir)/src/lib-settings -I$(incdir)/src/lib-test -I$(incdir)/src/lib-sasl -I$(incdir)/src/lib-stats" LIBDOVECOT_LDA_INCLUDE="-I$(incdir)/src/lib-lda -I$(incdir)/src/lda" LIBDOVECOT_DOVEADM_INCLUDE="-I$(incdir)/src/doveadm" LIBDOVECOT_STORAGE_INCLUDE="-I$(incdir)/src/lib-index -I$(incdir)/src/lib-storage -I$(incdir)/src/lib-storage/list -I$(incdir)/src/lib-storage/index -I$(incdir)/src/lib-storage/index/raw -I$(incdir)/src/lib-imap-storage -I$(incdir)/src/plugins/quota" From dovecot at dovecot.org Fri Apr 17 08:53:52 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 17 Apr 2015 08:53:52 +0000 Subject: dovecot-2.2: doveadm fs delete -R: Don't add extra '/' chars in ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3215ef3e3031 changeset: 18410:3215ef3e3031 user: Timo Sirainen date: Fri Apr 17 11:52:24 2015 +0300 description: doveadm fs delete -R: Don't add extra '/' chars in the middle of paths diffstat: src/doveadm/doveadm-fs.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 87ec8d53c9fa -r 3215ef3e3031 src/doveadm/doveadm-fs.c --- a/src/doveadm/doveadm-fs.c Fri Apr 17 11:20:23 2015 +0300 +++ b/src/doveadm/doveadm-fs.c Fri Apr 17 11:52:24 2015 +0300 @@ -263,7 +263,7 @@ t_array_init(&fnames, 8); iter = fs_iter_init(fs, path, FS_ITER_FLAG_DIRS); while ((fname = fs_iter_next(iter)) != NULL) { - fname = t_strconcat(fname, "/", NULL); + fname = t_strdup(fname); array_append(&fnames, &fname, 1); } if (fs_iter_deinit(&iter) < 0) { From dovecot at dovecot.org Mon Apr 20 12:30:50 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 20 Apr 2015 12:30:50 +0000 Subject: dovecot-2.2: fts: Code cleanup to do all index updates via fts_b... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/14e6d5289a87 changeset: 18411:14e6d5289a87 user: Timo Sirainen date: Mon Apr 20 15:02:04 2015 +0300 description: fts: Code cleanup to do all index updates via fts_build_data() This doesn't actually change the behavior. diffstat: src/plugins/fts/fts-build-mail.c | 68 +++++++++++++++++++++++---------------- 1 files changed, 40 insertions(+), 28 deletions(-) diffs (129 lines): diff -r 3215ef3e3031 -r 14e6d5289a87 src/plugins/fts/fts-build-mail.c --- a/src/plugins/fts/fts-build-mail.c Fri Apr 17 11:52:24 2015 +0300 +++ b/src/plugins/fts/fts-build-mail.c Mon Apr 20 15:02:04 2015 +0300 @@ -30,6 +30,9 @@ buffer_t *word_buf; }; +static int fts_build_data(struct fts_mail_build_context *ctx, + const unsigned char *data, size_t size, bool last); + static void fts_build_parse_content_type(struct fts_mail_build_context *ctx, const struct message_header_line *hdr) { @@ -91,8 +94,7 @@ buf[i] = data[i]; } } - (void)fts_backend_update_build_more(ctx->update_ctx, - data, hdr->full_value_len); + (void)fts_build_data(ctx, data, hdr->full_value_len, TRUE); i_free(buf); } @@ -133,9 +135,7 @@ str = t_str_new(hdr->full_value_len); message_address_write(str, addr); - (void)fts_backend_update_build_more(ctx->update_ctx, - str_data(str), - str_len(str)); + (void)fts_build_data(ctx, str_data(str), str_len(str), TRUE); } T_END; } @@ -192,31 +192,24 @@ return TRUE; } -static int fts_build_body_block(struct fts_mail_build_context *ctx, - struct message_block *block, bool last) +static int +fts_build_body_block_full_words(struct fts_mail_build_context *ctx, + const unsigned char *data, size_t size, bool last) { - unsigned int i; + size_t i; - i_assert(block->hdr == NULL); - - if ((ctx->update_ctx->backend->flags & - FTS_BACKEND_FLAG_BUILD_FULL_WORDS) == 0) { - return fts_backend_update_build_more(ctx->update_ctx, - block->data, block->size); - } /* we'll need to send only full words to the backend */ if (ctx->word_buf != NULL && ctx->word_buf->used > 0) { /* continuing previous word */ - for (i = 0; i < block->size; i++) { - if (IS_WORD_WHITESPACE(block->data[i])) + for (i = 0; i < size; i++) { + if (IS_WORD_WHITESPACE(data[i])) break; } - buffer_append(ctx->word_buf, block->data, i); - block->data += i; - block->size -= i; - if (block->size == 0 && ctx->word_buf->used < MAX_WORD_SIZE && - !last) { + buffer_append(ctx->word_buf, data, i); + data += i; + size -= i; + if (size == 0 && ctx->word_buf->used < MAX_WORD_SIZE && !last) { /* word is still not finished */ return 0; } @@ -230,27 +223,46 @@ /* find the boundary for last word */ if (last) - i = block->size; + i = size; else { - for (i = block->size; i > 0; i--) { - if (IS_WORD_WHITESPACE(block->data[i-1])) + for (i = size; i > 0; i--) { + if (IS_WORD_WHITESPACE(data[i-1])) break; } } - if (fts_backend_update_build_more(ctx->update_ctx, block->data, i) < 0) + if (fts_backend_update_build_more(ctx->update_ctx, data, i) < 0) return -1; - if (i < block->size) { + if (i < size) { if (ctx->word_buf == NULL) { ctx->word_buf = buffer_create_dynamic(default_pool, 128); } - buffer_append(ctx->word_buf, block->data + i, block->size - i); + buffer_append(ctx->word_buf, data + i, size - i); } return 0; } +static int fts_build_data(struct fts_mail_build_context *ctx, + const unsigned char *data, size_t size, bool last) +{ + if ((ctx->update_ctx->backend->flags & + FTS_BACKEND_FLAG_BUILD_FULL_WORDS) != 0) { + return fts_build_body_block_full_words(ctx, data, size, last); + } else { + return fts_backend_update_build_more(ctx->update_ctx, data, size); + } +} + +static int fts_build_body_block(struct fts_mail_build_context *ctx, + const struct message_block *block, bool last) +{ + i_assert(block->hdr == NULL); + + return fts_build_data(ctx, block->data, block->size, last); +} + static int fts_body_parser_finish(struct fts_mail_build_context *ctx) { struct message_block block; From dovecot at dovecot.org Mon Apr 20 12:30:51 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 20 Apr 2015 12:30:51 +0000 Subject: dovecot-2.2: configure: Stemmer and textcat checks are now done ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1c516f905152 changeset: 18412:1c516f905152 user: Timo Sirainen date: Mon Apr 20 15:12:14 2015 +0300 description: configure: Stemmer and textcat checks are now done even if CLucene isn't used. They'll be useful for other FTS backends as well. This patch doesn't change any actual functionality. diffstat: configure.ac | 60 +++++++++++++++-------------- src/plugins/fts-lucene/Makefile.am | 8 ++-- src/plugins/fts-lucene/fts-lucene-plugin.c | 4 +- src/plugins/fts-lucene/lucene-wrapper.cc | 10 ++-- 4 files changed, 42 insertions(+), 40 deletions(-) diffs (178 lines): diff -r 14e6d5289a87 -r 1c516f905152 configure.ac --- a/configure.ac Mon Apr 20 15:02:04 2015 +0300 +++ b/configure.ac Mon Apr 20 15:12:14 2015 +0300 @@ -160,12 +160,12 @@ AM_CONDITIONAL(BUILD_LUCENE, test "$want_lucene" = "yes") AC_ARG_WITH(stemmer, -AS_HELP_STRING([--with-stemmer], [Build with libstemmer support (for CLucene) (auto)]), +AS_HELP_STRING([--with-stemmer], [Build with libstemmer support (for FTS) (auto)]), TEST_WITH(stemmer, $withval), want_stemmer=auto) AC_ARG_WITH(textcat, -AS_HELP_STRING([--with-textcat], [Build with libtextcat support (for CLucene) (auto)]), +AS_HELP_STRING([--with-textcat], [Build with libtextcat support (for FTS) (auto)]), TEST_WITH(textcat, $withval), want_textcat=auto) @@ -2743,36 +2743,38 @@ # FIXME: we should verify here that this actually works.. CLUCENE_LIBS="-lclucene-shared -lclucene-core" ]) - if test $want_stemmer != no; then - AC_CHECK_LIB(stemmer, sb_stemmer_new, [ - have_lucene_stemmer=yes - AC_DEFINE(HAVE_LUCENE_STEMMER,, Define if you want stemming support for CLucene) - if test $want_textcat != no; then - AC_CHECK_LIB(textcat, special_textcat_Init, [ - have_lucene_textcat=yes - AC_DEFINE(HAVE_LUCENE_TEXTCAT,, Define if you want textcat support for CLucene) - ], [ - AC_CHECK_LIB(exttextcat, special_textcat_Init, [ - have_lucene_exttextcat=yes - AC_DEFINE(HAVE_LUCENE_EXTTEXTCAT,, Define if you want textcat (Debian version) support for CLucene) - ]) - ]) - if test $want_textcat = yes && test "$have_lucene_exttextcat" != yes && test "$have_lucene_textcat" != yes; then - AC_ERROR([Can't build with textcat support: libtextcat or libexttextcat not found]) - fi - fi - ], [ - if test $want_stemmer = yes; then - AC_ERROR([Can't build with stemmer support: libstemmer not found]) - fi - ]) - fi have_lucene=yes fts="$fts lucene" fi -AM_CONDITIONAL(BUILD_LUCENE_STEMMER, test "$have_lucene_stemmer" = "yes") -AM_CONDITIONAL(BUILD_LUCENE_TEXTCAT, test "$have_lucene_textcat" = "yes") -AM_CONDITIONAL(BUILD_LUCENE_EXTTEXTCAT, test "$have_lucene_exttextcat" = "yes") + +if test $want_stemmer != no; then + AC_CHECK_LIB(stemmer, sb_stemmer_new, [ + have_fts_stemmer=yes + AC_DEFINE(HAVE_FTS_STEMMER,, Define if you want stemming support for FTS) + ], [ + if test $want_stemmer = yes; then + AC_ERROR([Can't build with stemmer support: libstemmer not found]) + fi + ]) +fi +AM_CONDITIONAL(BUILD_FTS_STEMMER, test "$have_fts_stemmer" = "yes") + +if test $want_textcat != no; then + AC_CHECK_LIB(textcat, special_textcat_Init, [ + have_fts_textcat=yes + AC_DEFINE(HAVE_FTS_TEXTCAT,, Define if you want textcat support for FTS) + ], [ + AC_CHECK_LIB(exttextcat, special_textcat_Init, [ + have_fts_exttextcat=yes + AC_DEFINE(HAVE_FTS_EXTTEXTCAT,, Define if you want libexttextcat support for FTS) + ]) + ]) + if test $want_textcat = yes && test "$have_fts_exttextcat" != yes && test "$have_fts_textcat" != yes; then + AC_ERROR([Can't build with textcat support: libtextcat or libexttextcat not found]) + fi +fi +AM_CONDITIONAL(BUILD_FTS_TEXTCAT, test "$have_fts_textcat" = "yes") +AM_CONDITIONAL(BUILD_FTS_EXTTEXTCAT, test "$have_fts_exttextcat" = "yes") if test $have_lucene = no; then not_fts="$not_fts lucene" diff -r 14e6d5289a87 -r 1c516f905152 src/plugins/fts-lucene/Makefile.am --- a/src/plugins/fts-lucene/Makefile.am Mon Apr 20 15:02:04 2015 +0300 +++ b/src/plugins/fts-lucene/Makefile.am Mon Apr 20 15:12:14 2015 +0300 @@ -18,14 +18,14 @@ module_LTLIBRARIES = \ lib21_fts_lucene_plugin.la -if BUILD_LUCENE_STEMMER +if BUILD_FTS_STEMMER STEMMER_LIBS = -lstemmer SHOWBALL_SOURCES = Snowball.cc endif -if BUILD_LUCENE_TEXTCAT +if BUILD_FTS_TEXTCAT TEXTCAT_LIBS = -ltextcat else -if BUILD_LUCENE_EXTTEXTCAT +if BUILD_FTS_EXTTEXTCAT TEXTCAT_LIBS = -lexttextcat endif endif @@ -45,7 +45,7 @@ SnowballAnalyzer.h \ SnowballFilter.h -if BUILD_LUCENE_TEXTCAT +if BUILD_FTS_TEXTCAT exampledir = $(docdir)/example-config example_DATA = \ textcat.conf diff -r 14e6d5289a87 -r 1c516f905152 src/plugins/fts-lucene/fts-lucene-plugin.c --- a/src/plugins/fts-lucene/fts-lucene-plugin.c Mon Apr 20 15:02:04 2015 +0300 +++ b/src/plugins/fts-lucene/fts-lucene-plugin.c Mon Apr 20 15:12:14 2015 +0300 @@ -49,7 +49,7 @@ } if (set->whitespace_chars == NULL) set->whitespace_chars = ""; -#ifndef HAVE_LUCENE_STEMMER +#ifndef HAVE_FTS_STEMMER if (set->default_language != NULL) { i_error("fts_lucene: default_language set, " "but Dovecot built without stemmer support"); @@ -59,7 +59,7 @@ if (set->default_language == NULL) set->default_language = "english"; #endif -#ifndef HAVE_LUCENE_TEXTCAT +#ifndef HAVE_FTS_TEXTCAT if (set->textcat_conf != NULL) { i_error("fts_lucene: textcat_dir set, " "but Dovecot built without textcat support"); diff -r 14e6d5289a87 -r 1c516f905152 src/plugins/fts-lucene/lucene-wrapper.cc --- a/src/plugins/fts-lucene/lucene-wrapper.cc Mon Apr 20 15:02:04 2015 +0300 +++ b/src/plugins/fts-lucene/lucene-wrapper.cc Mon Apr 20 15:12:14 2015 +0300 @@ -19,10 +19,10 @@ #include "lucene-wrapper.h" #include -#ifdef HAVE_LUCENE_TEXTCAT +#ifdef HAVE_FTS_TEXTCAT # include #else -#ifdef HAVE_LUCENE_EXTTEXTCAT +#ifdef HAVE_FTS_EXTTEXTCAT # include #endif #endif @@ -124,7 +124,7 @@ /* this is valid only for doveadm dump, so it doesn't matter */ index->set.default_language = ""; } -#ifdef HAVE_LUCENE_STEMMER +#ifdef HAVE_FTS_STEMMER if (set == NULL || !set->no_snowball) { index->default_analyzer = _CLNEW snowball::SnowballAnalyzer(index->normalizer, @@ -180,7 +180,7 @@ } array_free(&index->analyzers); if (--textcat_refcount == 0 && textcat != NULL) { -#ifdef HAVE_LUCENE_TEXTCAT +#ifdef HAVE_FTS_TEXTCAT textcat_Done(textcat); #endif textcat = NULL; @@ -458,7 +458,7 @@ return 0; } -#ifdef HAVE_LUCENE_TEXTCAT +#ifdef HAVE_FTS_TEXTCAT static Analyzer *get_analyzer(struct lucene_index *index, const char *lang) { normalizer_func_t *normalizer = index->normalizer; From dovecot at dovecot.org Mon Apr 20 12:30:51 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 20 Apr 2015 12:30:51 +0000 Subject: dovecot-2.2: configure: Fixed/improved finding and using libextt... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/68c5e0db61db changeset: 18413:68c5e0db61db user: Timo Sirainen date: Mon Apr 20 15:27:02 2015 +0300 description: configure: Fixed/improved finding and using libexttextcat. libexttextcat wasn't actually being used at all by fts-lucene. Now we'll first prefer finding it via pkg-config, next finding exttextcat and only last look up textcat. diffstat: configure.ac | 26 ++++++++++++++++++-------- src/plugins/fts-lucene/Makefile.am | 10 ++++++---- src/plugins/fts-lucene/lucene-wrapper.cc | 10 +++++----- 3 files changed, 29 insertions(+), 17 deletions(-) diffs (89 lines): diff -r 1c516f905152 -r 68c5e0db61db configure.ac --- a/configure.ac Mon Apr 20 15:12:14 2015 +0300 +++ b/configure.ac Mon Apr 20 15:27:02 2015 +0300 @@ -2760,17 +2760,27 @@ AM_CONDITIONAL(BUILD_FTS_STEMMER, test "$have_fts_stemmer" = "yes") if test $want_textcat != no; then - AC_CHECK_LIB(textcat, special_textcat_Init, [ - have_fts_textcat=yes - AC_DEFINE(HAVE_FTS_TEXTCAT,, Define if you want textcat support for FTS) - ], [ + if test "$PKG_CONFIG" != "" && $PKG_CONFIG --exists libexttextcat 2>/dev/null; then + PKG_CHECK_MODULES(LIBEXTTEXTCAT, libexttextcat) + have_exttextcat=yes + else AC_CHECK_LIB(exttextcat, special_textcat_Init, [ have_fts_exttextcat=yes - AC_DEFINE(HAVE_FTS_EXTTEXTCAT,, Define if you want libexttextcat support for FTS) + AC_CHECK_HEADERS(libexttextcat/textcat.h) + LIBEXTTEXTCAT_LIBS=-lexttextcat + AC_SUBST(LIBEXTTEXTCAT_LIBS) + ], [ + AC_CHECK_LIB(textcat, special_textcat_Init, [ + have_fts_textcat=yes + AC_CHECK_HEADERS(libtextcat/textcat.h) + ]) ]) - ]) - if test $want_textcat = yes && test "$have_fts_exttextcat" != yes && test "$have_fts_textcat" != yes; then - AC_ERROR([Can't build with textcat support: libtextcat or libexttextcat not found]) + if test $want_textcat = yes && test "$have_fts_exttextcat" != yes && test "$have_fts_textcat" != yes; then + AC_ERROR([Can't build with textcat support: libtextcat or libexttextcat not found]) + fi + fi + if test "$have_fts_exttextcat" != yes || test "$have_fts_textcat" != yes; then + AC_DEFINE(HAVE_FTS_TEXTCAT,, Define if you want textcat support for FTS) fi fi AM_CONDITIONAL(BUILD_FTS_TEXTCAT, test "$have_fts_textcat" = "yes") diff -r 1c516f905152 -r 68c5e0db61db src/plugins/fts-lucene/Makefile.am --- a/src/plugins/fts-lucene/Makefile.am Mon Apr 20 15:12:14 2015 +0300 +++ b/src/plugins/fts-lucene/Makefile.am Mon Apr 20 15:27:02 2015 +0300 @@ -9,7 +9,8 @@ -I$(top_srcdir)/src/doveadm AM_CXXFLAGS = \ - $(CLUCENE_CFLAGS) + $(CLUCENE_CFLAGS) \ + $(LIBEXTTEXTCAT_CFLAGS) NOPLUGIN_LDFLAGS = lib21_fts_lucene_plugin_la_LDFLAGS = -module -avoid-version @@ -22,11 +23,12 @@ STEMMER_LIBS = -lstemmer SHOWBALL_SOURCES = Snowball.cc endif + +if BUILD_FTS_EXTTEXTCAT +TEXTCAT_LIBS = $(LIBEXTTEXTCAT_LIBS) +else if BUILD_FTS_TEXTCAT TEXTCAT_LIBS = -ltextcat -else -if BUILD_FTS_EXTTEXTCAT -TEXTCAT_LIBS = -lexttextcat endif endif diff -r 1c516f905152 -r 68c5e0db61db src/plugins/fts-lucene/lucene-wrapper.cc --- a/src/plugins/fts-lucene/lucene-wrapper.cc Mon Apr 20 15:12:14 2015 +0300 +++ b/src/plugins/fts-lucene/lucene-wrapper.cc Mon Apr 20 15:27:02 2015 +0300 @@ -19,12 +19,12 @@ #include "lucene-wrapper.h" #include -#ifdef HAVE_FTS_TEXTCAT +#ifdef HAVE_LIBEXTTEXTCAT_TEXTCAT_H +# include +#elif defined (HAVE_LIBTEXTCAT_TEXTCAT_H) # include -#else -#ifdef HAVE_FTS_EXTTEXTCAT -# include -#endif +#elif defined (HAVE_FTS_TEXTCAT) +# include #endif }; #include From dovecot at dovecot.org Mon Apr 20 13:24:08 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 20 Apr 2015 13:24:08 +0000 Subject: dovecot-2.2: Initial import for lib-fts. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/81e5b977e5c5 changeset: 18414:81e5b977e5c5 user: Timo Sirainen date: Mon Apr 20 16:19:07 2015 +0300 description: Initial import for lib-fts. Parts of what this code does was already implemented internally by fts-lucene. lib-fts is intended to be usable for all the FTS backends. The APIs are still going to change a bit, but hopefully not after v2.2.17 release. Mostly written by Teemu Huovila. diffstat: .hgignore | 4 + configure.ac | 17 + src/Makefile.am | 1 + src/lib-fts/Makefile.am | 111 +++++ src/lib-fts/fts-filter-normalizer.c | 318 ++++++++++++++ src/lib-fts/fts-filter-private.h | 31 + src/lib-fts/fts-filter-stemmer-snowball.c | 124 +++++ src/lib-fts/fts-filter-stopwords.c | 153 +++++++ src/lib-fts/fts-filter.c | 109 +++++ src/lib-fts/fts-filter.h | 61 ++ src/lib-fts/fts-language.c | 271 ++++++++++++ src/lib-fts/fts-language.h | 56 ++ src/lib-fts/fts-tokenizer-address.c | 356 ++++++++++++++++ src/lib-fts/fts-tokenizer-generic-private.h | 80 +++ src/lib-fts/fts-tokenizer-generic.c | 596 ++++++++++++++++++++++++++++ src/lib-fts/fts-tokenizer-private.h | 43 ++ src/lib-fts/fts-tokenizer.c | 173 ++++++++ src/lib-fts/fts-tokenizer.h | 60 ++ src/lib-fts/stopwords_en.txt | 54 ++ src/lib-fts/stopwords_fr.txt | 178 ++++++++ src/lib-fts/test-fts-filter.c | 551 +++++++++++++++++++++++++ src/lib-fts/test-fts-language.c | 228 ++++++++++ src/lib-fts/test-fts-tokenizer.c | 532 ++++++++++++++++++++++++ src/lib-fts/udhr_fra.txt | 217 ++++++++++ src/lib-fts/word-boundary-data.sh | 99 ++++ src/lib-fts/word-break-data.sh | 77 +++ 26 files changed, 4500 insertions(+), 0 deletions(-) diffs (truncated from 4636 to 300 lines): diff -r 68c5e0db61db -r 81e5b977e5c5 .hgignore --- a/.hgignore Mon Apr 20 15:27:02 2015 +0300 +++ b/.hgignore Mon Apr 20 16:19:07 2015 +0300 @@ -78,6 +78,10 @@ src/ipc/ipc src/lib/unicodemap.c src/lib/UnicodeData.txt +src/lib-fts/PropList.txt +src/lib-fts/WordBreakProperty.txt +src/lib-fts/word-boundary-data.c +src/lib-fts/word-break-data.c src/lib-dict/dict-drivers-register.c src/lib-sql/sql-drivers-register.c src/lib-storage/register/mail-storage-register.c diff -r 68c5e0db61db -r 81e5b977e5c5 configure.ac --- a/configure.ac Mon Apr 20 15:27:02 2015 +0300 +++ b/configure.ac Mon Apr 20 16:19:07 2015 +0300 @@ -169,6 +169,11 @@ TEST_WITH(textcat, $withval), want_textcat=auto) +AC_ARG_WITH(normalizer, +AS_HELP_STRING([--with-normalizer], [Build lib-fts with ICU normalization support (auto)]), + want_fts_normalizer=$withval, + want_fts_normalizer=auto) + AC_ARG_WITH(solr, AS_HELP_STRING([--with-solr], [Build with Solr full text search support]), TEST_WITH(solr, $withval), @@ -2786,6 +2791,17 @@ AM_CONDITIONAL(BUILD_FTS_TEXTCAT, test "$have_fts_textcat" = "yes") AM_CONDITIONAL(BUILD_FTS_EXTTEXTCAT, test "$have_fts_exttextcat" = "yes") +if test "$want_fts_normalizer" != "no"; then + if test "$PKG_CONFIG" != "" && $PKG_CONFIG --exists icu-i18n 2>/dev/null; then + PKG_CHECK_MODULES(FTS_NORMALIZER, icu-i18n) + have_fts_normalizer=yes + AC_DEFINE(HAVE_FTS_NORMALIZER,, Define if you want ICU normalization support for FTS) + elif test "$want_fts_normalizer" = "yes"; then + AC_ERROR([Can't build with normalizer support: libicu-i18n not found]) + fi +fi +AM_CONDITIONAL(BUILD_FTS_NORMALIZER, test "$have_fts_normalizer" = "yes") + if test $have_lucene = no; then not_fts="$not_fts lucene" fi @@ -2857,6 +2873,7 @@ src/lib-dict/Makefile src/lib-dns/Makefile src/lib-fs/Makefile +src/lib-fts/Makefile src/lib-http/Makefile src/lib-imap/Makefile src/lib-imap-storage/Makefile diff -r 68c5e0db61db -r 81e5b977e5c5 src/Makefile.am --- a/src/Makefile.am Mon Apr 20 15:27:02 2015 +0300 +++ b/src/Makefile.am Mon Apr 20 16:19:07 2015 +0300 @@ -19,6 +19,7 @@ SUBDIRS = \ $(LIBDOVECOT_SUBDIRS) \ lib-dovecot \ + lib-fts \ lib-imap-client \ lib-imap-urlauth \ lib-compression \ diff -r 68c5e0db61db -r 81e5b977e5c5 src/lib-fts/Makefile.am --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-fts/Makefile.am Mon Apr 20 16:19:07 2015 +0300 @@ -0,0 +1,111 @@ +noinst_LTLIBRARIES = libfts.la + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ + $(LIBEXTTEXTCAT_CFLAGS) \ + $(LIBFTS_NORMALIZER_CFLAGS) \ + -DUDHRDIR=\""$(top_srcdir)/src/lib-fts"\" \ + -DDATADIR=\"$(pkgdatadir)\" \ + -DTEST_TEXTCAT_DIR=\""$(top_srcdir)/ext/libexttextcat/langclass"\" \ + -DTEST_STOPWORDS_DIR=\""$(top_srcdir)/src/lib-fts"\" + +stopwordsdir = $(datadir)/${PACKAGE_TARNAME}/stopwords +dist_stopwords_DATA = stopwords_en.txt stopwords_fr.txt + +BUILT_SOURCES = word-boundary-data.c word-break-data.c + +EXTRA_DIST = \ + WordBreakProperty.txt \ + word-boundary-data.sh \ + word-boundary-data.c \ + word-break-data.sh \ + word-break-data.c + +WordBreakProperty.txt: + test -f WordBreakProperty.txt || wget http://www.unicode.org/Public/UNIDATA/auxiliary/WordBreakProperty.txt +$(srcdir)/word-boundary-data.c: word-boundary-data.sh WordBreakProperty.txt + $(srcdir)/word-boundary-data.sh < WordBreakProperty.txt > $@ + +PropList.txt: + test -f PropList.txt || wget http://www.unicode.org/Public/UNIDATA/PropList.txt +$(srcdir)/word-break-data.c: word-break-data.sh PropList.txt + $(srcdir)/word-break-data.sh < PropList.txt > $@ + + +if BUILD_FTS_STEMMER +STEMMER_LIBS = -lstemmer +endif + +if BUILD_FTS_EXTTEXTCAT +TEXTCAT_LIBS = $(LIBEXTTEXTCAT_LIBS) +else +if BUILD_FTS_TEXTCAT +TEXTCAT_LIBS = -ltextcat +endif +endif + +if BUILD_FTS_NORMALIZER +NORMALIZER_LIBS = $(LIBFTS_NORMALIZER_LIBS) +endif + +libfts_la_LIBADD = \ + $(STEMMER_LIBS) \ + $(TEXTCAT_LIBS) \ + $(NORMALIZER_LIBS) + +libfts_la_SOURCES = \ + fts-filter.c \ + fts-filter-normalizer.c \ + fts-filter-stopwords.c \ + fts-filter-stemmer-snowball.c \ + fts-language.c \ + fts-tokenizer.c \ + fts-tokenizer-address.c \ + fts-tokenizer-generic.c + +noinst_HEADERS = \ + fts-filter.h \ + fts-filter-private.h \ + fts-language.h \ + fts-tokenizer.h \ + fts-tokenizer-private.h \ + fts-tokenizer-generic-private.h + +test_programs = \ + test-fts-filter \ + $(TEST_FTS_LANGUAGE) \ + $(TEST_FTS_NORMALIZER) \ + test-fts-tokenizer + +noinst_PROGRAMS = $(test_programs) + +test_libs = \ + ../lib-test/libtest.la \ + ../lib/liblib.la +test_deps = $(noinst_LTLIBRARIES) $(test_libs) + +filter_deps = \ + fts-filter.lo fts-filter-stopwords.lo \ + fts-filter-stemmer-snowball.lo fts-filter-normalizer.lo + +test_fts_filter_SOURCES = test-fts-filter.c +test_fts_filter_LDADD = $(filter_deps) $(test_libs) $(STEMMER_LIBS) $(TEXTCAT_LIBS) $(NORMALIZER_LIBS) +test_fts_filter_DEPENDENCIES = $(test_deps) $(filter_deps) + +if BUILD_FTS_EXTTEXTCAT +TEST_FTS_LANGUAGE = test-fts-language +test_fts_language_SOURCES = test-fts-language.c +test_fts_language_LDADD = fts-language.lo $(test_libs) $(TEXTCAT_LIBS) +test_fts_language_DEPENDENCIES = $(test_deps) +endif + +test_fts_tokenizer_SOURCES = test-fts-tokenizer.c +test_fts_tokenizer_LDADD = fts-tokenizer.lo fts-tokenizer-generic.lo fts-tokenizer-address.lo $(test_libs) +test_fts_tokenizer_DEPENDENCIES = $(test_deps) + +check: check-am check-test +check-test: all-am + for bin in $(test_programs); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done diff -r 68c5e0db61db -r 81e5b977e5c5 src/lib-fts/fts-filter-normalizer.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-fts/fts-filter-normalizer.c Mon Apr 20 16:19:07 2015 +0300 @@ -0,0 +1,318 @@ +/* Copyright (c) 2014-2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "unichar.h" /* unicode replacement char */ +#include "fts-filter.h" +#include "fts-filter-private.h" +#include "fts-language.h" + +#ifdef HAVE_LIBFTS_NORMALIZER + +#include +#include +#include +#include +#include + +struct fts_filter_normalizer { + struct fts_filter filter; + const char *error; + pool_t pool; + UTransliterator *transliterator; +}; + +static void +icu_error(const char **error_r, const UErrorCode err, const char *func) +{ + if (error_r == NULL) + return; + + if (U_FAILURE(err)) { + *error_r = t_strdup_printf("Lib ICU function %s failed: %s\n", + func, u_errorName(err)); + } +} + +/* Thin wrapper for vprintf */ +static void ATTR_FORMAT(2, 3) +fts_filter_normalizer_error(const char **error_r, const char *format, ...) +{ + va_list args; + + if (error_r == NULL) + return; + + va_start(args, format); + *error_r = t_strdup_vprintf(format, args); + va_end(args); +} + +/* Helper to create UTF16, which libicu wants as input. Returns -1 on + error, 0 on success. + + On input, if *dst_uchars_r > 0, it indicates the number of UChar + sized units that should be allocated for the text. However, the + function will not use the number, if the text will not fit in that + amount. + + On return *dst_uchars_r will contain the number of UChar sized units + allocated for the dst. NOT the number of bytes nor the length of the + text. */ +static int make_uchar(const char *src, UChar **dst, int32_t *dst_uchars_r) +{ + UErrorCode err = U_ZERO_ERROR; + int32_t len = strlen(src); + int32_t ustr_len = 0; + int32_t ustr_len_actual = 0; + UChar *retp = NULL; + int32_t alloc_uchars = 0; + + i_assert(dst_uchars_r != NULL); + + /* Check length required for encoded dst. */ + retp = u_strFromUTF8(NULL, 0, &ustr_len, src, len, &err); + + /* When preflighting a successful call returns a buffer overflow + error. */ + if (U_BUFFER_OVERFLOW_ERROR != err && U_FAILURE(err)) { + i_panic("Failed to estimate allocation size with lib ICU" + " u_strFromUTF8(): %s",u_errorName(err)); + } + i_assert(NULL == retp); + + err = U_ZERO_ERROR; + if (*dst_uchars_r > 0 && *dst_uchars_r > ustr_len) + alloc_uchars = *dst_uchars_r; + else + alloc_uchars = ustr_len; + alloc_uchars++; /* room for null bytes(2) */ + *dst = t_malloc(alloc_uchars * sizeof(UChar)); + *dst_uchars_r = alloc_uchars; + retp = u_strFromUTF8(*dst, alloc_uchars, &ustr_len_actual, + src, len, &err); + + if (U_FAILURE(err)) + i_panic("Lib ICU u_strFromUTF8 failed: %s", u_errorName(err)); + i_assert(retp == *dst); + i_assert(ustr_len == ustr_len_actual); + return 0; +} + +static int make_utf8(const UChar *src, char **dst, const char **error_r) +{ + char *retp = NULL; + int32_t dsize = 0; + int32_t dsize_actual = 0; + int32_t sub_num = 0; + UErrorCode err = U_ZERO_ERROR; + int32_t usrc_len = u_strlen(src); /* libicu selects different codepaths + depending if srclen -1 or not */ + + retp = u_strToUTF8WithSub(NULL, 0, &dsize, src, usrc_len, + UNICODE_REPLACEMENT_CHAR, &sub_num, &err); + + /* Preflighting can cause buffer overflow to be reported */ + if (U_BUFFER_OVERFLOW_ERROR != err && U_FAILURE(err)) { From dovecot at dovecot.org Mon Apr 20 13:24:08 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 20 Apr 2015 13:24:08 +0000 Subject: dovecot-2.2: fts: Added FTS_BACKEND_FLAG_TOKENIZED_INPUT, which ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/914bdca67d1f changeset: 18415:914bdca67d1f user: Timo Sirainen date: Mon Apr 20 16:22:36 2015 +0300 description: fts: Added FTS_BACKEND_FLAG_TOKENIZED_INPUT, which is implemented via lib-fts. diffstat: src/plugins/fts/Makefile.am | 11 +- src/plugins/fts/fts-api-private.h | 9 +- src/plugins/fts/fts-build-mail.c | 148 ++++++++++++++++++++++++- src/plugins/fts/fts-plugin.c | 5 + src/plugins/fts/fts-search-args.c | 168 +++++++++++++++++++++++++++++ src/plugins/fts/fts-search-args.h | 7 + src/plugins/fts/fts-storage.c | 40 +++++- src/plugins/fts/fts-user.c | 218 ++++++++++++++++++++++++++++++++++++++ src/plugins/fts/fts-user.h | 22 +++ 9 files changed, 612 insertions(+), 16 deletions(-) diffs (truncated from 834 to 300 lines): diff -r 81e5b977e5c5 -r 914bdca67d1f src/plugins/fts/Makefile.am --- a/src/plugins/fts/Makefile.am Mon Apr 20 16:19:07 2015 +0300 +++ b/src/plugins/fts/Makefile.am Mon Apr 20 16:22:36 2015 +0300 @@ -4,6 +4,7 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-fts \ -I$(top_srcdir)/src/lib-http \ -I$(top_srcdir)/src/lib-mail \ -I$(top_srcdir)/src/lib-index \ @@ -18,6 +19,8 @@ module_LTLIBRARIES = \ lib20_fts_plugin.la +lib20_fts_plugin_la_LIBADD = ../../lib-fts/libfts.la + lib20_fts_plugin_la_SOURCES = \ fts-api.c \ fts-build-mail.c \ @@ -29,8 +32,10 @@ fts-parser-tika.c \ fts-plugin.c \ fts-search.c \ + fts-search-args.c \ fts-search-serialize.c \ - fts-storage.c + fts-storage.c \ + fts-user.c pkginc_libdir=$(pkgincludedir) pkginc_lib_HEADERS = \ @@ -44,8 +49,10 @@ doveadm-fts.h \ fts-build-mail.h \ fts-plugin.h \ + fts-search-args.h \ fts-search-serialize.h \ - fts-storage.h + fts-storage.h \ + fts-user.h pkglibexec_PROGRAMS = xml2text diff -r 81e5b977e5c5 -r 914bdca67d1f src/plugins/fts/fts-api-private.h --- a/src/plugins/fts/fts-api-private.h Mon Apr 20 16:19:07 2015 +0300 +++ b/src/plugins/fts/fts-api-private.h Mon Apr 20 16:22:36 2015 +0300 @@ -61,7 +61,12 @@ /* Send only fully indexable words rather than randomly sized blocks */ FTS_BACKEND_FLAG_BUILD_FULL_WORDS = 0x04, /* Fuzzy search works */ - FTS_BACKEND_FLAG_FUZZY_SEARCH = 0x08 + FTS_BACKEND_FLAG_FUZZY_SEARCH = 0x08, + /* Tokenize all the input. update_build_more() will be called a single + directly indexable token at a time. Searching will modify the search + args so that lookup() sees only tokens that can be directly + searched. */ + FTS_BACKEND_FLAG_TOKENIZED_INPUT = 0x10 }; struct fts_backend { @@ -71,6 +76,8 @@ struct fts_backend_vfuncs v; struct mail_namespace *ns; + struct fts_tokenizer *tokenizer; + unsigned int updating:1; }; diff -r 81e5b977e5c5 -r 914bdca67d1f src/plugins/fts/fts-build-mail.c --- a/src/plugins/fts/fts-build-mail.c Mon Apr 20 16:19:07 2015 +0300 +++ b/src/plugins/fts/fts-build-mail.c Mon Apr 20 16:22:36 2015 +0300 @@ -10,6 +10,10 @@ #include "message-decoder.h" #include "mail-storage.h" #include "fts-parser.h" +#include "fts-user.h" +#include "fts-language.h" +#include "fts-tokenizer.h" +#include "fts-filter.h" #include "fts-api-private.h" #include "fts-build-mail.h" @@ -27,7 +31,13 @@ char *content_type, *content_disposition; struct fts_parser *body_parser; - buffer_t *word_buf; + buffer_t *word_buf, *pending_input; + struct fts_user_language *cur_user_lang; +}; + +static struct fts_user_language fts_user_language_data = { + .lang = &fts_language_data, + .filter = NULL }; static int fts_build_data(struct fts_mail_build_context *ctx, @@ -62,6 +72,25 @@ i_strndup(hdr->full_value, hdr->full_value_len); } +static bool header_has_language(const char *name) +{ + /* FIXME: should email address headers be detected as different + languages? That mainly contains people's names.. */ + /*if (message_header_is_address(name)) + return TRUE;*/ + + /* Subject definitely contains language-specific data that can be + detected. Comment and Keywords headers also could contain, although + just about nobody uses those headers. + + For now we assume that other headers contain non-language specific + data that we don't want to filter in special ways. For example + it is good to be able to search for Message-IDs. */ + return strcasecmp(name, "Subject") == 0 || + strcasecmp(name, "Comments") == 0 || + strcasecmp(name, "Keywords") == 0; +} + static void fts_parse_mail_header(struct fts_mail_build_context *ctx, const struct message_block *raw_block) { @@ -116,6 +145,11 @@ key.part = block->part; key.hdr_name = hdr->name; + if (!header_has_language(key.hdr_name)) + ctx->cur_user_lang = &fts_user_language_data; + else + ctx->cur_user_lang = NULL; + if (!fts_backend_update_set_build_key(ctx->update_ctx, &key)) return; @@ -184,6 +218,7 @@ } key.body_content_type = content_type; key.body_content_disposition = ctx->content_disposition; + ctx->cur_user_lang = NULL; if (!fts_backend_update_set_build_key(ctx->update_ctx, &key)) { if (ctx->body_parser != NULL) (void)fts_parser_deinit(&ctx->body_parser); @@ -193,8 +228,104 @@ } static int -fts_build_body_block_full_words(struct fts_mail_build_context *ctx, - const unsigned char *data, size_t size, bool last) +fts_build_add_tokens_with_filter(struct fts_mail_build_context *ctx, + const unsigned char *data, size_t size) +{ + struct fts_tokenizer *tokenizer = ctx->update_ctx->backend->tokenizer; + struct fts_filter *filter = ctx->cur_user_lang->filter; + const char *token; + while ((token = fts_tokenizer_next(tokenizer, data, size)) != NULL) { + if (filter != NULL) { + token = fts_filter_filter(filter, token); + if (token == NULL) + continue; + } + if (fts_backend_update_build_more(ctx->update_ctx, + (const void *)token, + strlen(token)) < 0) + return -1; + } + return 0; +} + +static int +fts_detect_language(struct fts_mail_build_context *ctx, + const unsigned char *data, size_t size, bool last, + const struct fts_language **lang_r) +{ + struct mail_user *user = ctx->update_ctx->backend->ns->user; + struct fts_language_list *lang_list = fts_user_get_language_list(user); + const struct fts_language *lang; + + switch (fts_language_detect(lang_list, data, size, &lang)) { + case FTS_LANGUAGE_RESULT_SHORT: + /* save the input so far and try again later */ + buffer_append(ctx->pending_input, data, size); + if (last) { + /* we've run out of data. use the default language. */ + *lang_r = fts_language_list_get_first(lang_list); + return 1; + } + return 0; + case FTS_LANGUAGE_RESULT_UNKNOWN: + /* use the default language */ + *lang_r = fts_language_list_get_first(lang_list); + return 1; + case FTS_LANGUAGE_RESULT_OK: + *lang_r = lang; + return 1; + case FTS_LANGUAGE_RESULT_ERROR: + /* internal language detection library failure + (e.g. invalid config). don't index anything. */ + return -1; + default: + i_unreached(); + } +} + +static int +fts_build_tokenized(struct fts_mail_build_context *ctx, + const unsigned char *data, size_t size, bool last) +{ + struct mail_user *user = ctx->update_ctx->backend->ns->user; + const struct fts_language *lang; + const char *error; + int ret; + + if (ctx->cur_user_lang != NULL) { + /* we already have a language */ + } else if ((ret = fts_detect_language(ctx, data, size, last, &lang)) < 0) { + return -1; + } else if (ret == 0) { + /* wait for more data */ + return 0; + } else { + if (fts_user_language_get(user, lang, &ctx->cur_user_lang, + &error) < 0) { + i_error("fts-dovecot: Can't index input because of invalid language '%s' config: %s", + lang->name, error); + return -1; + } + if (ctx->pending_input->used > 0) { + if (fts_build_add_tokens_with_filter(ctx, + ctx->pending_input->data, + ctx->pending_input->used) < 0) + return -1; + buffer_set_used_size(ctx->pending_input, 0); + } + } + if (fts_build_add_tokens_with_filter(ctx, data, size) < 0) + return -1; + if (last) { + if (fts_build_add_tokens_with_filter(ctx, NULL, 0) < 0) + return -1; + } + return 0; +} + +static int +fts_build_full_words(struct fts_mail_build_context *ctx, + const unsigned char *data, size_t size, bool last) { size_t i; @@ -248,8 +379,11 @@ const unsigned char *data, size_t size, bool last) { if ((ctx->update_ctx->backend->flags & - FTS_BACKEND_FLAG_BUILD_FULL_WORDS) != 0) { - return fts_build_body_block_full_words(ctx, data, size, last); + FTS_BACKEND_FLAG_TOKENIZED_INPUT) != 0) { + return fts_build_tokenized(ctx, data, size, last); + } else if ((ctx->update_ctx->backend->flags & + FTS_BACKEND_FLAG_BUILD_FULL_WORDS) != 0) { + return fts_build_full_words(ctx, data, size, last); } else { return fts_backend_update_build_more(ctx->update_ctx, data, size); } @@ -308,6 +442,8 @@ memset(&ctx, 0, sizeof(ctx)); ctx.update_ctx = update_ctx; ctx.mail = mail; + if ((update_ctx->backend->flags & FTS_BACKEND_FLAG_TOKENIZED_INPUT) != 0) + ctx.pending_input = buffer_create_dynamic(default_pool, 128); prev_part = NULL; parser = message_parser_init(pool_datastack_create(), input, @@ -398,6 +534,8 @@ i_free(ctx.content_disposition); if (ctx.word_buf != NULL) buffer_free(&ctx.word_buf); + if (ctx.pending_input != NULL) + buffer_free(&ctx.pending_input); return ret < 0 ? -1 : 1; } diff -r 81e5b977e5c5 -r 914bdca67d1f src/plugins/fts/fts-plugin.c --- a/src/plugins/fts/fts-plugin.c Mon Apr 20 16:19:07 2015 +0300 +++ b/src/plugins/fts/fts-plugin.c Mon Apr 20 16:22:36 2015 +0300 @@ -2,8 +2,10 @@ #include "lib.h" #include "mail-storage-hooks.h" +#include "fts-filter.h" #include "fts-parser.h" #include "fts-storage.h" +#include "fts-user.h" #include "fts-plugin.h" #include @@ -11,6 +13,7 @@ const char *fts_plugin_version = DOVECOT_ABI_VERSION; From dovecot at dovecot.org Mon Apr 20 13:36:02 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 20 Apr 2015 13:36:02 +0000 Subject: dovecot-2.2: configure: Fixed linking with libexttextcat when it... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b93306938a88 changeset: 18416:b93306938a88 user: Timo Sirainen date: Mon Apr 20 16:34:31 2015 +0300 description: configure: Fixed linking with libexttextcat when it was found via pkg-config diffstat: configure.ac | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 914bdca67d1f -r b93306938a88 configure.ac --- a/configure.ac Mon Apr 20 16:22:36 2015 +0300 +++ b/configure.ac Mon Apr 20 16:34:31 2015 +0300 @@ -2767,7 +2767,7 @@ if test $want_textcat != no; then if test "$PKG_CONFIG" != "" && $PKG_CONFIG --exists libexttextcat 2>/dev/null; then PKG_CHECK_MODULES(LIBEXTTEXTCAT, libexttextcat) - have_exttextcat=yes + have_fts_exttextcat=yes else AC_CHECK_LIB(exttextcat, special_textcat_Init, [ have_fts_exttextcat=yes From dovecot at dovecot.org Mon Apr 20 14:02:44 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 20 Apr 2015 14:02:44 +0000 Subject: dovecot-2.2: lib-fts: Fixed default textcat datadir paths. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cf04173f3f69 changeset: 18417:cf04173f3f69 user: Timo Sirainen date: Mon Apr 20 17:01:12 2015 +0300 description: lib-fts: Fixed default textcat datadir paths. diffstat: configure.ac | 4 ++++ src/lib-fts/fts-language.c | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diffs (46 lines): diff -r b93306938a88 -r cf04173f3f69 configure.ac --- a/configure.ac Mon Apr 20 16:34:31 2015 +0300 +++ b/configure.ac Mon Apr 20 17:01:12 2015 +0300 @@ -2767,16 +2767,19 @@ if test $want_textcat != no; then if test "$PKG_CONFIG" != "" && $PKG_CONFIG --exists libexttextcat 2>/dev/null; then PKG_CHECK_MODULES(LIBEXTTEXTCAT, libexttextcat) + TEXTCAT_DATADIR=`$PKG_CONFIG --variable=pkgdatadir libexttextcat` have_fts_exttextcat=yes else AC_CHECK_LIB(exttextcat, special_textcat_Init, [ have_fts_exttextcat=yes AC_CHECK_HEADERS(libexttextcat/textcat.h) LIBEXTTEXTCAT_LIBS=-lexttextcat + TEXTCAT_DATADIR="/usr/share/libexttextcat" AC_SUBST(LIBEXTTEXTCAT_LIBS) ], [ AC_CHECK_LIB(textcat, special_textcat_Init, [ have_fts_textcat=yes + TEXTCAT_DATADIR="/usr/share/libtextcat" AC_CHECK_HEADERS(libtextcat/textcat.h) ]) ]) @@ -2786,6 +2789,7 @@ fi if test "$have_fts_exttextcat" != yes || test "$have_fts_textcat" != yes; then AC_DEFINE(HAVE_FTS_TEXTCAT,, Define if you want textcat support for FTS) + AC_DEFINE_UNQUOTED(TEXTCAT_DATADIR, "$TEXTCAT_DATADIR", Points to textcat pkgdatadir containing the language files) fi fi AM_CONDITIONAL(BUILD_FTS_TEXTCAT, test "$have_fts_textcat" = "yes") diff -r b93306938a88 -r cf04173f3f69 src/lib-fts/fts-language.c --- a/src/lib-fts/fts-language.c Mon Apr 20 16:34:31 2015 +0300 +++ b/src/lib-fts/fts-language.c Mon Apr 20 17:01:12 2015 +0300 @@ -191,9 +191,9 @@ return -1; config_path = list->textcat_config != NULL ? list->textcat_config : - DATADIR"/libexttextcat/fpdb.conf"; + TEXTCAT_DATADIR"/fpdb.conf"; data_dir = list->textcat_datadir != NULL ? list->textcat_datadir : - DATADIR"/libexttextcat/"; + TEXTCAT_DATADIR"/"; list->textcat_handle = special_textcat_Init(config_path, data_dir); if (list->textcat_handle == NULL) { i_error("special_textcat_Init(%s, %s) failed", From dovecot at dovecot.org Mon Apr 20 14:10:12 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 20 Apr 2015 14:10:12 +0000 Subject: dovecot-2.2: configure: Added Debian Wheezy workaround for findi... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e06740c8515a changeset: 18418:e06740c8515a user: Timo Sirainen date: Mon Apr 20 17:08:42 2015 +0300 description: configure: Added Debian Wheezy workaround for finding textcat.h diffstat: configure.ac | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r cf04173f3f69 -r e06740c8515a configure.ac --- a/configure.ac Mon Apr 20 17:01:12 2015 +0300 +++ b/configure.ac Mon Apr 20 17:08:42 2015 +0300 @@ -2769,6 +2769,8 @@ PKG_CHECK_MODULES(LIBEXTTEXTCAT, libexttextcat) TEXTCAT_DATADIR=`$PKG_CONFIG --variable=pkgdatadir libexttextcat` have_fts_exttextcat=yes + # Debian Wheezy workaround - LIBEXTTEXTCAT_CFLAGS include path is wrong: + AC_CHECK_HEADERS(libexttextcat/textcat.h) else AC_CHECK_LIB(exttextcat, special_textcat_Init, [ have_fts_exttextcat=yes From dovecot at dovecot.org Mon Apr 20 14:26:08 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 20 Apr 2015 14:26:08 +0000 Subject: dovecot-2.2: configure: Fixed building without textcat. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/56228757382d changeset: 18419:56228757382d user: Timo Sirainen date: Mon Apr 20 17:24:37 2015 +0300 description: configure: Fixed building without textcat. diffstat: configure.ac | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r e06740c8515a -r 56228757382d configure.ac --- a/configure.ac Mon Apr 20 17:08:42 2015 +0300 +++ b/configure.ac Mon Apr 20 17:24:37 2015 +0300 @@ -2789,7 +2789,7 @@ AC_ERROR([Can't build with textcat support: libtextcat or libexttextcat not found]) fi fi - if test "$have_fts_exttextcat" != yes || test "$have_fts_textcat" != yes; then + if test "$have_fts_exttextcat" = yes || test "$have_fts_textcat" = yes; then AC_DEFINE(HAVE_FTS_TEXTCAT,, Define if you want textcat support for FTS) AC_DEFINE_UNQUOTED(TEXTCAT_DATADIR, "$TEXTCAT_DATADIR", Points to textcat pkgdatadir containing the language files) fi From dovecot at dovecot.org Mon Apr 20 14:34:45 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 20 Apr 2015 14:34:45 +0000 Subject: dovecot-2.2: lib-fts: Fixed test-fts-language to use TEXTCAT_DAT... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9b777c7ede0d changeset: 18420:9b777c7ede0d user: Timo Sirainen date: Mon Apr 20 17:33:09 2015 +0300 description: lib-fts: Fixed test-fts-language to use TEXTCAT_DATADIR This may still make too many assumptions about what data exists where.. So we may need to remove this test from "make check". But for now leave it there. diffstat: src/lib-fts/Makefile.am | 1 - src/lib-fts/test-fts-language.c | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diffs (25 lines): diff -r 56228757382d -r 9b777c7ede0d src/lib-fts/Makefile.am --- a/src/lib-fts/Makefile.am Mon Apr 20 17:24:37 2015 +0300 +++ b/src/lib-fts/Makefile.am Mon Apr 20 17:33:09 2015 +0300 @@ -7,7 +7,6 @@ $(LIBFTS_NORMALIZER_CFLAGS) \ -DUDHRDIR=\""$(top_srcdir)/src/lib-fts"\" \ -DDATADIR=\"$(pkgdatadir)\" \ - -DTEST_TEXTCAT_DIR=\""$(top_srcdir)/ext/libexttextcat/langclass"\" \ -DTEST_STOPWORDS_DIR=\""$(top_srcdir)/src/lib-fts"\" stopwordsdir = $(datadir)/${PACKAGE_TARNAME}/stopwords diff -r 56228757382d -r 9b777c7ede0d src/lib-fts/test-fts-language.c --- a/src/lib-fts/test-fts-language.c Mon Apr 20 17:24:37 2015 +0300 +++ b/src/lib-fts/test-fts-language.c Mon Apr 20 17:33:09 2015 +0300 @@ -7,8 +7,8 @@ As such, they are not really unit test to be coupled with the build. */ const char *const settings[] = - {"fts_language_config", TEST_TEXTCAT_DIR"/fpdb.conf", - "fts_language_data", TEST_TEXTCAT_DIR"/LM/", NULL}; + {"fts_language_config", TEXTCAT_DATADIR"/fpdb.conf", + "fts_language_data", TEXTCAT_DATADIR"/", NULL}; /* Detect Finnish. fi--utf8 */ static void test_fts_language_detect_finnish(void) From dovecot at dovecot.org Mon Apr 20 14:47:07 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 20 Apr 2015 14:47:07 +0000 Subject: dovecot-2.2: lib-fts: Added missing stopwords_fi.txt Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/0ad7f8da2471 changeset: 18421:0ad7f8da2471 user: Timo Sirainen date: Mon Apr 20 17:40:12 2015 +0300 description: lib-fts: Added missing stopwords_fi.txt diffstat: src/lib-fts/Makefile.am | 5 +- src/lib-fts/stopwords_fi.txt | 89 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 1 deletions(-) diffs (108 lines): diff -r 9b777c7ede0d -r 0ad7f8da2471 src/lib-fts/Makefile.am --- a/src/lib-fts/Makefile.am Mon Apr 20 17:33:09 2015 +0300 +++ b/src/lib-fts/Makefile.am Mon Apr 20 17:40:12 2015 +0300 @@ -10,7 +10,10 @@ -DTEST_STOPWORDS_DIR=\""$(top_srcdir)/src/lib-fts"\" stopwordsdir = $(datadir)/${PACKAGE_TARNAME}/stopwords -dist_stopwords_DATA = stopwords_en.txt stopwords_fr.txt +dist_stopwords_DATA = \ + stopwords_en.txt \ + stopwords_fi.txt \ + stopwords_fr.txt BUILT_SOURCES = word-boundary-data.c word-break-data.c diff -r 9b777c7ede0d -r 0ad7f8da2471 src/lib-fts/stopwords_fi.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-fts/stopwords_fi.txt Mon Apr 20 17:40:12 2015 +0300 @@ -0,0 +1,89 @@ + +| forms of BE + +olla +olen +olet +on +olemme +olette +ovat +ole | negative form + +oli +olisi +olisit +olisin +olisimme +olisitte +olisivat +olit +olin +olimme +olitte +olivat +ollut +olleet + +en | negation +et +ei +emme +ette +eiv?t + +|Nom Gen Acc Part Iness Elat Illat Adess Ablat Allat Ess Trans +min? minun minut minua minussa minusta minuun minulla minulta minulle | I +sin? sinun sinut sinua sinussa sinusta sinuun sinulla sinulta sinulle | you +h?n h?nen h?net h?nt? h?ness? h?nest? h?neen h?nell? h?nelt? h?nelle | he she +me meid?n meid?t meit? meiss? meist? meihin meill? meilt? meille | we +te teid?n teid?t teit? teiss? teist? teihin teill? teilt? teille | you +he heid?n heid?t heit? heiss? heist? heihin heill? heilt? heille | they + +t?m? t?m?n t?t? t?ss? t?st? t?h?n t?ll? t?lt? t?lle t?n? t?ksi | this +tuo tuon tuota tuossa tuosta tuohon tuolla tuolta tuolle tuona tuoksi | that +se sen sit? siin? siit? siihen sill? silt? sille sin? siksi | it +n?m? n?iden n?it? n?iss? n?ist? n?ihin n?ill? n?ilt? n?ille n?in? n?iksi | these +nuo noiden noita noissa noista noihin noilla noilta noille noina noiksi | those +ne niiden niit? niiss? niist? niihin niill? niilt? niille niin? niiksi | they + +kuka kenen kenet ket? keness? kenest? keneen kenell? kenelt? kenelle kenen? keneksi| who +ketk? keiden ketk? keit? keiss? keist? keihin keill? keilt? keille kein? keiksi | (pl) +mik? mink? mink? mit? miss? mist? mihin mill? milt? mille min? miksi | which what +mitk? | (pl) + +joka jonka jota jossa josta johon jolla jolta jolle jona joksi | who which +jotka joiden joita joissa joista joihin joilla joilta joille joina joiksi | (pl) + +| conjunctions + +ett? | that +ja | and +jos | if +koska | because +kuin | than +mutta | but +niin | so +sek? | and +sill? | for +tai | or +vaan | but +vai | or +vaikka | although + + +| prepositions + +kanssa | with +mukaan | according to +noin | about +poikki | across +yli | over, across + +| other + +kun | when +niin | so +nyt | now +itse | self + From dovecot at dovecot.org Mon Apr 20 14:47:08 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 20 Apr 2015 14:47:08 +0000 Subject: dovecot-2.2: lib-fts: Fixed some wrong macro name checks. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/197c703f32da changeset: 18422:197c703f32da user: Timo Sirainen date: Mon Apr 20 17:43:32 2015 +0300 description: lib-fts: Fixed some wrong macro name checks. Forgot to change these ones while renaming them.. diffstat: src/lib-fts/fts-filter-normalizer.c | 2 +- src/lib-fts/fts-filter-stemmer-snowball.c | 2 +- src/lib-fts/test-fts-filter.c | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diffs (45 lines): diff -r 0ad7f8da2471 -r 197c703f32da src/lib-fts/fts-filter-normalizer.c --- a/src/lib-fts/fts-filter-normalizer.c Mon Apr 20 17:40:12 2015 +0300 +++ b/src/lib-fts/fts-filter-normalizer.c Mon Apr 20 17:43:32 2015 +0300 @@ -6,7 +6,7 @@ #include "fts-filter-private.h" #include "fts-language.h" -#ifdef HAVE_LIBFTS_NORMALIZER +#ifdef HAVE_FTS_NORMALIZER #include #include diff -r 0ad7f8da2471 -r 197c703f32da src/lib-fts/fts-filter-stemmer-snowball.c --- a/src/lib-fts/fts-filter-stemmer-snowball.c Mon Apr 20 17:40:12 2015 +0300 +++ b/src/lib-fts/fts-filter-stemmer-snowball.c Mon Apr 20 17:43:32 2015 +0300 @@ -5,7 +5,7 @@ #include "fts-filter.h" #include "fts-filter-private.h" -#ifdef HAVE_LIBFTS_STEMMER +#ifdef HAVE_FTS_STEMMER #include diff -r 0ad7f8da2471 -r 197c703f32da src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Mon Apr 20 17:40:12 2015 +0300 +++ b/src/lib-fts/test-fts-filter.c Mon Apr 20 17:43:32 2015 +0300 @@ -294,7 +294,7 @@ test_end(); } -#ifdef HAVE_LIBFTS_NORMALIZER +#ifdef HAVE_FTS_NORMALIZER static void test_fts_filter_normalizer_swedish_short(void) { const struct fts_filter *filter_class; @@ -533,7 +533,7 @@ test_fts_filter_stemmer_snowball_stem_english, test_fts_filter_stemmer_snowball_stem_french, test_fts_filter_stopwords_stemmer_eng, -#ifdef HAVE_LIBFTS_NORMALIZER +#ifdef HAVE_FTS_NORMALIZER test_fts_filter_normalizer_swedish_short, test_fts_filter_normalizer_swedish_short_default_id, test_fts_filter_normalizer_french, From dovecot at dovecot.org Mon Apr 20 14:47:18 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 20 Apr 2015 14:47:18 +0000 Subject: dovecot-2.2: lib-fts: Don't crash in test-fts-filter if libstemm... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/eff8774c6473 changeset: 18423:eff8774c6473 user: Timo Sirainen date: Mon Apr 20 17:44:21 2015 +0300 description: lib-fts: Don't crash in test-fts-filter if libstemmer support isn't built in diffstat: src/lib-fts/test-fts-filter.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (31 lines): diff -r 197c703f32da -r eff8774c6473 src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Mon Apr 20 17:43:32 2015 +0300 +++ b/src/lib-fts/test-fts-filter.c Mon Apr 20 17:44:21 2015 +0300 @@ -169,6 +169,7 @@ } +#ifdef HAVE_FTS_STEMMER static void test_fts_filter_stemmer_snowball_stem_english(void) { int ret; @@ -293,6 +294,7 @@ test_assert(filter == NULL); test_end(); } +#endif #ifdef HAVE_FTS_NORMALIZER static void test_fts_filter_normalizer_swedish_short(void) @@ -530,9 +532,11 @@ test_fts_filter_stopwords_fin, test_fts_filter_stopwords_fra, test_fts_filter_stopwords_fail_create, +#ifdef HAVE_FTS_STEMMER test_fts_filter_stemmer_snowball_stem_english, test_fts_filter_stemmer_snowball_stem_french, test_fts_filter_stopwords_stemmer_eng, +#endif #ifdef HAVE_FTS_NORMALIZER test_fts_filter_normalizer_swedish_short, test_fts_filter_normalizer_swedish_short_default_id, From dovecot at dovecot.org Mon Apr 20 14:47:23 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 20 Apr 2015 14:47:23 +0000 Subject: dovecot-2.2: lib-fts: Fixed using FTS_NORMALIZER_CFLAGS/LIBS. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/aa0424c40ced changeset: 18424:aa0424c40ced user: Timo Sirainen date: Mon Apr 20 17:45:32 2015 +0300 description: lib-fts: Fixed using FTS_NORMALIZER_CFLAGS/LIBS. diffstat: src/lib-fts/Makefile.am | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diffs (21 lines): diff -r eff8774c6473 -r aa0424c40ced src/lib-fts/Makefile.am --- a/src/lib-fts/Makefile.am Mon Apr 20 17:44:21 2015 +0300 +++ b/src/lib-fts/Makefile.am Mon Apr 20 17:45:32 2015 +0300 @@ -4,7 +4,7 @@ -I$(top_srcdir)/src/lib \ -I$(top_srcdir)/src/lib-test \ $(LIBEXTTEXTCAT_CFLAGS) \ - $(LIBFTS_NORMALIZER_CFLAGS) \ + $(FTS_NORMALIZER_CFLAGS) \ -DUDHRDIR=\""$(top_srcdir)/src/lib-fts"\" \ -DDATADIR=\"$(pkgdatadir)\" \ -DTEST_STOPWORDS_DIR=\""$(top_srcdir)/src/lib-fts"\" @@ -48,7 +48,7 @@ endif if BUILD_FTS_NORMALIZER -NORMALIZER_LIBS = $(LIBFTS_NORMALIZER_LIBS) +NORMALIZER_LIBS = $(FTS_NORMALIZER_LIBS) endif libfts_la_LIBADD = \ From dovecot at dovecot.org Mon Apr 20 15:04:01 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 20 Apr 2015 15:04:01 +0000 Subject: dovecot-2.2: lib-fts: Updated Finnish/French stopword lists from... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5b7e0fbd2565 changeset: 18425:5b7e0fbd2565 user: Timo Sirainen date: Mon Apr 20 18:02:31 2015 +0300 description: lib-fts: Updated Finnish/French stopword lists from Lucene code. Most importantly they added the licensing info to the files. diffstat: src/lib-fts/stopwords_fi.txt | 12 +++++++++--- src/lib-fts/stopwords_fr.txt | 10 ++++++++-- 2 files changed, 17 insertions(+), 5 deletions(-) diffs (50 lines): diff -r aa0424c40ced -r 5b7e0fbd2565 src/lib-fts/stopwords_fi.txt --- a/src/lib-fts/stopwords_fi.txt Mon Apr 20 17:45:32 2015 +0300 +++ b/src/lib-fts/stopwords_fi.txt Mon Apr 20 18:02:31 2015 +0300 @@ -1,4 +1,10 @@ - + | From svn.tartarus.org/snowball/trunk/website/algorithms/finnish/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | forms of BE olla @@ -40,8 +46,8 @@ te teid?n teid?t teit? teiss? teist? teihin teill? teilt? teille | you he heid?n heid?t heit? heiss? heist? heihin heill? heilt? heille | they -t?m? t?m?n t?t? t?ss? t?st? t?h?n t?ll? t?lt? t?lle t?n? t?ksi | this -tuo tuon tuota tuossa tuosta tuohon tuolla tuolta tuolle tuona tuoksi | that +t?m? t?m?n t?t? t?ss? t?st? t?h?n tall? t?lt? t?lle t?n? t?ksi | this +tuo tuon tuot? tuossa tuosta tuohon tuolla tuolta tuolle tuona tuoksi | that se sen sit? siin? siit? siihen sill? silt? sille sin? siksi | it n?m? n?iden n?it? n?iss? n?ist? n?ihin n?ill? n?ilt? n?ille n?in? n?iksi | these nuo noiden noita noissa noista noihin noilla noilta noille noina noiksi | those diff -r aa0424c40ced -r 5b7e0fbd2565 src/lib-fts/stopwords_fr.txt --- a/src/lib-fts/stopwords_fr.txt Mon Apr 20 17:45:32 2015 +0300 +++ b/src/lib-fts/stopwords_fr.txt Mon Apr 20 18:02:31 2015 +0300 @@ -1,3 +1,9 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/french/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. | A French stop word list. Comments begin with vertical bar. Each stop | word is at the start of a line. @@ -161,8 +167,8 @@ | Later additions (from Jean-Christophe Deschamps) ceci | this -cela | that (added 11 Apr 2012. Omission reported by Adrien Grand) -cel? | that (incorrect, though common) +cela | that +cel? | that cet | this cette | this ici | here From dovecot at dovecot.org Mon Apr 20 15:10:33 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 20 Apr 2015 15:10:33 +0000 Subject: dovecot-2.2: lib-fts requires libexttextcat actually - don't eve... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/50ef619ce58a changeset: 18426:50ef619ce58a user: Timo Sirainen date: Mon Apr 20 18:08:56 2015 +0300 description: lib-fts requires libexttextcat actually - don't even try to use textcat for it. I'm also not sure yet if some libexttextcats are too old for it. diffstat: configure.ac | 2 ++ src/lib-fts/fts-language.c | 15 +++++---------- 2 files changed, 7 insertions(+), 10 deletions(-) diffs (73 lines): diff -r 5b7e0fbd2565 -r 50ef619ce58a configure.ac --- a/configure.ac Mon Apr 20 18:02:31 2015 +0300 +++ b/configure.ac Mon Apr 20 18:08:56 2015 +0300 @@ -2768,6 +2768,7 @@ if test "$PKG_CONFIG" != "" && $PKG_CONFIG --exists libexttextcat 2>/dev/null; then PKG_CHECK_MODULES(LIBEXTTEXTCAT, libexttextcat) TEXTCAT_DATADIR=`$PKG_CONFIG --variable=pkgdatadir libexttextcat` + AC_DEFINE(HAVE_FTS_EXTTEXTCAT,, Define if you want exttextcat support for FTS) have_fts_exttextcat=yes # Debian Wheezy workaround - LIBEXTTEXTCAT_CFLAGS include path is wrong: AC_CHECK_HEADERS(libexttextcat/textcat.h) @@ -2776,6 +2777,7 @@ have_fts_exttextcat=yes AC_CHECK_HEADERS(libexttextcat/textcat.h) LIBEXTTEXTCAT_LIBS=-lexttextcat + AC_DEFINE(HAVE_FTS_EXTTEXTCAT,, Define if you want exttextcat support for FTS) TEXTCAT_DATADIR="/usr/share/libexttextcat" AC_SUBST(LIBEXTTEXTCAT_LIBS) ], [ diff -r 5b7e0fbd2565 -r 50ef619ce58a src/lib-fts/fts-language.c --- a/src/lib-fts/fts-language.c Mon Apr 20 18:02:31 2015 +0300 +++ b/src/lib-fts/fts-language.c Mon Apr 20 18:08:56 2015 +0300 @@ -8,13 +8,8 @@ #ifdef HAVE_LIBEXTTEXTCAT_TEXTCAT_H # include -# define HAVE_TEXTCAT -#elif defined (HAVE_LIBTEXTCAT_TEXTCAT_H) -# include -# define HAVE_TEXTCAT -#elif defined (HAVE_FTS_TEXTCAT) +#elif defined (HAVE_FTS_EXTTEXTCAT) # include -# define HAVE_TEXTCAT #endif #ifndef TEXTCAT_RESULT_UNKNOWN /* old textcat.h has typos */ @@ -99,7 +94,7 @@ struct fts_language_list *lp = *list; *list = NULL; -#ifdef HAVE_TEXTCAT +#ifdef HAVE_FTS_EXTTEXTCAT if (lp->textcat_handle != NULL) textcat_Done(lp->textcat_handle); #endif @@ -160,7 +155,7 @@ return *langp; } -#ifdef HAVE_TEXTCAT +#ifdef HAVE_FTS_EXTTEXTCAT static bool fts_language_match_lists(struct fts_language_list *list, candidate_t *candp, int candp_len, const struct fts_language **lang_r) @@ -178,7 +173,7 @@ } #endif -#ifdef HAVE_TEXTCAT +#ifdef HAVE_FTS_EXTTEXTCAT static int fts_language_textcat_init(struct fts_language_list *list) { const char *config_path; @@ -213,7 +208,7 @@ size_t size ATTR_UNUSED, const struct fts_language **lang_r ATTR_UNUSED) { -#ifdef HAVE_TEXTCAT +#ifdef HAVE_FTS_EXTTEXTCAT candidate_t *candp; /* textcat candidate result array pointer */ int cnt; bool match = FALSE; From dovecot at dovecot.org Mon Apr 20 16:59:13 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Mon, 20 Apr 2015 16:59:13 +0000 Subject: dovecot-2.2: lib-fts: Added PropList.txt to EXTRA_DIST Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a2f45d31d1c5 changeset: 18427:a2f45d31d1c5 user: Timo Sirainen date: Mon Apr 20 19:57:37 2015 +0300 description: lib-fts: Added PropList.txt to EXTRA_DIST diffstat: src/lib-fts/Makefile.am | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 50ef619ce58a -r a2f45d31d1c5 src/lib-fts/Makefile.am --- a/src/lib-fts/Makefile.am Mon Apr 20 18:08:56 2015 +0300 +++ b/src/lib-fts/Makefile.am Mon Apr 20 19:57:37 2015 +0300 @@ -18,6 +18,7 @@ BUILT_SOURCES = word-boundary-data.c word-break-data.c EXTRA_DIST = \ + PropList.txt \ WordBreakProperty.txt \ word-boundary-data.sh \ word-boundary-data.c \ From dovecot at dovecot.org Tue Apr 21 07:01:03 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 21 Apr 2015 07:01:03 +0000 Subject: dovecot-2.2: lib-fts: Added udhr_fra.txt to EXTRA_DIST Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9196358b6779 changeset: 18428:9196358b6779 user: Timo Sirainen date: Tue Apr 21 09:59:32 2015 +0300 description: lib-fts: Added udhr_fra.txt to EXTRA_DIST diffstat: src/lib-fts/Makefile.am | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r a2f45d31d1c5 -r 9196358b6779 src/lib-fts/Makefile.am --- a/src/lib-fts/Makefile.am Mon Apr 20 19:57:37 2015 +0300 +++ b/src/lib-fts/Makefile.am Tue Apr 21 09:59:32 2015 +0300 @@ -18,6 +18,7 @@ BUILT_SOURCES = word-boundary-data.c word-break-data.c EXTRA_DIST = \ + udhr_fra.txt \ PropList.txt \ WordBreakProperty.txt \ word-boundary-data.sh \ From dovecot at dovecot.org Tue Apr 21 08:15:22 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 21 Apr 2015 08:15:22 +0000 Subject: dovecot-2.2: stats: num_cmds was always 0 in global stats Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4f95a2b5a154 changeset: 18429:4f95a2b5a154 user: Timo Sirainen date: Tue Apr 21 11:13:51 2015 +0300 description: stats: num_cmds was always 0 in global stats diffstat: src/stats/mail-command.c | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diffs (11 lines): diff -r 9196358b6779 -r 4f95a2b5a154 src/stats/mail-command.c --- a/src/stats/mail-command.c Tue Apr 21 09:59:32 2015 +0300 +++ b/src/stats/mail-command.c Tue Apr 21 11:13:51 2015 +0300 @@ -155,6 +155,7 @@ session->user->domain->num_cmds++; if (session->ip != NULL) session->ip->num_cmds++; + mail_global_stats.num_cmds++; args += 5; } else { if (cmd == NULL) { From dovecot at dovecot.org Tue Apr 21 10:20:57 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 21 Apr 2015 10:20:57 +0000 Subject: dovecot-2.2: lda, lmtp: Added delivery_time and session_time var... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3ea5becbb56c changeset: 18430:3ea5becbb56c user: Timo Sirainen date: Tue Apr 21 13:19:24 2015 +0300 description: lda, lmtp: Added delivery_time and session_time variables to deliver_log_format session_time is meaningful only with LMTP. The delivery_time is separate for each mail delivery. The results are in milliseconds now. If needed we could add a new %modifier that converts it into secs.millisecs. diffstat: src/lib-lda/mail-deliver.c | 44 ++++++++++++++++++++++++++++++++++++++------ src/lib-lda/mail-deliver.h | 6 ++++++ src/lmtp/client.h | 2 ++ src/lmtp/commands.c | 9 +++++++++ 4 files changed, 55 insertions(+), 6 deletions(-) diffs (167 lines): diff -r 4f95a2b5a154 -r 3ea5becbb56c src/lib-lda/mail-deliver.c --- a/src/lib-lda/mail-deliver.c Tue Apr 21 11:13:51 2015 +0300 +++ b/src/lib-lda/mail-deliver.c Tue Apr 21 13:19:24 2015 +0300 @@ -5,6 +5,7 @@ #include "array.h" #include "str.h" #include "str-sanitize.h" +#include "time-util.h" #include "unichar.h" #include "var-expand.h" #include "message-address.h" @@ -38,8 +39,10 @@ NULL : t_strconcat(addr->mailbox, "@", addr->domain, NULL); } -const struct var_expand_table * -mail_deliver_get_log_var_expand_table(struct mail *mail, const char *message) +static const struct var_expand_table * +mail_deliver_get_log_var_expand_table_full(struct mail_deliver_context *ctx, + struct mail *mail, + const char *message) { static struct var_expand_table static_tab[] = { { '$', NULL, NULL }, @@ -49,6 +52,8 @@ { 'e', NULL, "from_envelope" }, { 'p', NULL, "size" }, { 'w', NULL, "vsize" }, + { '\0', NULL, "delivery_time" }, + { '\0', NULL, "session_time" }, { '\0', NULL, NULL } }; struct var_expand_table *tab; @@ -75,20 +80,44 @@ tab[5].value = dec2str(size); if (mail_get_virtual_size(mail, &size) == 0) tab[6].value = dec2str(size); + if (ctx != NULL) { + int delivery_time_msecs; + + io_loop_time_refresh(); + delivery_time_msecs = + timeval_diff_msecs(&ioloop_timeval, + &ctx->delivery_time_started); + tab[7].value = dec2str(delivery_time_msecs); + tab[8].value = dec2str(ctx->session_time_msecs); + } return tab; } +const struct var_expand_table * +mail_deliver_get_log_var_expand_table(struct mail *mail, const char *message) +{ + return mail_deliver_get_log_var_expand_table_full(NULL, mail, message); +} + +const struct var_expand_table * +mail_deliver_ctx_get_log_var_expand_table(struct mail_deliver_context *ctx, + const char *message) +{ + struct mail *mail; + + mail = ctx->dest_mail != NULL ? ctx->dest_mail : ctx->src_mail; + return mail_deliver_get_log_var_expand_table_full(ctx, mail, message); +} + static void mail_deliver_log_cache_var_expand_table(struct mail_deliver_context *ctx) { const struct var_expand_table *src; struct var_expand_table *dest; - struct mail *mail; unsigned int i, len; - mail = ctx->dest_mail != NULL ? ctx->dest_mail : ctx->src_mail; - src = mail_deliver_get_log_var_expand_table(mail, ""); - for (len = 0; src[len].key != '\0'; len++) ; + src = mail_deliver_ctx_get_log_var_expand_table(ctx, ""); + for (len = 0; src[len].key != '\0' || src[len].long_key != NULL; len++) ; dest = p_new(ctx->pool, struct var_expand_table, len + 1); for (i = 0; i < len; i++) { @@ -392,6 +421,9 @@ { int ret; + io_loop_time_refresh(); + ctx->delivery_time_started = ioloop_timeval; + *storage_r = NULL; if (deliver_mail == NULL) ret = -1; diff -r 4f95a2b5a154 -r 3ea5becbb56c src/lib-lda/mail-deliver.h --- a/src/lib-lda/mail-deliver.h Tue Apr 21 11:13:51 2015 +0300 +++ b/src/lib-lda/mail-deliver.h Tue Apr 21 13:19:24 2015 +0300 @@ -26,6 +26,9 @@ struct mail_deliver_session *session; unsigned int timeout_secs; + unsigned int session_time_msecs; + struct timeval delivery_time_started; + struct duplicate_context *dup_ctx; /* Session ID, used as log line prefix if non-NULL. */ @@ -78,6 +81,9 @@ extern deliver_mail_func_t *deliver_mail; const struct var_expand_table * +mail_deliver_ctx_get_log_var_expand_table(struct mail_deliver_context *ctx, + const char *message); +const struct var_expand_table * mail_deliver_get_log_var_expand_table(struct mail *mail, const char *message); void mail_deliver_log(struct mail_deliver_context *ctx, const char *fmt, ...) ATTR_FORMAT(2, 3); diff -r 4f95a2b5a154 -r 3ea5becbb56c src/lmtp/client.h --- a/src/lmtp/client.h Tue Apr 21 11:13:51 2015 +0300 +++ b/src/lmtp/client.h Tue Apr 21 13:19:24 2015 +0300 @@ -37,6 +37,8 @@ struct ostream *mail_data_output; const char *added_headers; + struct timeval mail_from_timeval, data_end_timeval; + struct mail *raw_mail; struct mail_user *dest_user; diff -r 4f95a2b5a154 -r 3ea5becbb56c src/lmtp/commands.c --- a/src/lmtp/commands.c Tue Apr 21 11:13:51 2015 +0300 +++ b/src/lmtp/commands.c Tue Apr 21 13:19:24 2015 +0300 @@ -12,6 +12,7 @@ #include "istream-dot.h" #include "safe-mkstemp.h" #include "hex-dec.h" +#include "time-util.h" #include "var-expand.h" #include "restrict-access.h" #include "settings-parser.h" @@ -212,6 +213,8 @@ p_array_init(&client->state.rcpt_to, client->state_pool, 64); client_send_line(client, "250 2.1.0 OK"); client_state_set(client, "MAIL FROM", client->state.mail_from); + + client->state.mail_from_timeval = ioloop_timeval; return 0; } @@ -809,6 +812,10 @@ dctx.src_mail = src_mail; dctx.src_envelope_sender = client->state.mail_from; dctx.dest_user = client->state.dest_user; + dctx.session_time_msecs = + timeval_diff_msecs(&client->state.data_end_timeval, + &client->state.mail_from_timeval); + if (orcpt_get_valid_rfc822(rcpt->params.dsn_orcpt, &dctx.dest_addr)) { /* used ORCPT */ } else if (*dctx.set->lda_original_recipient_header != '\0') { @@ -1093,6 +1100,8 @@ io_remove(&client->io); i_stream_destroy(&client->dot_input); + client->state.data_end_timeval = ioloop_timeval; + input = client_get_input(client); if (array_count(&client->state.rcpt_to) != 0) client_input_data_write_local(client, input); From dovecot at dovecot.org Tue Apr 21 11:05:49 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 21 Apr 2015 11:05:49 +0000 Subject: dovecot-2.2: imap: Small code cleanup. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7a09ddcd62b4 changeset: 18431:7a09ddcd62b4 user: Timo Sirainen date: Tue Apr 21 14:03:52 2015 +0300 description: imap: Small code cleanup. Removed all the checks of CLIENT_COMMAND_STATE_DONE to command_exec() itself. diffstat: src/imap/cmd-append.c | 2 +- src/imap/imap-client.c | 7 +++---- src/imap/imap-commands.c | 8 +++++--- 3 files changed, 9 insertions(+), 8 deletions(-) diffs (66 lines): diff -r 3ea5becbb56c -r 7a09ddcd62b4 src/imap/cmd-append.c --- a/src/imap/cmd-append.c Tue Apr 21 13:19:24 2015 +0300 +++ b/src/imap/cmd-append.c Tue Apr 21 14:03:52 2015 +0300 @@ -114,7 +114,7 @@ o_stream_cork(client->output); finished = command_exec(cmd); - if (!finished && cmd->state != CLIENT_COMMAND_STATE_DONE) + if (!finished) (void)client_handle_unfinished_cmd(cmd); else client_command_free(&cmd); diff -r 3ea5becbb56c -r 7a09ddcd62b4 src/imap/imap-client.c --- a/src/imap/imap-client.c Tue Apr 21 13:19:24 2015 +0300 +++ b/src/imap/imap-client.c Tue Apr 21 14:03:52 2015 +0300 @@ -187,7 +187,7 @@ cmd_ret = !cmd->cancel || cmd->func == NULL ? TRUE : command_exec(cmd); - if (!cmd_ret && cmd->state != CLIENT_COMMAND_STATE_DONE) { + if (!cmd_ret) { if (cmd->client->output->closed) i_panic("command didn't cancel itself: %s", cmd->name); } else { @@ -779,8 +779,7 @@ if (cmd->func != NULL) { /* command is being executed - continue it */ - if (command_exec(cmd) || - cmd->state == CLIENT_COMMAND_STATE_DONE) { + if (command_exec(cmd)) { /* command execution was finished */ client_command_free(&cmd); client_add_missing_io(client); @@ -959,7 +958,7 @@ bool finished; /* continue processing command */ - finished = command_exec(cmd) || cmd->state == CLIENT_COMMAND_STATE_DONE; + finished = command_exec(cmd); if (!finished) (void)client_handle_unfinished_cmd(cmd); diff -r 3ea5becbb56c -r 7a09ddcd62b4 src/imap/imap-commands.c --- a/src/imap/imap-commands.c Tue Apr 21 13:19:24 2015 +0300 +++ b/src/imap/imap-commands.c Tue Apr 21 14:03:52 2015 +0300 @@ -151,14 +151,16 @@ bool command_exec(struct client_command_context *cmd) { const struct command_hook *hook; - bool ret; + bool finished; array_foreach(&command_hooks, hook) hook->pre(cmd); - ret = cmd->func(cmd); + finished = cmd->func(cmd); array_foreach(&command_hooks, hook) hook->post(cmd); - return ret; + if (cmd->state == CLIENT_COMMAND_STATE_DONE) + finished = TRUE; + return finished; } static int command_cmp(const struct command *c1, const struct command *c2) From dovecot at dovecot.org Tue Apr 21 11:08:13 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 21 Apr 2015 11:08:13 +0000 Subject: dovecot-2.2: imap: Include in tagged reply how much running time... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/abf2727d469a changeset: 18432:abf2727d469a user: Timo Sirainen date: Tue Apr 21 14:06:22 2015 +0300 description: imap: Include in tagged reply how much running time each IMAP command takes. The running time doesn't include time spent waiting for the client on ioloop. diffstat: src/imap/imap-client.c | 24 ++++++++++++++++++++---- src/imap/imap-client.h | 8 ++++++++ src/imap/imap-commands.c | 13 +++++++++++++ 3 files changed, 41 insertions(+), 4 deletions(-) diffs (105 lines): diff -r 7a09ddcd62b4 -r abf2727d469a src/imap/imap-client.c --- a/src/imap/imap-client.c Tue Apr 21 14:03:52 2015 +0300 +++ b/src/imap/imap-client.c Tue Apr 21 14:06:22 2015 +0300 @@ -379,6 +379,7 @@ { struct client *client = cmd->client; const char *tag = cmd->tag; + int time_msecs; if (client->output->closed || cmd->cancel) return; @@ -389,10 +390,24 @@ if (tag == NULL || *tag == '\0') tag = "*"; - o_stream_nsend_str(client->output, tag); - o_stream_nsend(client->output, " ", 1); - o_stream_nsend_str(client->output, data); - o_stream_nsend(client->output, "\r\n", 2); + T_BEGIN { + string_t *str = t_str_new(256); + str_printfa(str, "%s %s", tag, data); + if (cmd->start_time.tv_sec != 0) { + if (str_data(str)[str_len(str)-1] == '.') + str_truncate(str, str_len(str)-1); + io_loop_time_refresh(); + time_msecs = timeval_diff_msecs(&ioloop_timeval, + &cmd->start_time); + time_msecs -= cmd->usecs_in_ioloop/1000; + if (time_msecs >= 0) { + str_printfa(str, " (%d.%03d secs).", + time_msecs/1000, time_msecs%1000); + } + } + str_append(str, "\r\n"); + o_stream_nsend(client->output, str_data(str), str_len(str)); + } T_END; client->last_output = ioloop_time; } @@ -459,6 +474,7 @@ str = t_str_new(256); imap_write_args(str, *args_r); cmd->args = p_strdup(cmd->pool, str_c(str)); + cmd->start_time = ioloop_timeval; cmd->client->input_lock = NULL; return TRUE; diff -r 7a09ddcd62b4 -r abf2727d469a src/imap/imap-client.h --- a/src/imap/imap-client.h Tue Apr 21 14:03:52 2015 +0300 +++ b/src/imap/imap-client.h Tue Apr 21 14:06:22 2015 +0300 @@ -74,6 +74,14 @@ struct imap_parser *parser; enum client_command_state state; + /* time when command handling was started - typically this is after + reading all the parameters. */ + struct timeval start_time; + /* time when an unfinished command handling entered back to ioloop. + used for calculating usecs_in_ioloop */ + struct timeval last_ioloop_time; + /* how much time was spent waiting for the client in ioloop */ + uint64_t usecs_in_ioloop; struct client_sync_context *sync; diff -r 7a09ddcd62b4 -r abf2727d469a src/imap/imap-commands.c --- a/src/imap/imap-commands.c Tue Apr 21 14:03:52 2015 +0300 +++ b/src/imap/imap-commands.c Tue Apr 21 14:06:22 2015 +0300 @@ -3,6 +3,8 @@ #include "imap-common.h" #include "array.h" #include "buffer.h" +#include "ioloop.h" +#include "time-util.h" #include "imap-commands.h" #include @@ -151,8 +153,15 @@ bool command_exec(struct client_command_context *cmd) { const struct command_hook *hook; + long long diff; bool finished; + if (cmd->last_ioloop_time.tv_sec != 0) { + diff = timeval_diff_usecs(&ioloop_timeval, &cmd->last_ioloop_time); + if (diff > 0) + cmd->usecs_in_ioloop += diff; + } + array_foreach(&command_hooks, hook) hook->pre(cmd); finished = cmd->func(cmd); @@ -160,6 +169,10 @@ hook->post(cmd); if (cmd->state == CLIENT_COMMAND_STATE_DONE) finished = TRUE; + if (!finished) { + io_loop_time_refresh(); + cmd->last_ioloop_time = ioloop_timeval; + } return finished; } From dovecot at dovecot.org Tue Apr 21 11:08:19 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 21 Apr 2015 11:08:19 +0000 Subject: dovecot-2.2: imap: Removed separate time counting from SELECT an... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b651fba04408 changeset: 18433:b651fba04408 user: Timo Sirainen date: Tue Apr 21 14:06:41 2015 +0300 description: imap: Removed separate time counting from SELECT and SEARCH commands. diffstat: src/imap/cmd-select.c | 10 ++-------- src/imap/imap-search.c | 13 ++----------- src/imap/imap-search.h | 1 - 3 files changed, 4 insertions(+), 20 deletions(-) diffs (84 lines): diff -r abf2727d469a -r b651fba04408 src/imap/cmd-select.c --- a/src/imap/cmd-select.c Tue Apr 21 14:06:22 2015 +0300 +++ b/src/imap/cmd-select.c Tue Apr 21 14:06:41 2015 +0300 @@ -201,8 +201,6 @@ static void cmd_select_finish(struct imap_select_context *ctx, int ret) { const char *resp_code; - struct timeval end_time; - int time_msecs; if (ret < 0) { if (ctx->box != NULL) @@ -211,13 +209,9 @@ } else { resp_code = mailbox_is_readonly(ctx->box) ? "READ-ONLY" : "READ-WRITE"; - if (gettimeofday(&end_time, NULL) < 0) - memset(&end_time, 0, sizeof(end_time)); - time_msecs = timeval_diff_msecs(&end_time, &ctx->start_time); client_send_tagline(ctx->cmd, t_strdup_printf( - "OK [%s] %s completed (%d.%03d secs).", resp_code, - ctx->cmd->client->mailbox_examined ? "Examine" : "Select", - time_msecs/1000, time_msecs%1000)); + "OK [%s] %s completed", resp_code, + ctx->cmd->client->mailbox_examined ? "Examine" : "Select")); } select_context_free(ctx); } diff -r abf2727d469a -r b651fba04408 src/imap/imap-search.c --- a/src/imap/imap-search.c Tue Apr 21 14:06:22 2015 +0300 +++ b/src/imap/imap-search.c Tue Apr 21 14:06:41 2015 +0300 @@ -415,12 +415,10 @@ enum search_return_options opts = ctx->return_options; struct mail *mail; enum mailbox_sync_flags sync_flags; - struct timeval end_time; const struct seq_range *range; unsigned int count; uint32_t id, id_min, id_max; const char *ok_reply; - int time_msecs; bool tryagain, minmax, lost_data; if (cmd->cancel) { @@ -503,18 +501,12 @@ return TRUE; } - if (gettimeofday(&end_time, NULL) < 0) - memset(&end_time, 0, sizeof(end_time)); - - time_msecs = timeval_diff_msecs(&end_time, &ctx->start_time); - sync_flags = MAILBOX_SYNC_FLAG_FAST; if (!cmd->uid || ctx->have_seqsets) sync_flags |= MAILBOX_SYNC_FLAG_NO_EXPUNGES; - ok_reply = t_strdup_printf("OK %s%s completed (%d.%03d secs).", + ok_reply = t_strdup_printf("OK %s%s completed", lost_data ? "["IMAP_RESP_CODE_EXPUNGEISSUED"] " : "", - !ctx->sorting ? "Search" : "Sort", - time_msecs/1000, time_msecs%1000); + !ctx->sorting ? "Search" : "Sort"); return cmd_sync(cmd, sync_flags, 0, ok_reply); } @@ -594,7 +586,6 @@ ctx->search_ctx = mailbox_search_init(ctx->trans, sargs, sort_program, 0, NULL); ctx->sorting = sort_program != NULL; - (void)gettimeofday(&ctx->start_time, NULL); i_array_init(&ctx->result, 128); if ((ctx->return_options & SEARCH_RETURN_UPDATE) != 0) imap_search_result_save(ctx); diff -r abf2727d469a -r b651fba04408 src/imap/imap-search.h --- a/src/imap/imap-search.h Tue Apr 21 14:06:22 2015 +0300 +++ b/src/imap/imap-search.h Tue Apr 21 14:06:41 2015 +0300 @@ -41,7 +41,6 @@ float min_relevancy, max_relevancy; uint64_t highest_seen_modseq; - struct timeval start_time; unsigned int have_seqsets:1; unsigned int have_modseqs:1; From dovecot at dovecot.org Tue Apr 21 11:09:44 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 21 Apr 2015 11:09:44 +0000 Subject: dovecot-2.2: imap: Finishing previous commit by removing now-unn... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/881ab9a0dc2e changeset: 18434:881ab9a0dc2e user: Timo Sirainen date: Tue Apr 21 14:08:12 2015 +0300 description: imap: Finishing previous commit by removing now-unnecessary code. diffstat: src/imap/cmd-select.c | 2 -- 1 files changed, 0 insertions(+), 2 deletions(-) diffs (19 lines): diff -r b651fba04408 -r 881ab9a0dc2e src/imap/cmd-select.c --- a/src/imap/cmd-select.c Tue Apr 21 14:06:41 2015 +0300 +++ b/src/imap/cmd-select.c Tue Apr 21 14:08:12 2015 +0300 @@ -17,7 +17,6 @@ struct mail_namespace *ns; struct mailbox *box; - struct timeval start_time; struct imap_fetch_context *fetch_ctx; uint32_t qresync_uid_validity; @@ -405,7 +404,6 @@ client_send_tagline(cmd, error); return TRUE; } - (void)gettimeofday(&ctx->start_time, NULL); if (imap_arg_get_list(&args[1], &list_args)) { if (!select_parse_options(ctx, list_args)) { From dovecot at dovecot.org Tue Apr 21 14:03:11 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 21 Apr 2015 14:03:11 +0000 Subject: dovecot-2.2: lib-lda: Added lmtp-client to track timestamps when... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/860af8c30c59 changeset: 18435:860af8c30c59 user: Timo Sirainen date: Tue Apr 21 16:50:43 2015 +0300 description: lib-lda: Added lmtp-client to track timestamps when various events happen. diffstat: src/lib-lda/lmtp-client.c | 13 +++++++++++++ src/lib-lda/lmtp-client.h | 10 ++++++++++ 2 files changed, 23 insertions(+), 0 deletions(-) diffs (81 lines): diff -r 881ab9a0dc2e -r 860af8c30c59 src/lib-lda/lmtp-client.c --- a/src/lib-lda/lmtp-client.c Tue Apr 21 14:08:12 2015 +0300 +++ b/src/lib-lda/lmtp-client.c Tue Apr 21 16:50:43 2015 +0300 @@ -71,6 +71,7 @@ unsigned int rcpt_next_send_idx; struct istream *data_input; unsigned char output_last; + struct lmtp_client_times times; unsigned int running:1; unsigned int xclient_sent:1; @@ -392,6 +393,8 @@ } o_stream_nsend(client->output, ".\r\n", 3); client->output_finished = TRUE; + io_loop_time_refresh(); + client->times.data_sent = ioloop_timeval; return 0; } @@ -507,6 +510,7 @@ "451 4.5.0 Received invalid greeting: %s", line)); return -1; } + client->times.banner_received = ioloop_timeval; lmtp_client_send_handshake(client); client->input_state = LMTP_INPUT_STATE_LHLO; break; @@ -561,6 +565,7 @@ return -1; } client->input_state++; + client->times.data_started = ioloop_timeval; if (client->data_header != NULL) o_stream_nsend_str(client->output, client->data_header); if (lmtp_client_send_data(client) < 0) @@ -662,6 +667,8 @@ { i_assert(client->fd == -1); + client->times.connect_started = ioloop_timeval; + client->fd = net_connect_ip(&client->ip, client->port, NULL); if (client->fd == -1) { i_error("lmtp client: connect(%s, %u) failed: %m", @@ -879,3 +886,9 @@ client->data_output_callback = callback; client->data_output_context = context; } + +const struct lmtp_client_times * +lmtp_client_get_times(struct lmtp_client *client) +{ + return &client->times; +} diff -r 881ab9a0dc2e -r 860af8c30c59 src/lib-lda/lmtp-client.h --- a/src/lib-lda/lmtp-client.h Tue Apr 21 14:08:12 2015 +0300 +++ b/src/lib-lda/lmtp-client.h Tue Apr 21 16:50:43 2015 +0300 @@ -26,6 +26,13 @@ const char *dsn_orcpt; }; +struct lmtp_client_times { + struct timeval connect_started; + struct timeval banner_received; + struct timeval data_started; + struct timeval data_sent; +}; + struct lmtp_client_settings { const char *my_hostname; /* The whole MAIL FROM line, including parameters */ @@ -93,5 +100,8 @@ void lmtp_client_set_data_output_callback(struct lmtp_client *client, void (*callback)(void *), void *context); +/* Return LMTP client statistics. */ +const struct lmtp_client_times * +lmtp_client_get_times(struct lmtp_client *client); #endif From dovecot at dovecot.org Tue Apr 21 14:03:22 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 21 Apr 2015 14:03:22 +0000 Subject: dovecot-2.2: lib-lda: mail_deliver_log() now updates the %{deliv... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6ecca5308232 changeset: 18436:6ecca5308232 user: Timo Sirainen date: Tue Apr 21 16:52:59 2015 +0300 description: lib-lda: mail_deliver_log() now updates the %{delivery_time} instead of using the first cached value. The first time the cached values are set the transaction isn't committed and the mailbox isn't synced. diffstat: src/lib-lda/mail-deliver.c | 23 ++++++++++++++++------- 1 files changed, 16 insertions(+), 7 deletions(-) diffs (49 lines): diff -r 860af8c30c59 -r 6ecca5308232 src/lib-lda/mail-deliver.c --- a/src/lib-lda/mail-deliver.c Tue Apr 21 16:50:43 2015 +0300 +++ b/src/lib-lda/mail-deliver.c Tue Apr 21 16:52:59 2015 +0300 @@ -39,6 +39,19 @@ NULL : t_strconcat(addr->mailbox, "@", addr->domain, NULL); } +static void +mail_deliver_log_var_expand_table_update_times(struct mail_deliver_context *ctx, + struct var_expand_table *tab) +{ +#define VAR_EXPAND_SESSION_TIME_IDX 7 + int delivery_time_msecs; + + io_loop_time_refresh(); + delivery_time_msecs = timeval_diff_msecs(&ioloop_timeval, + &ctx->delivery_time_started); + tab[VAR_EXPAND_SESSION_TIME_IDX].value = dec2str(delivery_time_msecs); +} + static const struct var_expand_table * mail_deliver_get_log_var_expand_table_full(struct mail_deliver_context *ctx, struct mail *mail, @@ -81,13 +94,7 @@ if (mail_get_virtual_size(mail, &size) == 0) tab[6].value = dec2str(size); if (ctx != NULL) { - int delivery_time_msecs; - - io_loop_time_refresh(); - delivery_time_msecs = - timeval_diff_msecs(&ioloop_timeval, - &ctx->delivery_time_started); - tab[7].value = dec2str(delivery_time_msecs); + mail_deliver_log_var_expand_table_update_times(ctx, tab); tab[8].value = dec2str(ctx->session_time_msecs); } return tab; @@ -147,8 +154,10 @@ mail_deliver_log_cache_var_expand_table(ctx); /* update %$ */ ctx->var_expand_table[0].value = msg; + mail_deliver_log_var_expand_table_update_times(ctx, ctx->var_expand_table); var_expand(str, ctx->set->deliver_log_format, ctx->var_expand_table); ctx->var_expand_table[0].value = ""; + ctx->var_expand_table[VAR_EXPAND_SESSION_TIME_IDX].value = ""; i_info("%s", str_c(str)); va_end(args); From dovecot at dovecot.org Tue Apr 21 14:03:27 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 21 Apr 2015 14:03:27 +0000 Subject: dovecot-2.2: lib-lda: Assume that the caller sets delivery_time_... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7ef1722d8f7a changeset: 18437:7ef1722d8f7a user: Timo Sirainen date: Tue Apr 21 16:54:51 2015 +0300 description: lib-lda: Assume that the caller sets delivery_time_started This is required because parts of the delivery time may be done at the time the user is being initialized, so it needs to be set before the user init is done. diffstat: src/lda/main.c | 4 ++++ src/lib-lda/mail-deliver.c | 3 --- src/lmtp/commands.c | 6 ++++++ 3 files changed, 10 insertions(+), 3 deletions(-) diffs (64 lines): diff -r 6ecca5308232 -r 7ef1722d8f7a src/lda/main.c --- a/src/lda/main.c Tue Apr 21 16:52:59 2015 +0300 +++ b/src/lda/main.c Tue Apr 21 16:54:51 2015 +0300 @@ -2,6 +2,7 @@ #include "lib.h" #include "lib-signals.h" +#include "ioloop.h" #include "env-util.h" #include "fd-set-nonblock.h" #include "istream.h" @@ -409,6 +410,9 @@ MAIL_STORAGE_SERVICE_FLAG_USE_SYSEXITS; storage_service = mail_storage_service_init(master_service, set_roots, service_flags); + /* set before looking up the user (or ideally we'd do this between + _lookup() and _next(), but don't bother) */ + ctx.delivery_time_started = ioloop_timeval; ret = mail_storage_service_lookup_next(storage_service, &service_input, &service_user, &ctx.dest_user, &errstr); diff -r 6ecca5308232 -r 7ef1722d8f7a src/lib-lda/mail-deliver.c --- a/src/lib-lda/mail-deliver.c Tue Apr 21 16:52:59 2015 +0300 +++ b/src/lib-lda/mail-deliver.c Tue Apr 21 16:54:51 2015 +0300 @@ -430,9 +430,6 @@ { int ret; - io_loop_time_refresh(); - ctx->delivery_time_started = ioloop_timeval; - *storage_r = NULL; if (deliver_mail == NULL) ret = -1; diff -r 6ecca5308232 -r 7ef1722d8f7a src/lmtp/commands.c --- a/src/lmtp/commands.c Tue Apr 21 16:52:59 2015 +0300 +++ b/src/lmtp/commands.c Tue Apr 21 16:54:51 2015 +0300 @@ -750,6 +750,7 @@ struct lda_settings *lda_set; struct mail_namespace *ns; struct setting_parser_context *set_parser; + struct timeval delivery_time_started; void **sets; const char *line, *error, *username; string_t *str; @@ -785,6 +786,10 @@ i_unreached(); } + /* get the timestamp before user is created, since it starts the I/O */ + io_loop_time_refresh(); + delivery_time_started = ioloop_timeval; + client_state_set(client, "DATA", username); i_set_failure_prefix("lmtp(%s, %s): ", my_pid, username); if (mail_storage_service_next(storage_service, rcpt->service_user, @@ -815,6 +820,7 @@ dctx.session_time_msecs = timeval_diff_msecs(&client->state.data_end_timeval, &client->state.mail_from_timeval); + dctx.delivery_time_started = delivery_time_started; if (orcpt_get_valid_rfc822(rcpt->params.dsn_orcpt, &dctx.dest_addr)) { /* used ORCPT */ From dovecot at dovecot.org Tue Apr 21 14:03:27 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 21 Apr 2015 14:03:27 +0000 Subject: dovecot-2.2: lmtp proxy: Include in log messages how long it too... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b64bcee2ffa0 changeset: 18438:b64bcee2ffa0 user: Timo Sirainen date: Tue Apr 21 17:01:39 2015 +0300 description: lmtp proxy: Include in log messages how long it took to deliver the mail. In multi-RCPT TO sessions it's a bit unreliable to split the single DATA reply's times to multiple lines, so instead each line just says "(2/3 at 123 ms)" meaning that the second mail was delivered at the time when 123 milliseconds had elapsed since the delivery was started. diffstat: src/lmtp/lmtp-proxy.c | 33 +++++++++++++++++++++++---------- 1 files changed, 23 insertions(+), 10 deletions(-) diffs (82 lines): diff -r 7ef1722d8f7a -r b64bcee2ffa0 src/lmtp/lmtp-proxy.c --- a/src/lmtp/lmtp-proxy.c Tue Apr 21 16:54:51 2015 +0300 +++ b/src/lmtp/lmtp-proxy.c Tue Apr 21 17:01:39 2015 +0300 @@ -6,6 +6,8 @@ #include "istream.h" #include "istream-sized.h" #include "ostream.h" +#include "str.h" +#include "time-util.h" #include "lmtp-client.h" #include "lmtp-proxy.h" @@ -15,6 +17,7 @@ struct lmtp_proxy_connection *conn; const char *address; const char *reply; + unsigned int idx; unsigned int rcpt_to_failed:1; unsigned int data_reply_received:1; @@ -238,6 +241,9 @@ { struct lmtp_proxy_recipient *rcpt = context; struct lmtp_proxy_connection *conn = rcpt->conn; + const struct lmtp_client_times *times = + lmtp_client_get_times(conn->client); + string_t *msg; i_assert(!rcpt->rcpt_to_failed); i_assert(rcpt->reply != NULL); @@ -249,27 +255,33 @@ rcpt->reply = p_strdup(conn->proxy->pool, reply); rcpt->data_reply_received = TRUE; + msg = t_str_new(128); + str_printfa(msg, "%s: ", conn->proxy->set.session_id); switch (result) { case LMTP_CLIENT_RESULT_OK: - i_info("%s: Sent message to <%s> at %s:%u: %s", - conn->proxy->set.session_id, rcpt->address, - conn->set.host, conn->set.port, reply); + str_append(msg, "Sent message to"); break; case LMTP_CLIENT_RESULT_REMOTE_ERROR: + case LMTP_CLIENT_RESULT_INTERNAL_ERROR: + str_append(msg, "Failed to send message to"); + break; + } + str_printfa(msg, " <%s> at %s:%u: %s (%u/%u at %u ms)", + rcpt->address, conn->set.host, conn->set.port, reply, + rcpt->idx + 1, array_count(&conn->proxy->rcpt_to), + timeval_diff_msecs(&ioloop_timeval, ×->connect_started)); + switch (result) { + case LMTP_CLIENT_RESULT_OK: + case LMTP_CLIENT_RESULT_REMOTE_ERROR: /* the problem isn't with the proxy, it's with the remote side. so the remote side will log an error, while for us this is just an info event */ - i_info("%s: Failed to send message to <%s> at %s:%u: %s", - conn->proxy->set.session_id, rcpt->address, - conn->set.host, conn->set.port, reply); + i_info("%s", str_c(msg)); break; case LMTP_CLIENT_RESULT_INTERNAL_ERROR: - i_error("%s: Failed to send message to <%s> at %s:%u: %s", - conn->proxy->set.session_id, rcpt->address, - conn->set.host, conn->set.port, reply); + i_error("%s", str_c(msg)); break; } - lmtp_proxy_try_finish(conn->proxy); } @@ -284,6 +296,7 @@ return -1; rcpt = p_new(proxy->pool, struct lmtp_proxy_recipient, 1); + rcpt->idx = array_count(&proxy->rcpt_to); rcpt->conn = conn; rcpt->address = p_strdup(proxy->pool, address); array_append(&proxy->rcpt_to, &rcpt, 1); From dovecot at dovecot.org Tue Apr 21 16:33:03 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 21 Apr 2015 16:33:03 +0000 Subject: dovecot-2.2: configure: s/normalizer/libicu/ since we it could b... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b179bbd226e5 changeset: 18439:b179bbd226e5 user: Timo Sirainen date: Tue Apr 21 19:31:14 2015 +0300 description: configure: s/normalizer/libicu/ since we it could be used for something else as well. diffstat: configure.ac | 22 +++++++++++----------- src/lib-fts/Makefile.am | 7 +++---- src/lib-fts/fts-filter-normalizer.c | 2 +- 3 files changed, 15 insertions(+), 16 deletions(-) diffs (85 lines): diff -r b64bcee2ffa0 -r b179bbd226e5 configure.ac --- a/configure.ac Tue Apr 21 17:01:39 2015 +0300 +++ b/configure.ac Tue Apr 21 19:31:14 2015 +0300 @@ -169,10 +169,10 @@ TEST_WITH(textcat, $withval), want_textcat=auto) -AC_ARG_WITH(normalizer, -AS_HELP_STRING([--with-normalizer], [Build lib-fts with ICU normalization support (auto)]), - want_fts_normalizer=$withval, - want_fts_normalizer=auto) +AC_ARG_WITH(icu, +AS_HELP_STRING([--with-icu], [Build with libicu support (for FTS normalization) (auto)]), + want_icu=$withval, + want_icu=auto) AC_ARG_WITH(solr, AS_HELP_STRING([--with-solr], [Build with Solr full text search support]), @@ -2799,16 +2799,16 @@ AM_CONDITIONAL(BUILD_FTS_TEXTCAT, test "$have_fts_textcat" = "yes") AM_CONDITIONAL(BUILD_FTS_EXTTEXTCAT, test "$have_fts_exttextcat" = "yes") -if test "$want_fts_normalizer" != "no"; then +if test "$want_icu" != "no"; then if test "$PKG_CONFIG" != "" && $PKG_CONFIG --exists icu-i18n 2>/dev/null; then - PKG_CHECK_MODULES(FTS_NORMALIZER, icu-i18n) - have_fts_normalizer=yes - AC_DEFINE(HAVE_FTS_NORMALIZER,, Define if you want ICU normalization support for FTS) - elif test "$want_fts_normalizer" = "yes"; then - AC_ERROR([Can't build with normalizer support: libicu-i18n not found]) + PKG_CHECK_MODULES(LIBICU, icu-i18n) + have_icu=yes + AC_DEFINE(HAVE_LIBICU,, Define if you want ICU normalization support for FTS) + elif test "$want_icu" = "yes"; then + AC_ERROR([Can't build with libicu support: libicu-i18n not found]) fi fi -AM_CONDITIONAL(BUILD_FTS_NORMALIZER, test "$have_fts_normalizer" = "yes") +AM_CONDITIONAL(BUILD_LIBICU, test "$have_icu" = "yes") if test $have_lucene = no; then not_fts="$not_fts lucene" diff -r b64bcee2ffa0 -r b179bbd226e5 src/lib-fts/Makefile.am --- a/src/lib-fts/Makefile.am Tue Apr 21 17:01:39 2015 +0300 +++ b/src/lib-fts/Makefile.am Tue Apr 21 19:31:14 2015 +0300 @@ -4,7 +4,7 @@ -I$(top_srcdir)/src/lib \ -I$(top_srcdir)/src/lib-test \ $(LIBEXTTEXTCAT_CFLAGS) \ - $(FTS_NORMALIZER_CFLAGS) \ + $(LIBICU_CFLAGS) \ -DUDHRDIR=\""$(top_srcdir)/src/lib-fts"\" \ -DDATADIR=\"$(pkgdatadir)\" \ -DTEST_STOPWORDS_DIR=\""$(top_srcdir)/src/lib-fts"\" @@ -49,8 +49,8 @@ endif endif -if BUILD_FTS_NORMALIZER -NORMALIZER_LIBS = $(FTS_NORMALIZER_LIBS) +if BUILD_LIBICU +NORMALIZER_LIBS = $(LIBICU_LIBS) endif libfts_la_LIBADD = \ @@ -79,7 +79,6 @@ test_programs = \ test-fts-filter \ $(TEST_FTS_LANGUAGE) \ - $(TEST_FTS_NORMALIZER) \ test-fts-tokenizer noinst_PROGRAMS = $(test_programs) diff -r b64bcee2ffa0 -r b179bbd226e5 src/lib-fts/fts-filter-normalizer.c --- a/src/lib-fts/fts-filter-normalizer.c Tue Apr 21 17:01:39 2015 +0300 +++ b/src/lib-fts/fts-filter-normalizer.c Tue Apr 21 19:31:14 2015 +0300 @@ -6,7 +6,7 @@ #include "fts-filter-private.h" #include "fts-language.h" -#ifdef HAVE_FTS_NORMALIZER +#ifdef HAVE_LIBICU #include #include From dovecot at dovecot.org Tue Apr 21 16:38:05 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 21 Apr 2015 16:38:05 +0000 Subject: dovecot-2.2: lib-fts: Renamed normalizer to icu-normalizer, incl... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/41abc9c7eca2 changeset: 18440:41abc9c7eca2 user: Timo Sirainen date: Tue Apr 21 19:36:27 2015 +0300 description: lib-fts: Renamed normalizer to icu-normalizer, including the source code. diffstat: src/lib-fts/Makefile.am | 4 +- src/lib-fts/fts-filter-normalizer-icu.c | 318 ++++++++++++++++++++++++++++++++ src/lib-fts/fts-filter-normalizer.c | 318 -------------------------------- src/lib-fts/fts-filter.c | 2 +- src/lib-fts/fts-filter.h | 4 +- 5 files changed, 323 insertions(+), 323 deletions(-) diffs (truncated from 691 to 300 lines): diff -r b179bbd226e5 -r 41abc9c7eca2 src/lib-fts/Makefile.am --- a/src/lib-fts/Makefile.am Tue Apr 21 19:31:14 2015 +0300 +++ b/src/lib-fts/Makefile.am Tue Apr 21 19:36:27 2015 +0300 @@ -60,7 +60,7 @@ libfts_la_SOURCES = \ fts-filter.c \ - fts-filter-normalizer.c \ + fts-filter-normalizer-icu.c \ fts-filter-stopwords.c \ fts-filter-stemmer-snowball.c \ fts-language.c \ @@ -90,7 +90,7 @@ filter_deps = \ fts-filter.lo fts-filter-stopwords.lo \ - fts-filter-stemmer-snowball.lo fts-filter-normalizer.lo + fts-filter-stemmer-snowball.lo fts-filter-normalizer-icu.lo test_fts_filter_SOURCES = test-fts-filter.c test_fts_filter_LDADD = $(filter_deps) $(test_libs) $(STEMMER_LIBS) $(TEXTCAT_LIBS) $(NORMALIZER_LIBS) diff -r b179bbd226e5 -r 41abc9c7eca2 src/lib-fts/fts-filter-normalizer-icu.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-fts/fts-filter-normalizer-icu.c Tue Apr 21 19:36:27 2015 +0300 @@ -0,0 +1,318 @@ +/* Copyright (c) 2014-2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "unichar.h" /* unicode replacement char */ +#include "fts-filter.h" +#include "fts-filter-private.h" +#include "fts-language.h" + +#ifdef HAVE_LIBICU + +#include +#include +#include +#include +#include + +struct fts_filter_normalizer { + struct fts_filter filter; + const char *error; + pool_t pool; + UTransliterator *transliterator; +}; + +static void +icu_error(const char **error_r, const UErrorCode err, const char *func) +{ + if (error_r == NULL) + return; + + if (U_FAILURE(err)) { + *error_r = t_strdup_printf("Lib ICU function %s failed: %s\n", + func, u_errorName(err)); + } +} + +/* Thin wrapper for vprintf */ +static void ATTR_FORMAT(2, 3) +fts_filter_normalizer_icu_error(const char **error_r, const char *format, ...) +{ + va_list args; + + if (error_r == NULL) + return; + + va_start(args, format); + *error_r = t_strdup_vprintf(format, args); + va_end(args); +} + +/* Helper to create UTF16, which libicu wants as input. Returns -1 on + error, 0 on success. + + On input, if *dst_uchars_r > 0, it indicates the number of UChar + sized units that should be allocated for the text. However, the + function will not use the number, if the text will not fit in that + amount. + + On return *dst_uchars_r will contain the number of UChar sized units + allocated for the dst. NOT the number of bytes nor the length of the + text. */ +static int make_uchar(const char *src, UChar **dst, int32_t *dst_uchars_r) +{ + UErrorCode err = U_ZERO_ERROR; + int32_t len = strlen(src); + int32_t ustr_len = 0; + int32_t ustr_len_actual = 0; + UChar *retp = NULL; + int32_t alloc_uchars = 0; + + i_assert(dst_uchars_r != NULL); + + /* Check length required for encoded dst. */ + retp = u_strFromUTF8(NULL, 0, &ustr_len, src, len, &err); + + /* When preflighting a successful call returns a buffer overflow + error. */ + if (U_BUFFER_OVERFLOW_ERROR != err && U_FAILURE(err)) { + i_panic("Failed to estimate allocation size with lib ICU" + " u_strFromUTF8(): %s",u_errorName(err)); + } + i_assert(NULL == retp); + + err = U_ZERO_ERROR; + if (*dst_uchars_r > 0 && *dst_uchars_r > ustr_len) + alloc_uchars = *dst_uchars_r; + else + alloc_uchars = ustr_len; + alloc_uchars++; /* room for null bytes(2) */ + *dst = t_malloc(alloc_uchars * sizeof(UChar)); + *dst_uchars_r = alloc_uchars; + retp = u_strFromUTF8(*dst, alloc_uchars, &ustr_len_actual, + src, len, &err); + + if (U_FAILURE(err)) + i_panic("Lib ICU u_strFromUTF8 failed: %s", u_errorName(err)); + i_assert(retp == *dst); + i_assert(ustr_len == ustr_len_actual); + return 0; +} + +static int make_utf8(const UChar *src, char **dst, const char **error_r) +{ + char *retp = NULL; + int32_t dsize = 0; + int32_t dsize_actual = 0; + int32_t sub_num = 0; + UErrorCode err = U_ZERO_ERROR; + int32_t usrc_len = u_strlen(src); /* libicu selects different codepaths + depending if srclen -1 or not */ + + retp = u_strToUTF8WithSub(NULL, 0, &dsize, src, usrc_len, + UNICODE_REPLACEMENT_CHAR, &sub_num, &err); + + /* Preflighting can cause buffer overflow to be reported */ + if (U_BUFFER_OVERFLOW_ERROR != err && U_FAILURE(err)) { + i_panic("Failed to estimate allocation size with lib ICU" + " u_strToUTF8(): %s",u_errorName(err)); + } + i_assert(0 == sub_num); + i_assert(NULL == retp); + + dsize++; /* room for '\0' byte */ + *dst = t_malloc(dsize); + err = U_ZERO_ERROR; + retp = u_strToUTF8WithSub(*dst, dsize, &dsize_actual, src, usrc_len, + UNICODE_REPLACEMENT_CHAR, &sub_num, &err); + if (U_FAILURE(err)) + i_panic("Lib ICU u_strToUTF8WithSub() failed: %s", + u_errorName(err)); + if (dsize_actual >= dsize) { + i_panic("Produced UTF8 string length (%d) does not fit in " + "preflighted(%d). Buffer overflow?", + dsize_actual, dsize); + } + if (0 != sub_num) { + fts_filter_normalizer_icu_error(error_r, "UTF8 string not well formed." + " Substitutions (%d) were made.", sub_num); + return -1; + } + i_assert(retp == *dst); + + return 0; +} + +static bool fts_filter_normalizer_icu_supports(const struct fts_language *lang) +{ + if (lang == NULL || lang->name == NULL) + return FALSE; + return TRUE; +} + +static void fts_filter_normalizer_icu_destroy(struct fts_filter *filter) +{ + struct fts_filter_normalizer *np = + (struct fts_filter_normalizer *)filter; + + if (np->transliterator != NULL) + utrans_close(np->transliterator); + pool_unref(&np->pool); + return; +} + +static int +fts_filter_normalizer_icu_create(const struct fts_language *lang ATTR_UNUSED, + const char *const *settings, + struct fts_filter **filter_r, + const char **error_r) +{ + struct fts_filter_normalizer *np; + pool_t pp; + UErrorCode err = U_ZERO_ERROR; + UParseError perr; + UChar *id_uchar = NULL; + int32_t id_len_uchar = 0; + unsigned int i; + const char *id = "Any-Lower; NFKD; [: Nonspacing Mark :] Remove; NFC"; + + memset(&perr, 0, sizeof(perr)); + + for (i = 0; settings[i] != NULL; i += 2) { + const char *key = settings[i], *value = settings[i+1]; + + if (strcmp(key, "id") == 0) { + id = value; + } else { + *error_r = t_strdup_printf("Unknown setting: %s", key); + return -1; + } + } + + pp = pool_alloconly_create(MEMPOOL_GROWING"fts_filter_normalizer", + sizeof(struct fts_filter_normalizer)); + np = p_new(pp, struct fts_filter_normalizer, 1); + np->pool = pp; + np->filter = *fts_filter_normalizer_icu; + if (make_uchar(id, &id_uchar, &id_len_uchar) < 0) { + + } + np->transliterator = utrans_openU(id_uchar, u_strlen(id_uchar), UTRANS_FORWARD, + NULL, 0, &perr, &err); + if (U_FAILURE(err)) { + if (perr.line >= 1) { + fts_filter_normalizer_icu_error(error_r, "Failed to open transliterator for id: %s. Lib ICU error: %s. Parse error on line %u offset %u.", id, u_errorName(err), perr.line, perr.offset); + } + else { + fts_filter_normalizer_icu_error(error_r, "Failed to open transliterator for id: %s. Lib ICU error: %s.", id, u_errorName(err)); + } + fts_filter_normalizer_icu_destroy(&np->filter); + return -1; + } + *filter_r = &np->filter; + return 0; +} + +/* Returns 0 on success and -1 on error. */ +/* TODO: delay errors until _deinit() and return some other values? */ +static const char * +fts_filter_normalizer_icu_filter(struct fts_filter *filter, const char *token) +{ + UErrorCode err = U_ZERO_ERROR; + UChar *utext = NULL; + int32_t utext_cap = 0; + int32_t utext_len = -1; + int32_t utext_limit; + char *normalized = NULL; + struct fts_filter_normalizer *np = + (struct fts_filter_normalizer *)filter; + + /* TODO: fix error handling */ + if (np->error != NULL) + return NULL; + + if (make_uchar(token, &utext, &utext_cap) < 0) { + fts_filter_normalizer_icu_error(&np->error, "Conversion to UChar failed"); + return NULL; + } + /* + TODO: Some problems here. How much longer can the result + be, than the source? Can it be calculated? Preflighted? + */ + utext_limit = u_strlen(utext); + utrans_transUChars(np->transliterator, utext, &utext_len, + utext_cap, 0, &utext_limit, &err); + + /* Data did not fit into utext. */ + if (utext_len > utext_cap || err == U_BUFFER_OVERFLOW_ERROR) { + + /* This is a crude retry fix... Make a new utext of the + size utrans_transUChars indicated */ + utext_len++; /* room for '\0' bytes(2) */ + utext_cap = utext_len; + if (make_uchar(token, &utext, &utext_cap) < 0) + return NULL; + i_assert(utext_cap == utext_len); + utext_limit = u_strlen(utext); + utext_len = -1; + err = U_ZERO_ERROR; + utrans_transUChars(np->transliterator, utext, + &utext_len, utext_cap, 0, + &utext_limit, &err); + } + + if (U_FAILURE(err)) { + icu_error(&np->error, err, "utrans_transUChars()"); + return NULL; + } + + if (make_utf8(utext, &normalized, &np->error) < 0) + return NULL; + + return normalized; +} + +#else + From dovecot at dovecot.org Tue Apr 21 16:42:19 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 21 Apr 2015 16:42:19 +0000 Subject: dovecot-2.2: lib-fts: Fixed test-fts-filter to work again after ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/782594d44002 changeset: 18441:782594d44002 user: Timo Sirainen date: Tue Apr 21 19:40:42 2015 +0300 description: lib-fts: Fixed test-fts-filter to work again after previous normalizer changes. diffstat: src/lib-fts/test-fts-filter.c | 14 +++++++------- 1 files changed, 7 insertions(+), 7 deletions(-) diffs (66 lines): diff -r 41abc9c7eca2 -r 782594d44002 src/lib-fts/test-fts-filter.c --- a/src/lib-fts/test-fts-filter.c Tue Apr 21 19:36:27 2015 +0300 +++ b/src/lib-fts/test-fts-filter.c Tue Apr 21 19:40:42 2015 +0300 @@ -296,7 +296,7 @@ } #endif -#ifdef HAVE_FTS_NORMALIZER +#ifdef HAVE_LIBICU static void test_fts_filter_normalizer_swedish_short(void) { const struct fts_filter *filter_class; @@ -328,7 +328,7 @@ test_begin("fts filter normalizer Swedish short text"); T_BEGIN { - filter_class = fts_filter_find(NORMALIZER_FILTER_NAME); + filter_class = fts_filter_find(ICU_NORMALIZER_FILTER_NAME); ret = fts_filter_create(filter_class, NULL, NULL, settings, &norm, &error); test_assert(ret == 0); for (i = 0; i < N_ELEMENTS(input); i++) { @@ -372,7 +372,7 @@ test_begin("fts filter normalizer Swedish short text using default ID"); T_BEGIN { - filter_class = fts_filter_find(NORMALIZER_FILTER_NAME); + filter_class = fts_filter_find(ICU_NORMALIZER_FILTER_NAME); ret = fts_filter_create(filter_class, NULL, NULL, NULL, &norm, &error); test_assert(ret == 0); for (i = 0; i < N_ELEMENTS(input); i++) { @@ -417,7 +417,7 @@ T_BEGIN { udhr_path = t_strconcat(UDHRDIR, UDHR_FRA_NAME, NULL); - filter_class = fts_filter_find(NORMALIZER_FILTER_NAME); + filter_class = fts_filter_find(ICU_NORMALIZER_FILTER_NAME); ret = fts_filter_create(filter_class, NULL, NULL, settings, &norm, &error); test_assert(ret == 0); input = fopen(udhr_path, "r"); @@ -451,7 +451,7 @@ int ret; test_begin("fts filter normalizer invalid id"); - filter_class = fts_filter_find(NORMALIZER_FILTER_NAME); + filter_class = fts_filter_find(ICU_NORMALIZER_FILTER_NAME); ret = fts_filter_create(filter_class, NULL, NULL, settings, &norm, &error); test_assert(ret < 0 && error != NULL); test_assert(norm == NULL); @@ -489,7 +489,7 @@ test_begin("fts filters normalizer, stopwords and stemming chained, English"); - filter_class = fts_filter_find(NORMALIZER_FILTER_NAME); + filter_class = fts_filter_find(ICU_NORMALIZER_FILTER_NAME); ret = fts_filter_create(filter_class, NULL, NULL, id_settings, &normalizer, &error); test_assert(ret == 0); @@ -537,7 +537,7 @@ test_fts_filter_stemmer_snowball_stem_french, test_fts_filter_stopwords_stemmer_eng, #endif -#ifdef HAVE_FTS_NORMALIZER +#ifdef HAVE_LIBICU test_fts_filter_normalizer_swedish_short, test_fts_filter_normalizer_swedish_short_default_id, test_fts_filter_normalizer_french, From dovecot at dovecot.org Tue Apr 21 17:18:10 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 21 Apr 2015 17:18:10 +0000 Subject: dovecot-2.2: lib-fts: Added normalizer-simple for doing normaliz... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9aaacd260a7d changeset: 18442:9aaacd260a7d user: Timo Sirainen date: Tue Apr 21 20:16:36 2015 +0300 description: lib-fts: Added normalizer-simple for doing normalization without libicu. diffstat: src/lib-fts/Makefile.am | 9 +-- src/lib-fts/fts-filter-normalizer-simple.c | 77 ++++++++++++++++++++++++++++++ src/lib-fts/fts-filter.c | 1 + src/lib-fts/fts-filter.h | 4 + 4 files changed, 85 insertions(+), 6 deletions(-) diffs (133 lines): diff -r 782594d44002 -r 9aaacd260a7d src/lib-fts/Makefile.am --- a/src/lib-fts/Makefile.am Tue Apr 21 19:40:42 2015 +0300 +++ b/src/lib-fts/Makefile.am Tue Apr 21 20:16:36 2015 +0300 @@ -61,6 +61,7 @@ libfts_la_SOURCES = \ fts-filter.c \ fts-filter-normalizer-icu.c \ + fts-filter-normalizer-simple.c \ fts-filter-stopwords.c \ fts-filter-stemmer-snowball.c \ fts-language.c \ @@ -88,13 +89,9 @@ ../lib/liblib.la test_deps = $(noinst_LTLIBRARIES) $(test_libs) -filter_deps = \ - fts-filter.lo fts-filter-stopwords.lo \ - fts-filter-stemmer-snowball.lo fts-filter-normalizer-icu.lo - test_fts_filter_SOURCES = test-fts-filter.c -test_fts_filter_LDADD = $(filter_deps) $(test_libs) $(STEMMER_LIBS) $(TEXTCAT_LIBS) $(NORMALIZER_LIBS) -test_fts_filter_DEPENDENCIES = $(test_deps) $(filter_deps) +test_fts_filter_LDADD = libfts.la $(test_libs) +test_fts_filter_DEPENDENCIES = libfts.la $(test_deps) if BUILD_FTS_EXTTEXTCAT TEST_FTS_LANGUAGE = test-fts-language diff -r 782594d44002 -r 9aaacd260a7d src/lib-fts/fts-filter-normalizer-simple.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-fts/fts-filter-normalizer-simple.c Tue Apr 21 20:16:36 2015 +0300 @@ -0,0 +1,77 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "unichar.h" +#include "str.h" +#include "fts-filter.h" +#include "fts-filter-private.h" +#include "fts-language.h" + +struct fts_filter_normalizer_simple { + struct fts_filter filter; + string_t *str; +}; + +static bool +fts_filter_normalizer_simple_supports(const struct fts_language *lang ATTR_UNUSED) +{ + return TRUE; +} + +static void +fts_filter_normalizer_simple_destroy(struct fts_filter *_filter) +{ + struct fts_filter_normalizer_simple *filter = + (struct fts_filter_normalizer_simple *)_filter; + + str_free(&filter->str); + i_free(filter); +} + +static int +fts_filter_normalizer_simple_create(const struct fts_language *lang ATTR_UNUSED, + const char *const *settings, + struct fts_filter **filter_r, + const char **error_r) +{ + struct fts_filter_normalizer_simple *filter; + + if (settings[0] != NULL) { + *error_r = t_strdup_printf("Unknown setting: %s", settings[0]); + return -1; + } + filter = i_new(struct fts_filter_normalizer_simple, 1); + filter->filter = *fts_filter_normalizer_simple; + filter->str = str_new(default_pool, 128); + + *filter_r = &filter->filter; + return 0; +} + +static const char * +fts_filter_normalizer_simple_filter(struct fts_filter *_filter, + const char *token) +{ + struct fts_filter_normalizer_simple *filter = + (struct fts_filter_normalizer_simple *)_filter; + + str_truncate(filter->str, 0); + if (uni_utf8_to_decomposed_titlecase(token, strlen(token), + filter->str) < 0) + return NULL; + return str_c(filter->str); +} + +static const struct fts_filter_vfuncs normalizer_filter_vfuncs = { + fts_filter_normalizer_simple_supports, + fts_filter_normalizer_simple_create, + fts_filter_normalizer_simple_filter, + fts_filter_normalizer_simple_destroy +}; + +static const struct fts_filter fts_filter_normalizer_simple_real = { + .class_name = SIMPLE_NORMALIZER_FILTER_NAME, + .v = &normalizer_filter_vfuncs +}; + +const struct fts_filter *fts_filter_normalizer_simple = &fts_filter_normalizer_simple_real; diff -r 782594d44002 -r 9aaacd260a7d src/lib-fts/fts-filter.c --- a/src/lib-fts/fts-filter.c Tue Apr 21 19:40:42 2015 +0300 +++ b/src/lib-fts/fts-filter.c Tue Apr 21 20:16:36 2015 +0300 @@ -15,6 +15,7 @@ fts_filter_register(fts_filter_stopwords); fts_filter_register(fts_filter_stemmer_snowball); fts_filter_register(fts_filter_normalizer_icu); + fts_filter_register(fts_filter_normalizer_simple); } void fts_filters_deinit(void) diff -r 782594d44002 -r 9aaacd260a7d src/lib-fts/fts-filter.h --- a/src/lib-fts/fts-filter.h Tue Apr 21 19:40:42 2015 +0300 +++ b/src/lib-fts/fts-filter.h Tue Apr 21 20:16:36 2015 +0300 @@ -33,6 +33,10 @@ extern const struct fts_filter *fts_filter_normalizer_icu; #define ICU_NORMALIZER_FILTER_NAME "normalizer-icu" +/* Normalization using i;unicode-casemap (RFC 5051) */ +extern const struct fts_filter *fts_filter_normalizer_simple; +#define SIMPLE_NORMALIZER_FILTER_NAME "normalizer-simple" + /* Register all built-in filters. */ void fts_filters_init(void); void fts_filters_deinit(void); From dovecot at dovecot.org Tue Apr 21 18:01:32 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 21 Apr 2015 18:01:32 +0000 Subject: dovecot-2.2: lib-fs: Fixed fs-posix to work with fs_iter_init(pa... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5b8959b2ff5e changeset: 18443:5b8959b2ff5e user: Timo Sirainen date: Tue Apr 21 20:59:48 2015 +0300 description: lib-fs: Fixed fs-posix to work with fs_iter_init(path="") when fs prefix wasn't set. diffstat: src/lib-fs/fs-posix.c | 4 ++++ 1 files changed, 4 insertions(+), 0 deletions(-) diffs (14 lines): diff -r 9aaacd260a7d -r 5b8959b2ff5e src/lib-fs/fs-posix.c --- a/src/lib-fs/fs-posix.c Tue Apr 21 20:16:36 2015 +0300 +++ b/src/lib-fs/fs-posix.c Tue Apr 21 20:59:48 2015 +0300 @@ -707,6 +707,10 @@ iter->iter.flags = flags; iter->path = fs->path_prefix == NULL ? i_strdup(path) : i_strconcat(fs->path_prefix, path, NULL); + if (iter->path[0] == '\0') { + i_free(iter->path); + iter->path = i_strdup("."); + } iter->dir = opendir(iter->path); if (iter->dir == NULL && errno != ENOENT) { iter->err = errno; From dovecot at dovecot.org Tue Apr 21 18:31:15 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 21 Apr 2015 18:31:15 +0000 Subject: dovecot-2.2: fts: If backend has FTS_BACKEND_FLAG_TOKENIZED_INPU... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3f24c8cab32d changeset: 18444:3f24c8cab32d user: Timo Sirainen date: Tue Apr 21 21:28:41 2015 +0300 description: fts: If backend has FTS_BACKEND_FLAG_TOKENIZED_INPUT set, index also the header name. We were sending the hdr_name to the backend without tokenizing it or filtering it, so the backend couldn't have done anything useful about it. diffstat: src/plugins/fts/fts-build-mail.c | 10 ++++++++++ 1 files changed, 10 insertions(+), 0 deletions(-) diffs (20 lines): diff -r 5b8959b2ff5e -r 3f24c8cab32d src/plugins/fts/fts-build-mail.c --- a/src/plugins/fts/fts-build-mail.c Tue Apr 21 20:59:48 2015 +0300 +++ b/src/plugins/fts/fts-build-mail.c Tue Apr 21 21:28:41 2015 +0300 @@ -171,6 +171,16 @@ (void)fts_build_data(ctx, str_data(str), str_len(str), TRUE); } T_END; + + if ((ctx->update_ctx->backend->flags & + FTS_BACKEND_FLAG_TOKENIZED_INPUT) != 0) { + /* index the header name itself */ + key.hdr_name = ""; + if (fts_backend_update_set_build_key(ctx->update_ctx, &key)) { + (void)fts_build_data(ctx, (const void *)hdr->name, + strlen(hdr->name), TRUE); + } + } } static bool From dovecot at dovecot.org Tue Apr 21 18:31:15 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 21 Apr 2015 18:31:15 +0000 Subject: dovecot-2.2: fts-lucene: Optionally use lib-fts instead of CLuce... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1ba63410b902 changeset: 18445:1ba63410b902 user: Timo Sirainen date: Tue Apr 21 21:29:42 2015 +0300 description: fts-lucene: Optionally use lib-fts instead of CLucene's own analyzers. fts_lucene = use_libfts enables this. diffstat: src/plugins/fts-lucene/fts-backend-lucene.c | 7 +++- src/plugins/fts-lucene/fts-lucene-plugin.c | 5 ++ src/plugins/fts-lucene/fts-lucene-plugin.h | 1 + src/plugins/fts-lucene/lucene-wrapper.cc | 53 ++++++++++++++++++++-------- 4 files changed, 49 insertions(+), 17 deletions(-) diffs (195 lines): diff -r 3f24c8cab32d -r 1ba63410b902 src/plugins/fts-lucene/fts-backend-lucene.c --- a/src/plugins/fts-lucene/fts-backend-lucene.c Tue Apr 21 21:28:41 2015 +0300 +++ b/src/plugins/fts-lucene/fts-backend-lucene.c Tue Apr 21 21:29:42 2015 +0300 @@ -161,8 +161,13 @@ *error_r = "Invalid fts_lucene settings"; return -1; } + /* fts already checked that index exists */ - /* fts already checked that index exists */ + if (fuser->set.use_libfts) { + /* change our flags so we get proper input */ + _backend->flags &= ~FTS_BACKEND_FLAG_FUZZY_SEARCH; + _backend->flags |= FTS_BACKEND_FLAG_TOKENIZED_INPUT; + } return 0; } diff -r 3f24c8cab32d -r 1ba63410b902 src/plugins/fts-lucene/fts-lucene-plugin.c --- a/src/plugins/fts-lucene/fts-lucene-plugin.c Tue Apr 21 21:28:41 2015 +0300 +++ b/src/plugins/fts-lucene/fts-lucene-plugin.c Tue Apr 21 21:29:42 2015 +0300 @@ -34,6 +34,8 @@ set->no_snowball = TRUE; } else if (strcmp(*tmp, "mime_parts") == 0) { set->mime_parts = TRUE; + } else if (strcmp(*tmp, "use_libfts") == 0) { + set->use_libfts = TRUE; } else { i_error("fts_lucene: Invalid setting: %s", *tmp); return -1; @@ -73,6 +75,9 @@ { uint32_t crc; + if (set->use_libfts) + return crc32_str("l"); + /* checksum is always different when compiling with/without stemmer */ crc = set->default_language == NULL ? 0 : crc32_str(set->default_language); diff -r 3f24c8cab32d -r 1ba63410b902 src/plugins/fts-lucene/fts-lucene-plugin.h --- a/src/plugins/fts-lucene/fts-lucene-plugin.h Tue Apr 21 21:28:41 2015 +0300 +++ b/src/plugins/fts-lucene/fts-lucene-plugin.h Tue Apr 21 21:29:42 2015 +0300 @@ -15,6 +15,7 @@ bool normalize; bool no_snowball; bool mime_parts; + bool use_libfts; }; struct fts_lucene_user { diff -r 3f24c8cab32d -r 1ba63410b902 src/plugins/fts-lucene/lucene-wrapper.cc --- a/src/plugins/fts-lucene/lucene-wrapper.cc Tue Apr 21 21:28:41 2015 +0300 +++ b/src/plugins/fts-lucene/lucene-wrapper.cc Tue Apr 21 21:29:42 2015 +0300 @@ -77,6 +77,7 @@ Document *doc; uint32_t prev_uid, prev_part_idx; + bool no_analyzer; }; struct rescan_context { @@ -124,6 +125,9 @@ /* this is valid only for doveadm dump, so it doesn't matter */ index->set.default_language = ""; } + if (index->set.use_libfts) { + index->default_analyzer = _CLNEW KeywordAnalyzer(); + } else #ifdef HAVE_FTS_STEMMER if (set == NULL || !set->no_snowball) { index->default_analyzer = @@ -198,7 +202,7 @@ const char *whitespace_chars = index->set.whitespace_chars; unsigned int i; - if (*whitespace_chars == '\0') + if (*whitespace_chars == '\0' || index->set.use_libfts) return; for (i = 0; i < len; i++) { @@ -224,8 +228,7 @@ } static const wchar_t * -t_lucene_utf8_to_tchar(struct lucene_index *index, - const char *str, bool translate) +t_lucene_utf8_to_tchar(struct lucene_index *index, const char *str) { ARRAY_TYPE(unichars) dest_arr; const unichar_t *chars; @@ -540,10 +543,13 @@ return 0; try { - index->writer->addDocument(index->doc, - index->cur_analyzer != NULL ? - index->cur_analyzer : - index->default_analyzer); + CL_NS(analysis)::Analyzer *analyzer = NULL; + + if (!index->set.use_libfts) { + analyzer = index->cur_analyzer != NULL ? + index->cur_analyzer : index->default_analyzer; + } + index->writer->addDocument(index->doc, analyzer); } catch (CLuceneError &err) { lucene_handle_error(index, err, "IndexWriter::addDocument()"); ret = -1; @@ -578,7 +584,7 @@ index->doc->add(*_CLNEW Field(_T("box"), index->mailbox_guid, Field::STORE_YES | Field::INDEX_UNTOKENIZED)); } - if (index->normalizer_buf != NULL) { + if (index->normalizer_buf != NULL && !index->set.use_libfts) { buffer_set_used_size(index->normalizer_buf, 0); index->normalizer(data, size, index->normalizer_buf); data = (const unsigned char *)index->normalizer_buf->data; @@ -594,6 +600,8 @@ lucene_utf8_n_to_tchar(data, size, dest, datasize); lucene_data_translate(index, dest, datasize-1); + int token_flag = index->set.use_libfts ? + Field::INDEX_UNTOKENIZED : Field::INDEX_TOKENIZED; if (hdr_name != NULL) { /* hdr_name should be ASCII, but don't break in case it isn't */ hdr_name = t_str_lcase(hdr_name); @@ -601,15 +609,16 @@ wchar_t wname[namesize]; lucene_utf8_n_to_tchar((const unsigned char *)hdr_name, strlen(hdr_name), wname, namesize); - index->doc->add(*_CLNEW Field(_T("hdr"), wname, Field::STORE_NO | Field::INDEX_TOKENIZED)); - index->doc->add(*_CLNEW Field(_T("hdr"), dest, Field::STORE_NO | Field::INDEX_TOKENIZED)); + if (index->set.use_libfts) + index->doc->add(*_CLNEW Field(_T("hdr"), wname, Field::STORE_NO | token_flag)); + index->doc->add(*_CLNEW Field(_T("hdr"), dest, Field::STORE_NO | token_flag)); if (fts_header_want_indexed(hdr_name)) - index->doc->add(*_CLNEW Field(wname, dest, Field::STORE_NO | Field::INDEX_TOKENIZED)); + index->doc->add(*_CLNEW Field(wname, dest, Field::STORE_NO | token_flag)); } else if (size > 0) { - if (index->cur_analyzer == NULL) + if (index->cur_analyzer == NULL && !index->set.use_libfts) index->cur_analyzer = guess_analyzer(index, data, size); - index->doc->add(*_CLNEW Field(_T("body"), dest, Field::STORE_NO | Field::INDEX_TOKENIZED)); + index->doc->add(*_CLNEW Field(_T("body"), dest, Field::STORE_NO | token_flag)); } i_free(dest_free); return 0; @@ -1130,6 +1139,18 @@ const TCHAR *wvalue; Analyzer *analyzer; + if (index->set.use_libfts) { + const wchar_t *wstr = t_lucene_utf8_to_tchar(index, str); + Term* tm = _CLNEW Term(key, wstr); + Query* ret; + if (fuzzy) + ret = _CLNEW FuzzyQuery( tm ); + else + ret = _CLNEW TermQuery( tm ); + _CLDECDELETE(tm); + return ret; + } + if (index->normalizer_buf != NULL) { buffer_set_used_size(index->normalizer_buf, 0); index->normalizer(str, strlen(str), index->normalizer_buf); @@ -1137,7 +1158,7 @@ str = (const char *)index->normalizer_buf->data; } - wvalue = t_lucene_utf8_to_tchar(index, str, TRUE); + wvalue = t_lucene_utf8_to_tchar(index, str); analyzer = guess_analyzer(index, str, strlen(str)); if (analyzer == NULL) analyzer = index->default_analyzer; @@ -1194,7 +1215,7 @@ return false; q = lucene_get_query(index, - t_lucene_utf8_to_tchar(index, t_str_lcase(arg->hdr_field_name), FALSE), + t_lucene_utf8_to_tchar(index, t_str_lcase(arg->hdr_field_name)), arg); break; default: @@ -1236,7 +1257,7 @@ case SEARCH_HEADER: case SEARCH_HEADER_ADDRESS: case SEARCH_HEADER_COMPRESS_LWSP: - if (*arg->value.str == '\0') { + if (*arg->value.str == '\0' && !index->set.use_libfts) { /* checking potential existence of the header name */ q = lucene_get_query_str(index, _T("hdr"), t_str_lcase(arg->hdr_field_name), FALSE); From dovecot at dovecot.org Tue Apr 21 20:01:43 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 21 Apr 2015 20:01:43 +0000 Subject: dovecot-2.2: fts-lucene: Fix to earlier commit: Header names mus... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/7a0d3453600a changeset: 18446:7a0d3453600a user: Timo Sirainen date: Tue Apr 21 23:00:08 2015 +0300 description: fts-lucene: Fix to earlier commit: Header names must be indexed without libfts, not with. diffstat: src/plugins/fts-lucene/lucene-wrapper.cc | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 1ba63410b902 -r 7a0d3453600a src/plugins/fts-lucene/lucene-wrapper.cc --- a/src/plugins/fts-lucene/lucene-wrapper.cc Tue Apr 21 21:29:42 2015 +0300 +++ b/src/plugins/fts-lucene/lucene-wrapper.cc Tue Apr 21 23:00:08 2015 +0300 @@ -609,7 +609,7 @@ wchar_t wname[namesize]; lucene_utf8_n_to_tchar((const unsigned char *)hdr_name, strlen(hdr_name), wname, namesize); - if (index->set.use_libfts) + if (!index->set.use_libfts) index->doc->add(*_CLNEW Field(_T("hdr"), wname, Field::STORE_NO | token_flag)); index->doc->add(*_CLNEW Field(_T("hdr"), dest, Field::STORE_NO | token_flag)); From dovecot at dovecot.org Wed Apr 22 15:18:46 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 22 Apr 2015 15:18:46 +0000 Subject: dovecot-2.2: lib-storage: Added support for subscriptions file v... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/faafabb58036 changeset: 18447:faafabb58036 user: Timo Sirainen date: Wed Apr 22 18:17:10 2015 +0300 description: lib-storage: Added support for subscriptions file version 2 format. The v2 format's main benefit is that it doesn't write the hierarchy separator to the subscriptions file, which allows the separator to be changed without breaking subscriptions. This mainly affects LAYOUT=INDEX, which doesn't have a native hardcoded separator. The plan is to start writing v2 subscription files in Dovecot v2.3. So for now we simply read such files and if we find v2 file we also modify it in v2 format, but we never create new v2 format files or convert v0 to v2. diffstat: src/lib-storage/list/subscription-file.c | 91 +++++++++++++++++++++++++++++-- 1 files changed, 83 insertions(+), 8 deletions(-) diffs (184 lines): diff -r 7a0d3453600a -r faafabb58036 src/lib-storage/list/subscription-file.c --- a/src/lib-storage/list/subscription-file.c Tue Apr 21 23:00:08 2015 +0300 +++ b/src/lib-storage/list/subscription-file.c Wed Apr 22 18:17:10 2015 +0300 @@ -1,6 +1,8 @@ /* Copyright (c) 2002-2015 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "str.h" +#include "strescape.h" #include "istream.h" #include "ostream.h" #include "nfs-workarounds.h" @@ -20,10 +22,14 @@ struct mailbox_list *list; struct istream *input; char *path; + string_t *name; + unsigned int version; bool failed; }; +static const char version2_header[] = "V\t2\n\n"; + static void subsread_set_syscall_error(struct mailbox_list *list, const char *function, const char *path) { @@ -50,6 +56,29 @@ } } +static void +subsfile_list_read_header(struct mailbox_list *list, struct istream *input, + unsigned int *version_r) +{ + const unsigned char version2_header_len = strlen(version2_header); + const unsigned char *data; + size_t size; + int ret; + + *version_r = 0; + + ret = i_stream_read_data(input, &data, &size, version2_header_len-1); + if (ret < 0) { + i_assert(ret == -1); + subswrite_set_syscall_error(list, "read()", i_stream_get_name(input)); + return; + } + if (memcmp(data, version2_header, version2_header_len) == 0) { + *version_r = 2; + i_stream_skip(input, version2_header_len); + } +} + static const char *next_line(struct mailbox_list *list, const char *path, struct istream *input, bool *failed_r, bool ignore_estale) @@ -83,18 +112,20 @@ } int subsfile_set_subscribed(struct mailbox_list *list, const char *path, - const char *temp_prefix, const char *name, bool set) + const char *temp_prefix, const char *name, + bool set) { const struct mail_storage_settings *mail_set = list->mail_set; struct dotlock_settings dotlock_set; struct dotlock *dotlock; struct mailbox_permissions perm; - const char *line, *dir, *fname; - struct istream *input; + const char *line, *dir, *fname, *escaped_name; + struct istream *input = NULL; struct ostream *output; int fd_in, fd_out; enum mailbox_list_path_type type; bool found, changed = FALSE, failed = FALSE; + unsigned int version = 0; if (strcasecmp(name, "INBOX") == 0) name = "INBOX"; @@ -145,15 +176,38 @@ file_dotlock_delete(&dotlock); return -1; } + if (fd_in != -1) { + input = i_stream_create_fd_autoclose(&fd_in, list->mailbox_name_max_length+1); + i_stream_set_return_partial_line(input, TRUE); + subsfile_list_read_header(list, input, &version); + } found = FALSE; output = o_stream_create_fd_file(fd_out, 0, FALSE); o_stream_cork(output); - if (fd_in != -1) { - input = i_stream_create_fd_autoclose(&fd_in, list->mailbox_name_max_length+1); + if (version >= 2) + o_stream_send_str(output, version2_header); + if (version < 2 || name[0] == '\0') + escaped_name = name; + else { + const char *const *tmp; + char separators[2]; + string_t *str = t_str_new(64); + + separators[0] = mailbox_list_get_hierarchy_sep(list); + separators[1] = '\0'; + tmp = t_strsplit(name, separators); + str_append_tabescaped(str, *tmp); + for (tmp++; *tmp != NULL; tmp++) { + str_append_c(str, '\t'); + str_append_tabescaped(str, *tmp); + } + escaped_name = str_c(str); + } + if (input != NULL) { while ((line = next_line(list, path, input, &failed, FALSE)) != NULL) { - if (strcmp(line, name) == 0) { + if (strcmp(line, escaped_name) == 0) { found = TRUE; if (!set) { changed = TRUE; @@ -169,7 +223,7 @@ if (!failed && set && !found) { /* append subscription */ - line = t_strconcat(name, "\n", NULL); + line = t_strconcat(escaped_name, "\n", NULL); o_stream_nsend_str(output, line); changed = TRUE; } @@ -227,8 +281,10 @@ ctx->input = i_stream_create_fd_autoclose(&fd, list->mailbox_name_max_length+1); i_stream_set_return_partial_line(ctx->input, TRUE); + subsfile_list_read_header(ctx->list, ctx->input, &ctx->version); } ctx->path = i_strdup(path); + ctx->name = str_new(default_pool, 128); return ctx; } @@ -241,6 +297,7 @@ if (ctx->input != NULL) i_stream_destroy(&ctx->input); + str_free(&ctx->name); i_free(ctx->path); i_free(ctx); return ret; @@ -261,6 +318,21 @@ return 0; } +static const char * +subsfile_list_unescaped(struct subsfile_list_context *ctx, const char *line) +{ + const char *p; + + str_truncate(ctx->name, 0); + while ((p = strchr(line, '\t')) != NULL) { + str_append_tabunescaped(ctx->name, line, p-line); + str_append_c(ctx->name, mailbox_list_get_hierarchy_sep(ctx->list)); + line = p+1; + } + str_append_tabunescaped(ctx->name, line, strlen(line)); + return str_c(ctx->name); +} + const char *subsfile_list_next(struct subsfile_list_context *ctx) { const char *line; @@ -299,6 +371,9 @@ ctx->input = i_stream_create_fd_autoclose(&fd, ctx->list->mailbox_name_max_length+1); i_stream_set_return_partial_line(ctx->input, TRUE); - } + } + + if (ctx->version > 1 && line != NULL) + line = subsfile_list_unescaped(ctx, line); return line; } From pigeonhole at rename-it.nl Wed Apr 22 19:44:54 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 22 Apr 2015 21:44:54 +0200 Subject: dovecot-2.2-pigeonhole: LDA Sieve plugin: Changed error handling... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/c9c16230501e changeset: 2037:c9c16230501e user: Stephan Bosch date: Wed Apr 22 21:36:29 2015 +0200 description: LDA Sieve plugin: Changed error handling so that each action log message can have up-to-date time stamps. This change matches a change in LDA that is aimed at logging timing statistics. diffstat: src/lib-sieve/plugins/enotify/cmd-notify.c | 2 +- src/lib-sieve/sieve-result.c | 61 +++--------------- src/lib-sieve/sieve-result.h | 15 +-- src/lib-sieve/sieve-types.h | 3 - src/lib-sieve/sieve.c | 64 ++++++++---------- src/lib-sieve/sieve.h | 18 +++-- src/plugins/lda-sieve/Makefile.am | 2 + src/plugins/lda-sieve/lda-sieve-log.c | 97 ++++++++++++++++++++++++++++++ src/plugins/lda-sieve/lda-sieve-log.h | 11 +++ src/plugins/lda-sieve/lda-sieve-plugin.c | 52 +++++++++++---- src/sieve-tools/sieve-filter.c | 10 ++- src/sieve-tools/sieve-test.c | 31 ++++++--- src/testsuite/testsuite-result.c | 7 +- src/testsuite/testsuite-script.c | 4 +- 14 files changed, 241 insertions(+), 136 deletions(-) diffs (truncated from 868 to 300 lines): diff -r 2ebd3bd76e1e -r c9c16230501e src/lib-sieve/plugins/enotify/cmd-notify.c --- a/src/lib-sieve/plugins/enotify/cmd-notify.c Sat Mar 28 10:49:23 2015 +0100 +++ b/src/lib-sieve/plugins/enotify/cmd-notify.c Wed Apr 22 21:36:29 2015 +0200 @@ -523,7 +523,7 @@ nenv.svinst = renv->svinst; nenv.method = nact->method; nenv.ehandler = sieve_prefix_ehandler_create - (sieve_result_get_error_handler(renv->result), act->location, "notify"); + (renv->ehandler, act->location, "notify"); result = nmth_def->action_check_duplicates(&nenv, nact, nact_other); diff -r 2ebd3bd76e1e -r c9c16230501e src/lib-sieve/sieve-result.c --- a/src/lib-sieve/sieve-result.c Sat Mar 28 10:49:23 2015 +0100 +++ b/src/lib-sieve/sieve-result.c Wed Apr 22 21:36:29 2015 +0200 @@ -26,12 +26,6 @@ #include /* - * Defaults - */ - -#define DEFAULT_ACTION_LOG_FORMAT "msgid=%m: %$" - -/* * Types */ @@ -79,8 +73,6 @@ /* Context data for extensions */ ARRAY(void *) ext_contexts; - struct sieve_error_handler *ehandler; - struct sieve_action_exec_env action_env; struct sieve_action keep_action; @@ -100,7 +92,7 @@ struct sieve_result *sieve_result_create (struct sieve_instance *svinst, const struct sieve_message_data *msgdata, - const struct sieve_script_env *senv, struct sieve_error_handler *ehandler) + const struct sieve_script_env *senv) { pool_t pool; struct sieve_result *result; @@ -113,10 +105,6 @@ p_array_init(&result->ext_contexts, pool, 4); - if ( ehandler != NULL ) - sieve_error_handler_ref(ehandler); - result->ehandler = ehandler; - result->action_env.svinst = svinst; result->action_env.result = result; result->action_env.scriptenv = senv; @@ -153,9 +141,6 @@ if ( hash_table_is_created((*result)->action_contexts) ) hash_table_destroy(&(*result)->action_contexts); - if ( (*result)->ehandler != NULL ) - sieve_error_handler_unref(&(*result)->ehandler); - if ( (*result)->action_env.ehandler != NULL ) sieve_error_handler_unref(&(*result)->action_env.ehandler); @@ -173,11 +158,6 @@ * Getters/Setters */ -struct sieve_error_handler *sieve_result_get_error_handler -(struct sieve_result *result) -{ - return result->ehandler; -} const struct sieve_script_env *sieve_result_get_script_env (struct sieve_result *result) { @@ -196,16 +176,6 @@ return result->action_env.msgctx; } -void sieve_result_set_error_handler -(struct sieve_result *result, struct sieve_error_handler *ehandler) -{ - if ( result->ehandler != ehandler ) { - sieve_error_handler_ref(ehandler); - sieve_error_handler_unref(&result->ehandler); - result->ehandler = ehandler; - } -} - /* * Extension support */ @@ -939,27 +909,15 @@ * Result execution */ -static void _sieve_result_prepare_execution(struct sieve_result *result) +static void _sieve_result_prepare_execution(struct sieve_result *result, + struct sieve_error_handler *ehandler) { - const struct sieve_message_data *msgdata = result->action_env.msgdata; const struct sieve_script_env *senv = result->action_env.scriptenv; - const struct var_expand_table *tab; - tab = mail_deliver_get_log_var_expand_table(msgdata->mail, NULL); + result->action_env.ehandler = ehandler; result->action_env.exec_status = ( senv->exec_status == NULL ? t_new(struct sieve_exec_status, 1) : senv->exec_status ); - - if ( result->action_env.ehandler != NULL ) - sieve_error_handler_unref(&result->action_env.ehandler); - - if ( senv->action_log_format != NULL ) { - result->action_env.ehandler = sieve_varexpand_ehandler_create - (result->ehandler, senv->action_log_format, tab); - } else { - result->action_env.ehandler = sieve_varexpand_ehandler_create - (result->ehandler, DEFAULT_ACTION_LOG_FORMAT, tab); - } } static int _sieve_result_implicit_keep @@ -1072,9 +1030,10 @@ } int sieve_result_implicit_keep -(struct sieve_result *result) +(struct sieve_result *result, + struct sieve_error_handler *ehandler) { - _sieve_result_prepare_execution(result); + _sieve_result_prepare_execution(result, ehandler); return _sieve_result_implicit_keep(result, TRUE); } @@ -1357,7 +1316,8 @@ } int sieve_result_execute -(struct sieve_result *result, bool *keep) +(struct sieve_result *result, bool *keep, + struct sieve_error_handler *ehandler) { int status = SIEVE_EXEC_OK, result_status; struct sieve_result_action *first_action, *last_action; @@ -1368,7 +1328,7 @@ /* Prepare environment */ - _sieve_result_prepare_execution(result); + _sieve_result_prepare_execution(result, ehandler); /* Make notice of this attempt */ @@ -1424,6 +1384,7 @@ sieve_result_transaction_finish (result, first_action, status); + result->action_env.ehandler = NULL; return result_status; } diff -r 2ebd3bd76e1e -r c9c16230501e src/lib-sieve/sieve-result.h --- a/src/lib-sieve/sieve-result.h Sat Mar 28 10:49:23 2015 +0100 +++ b/src/lib-sieve/sieve-result.h Wed Apr 22 21:36:29 2015 +0200 @@ -21,8 +21,7 @@ struct sieve_result *sieve_result_create (struct sieve_instance *svinst, const struct sieve_message_data *msgdata, - const struct sieve_script_env *senv, - struct sieve_error_handler *ehandler); + const struct sieve_script_env *senv); void sieve_result_ref(struct sieve_result *result); @@ -34,11 +33,6 @@ * Getters/Setters */ -struct sieve_error_handler *sieve_result_get_error_handler - (struct sieve_result *result); -void sieve_result_set_error_handler - (struct sieve_result *result, struct sieve_error_handler *ehandler); - const struct sieve_script_env *sieve_result_get_script_env (struct sieve_result *result); const struct sieve_message_data *sieve_result_get_message_data @@ -148,11 +142,14 @@ * Result execution */ -int sieve_result_implicit_keep(struct sieve_result *result); +int sieve_result_implicit_keep(struct sieve_result *result, + struct sieve_error_handler *ehandler); void sieve_result_mark_executed(struct sieve_result *result); -int sieve_result_execute(struct sieve_result *result, bool *keep); +int sieve_result_execute + (struct sieve_result *result, bool *keep, + struct sieve_error_handler *ehandler); bool sieve_result_executed(struct sieve_result *result); diff -r 2ebd3bd76e1e -r c9c16230501e src/lib-sieve/sieve-types.h --- a/src/lib-sieve/sieve-types.h Sat Mar 28 10:49:23 2015 +0100 +++ b/src/lib-sieve/sieve-types.h Wed Apr 22 21:36:29 2015 +0200 @@ -173,9 +173,6 @@ */ struct sieve_script_env { - /* Logging related */ - const char *action_log_format; - /* Mail-related */ struct mail_user *user; const char *default_mailbox; diff -r 2ebd3bd76e1e -r c9c16230501e src/lib-sieve/sieve.c --- a/src/lib-sieve/sieve.c Sat Mar 28 10:49:23 2015 +0100 +++ b/src/lib-sieve/sieve.c Wed Apr 22 21:36:29 2015 +0200 @@ -333,11 +333,9 @@ memset(senv->exec_status, 0, sizeof(*senv->exec_status)); /* Create result object */ - if ( *result == NULL ) + if ( *result == NULL ) { *result = sieve_result_create - (sieve_binary_svinst(sbin), msgdata, senv, ehandler); - else { - sieve_result_set_error_handler(*result, ehandler); + (sieve_binary_svinst(sbin), msgdata, senv); } /* Run the interpreter */ @@ -539,7 +537,9 @@ int sieve_execute (struct sieve_binary *sbin, const struct sieve_message_data *msgdata, - const struct sieve_script_env *senv, struct sieve_error_handler *ehandler, + const struct sieve_script_env *senv, + struct sieve_error_handler *exec_ehandler, + struct sieve_error_handler *action_ehandler, enum sieve_runtime_flags flags, bool *keep) { struct sieve_result *result = NULL; @@ -548,7 +548,7 @@ if ( keep != NULL ) *keep = FALSE; /* Run the script */ - ret = sieve_run(sbin, &result, msgdata, senv, ehandler, flags); + ret = sieve_run(sbin, &result, msgdata, senv, exec_ehandler, flags); /* Evaluate status and execute the result: * Strange situations, e.g. currupt binaries, must be handled by the caller. @@ -557,10 +557,11 @@ */ if ( ret > 0 ) { /* Execute result */ - ret = sieve_result_execute(result, keep); + ret = sieve_result_execute(result, keep, action_ehandler); } else if ( ret == SIEVE_EXEC_FAILURE ) { /* Perform implicit keep if script failed with a normal runtime error */ - switch ( sieve_result_implicit_keep(result) ) { + switch ( sieve_result_implicit_keep + (result, action_ehandler) ) { case SIEVE_EXEC_OK: if ( keep != NULL ) *keep = TRUE; break; @@ -604,7 +605,7 @@ struct sieve_result *result; struct sieve_multiscript *mscript; - result = sieve_result_create(svinst, msgdata, senv, NULL); + result = sieve_result_create(svinst, msgdata, senv); pool = sieve_result_pool(result); sieve_result_set_keep_action(result, NULL, NULL); @@ -634,11 +635,8 @@ } static void sieve_multiscript_test -(struct sieve_multiscript *mscript, struct sieve_error_handler *ehandler, - bool *keep) +(struct sieve_multiscript *mscript, bool *keep) { - sieve_result_set_error_handler(mscript->result, ehandler); - if ( mscript->status > 0 ) { mscript->status = sieve_result_print (mscript->result, mscript->scriptenv, mscript->teststream, keep); @@ -653,12 +651,10 @@ (struct sieve_multiscript *mscript, struct sieve_error_handler *ehandler, bool *keep) { - sieve_result_set_error_handler(mscript->result, ehandler); From pigeonhole at rename-it.nl Wed Apr 22 19:44:55 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Wed, 22 Apr 2015 21:44:55 +0200 Subject: dovecot-2.2-pigeonhole: Merged concurrent changes. Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/6926e54e8817 changeset: 2038:6926e54e8817 user: Stephan Bosch date: Wed Apr 22 21:43:33 2015 +0200 description: Merged concurrent changes. diffstat: INSTALL | 60 ++++++++++++++--------- doc/example-config/conf.d/90-sieve.conf | 2 +- src/lib-sieve/plugins/editheader/cmd-addheader.c | 2 +- 3 files changed, 39 insertions(+), 25 deletions(-) diffs (102 lines): diff -r c9c16230501e -r 6926e54e8817 INSTALL --- a/INSTALL Wed Apr 22 21:36:29 2015 +0200 +++ b/INSTALL Wed Apr 22 21:43:33 2015 +0200 @@ -565,28 +565,43 @@ plugin section of the config file. These settings are the same ones used by the LDA Sieve plugin. - sieve_dir = ~/sieve - This specifies the path to the directory where the uploaded scripts are - stored. Scripts are stored as separate files with extension '.sieve'. All - other files are ignored when scripts are listed by a ManageSieve client. The - Sieve interpreter also uses this setting to locate the user's personal - scripts for use with the Sieve include extension. A storage location - specified by sieve_dir is always generated automatically if it does not exist - (as far as the system permits the user to do so; no root privileges are - used). This is similar to the behavior of the mail daemons regarding the - mail_location configuration. + sieve = file:~/sieve;active=~/.dovecot.sieve + This specifies the location where the scripts that are uploaded through + ManageSieve are stored. During delivery, the LDA Sieve plugin uses this + location setting to find the active script for Sieve filtering. The Sieve + "include" extension uses this location for retrieving ":personal" scripts. + If the location type does not allow uploading scripts, the ManageSieve + service cannot be used. Currently, only the 'file' location type supports + ManageSieve. - sieve = ~/.dovecot.sieve - This specifies the location of the symbolic link pointing to the active - script in the Sieve storage directory. The Sieve interpreter uses this - setting to locate the main script file that needs to be executed upon - delivery. When using ManageSieve, this is a symbolic link managed by the - ManageSieve service. ManageSieve thereby determines which script (if any) in - the sieve_dir directory is executed for incoming messages. If a regular file - already exists at the location specified by the sieve setting, it is moved to - the sieve_dir location before the symbolic link is installed. It is renamed - to dovecot.orig.sieve and therefore listed as `dovecot.orig' by a ManageSieve - client. + For the 'file' location type: + + * The location is the path to the storage directory for all the user's + personal Sieve scripts. Scripts are stored as separate files with + extension .sieve. All other files are ignored when scripts are listed + by a ManageSieve client. The storage directory is always generated + automatically if it does not exist (as far as the system permits the + user to do so; no root privileges are used). This is similar to the + behavior of the mail daemons regarding the mail_location configuration. + + * ManageSieve maintains a symbolic link pointing to the currently active + script (the script executed at delivery). The location of this symbolic + link can be configured using the ;active= option. If a regular + file already exists at the location specified by in the ;active= + location option, it is moved to the storage directory before the + symbolic link is installed. It is renamed to dovecot.orig.sieve and + therefore listed as dovecot.orig by a ManageSieve client. + + NOTE: It is not wise to place this active symbolic link inside your + mail store, as it may be mistaken for a mail folder. Inside a + maildir for instance, the default .dovecot.sieve would show up + as phantom folder /dovecot/sieve in your IMAP tree. + + For Pigeonhole versions before v0.3.1, this setting can only be a + filesystem path pointing to a script file, or - when ManageSieve is used - + it is the location of the symbolic link pointing to the active script in + the storage directory. That storage directory is then configured using the + deprecated `sieve_dir' setting. The following provides an example configuration for Sieve and ManageSieve. Only sections and settings relevant to ManageSieve are shown. Refer to @@ -614,8 +629,7 @@ # *** conf.d/90-sieve.conf *** plugin { - sieve=~/.dovecot.sieve - sieve_dir=~/sieve + sieve=file:~/sieve;active=~/.dovecot.sieve } # *** dovecot.conf *** diff -r c9c16230501e -r 6926e54e8817 doc/example-config/conf.d/90-sieve.conf --- a/doc/example-config/conf.d/90-sieve.conf Wed Apr 22 21:36:29 2015 +0200 +++ b/doc/example-config/conf.d/90-sieve.conf Wed Apr 22 21:43:33 2015 +0200 @@ -9,7 +9,7 @@ # locations. The default `file' location type is a local filesystem path # pointing to a Sieve script file or a directory containing multiple Sieve # script files. More complex setups can use other location types such as -# `ldap'or `dict' to fetch Sieve scripts from remote databases. +# `ldap' or `dict' to fetch Sieve scripts from remote databases. # # All settings that specify the location of one ore more Sieve scripts accept # the following syntax: diff -r c9c16230501e -r 6926e54e8817 src/lib-sieve/plugins/editheader/cmd-addheader.c --- a/src/lib-sieve/plugins/editheader/cmd-addheader.c Wed Apr 22 21:36:29 2015 +0200 +++ b/src/lib-sieve/plugins/editheader/cmd-addheader.c Wed Apr 22 21:43:33 2015 +0200 @@ -76,7 +76,7 @@ (const struct sieve_runtime_env *renv, sieve_size_t *address); const struct sieve_operation_def addheader_operation = { - "addheader", + "ADDHEADER", &editheader_extension, EXT_EDITHEADER_OPERATION_ADDHEADER, cmd_addheader_operation_dump, From pigeonhole at rename-it.nl Thu Apr 23 00:04:48 2015 From: pigeonhole at rename-it.nl (pigeonhole at rename-it.nl) Date: Thu, 23 Apr 2015 02:04:48 +0200 Subject: dovecot-2.2-pigeonhole: lib-sieve: util: Fixed bug in t_realpath... Message-ID: details: http://hg.rename-it.nl/dovecot-2.2-pigeonhole/rev/dd430dfa2ddb changeset: 2039:dd430dfa2ddb user: Stephan Bosch date: Thu Apr 23 02:04:38 2015 +0200 description: lib-sieve: util: Fixed bug in t_realpath() normalization occuring with relative symlinks below root. Also contains a few small code cleanups. diffstat: src/lib-sieve/util/realpath.c | 16 ++++++++-------- 1 files changed, 8 insertions(+), 8 deletions(-) diffs (60 lines): diff -r 6926e54e8817 -r dd430dfa2ddb src/lib-sieve/util/realpath.c --- a/src/lib-sieve/util/realpath.c Wed Apr 22 21:43:33 2015 +0200 +++ b/src/lib-sieve/util/realpath.c Thu Apr 23 02:04:38 2015 +0200 @@ -42,13 +42,13 @@ const char *p; size_t asize; - if (path[0] != '/') { + if (path[0] != '/') { /* relative; initialize npath with current directory */ if (t_getcwd_alloc(&npath, &asize) < 0) return -1; npath_pos = npath + strlen(npath); i_assert(npath[0] == '/'); - } else { + } else { /* absolute; initialize npath with root */ asize = 128; npath = t_buffer_get(asize); @@ -77,9 +77,9 @@ } else if (seglen == 2 && p[0] == '.' && p[1] == '.') { /* a reference to parent segment; back up to previous slash */ if (npath_pos > npath + 1) { - if (*npath_pos == '/') + if (*(npath_pos-1) == '/') npath_pos--; - for (; *npath_pos != '/'; npath_pos--); + for (; *(npath_pos-1) != '/'; npath_pos--); } } else { /* make sure npath now ends in slash */ @@ -95,7 +95,7 @@ } /* copy segment to normalized path */ - (void)memmove(npath_pos, p, segend - p); + (void)memmove(npath_pos, p, seglen); npath_pos += seglen; } @@ -175,16 +175,16 @@ /* use as new source path */ path = segend = npath_link; - + if (path[0] == '/') { /* absolute symlink; start over at root */ npath_pos = npath + 1; } else { /* relative symlink; back up to previous segment */ if (npath_pos > npath + 1) { - if (*npath_pos == '/') + if (*(npath_pos-1) == '/') npath_pos--; - for (; *npath_pos != '/'; npath_pos--); + for (; *(npath_pos-1) != '/'; npath_pos--); } } From dovecot at dovecot.org Thu Apr 23 08:22:32 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 23 Apr 2015 08:22:32 +0000 Subject: dovecot-2.2: lib-storage: Fixed subscription file reading when f... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fedd88ce9315 changeset: 18448:fedd88ce9315 user: Timo Sirainen date: Thu Apr 23 11:20:53 2015 +0300 description: lib-storage: Fixed subscription file reading when file was (nearly) empty. Broken by the earlier changes. diffstat: src/lib-storage/list/subscription-file.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diffs (18 lines): diff -r faafabb58036 -r fedd88ce9315 src/lib-storage/list/subscription-file.c --- a/src/lib-storage/list/subscription-file.c Wed Apr 22 18:17:10 2015 +0300 +++ b/src/lib-storage/list/subscription-file.c Thu Apr 23 11:20:53 2015 +0300 @@ -70,10 +70,12 @@ ret = i_stream_read_data(input, &data, &size, version2_header_len-1); if (ret < 0) { i_assert(ret == -1); - subswrite_set_syscall_error(list, "read()", i_stream_get_name(input)); + if (input->stream_errno != 0) + subswrite_set_syscall_error(list, "read()", i_stream_get_name(input)); return; } - if (memcmp(data, version2_header, version2_header_len) == 0) { + if (ret > 0 && + memcmp(data, version2_header, version2_header_len) == 0) { *version_r = 2; i_stream_skip(input, version2_header_len); } From dovecot at dovecot.org Thu Apr 23 08:41:38 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 23 Apr 2015 08:41:38 +0000 Subject: dovecot-2.2: lib: iostream_rawlog_create_from_stream() now allow... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/49dd781d8f26 changeset: 18449:49dd781d8f26 user: Timo Sirainen date: Thu Apr 23 11:40:03 2015 +0300 description: lib: iostream_rawlog_create_from_stream() now allows input or output to be NULL. diffstat: src/lib/iostream-rawlog.c | 20 ++++++++++++-------- 1 files changed, 12 insertions(+), 8 deletions(-) diffs (29 lines): diff -r fedd88ce9315 -r 49dd781d8f26 src/lib/iostream-rawlog.c --- a/src/lib/iostream-rawlog.c Thu Apr 23 11:20:53 2015 +0300 +++ b/src/lib/iostream-rawlog.c Thu Apr 23 11:40:03 2015 +0300 @@ -285,13 +285,17 @@ struct istream *old_input; struct ostream *old_output; - old_input = *input; - old_output = *output; + if (input != NULL) { + old_input = *input; + *input = i_stream_create_rawlog_from_stream(old_input, + rawlog_output, rawlog_flags); + i_stream_unref(&old_input); + } + if (output != NULL) { + old_output = *output; + *output = o_stream_create_rawlog_from_stream(old_output, + rawlog_output, rawlog_flags); + o_stream_unref(&old_output); + } o_stream_ref(rawlog_output); - *input = i_stream_create_rawlog_from_stream(old_input, rawlog_output, - rawlog_flags); - *output = o_stream_create_rawlog_from_stream(old_output, rawlog_output, - rawlog_flags); - i_stream_unref(&old_input); - o_stream_unref(&old_output); } From dovecot at dovecot.org Thu Apr 23 10:03:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 23 Apr 2015 10:03:25 +0000 Subject: dovecot-2.2: lib: Fixed reference counting in iostream-rawlog if... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c3acdfc64a0b changeset: 18450:c3acdfc64a0b user: Timo Sirainen date: Thu Apr 23 13:01:50 2015 +0300 description: lib: Fixed reference counting in iostream-rawlog if either i/ostream wasn't used. diffstat: src/lib/iostream-rawlog.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (11 lines): diff -r 49dd781d8f26 -r c3acdfc64a0b src/lib/iostream-rawlog.c --- a/src/lib/iostream-rawlog.c Thu Apr 23 11:40:03 2015 +0300 +++ b/src/lib/iostream-rawlog.c Thu Apr 23 13:01:50 2015 +0300 @@ -297,5 +297,6 @@ rawlog_output, rawlog_flags); o_stream_unref(&old_output); } - o_stream_ref(rawlog_output); + if (input != NULL && output != NULL) + o_stream_ref(rawlog_output); } From dovecot at dovecot.org Thu Apr 23 10:28:52 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 23 Apr 2015 10:28:52 +0000 Subject: dovecot-2.2: lib-storage: Small code cleanup for index_mail_get_... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/de2023e630cb changeset: 18451:de2023e630cb user: Timo Sirainen date: Thu Apr 23 13:27:17 2015 +0300 description: lib-storage: Small code cleanup for index_mail_get_binary_stream() stream_r is always non-NULL here. Also move unreferencing cache istream to be last in case it might fix data.stream=NULL being here in some situations. diffstat: src/lib-storage/index/index-mail-binary.c | 23 ++++++++++------------- 1 files changed, 10 insertions(+), 13 deletions(-) diffs (37 lines): diff -r c3acdfc64a0b -r de2023e630cb src/lib-storage/index/index-mail-binary.c --- a/src/lib-storage/index/index-mail-binary.c Thu Apr 23 13:01:50 2015 +0300 +++ b/src/lib-storage/index/index-mail-binary.c Thu Apr 23 13:27:17 2015 +0300 @@ -571,23 +571,20 @@ } *size_r = cache->size; *binary_r = binary; - if (stream_r != NULL) { - i_stream_ref(cache->input); - *stream_r = cache->input; - } if (!converted) { /* don't keep this cached. it's exactly the same as the original stream */ + i_assert(mail->data.stream != NULL); + i_stream_seek(mail->data.stream, part->physical_pos + + (include_hdr ? 0 : + part->header_size.physical_size)); + input = i_stream_create_crlf(mail->data.stream); + *stream_r = i_stream_create_limit(input, *size_r); + i_stream_unref(&input); mail_storage_free_binary_cache(_mail->box->storage); - if (stream_r != NULL) { - i_stream_unref(stream_r); - i_stream_seek(mail->data.stream, part->physical_pos + - (include_hdr ? 0 : - part->header_size.physical_size)); - input = i_stream_create_crlf(mail->data.stream); - *stream_r = i_stream_create_limit(input, *size_r); - i_stream_unref(&input); - } + } else { + *stream_r = cache->input; + i_stream_ref(cache->input); } return 0; } From dovecot at dovecot.org Thu Apr 23 10:38:57 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 23 Apr 2015 10:38:57 +0000 Subject: dovecot-2.2: imapc: After auth failed, return failure immediatel... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/d64fbe56a742 changeset: 18452:d64fbe56a742 user: Timo Sirainen date: Thu Apr 23 13:37:22 2015 +0300 description: imapc: After auth failed, return failure immediately when opening a mailbox. Instead of trying to re-authenticate which again will most likely fail. Based on patch by Michael M Slusarz diffstat: src/lib-storage/index/imapc/imapc-storage.c | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diffs (16 lines): diff -r de2023e630cb -r d64fbe56a742 src/lib-storage/index/imapc/imapc-storage.c --- a/src/lib-storage/index/imapc/imapc-storage.c Thu Apr 23 13:27:17 2015 +0300 +++ b/src/lib-storage/index/imapc/imapc-storage.c Thu Apr 23 13:37:22 2015 +0300 @@ -509,6 +509,12 @@ i_assert(mbox->client_box == NULL); + /* If authentication failed, don't check again. */ + if (mbox->storage->client->auth_failed) { + mail_storage_set_internal_error(&mbox->storage->storage); + return -1; + } + mbox->client_box = imapc_client_mailbox_open(mbox->storage->client->client, mbox); imapc_client_mailbox_set_reopen_cb(mbox->client_box, From dovecot at dovecot.org Thu Apr 23 13:34:54 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 23 Apr 2015 13:34:54 +0000 Subject: dovecot-2.2: lib-storage: Fixed assert-crash with mailbox_list_i... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b4a81e123f1a changeset: 18453:b4a81e123f1a user: Timo Sirainen date: Thu Apr 23 16:33:19 2015 +0300 description: lib-storage: Fixed assert-crash with mailbox_list_index=yes if root mailbox name was empty. This happened if the index was used for a non-listable namespace whose prefix started with the namespace separator (e.g. prefix="/expunged/") diffstat: src/lib-storage/list/mailbox-list-index-iter.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (15 lines): diff -r d64fbe56a742 -r b4a81e123f1a src/lib-storage/list/mailbox-list-index-iter.c --- a/src/lib-storage/list/mailbox-list-index-iter.c Thu Apr 23 13:37:22 2015 +0300 +++ b/src/lib-storage/list/mailbox-list-index-iter.c Thu Apr 23 16:33:19 2015 +0300 @@ -73,7 +73,10 @@ p_clear(ctx->info_pool); str_truncate(ctx->path, ctx->parent_len); - if (str_len(ctx->path) > 0) { + /* the root directory may have an empty name. in that case we'll still + want to insert the separator, so check for non-NULL parent rather + than non-empty path. */ + if (node->parent != NULL) { str_append_c(ctx->path, mailbox_list_get_hierarchy_sep(ctx->ctx.list)); } From dovecot at dovecot.org Thu Apr 23 16:21:35 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 23 Apr 2015 16:21:35 +0000 Subject: dovecot-2.2: lib-imap: Added imap_to_date() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/274b42b7d0de changeset: 18454:274b42b7d0de user: Timo Sirainen date: Thu Apr 23 19:19:10 2015 +0300 description: lib-imap: Added imap_to_date() diffstat: src/lib-imap/imap-date.c | 30 ++++++++++++++++++++++++------ src/lib-imap/imap-date.h | 5 +++++ 2 files changed, 29 insertions(+), 6 deletions(-) diffs (67 lines): diff -r b4a81e123f1a -r 274b42b7d0de src/lib-imap/imap-date.c --- a/src/lib-imap/imap-date.c Thu Apr 23 16:33:19 2015 +0300 +++ b/src/lib-imap/imap-date.c Thu Apr 23 19:19:10 2015 +0300 @@ -158,15 +158,10 @@ return TRUE; } -static const char * -imap_to_datetime_tm(const struct tm *tm, int timezone_offset) +static void imap_to_date_tm(char buf[11], const struct tm *tm) { - char *buf; int year; - /* @UNSAFE: but faster than t_strdup_printf() call.. */ - buf = t_malloc(27); - /* dd-mon- */ buf[0] = (tm->tm_mday / 10) + '0'; buf[1] = (tm->tm_mday % 10) + '0'; @@ -180,6 +175,16 @@ buf[8] = ((year / 100) % 10) + '0'; buf[9] = ((year / 10) % 10) + '0'; buf[10] = (year % 10) + '0'; +} + +static const char * +imap_to_datetime_tm(const struct tm *tm, int timezone_offset) +{ + char *buf; + + /* @UNSAFE: but faster than t_strdup_printf() call.. */ + buf = t_malloc(27); + imap_to_date_tm(buf, tm); buf[11] = ' '; /* hh:mi:ss */ @@ -227,3 +232,16 @@ tm = gmtime(&adjusted); return imap_to_datetime_tm(tm, timezone_offset); } + +bool imap_to_date(time_t timestamp, const char **str_r) +{ + const struct tm *tm; + char *buf; + + tm = localtime(×tamp); + + buf = t_malloc0(12); + imap_to_date_tm(buf, tm); + *str_r = buf; + return tm->tm_hour == 0 && tm->tm_min == 0 && tm->tm_sec == 0; +} diff -r b4a81e123f1a -r 274b42b7d0de src/lib-imap/imap-date.h --- a/src/lib-imap/imap-date.h Thu Apr 23 16:33:19 2015 +0300 +++ b/src/lib-imap/imap-date.h Thu Apr 23 19:19:10 2015 +0300 @@ -17,4 +17,9 @@ /* Returns given UTC timestamp as IMAP date-time string in given timezone. */ const char *imap_to_datetime_tz(time_t timestamp, int timezone_offset); +/* Returns TRUE if timestamp was successfully converted to a date, + FALSE if time would have been required as well (but the string is still + returned). */ +bool imap_to_date(time_t timestamp, const char **str_r); + #endif From dovecot at dovecot.org Thu Apr 23 16:21:36 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 23 Apr 2015 16:21:36 +0000 Subject: dovecot-2.2: lib-storage: Added mail_thread_type_to_str() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3d7b8a09d85d changeset: 18455:3d7b8a09d85d user: Timo Sirainen date: Thu Apr 23 19:20:00 2015 +0300 description: lib-storage: Added mail_thread_type_to_str() diffstat: src/lib-storage/mail-thread.c | 38 +++++++++++++++++++++++++++++--------- src/lib-storage/mail-thread.h | 2 ++ 2 files changed, 31 insertions(+), 9 deletions(-) diffs (60 lines): diff -r 274b42b7d0de -r 3d7b8a09d85d src/lib-storage/mail-thread.c --- a/src/lib-storage/mail-thread.c Thu Apr 23 19:19:10 2015 +0300 +++ b/src/lib-storage/mail-thread.c Thu Apr 23 19:20:00 2015 +0300 @@ -3,15 +3,35 @@ #include "lib.h" #include "mail-thread.h" +struct { + const char *name; + enum mail_thread_type type; +} mail_thread_type_strings[] = { + { "REFERENCES", MAIL_THREAD_REFERENCES }, + { "REFS", MAIL_THREAD_REFS }, + { "ORDEREDSUBJECT", MAIL_THREAD_ORDEREDSUBJECT } +}; + bool mail_thread_type_parse(const char *str, enum mail_thread_type *type_r) { - if (strcasecmp(str, "REFERENCES") == 0) - *type_r = MAIL_THREAD_REFERENCES; - else if (strcasecmp(str, "REFS") == 0) - *type_r = MAIL_THREAD_REFS; - else if (strcasecmp(str, "ORDEREDSUBJECT") == 0) - *type_r = MAIL_THREAD_ORDEREDSUBJECT; - else - return FALSE; - return TRUE; + unsigned int i; + + for (i = 0; i < N_ELEMENTS(mail_thread_type_strings); i++) { + if (strcasecmp(str, mail_thread_type_strings[i].name) == 0) { + *type_r = mail_thread_type_strings[i].type; + return TRUE; + } + } + return FALSE; } + +const char *mail_thread_type_to_str(enum mail_thread_type type) +{ + unsigned int i; + + for (i = 0; i < N_ELEMENTS(mail_thread_type_strings); i++) { + if (mail_thread_type_strings[i].type == type) + return mail_thread_type_strings[i].name; + } + i_panic("Unknown mail_thread_type %d", type); +} diff -r 274b42b7d0de -r 3d7b8a09d85d src/lib-storage/mail-thread.h --- a/src/lib-storage/mail-thread.h Thu Apr 23 19:19:10 2015 +0300 +++ b/src/lib-storage/mail-thread.h Thu Apr 23 19:20:00 2015 +0300 @@ -25,6 +25,8 @@ /* Convert thread type string to enum. Returns TRUE if ok, FALSE if type is unknown. */ bool mail_thread_type_parse(const char *str, enum mail_thread_type *type_r); +/* Return thread type as string. */ +const char *mail_thread_type_to_str(enum mail_thread_type type); /* Build thread from given search arguments. args=NULL searches everything. */ int mail_thread_init(struct mailbox *box, struct mail_search_args *args, From dovecot at dovecot.org Thu Apr 23 16:29:40 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 23 Apr 2015 16:29:40 +0000 Subject: dovecot-2.2: lib-storage: Added mail_search_args_to_imap() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/fdd0f4d50256 changeset: 18456:fdd0f4d50256 user: Timo Sirainen date: Thu Apr 23 19:24:50 2015 +0300 description: lib-storage: Added mail_search_args_to_imap() Useful for writing IMAP SEARCH parameters from struct mail_search_arg. diffstat: src/lib-storage/Makefile.am | 10 + src/lib-storage/mail-search-args-imap.c | 304 +++++++++++++++++++++++++++ src/lib-storage/mail-search.h | 9 + src/lib-storage/test-mail-search-args-imap.c | 133 +++++++++++ 4 files changed, 456 insertions(+), 0 deletions(-) diffs (truncated from 498 to 300 lines): diff -r 3d7b8a09d85d -r fdd0f4d50256 src/lib-storage/Makefile.am --- a/src/lib-storage/Makefile.am Thu Apr 23 19:20:00 2015 +0300 +++ b/src/lib-storage/Makefile.am Thu Apr 23 19:24:50 2015 +0300 @@ -28,6 +28,7 @@ mail-error.c \ mail-namespace.c \ mail-search.c \ + mail-search-args-imap.c \ mail-search-build.c \ mail-search-parser.c \ mail-search-parser-imap.c \ @@ -96,6 +97,7 @@ libdovecot_storage_la_LDFLAGS = -export-dynamic test_programs = \ + test-mail-search-args-imap \ test-mailbox-get noinst_PROGRAMS = $(test_programs) @@ -104,6 +106,14 @@ $(top_builddir)/src/lib-test/libtest.la \ $(top_builddir)/src/lib/liblib.la +test_mail_search_args_imap_SOURCES = test-mail-search-args-imap.c +test_mail_search_args_imap_LDADD = \ + $(LIBDOVECOT_STORAGE) \ + $(LIBDOVECOT) +test_mail_search_args_imap_DEPENDENCIES = \ + libstorage.la \ + $(LIBDOVECOT_DEPS) + test_mailbox_get_SOURCES = test-mailbox-get.c test_mailbox_get_LDADD = mailbox-get.lo $(test_libs) test_mailbox_get_DEPENDENCIES = $(noinst_LTLIBRARIES) $(test_libs) diff -r 3d7b8a09d85d -r fdd0f4d50256 src/lib-storage/mail-search-args-imap.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/mail-search-args-imap.c Thu Apr 23 19:24:50 2015 +0300 @@ -0,0 +1,304 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "str.h" +#include "utc-offset.h" +#include "mail-index.h" +#include "imap-date.h" +#include "imap-util.h" +#include "imap-quote.h" +#include "mail-search.h" + +#include + +static bool +mail_search_subargs_to_imap(string_t *dest, const struct mail_search_arg *args, + const char *prefix, const char **error_r) +{ + const struct mail_search_arg *arg; + + str_append_c(dest, '('); + for (arg = args; arg != NULL; arg = arg->next) { + if (arg->next != NULL) + str_append(dest, prefix); + if (!mail_search_arg_to_imap(dest, arg, error_r)) + return FALSE; + if (arg->next != NULL) + str_append_c(dest, ' '); + } + str_append_c(dest, ')'); + return TRUE; +} + +static bool +mail_search_arg_to_imap_date(string_t *dest, const struct mail_search_arg *arg) +{ + time_t timestamp = arg->value.time; + const char *str; + + if ((arg->value.search_flags & + MAIL_SEARCH_ARG_FLAG_USE_TZ) == 0) { + struct tm *tm = localtime(×tamp); + int tz_offset = utc_offset(tm, timestamp); + timestamp -= tz_offset * 60; + } + if (!imap_to_date(timestamp, &str)) + return FALSE; + str_printfa(dest, " \"%s\"", str); + return TRUE; +} + +bool mail_search_arg_to_imap(string_t *dest, const struct mail_search_arg *arg, + const char **error_r) +{ + unsigned int start_pos; + + if (arg->match_not) + str_append(dest, "NOT "); + start_pos = str_len(dest); + switch (arg->type) { + case SEARCH_OR: + if (!mail_search_subargs_to_imap(dest, arg->value.subargs, + "OR ", error_r)) + return FALSE; + break; + case SEARCH_SUB: + if (!mail_search_subargs_to_imap(dest, arg->value.subargs, + "", error_r)) + return FALSE; + break; + case SEARCH_ALL: + str_append(dest, "ALL"); + break; + case SEARCH_SEQSET: + imap_write_seq_range(dest, &arg->value.seqset); + break; + case SEARCH_UIDSET: + str_append(dest, "UID "); + imap_write_seq_range(dest, &arg->value.seqset); + break; + case SEARCH_FLAGS: + i_assert((arg->value.flags & MAIL_FLAGS_MASK) != 0); + str_append_c(dest, '('); + if ((arg->value.flags & MAIL_ANSWERED) != 0) + str_append(dest, "ANSWERED "); + if ((arg->value.flags & MAIL_FLAGGED) != 0) + str_append(dest, "FLAGGED "); + if ((arg->value.flags & MAIL_DELETED) != 0) + str_append(dest, "DELETED "); + if ((arg->value.flags & MAIL_SEEN) != 0) + str_append(dest, "SEEN "); + if ((arg->value.flags & MAIL_DRAFT) != 0) + str_append(dest, "DRAFT "); + if ((arg->value.flags & MAIL_RECENT) != 0) + str_append(dest, "RECENT "); + str_truncate(dest, str_len(dest)-1); + str_append_c(dest, ')'); + break; + case SEARCH_KEYWORDS: { + const struct mail_keywords *kw = arg->value.keywords; + const ARRAY_TYPE(keywords) *names_arr; + const char *const *namep; + unsigned int i; + + if (kw == NULL) { + /* uninitialized */ + str_printfa(dest, "KEYWORD %s", arg->value.str); + break; + } + + names_arr = mail_index_get_keywords(kw->index); + + str_append_c(dest, '('); + for (i = 0; i < kw->count; i++) { + namep = array_idx(names_arr, kw->idx[i]); + if (i > 0) + str_append_c(dest, ' '); + str_printfa(dest, "KEYWORD %s", *namep); + } + str_append_c(dest, ')'); + break; + } + + case SEARCH_BEFORE: + switch (arg->value.date_type) { + case MAIL_SEARCH_DATE_TYPE_SENT: + str_append(dest, "SENTBEFORE"); + break; + case MAIL_SEARCH_DATE_TYPE_RECEIVED: + str_append(dest, "BEFORE"); + break; + case MAIL_SEARCH_DATE_TYPE_SAVED: + str_append(dest, "X-SAVEDBEFORE"); + break; + } + if (mail_search_arg_to_imap_date(dest, arg)) + ; + else if (arg->value.date_type != MAIL_SEARCH_DATE_TYPE_RECEIVED || + arg->value.time > ioloop_time) { + *error_r = t_strdup_printf( + "SEARCH_BEFORE can't be written as IMAP for timestamp %ld (type=%d, use_tz=%d)", + (long)arg->value.time, arg->value.date_type, + (arg->value.search_flags & MAIL_SEARCH_ARG_FLAG_USE_TZ) != 0); + return FALSE; + } else { + str_truncate(dest, start_pos); + str_printfa(dest, "OLDER %u", + (unsigned int)(ioloop_time - arg->value.time + 1)); + } + break; + case SEARCH_ON: + switch (arg->value.date_type) { + case MAIL_SEARCH_DATE_TYPE_SENT: + str_append(dest, "SENTON"); + break; + case MAIL_SEARCH_DATE_TYPE_RECEIVED: + str_append(dest, "ON"); + break; + case MAIL_SEARCH_DATE_TYPE_SAVED: + str_append(dest, "X-SAVEDON"); + break; + } + if (!mail_search_arg_to_imap_date(dest, arg)) { + *error_r = t_strdup_printf( + "SEARCH_ON can't be written as IMAP for timestamp %ld (type=%d, use_tz=%d)", + (long)arg->value.time, arg->value.date_type, + (arg->value.search_flags & MAIL_SEARCH_ARG_FLAG_USE_TZ) != 0); + return FALSE; + } + break; + case SEARCH_SINCE: + switch (arg->value.date_type) { + case MAIL_SEARCH_DATE_TYPE_SENT: + str_append(dest, "SENTSINCE"); + break; + case MAIL_SEARCH_DATE_TYPE_RECEIVED: + str_append(dest, "SINCE"); + break; + case MAIL_SEARCH_DATE_TYPE_SAVED: + str_append(dest, "X-SAVEDSINCE"); + break; + } + if (mail_search_arg_to_imap_date(dest, arg)) + ; + else if (arg->value.date_type != MAIL_SEARCH_DATE_TYPE_RECEIVED || + arg->value.time >= ioloop_time) { + *error_r = t_strdup_printf( + "SEARCH_SINCE can't be written as IMAP for timestamp %ld (type=%d, use_tz=%d)", + (long)arg->value.time, arg->value.date_type, + (arg->value.search_flags & MAIL_SEARCH_ARG_FLAG_USE_TZ) != 0); + return FALSE; + } else { + str_truncate(dest, start_pos); + str_printfa(dest, "YOUNGER %u", + (unsigned int)(ioloop_time - arg->value.time)); + } + break; + case SEARCH_SMALLER: + str_printfa(dest, "SMALLER %llu", (unsigned long long)arg->value.size); + break; + case SEARCH_LARGER: + str_printfa(dest, "LARGER %llu", (unsigned long long)arg->value.size); + break; + case SEARCH_HEADER: + case SEARCH_HEADER_ADDRESS: + case SEARCH_HEADER_COMPRESS_LWSP: + if (strcasecmp(arg->hdr_field_name, "From") == 0 || + strcasecmp(arg->hdr_field_name, "To") == 0 || + strcasecmp(arg->hdr_field_name, "Cc") == 0 || + strcasecmp(arg->hdr_field_name, "Bcc") == 0 || + strcasecmp(arg->hdr_field_name, "Subject") == 0) + str_append(dest, t_str_ucase(arg->hdr_field_name)); + else { + str_append(dest, "HEADER "); + imap_append_astring(dest, arg->hdr_field_name); + } + str_append_c(dest, ' '); + imap_append_astring(dest, arg->value.str); + break; + + case SEARCH_BODY: + str_append(dest, "BODY "); + imap_append_astring(dest, arg->value.str); + break; + case SEARCH_TEXT: + str_append(dest, "TEXT "); + imap_append_astring(dest, arg->value.str); + break; + + /* extensions */ + case SEARCH_MODSEQ: { + bool extended_output = FALSE; + + str_append(dest, "MODSEQ "); + if (arg->value.str != NULL) { + str_printfa(dest, "/flags/%s", arg->value.str); + extended_output = TRUE; + } else if (arg->value.flags != 0) { + str_append(dest, "/flags/"); + imap_write_flags(dest, arg->value.flags, NULL); + extended_output = TRUE; + } + if (extended_output) { + str_append_c(dest, ' '); + switch (arg->value.modseq->type) { + case MAIL_SEARCH_MODSEQ_TYPE_ANY: + str_append(dest, "all"); + break; + case MAIL_SEARCH_MODSEQ_TYPE_PRIVATE: + str_append(dest, "priv"); + break; + case MAIL_SEARCH_MODSEQ_TYPE_SHARED: + str_append(dest, "shared"); + break; + } + str_append_c(dest, ' '); + } + str_printfa(dest, "%llu", (unsigned long long)arg->value.modseq->modseq); + break; + } + case SEARCH_INTHREAD: From dovecot at dovecot.org Thu Apr 23 16:29:40 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 23 Apr 2015 16:29:40 +0000 Subject: dovecot-2.2: imapc: Use mail_search_arg_to_imap() to write IMAP ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/de60c42e6ad6 changeset: 18457:de60c42e6ad6 user: Timo Sirainen date: Thu Apr 23 19:28:04 2015 +0300 description: imapc: Use mail_search_arg_to_imap() to write IMAP SEARCH parameters. This also fixes a few bugs, since some of the search parameters weren't written entirely correctly. diffstat: src/lib-storage/index/imapc/imapc-search.c | 100 ++++------------------------ 1 files changed, 15 insertions(+), 85 deletions(-) diffs (159 lines): diff -r fdd0f4d50256 -r de60c42e6ad6 src/lib-storage/index/imapc/imapc-search.c --- a/src/lib-storage/index/imapc/imapc-search.c Thu Apr 23 19:24:50 2015 +0300 +++ b/src/lib-storage/index/imapc/imapc-search.c Thu Apr 23 19:28:04 2015 +0300 @@ -3,8 +3,6 @@ #include "lib.h" #include "str.h" #include "imap-arg.h" -#include "imap-date.h" -#include "imap-quote.h" #include "imap-seqset.h" #include "imap-util.h" #include "mail-search.h" @@ -69,24 +67,25 @@ { enum imapc_capability capa = imapc_client_get_capabilities(mbox->storage->client->client); + struct mail_search_arg arg2 = *arg; + const char *error; if (arg->match_not) str_append(str, "NOT "); + arg2.match_not = FALSE; + arg = &arg2; + switch (arg->type) { case SEARCH_OR: str_append_c(str, '('); imapc_build_search_query_args(mbox, arg->value.subargs, TRUE, str); str_append_c(str, ')'); - break; + return TRUE; case SEARCH_SUB: str_append_c(str, '('); imapc_build_search_query_args(mbox, arg->value.subargs, FALSE, str); str_append_c(str, ')'); - break; - - case SEARCH_ALL: - str_append(str, "ALL"); - break; + return TRUE; case SEARCH_SEQSET: /* translate to UIDs */ T_BEGIN { @@ -98,106 +97,37 @@ str_append(str, "UID "); imap_write_seq_range(str, &uids); } T_END; - break; + return TRUE; + case SEARCH_ALL: case SEARCH_UIDSET: - str_append(str, "UID "); - imap_write_seq_range(str, &arg->value.seqset); - break; case SEARCH_FLAGS: - i_assert((arg->value.flags & MAIL_FLAGS_MASK) != 0); - str_append_c(str, '('); - if ((arg->value.flags & MAIL_ANSWERED) != 0) - str_append(str, "ANSWERED "); - if ((arg->value.flags & MAIL_FLAGGED) != 0) - str_append(str, "FLAGGED "); - if ((arg->value.flags & MAIL_DELETED) != 0) - str_append(str, "DELETED "); - if ((arg->value.flags & MAIL_SEEN) != 0) - str_append(str, "SEEN "); - if ((arg->value.flags & MAIL_DRAFT) != 0) - str_append(str, "DRAFT "); - if ((arg->value.flags & MAIL_RECENT) != 0) - str_append(str, "RECENT "); - str_truncate(str, str_len(str)-1); - str_append_c(str, ')'); - break; - case SEARCH_KEYWORDS: { - const struct mail_keywords *kw = arg->value.keywords; - const ARRAY_TYPE(keywords) *names_arr; - const char *const *namep; - unsigned int i; - - names_arr = mail_index_get_keywords(kw->index); - - str_append_c(str, '('); - for (i = 0; i < kw->count; i++) { - namep = array_idx(names_arr, kw->idx[i]); - if (i > 0) - str_append_c(str, ' '); - str_printfa(str, "KEYWORD %s", *namep); - } - str_append_c(str, ')'); - break; - } - + case SEARCH_KEYWORDS: case SEARCH_BEFORE: - str_printfa(str, "BEFORE \"%s\"", imap_to_datetime(arg->value.time)); - break; case SEARCH_ON: - str_printfa(str, "ON \"%s\"", imap_to_datetime(arg->value.time)); - break; case SEARCH_SINCE: - str_printfa(str, "SINCE \"%s\"", imap_to_datetime(arg->value.time)); - break; case SEARCH_SMALLER: - str_printfa(str, "SMALLER %llu", (unsigned long long)arg->value.size); - break; case SEARCH_LARGER: - str_printfa(str, "LARGER %llu", (unsigned long long)arg->value.size); - break; case SEARCH_HEADER: case SEARCH_HEADER_ADDRESS: case SEARCH_HEADER_COMPRESS_LWSP: - if (strcasecmp(arg->hdr_field_name, "From") == 0 || - strcasecmp(arg->hdr_field_name, "To") == 0 || - strcasecmp(arg->hdr_field_name, "Cc") == 0 || - strcasecmp(arg->hdr_field_name, "Bcc") == 0 || - strcasecmp(arg->hdr_field_name, "Subject") == 0) - str_append(str, arg->hdr_field_name); - else { - str_append(str, "HEADER "); - imap_append_astring(str, arg->hdr_field_name); - } - str_append_c(str, ' '); - imap_append_astring(str, arg->value.str); - break; - case SEARCH_BODY: - str_append(str, "BODY "); - imap_append_astring(str, arg->value.str); - break; case SEARCH_TEXT: - str_append(str, "TEXT "); - imap_append_astring(str, arg->value.str); - break; - + return mail_search_arg_to_imap(str, arg, &error); /* extensions */ case SEARCH_MODSEQ: if ((capa & IMAPC_CAPABILITY_CONDSTORE) == 0) return FALSE; - str_printfa(str, "MODSEQ %llu", (unsigned long long)arg->value.modseq->modseq); - break; + return mail_search_arg_to_imap(str, arg, &error); case SEARCH_INTHREAD: case SEARCH_GUID: case SEARCH_MAILBOX: case SEARCH_MAILBOX_GUID: case SEARCH_MAILBOX_GLOB: case SEARCH_REAL_UID: - return FALSE; - default: - return FALSE; + /* not supported for now */ + break; } - return TRUE; + return FALSE; } static bool From dovecot at dovecot.org Thu Apr 23 16:31:23 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 23 Apr 2015 16:31:23 +0000 Subject: dovecot-2.2: lib-storage: Moved mail_search_args_simplify() to i... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/faa7d6c473f7 changeset: 18458:faa7d6c473f7 user: Timo Sirainen date: Thu Apr 23 19:29:43 2015 +0300 description: lib-storage: Moved mail_search_args_simplify() to its own file and added unit tests. No functional changes. diffstat: src/lib-storage/Makefile.am | 10 + src/lib-storage/mail-search-args-simplify.c | 156 +++++++++++++++++++++++ src/lib-storage/mail-search.c | 152 ---------------------- src/lib-storage/test-mail-search-args-simplify.c | 80 +++++++++++ 4 files changed, 246 insertions(+), 152 deletions(-) diffs (truncated from 440 to 300 lines): diff -r de60c42e6ad6 -r faa7d6c473f7 src/lib-storage/Makefile.am --- a/src/lib-storage/Makefile.am Thu Apr 23 19:28:04 2015 +0300 +++ b/src/lib-storage/Makefile.am Thu Apr 23 19:29:43 2015 +0300 @@ -29,6 +29,7 @@ mail-namespace.c \ mail-search.c \ mail-search-args-imap.c \ + mail-search-args-simplify.c \ mail-search-build.c \ mail-search-parser.c \ mail-search-parser-imap.c \ @@ -98,6 +99,7 @@ test_programs = \ test-mail-search-args-imap \ + test-mail-search-args-simplify \ test-mailbox-get noinst_PROGRAMS = $(test_programs) @@ -114,6 +116,14 @@ libstorage.la \ $(LIBDOVECOT_DEPS) +test_mail_search_args_simplify_SOURCES = test-mail-search-args-simplify.c +test_mail_search_args_simplify_LDADD = \ + $(LIBDOVECOT_STORAGE) \ + $(LIBDOVECOT) +test_mail_search_args_simplify_DEPENDENCIES = \ + libstorage.la \ + $(LIBDOVECOT_DEPS) + test_mailbox_get_SOURCES = test-mailbox-get.c test_mailbox_get_LDADD = mailbox-get.lo $(test_libs) test_mailbox_get_DEPENDENCIES = $(noinst_LTLIBRARIES) $(test_libs) diff -r de60c42e6ad6 -r faa7d6c473f7 src/lib-storage/mail-search-args-simplify.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-storage/mail-search-args-simplify.c Thu Apr 23 19:29:43 2015 +0300 @@ -0,0 +1,156 @@ +/* Copyright (c) 2002-2015 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "mail-search.h" + +static void +mail_search_args_simplify_sub(struct mailbox *box, + struct mail_search_arg *args, bool parent_and) +{ + struct mail_search_arg *sub, *prev = NULL; + struct mail_search_arg *prev_flags_arg, *prev_not_flags_arg; + + prev_flags_arg = prev_not_flags_arg = NULL; + while (args != NULL) { + if (args->match_not && (args->type == SEARCH_SUB || + args->type == SEARCH_OR)) { + /* neg(p and q and ..) == neg(p) or neg(q) or .. + neg(p or q or ..) == neg(p) and neg(q) and .. */ + args->type = args->type == SEARCH_SUB ? + SEARCH_OR : SEARCH_SUB; + args->match_not = FALSE; + sub = args->value.subargs; + do { + sub->match_not = !sub->match_not; + sub = sub->next; + } while (sub != NULL); + } + + if ((args->type == SEARCH_SUB && parent_and) || + (args->type == SEARCH_OR && !parent_and) || + ((args->type == SEARCH_SUB || args->type == SEARCH_OR) && + args->value.subargs->next == NULL)) { + /* p and (q and ..) == p and q and .. + p or (q or ..) == p or q or .. + (p) = p */ + sub = args->value.subargs; + for (; sub->next != NULL; sub = sub->next) ; + sub->next = args->next; + *args = *args->value.subargs; + continue; + } + + if (args->type == SEARCH_SUB || + args->type == SEARCH_OR || + args->type == SEARCH_INTHREAD) { + mail_search_args_simplify_sub(box, args->value.subargs, + args->type != SEARCH_OR); + } + + /* merge all flags arguments */ + if (args->type == SEARCH_FLAGS && + !args->match_not && parent_and) { + if (prev_flags_arg == NULL) + prev_flags_arg = args; + else { + prev_flags_arg->value.flags |= + args->value.flags; + prev->next = args->next; + args = args->next; + continue; + } + } else if (args->type == SEARCH_FLAGS && args->match_not && + !parent_and) { + if (prev_not_flags_arg == NULL) + prev_not_flags_arg = args; + else { + prev_not_flags_arg->value.flags |= + args->value.flags; + prev->next = args->next; + args = args->next; + continue; + } + } + + prev = args; + args = args->next; + } +} + +static bool +mail_search_args_unnest_inthreads(struct mail_search_args *args, + struct mail_search_arg **argp, + bool parent_inthreads, bool parent_and) +{ + struct mail_search_arg *arg, *thread_arg, *or_arg; + bool child_inthreads = FALSE, non_inthreads = FALSE; + + for (arg = *argp; arg != NULL; arg = arg->next) { + switch (arg->type) { + case SEARCH_SUB: + case SEARCH_OR: + if (!mail_search_args_unnest_inthreads(args, + &arg->value.subargs, parent_inthreads, + arg->type != SEARCH_OR)) { + arg->result = 1; + child_inthreads = TRUE; + } else { + arg->result = 0; + non_inthreads = TRUE; + } + break; + case SEARCH_INTHREAD: + if (mail_search_args_unnest_inthreads(args, + &arg->value.subargs, TRUE, TRUE)) { + /* children converted to SEARCH_INTHREADs */ + arg->type = SEARCH_SUB; + } + args->have_inthreads = TRUE; + arg->result = 1; + child_inthreads = TRUE; + break; + default: + arg->result = 0; + non_inthreads = TRUE; + break; + } + } + + if (!parent_inthreads || !child_inthreads || !non_inthreads) + return FALSE; + + /* put all non-INTHREADs under a single INTHREAD */ + thread_arg = p_new(args->pool, struct mail_search_arg, 1); + thread_arg->type = SEARCH_INTHREAD; + + while (*argp != NULL) { + arg = *argp; + argp = &(*argp)->next; + + if (arg->result == 0) { + /* not an INTHREAD or a SUB/OR with only INTHREADs */ + arg->next = thread_arg->value.subargs; + thread_arg->value.subargs = arg; + } + } + if (!parent_and) { + /* We want to OR the args */ + or_arg = p_new(args->pool, struct mail_search_arg, 1); + or_arg->type = SEARCH_OR; + or_arg->value.subargs = thread_arg->value.subargs; + thread_arg->value.subargs = or_arg; + } + return TRUE; +} + +void mail_search_args_simplify(struct mail_search_args *args) +{ + args->simplified = TRUE; + + mail_search_args_simplify_sub(args->box, args->args, TRUE); + if (mail_search_args_unnest_inthreads(args, &args->args, + FALSE, TRUE)) { + /* we may have added some extra SUBs that could be dropped */ + mail_search_args_simplify_sub(args->box, args->args, TRUE); + } +} diff -r de60c42e6ad6 -r faa7d6c473f7 src/lib-storage/mail-search.c --- a/src/lib-storage/mail-search.c Thu Apr 23 19:28:04 2015 +0300 +++ b/src/lib-storage/mail-search.c Thu Apr 23 19:29:43 2015 +0300 @@ -587,158 +587,6 @@ return TRUE; } -static void -mail_search_args_simplify_sub(struct mailbox *box, - struct mail_search_arg *args, bool parent_and) -{ - struct mail_search_arg *sub, *prev = NULL; - struct mail_search_arg *prev_flags_arg, *prev_not_flags_arg; - - prev_flags_arg = prev_not_flags_arg = NULL; - while (args != NULL) { - if (args->match_not && (args->type == SEARCH_SUB || - args->type == SEARCH_OR)) { - /* neg(p and q and ..) == neg(p) or neg(q) or .. - neg(p or q or ..) == neg(p) and neg(q) and .. */ - args->type = args->type == SEARCH_SUB ? - SEARCH_OR : SEARCH_SUB; - args->match_not = FALSE; - sub = args->value.subargs; - do { - sub->match_not = !sub->match_not; - sub = sub->next; - } while (sub != NULL); - } - - if ((args->type == SEARCH_SUB && parent_and) || - (args->type == SEARCH_OR && !parent_and) || - ((args->type == SEARCH_SUB || args->type == SEARCH_OR) && - args->value.subargs->next == NULL)) { - /* p and (q and ..) == p and q and .. - p or (q or ..) == p or q or .. - (p) = p */ - sub = args->value.subargs; - for (; sub->next != NULL; sub = sub->next) ; - sub->next = args->next; - *args = *args->value.subargs; - continue; - } - - if (args->type == SEARCH_SUB || - args->type == SEARCH_OR || - args->type == SEARCH_INTHREAD) { - mail_search_args_simplify_sub(box, args->value.subargs, - args->type != SEARCH_OR); - } - - /* merge all flags arguments */ - if (args->type == SEARCH_FLAGS && - !args->match_not && parent_and) { - if (prev_flags_arg == NULL) - prev_flags_arg = args; - else { - prev_flags_arg->value.flags |= - args->value.flags; - prev->next = args->next; - args = args->next; - continue; - } - } else if (args->type == SEARCH_FLAGS && args->match_not && - !parent_and) { - if (prev_not_flags_arg == NULL) - prev_not_flags_arg = args; - else { - prev_not_flags_arg->value.flags |= - args->value.flags; - prev->next = args->next; - args = args->next; - continue; - } - } - - prev = args; - args = args->next; - } -} - -static bool -mail_search_args_unnest_inthreads(struct mail_search_args *args, - struct mail_search_arg **argp, - bool parent_inthreads, bool parent_and) -{ - struct mail_search_arg *arg, *thread_arg, *or_arg; - bool child_inthreads = FALSE, non_inthreads = FALSE; - - for (arg = *argp; arg != NULL; arg = arg->next) { - switch (arg->type) { - case SEARCH_SUB: - case SEARCH_OR: - if (!mail_search_args_unnest_inthreads(args, - &arg->value.subargs, parent_inthreads, - arg->type != SEARCH_OR)) { - arg->result = 1; - child_inthreads = TRUE; - } else { - arg->result = 0; - non_inthreads = TRUE; - } - break; - case SEARCH_INTHREAD: - if (mail_search_args_unnest_inthreads(args, - &arg->value.subargs, TRUE, TRUE)) { From dovecot at dovecot.org Thu Apr 23 16:39:00 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 23 Apr 2015 16:39:00 +0000 Subject: dovecot-2.2: imapc: Don't write SEARCH YOUNGER/OLDER queries if ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3654b56b6fd2 changeset: 18459:3654b56b6fd2 user: Timo Sirainen date: Thu Apr 23 19:37:25 2015 +0300 description: imapc: Don't write SEARCH YOUNGER/OLDER queries if server doesn't support WITHIN extension diffstat: src/lib-imap-client/imapc-client.c | 1 + src/lib-imap-client/imapc-client.h | 1 + src/lib-storage/index/imapc/imapc-search.c | 15 +++++++++++++-- 3 files changed, 15 insertions(+), 2 deletions(-) diffs (52 lines): diff -r faa7d6c473f7 -r 3654b56b6fd2 src/lib-imap-client/imapc-client.c --- a/src/lib-imap-client/imapc-client.c Thu Apr 23 19:29:43 2015 +0300 +++ b/src/lib-imap-client/imapc-client.c Thu Apr 23 19:37:25 2015 +0300 @@ -25,6 +25,7 @@ { "NAMESPACE", IMAPC_CAPABILITY_NAMESPACE }, { "UNSELECT", IMAPC_CAPABILITY_UNSELECT }, { "ESEARCH", IMAPC_CAPABILITY_ESEARCH }, + { "WITHIN", IMAPC_CAPABILITY_WITHIN }, { "IMAP4REV1", IMAPC_CAPABILITY_IMAP4REV1 }, { NULL, 0 } diff -r faa7d6c473f7 -r 3654b56b6fd2 src/lib-imap-client/imapc-client.h --- a/src/lib-imap-client/imapc-client.h Thu Apr 23 19:29:43 2015 +0300 +++ b/src/lib-imap-client/imapc-client.h Thu Apr 23 19:37:25 2015 +0300 @@ -24,6 +24,7 @@ IMAPC_CAPABILITY_NAMESPACE = 0x200, IMAPC_CAPABILITY_UNSELECT = 0x400, IMAPC_CAPABILITY_ESEARCH = 0x800, + IMAPC_CAPABILITY_WITHIN = 0x1000, IMAPC_CAPABILITY_IMAP4REV1 = 0x40000000 }; diff -r faa7d6c473f7 -r 3654b56b6fd2 src/lib-storage/index/imapc/imapc-search.c --- a/src/lib-storage/index/imapc/imapc-search.c Thu Apr 23 19:29:43 2015 +0300 +++ b/src/lib-storage/index/imapc/imapc-search.c Thu Apr 23 19:37:25 2015 +0300 @@ -98,13 +98,24 @@ imap_write_seq_range(str, &uids); } T_END; return TRUE; + case SEARCH_BEFORE: + case SEARCH_SINCE: + if ((capa & IMAPC_CAPABILITY_WITHIN) == 0) { + /* a bit kludgy way to check this.. */ + unsigned int pos = str_len(str); + if (!mail_search_arg_to_imap(str, arg, &error)) + return FALSE; + if (strncasecmp(str_c(str) + pos, "OLDER", 5) == 0 || + strncasecmp(str_c(str) + pos, "YOUNGER", 7) == 0) + return FALSE; + return TRUE; + } + /* fall thrugh */ case SEARCH_ALL: case SEARCH_UIDSET: case SEARCH_FLAGS: case SEARCH_KEYWORDS: - case SEARCH_BEFORE: case SEARCH_ON: - case SEARCH_SINCE: case SEARCH_SMALLER: case SEARCH_LARGER: case SEARCH_HEADER: From dovecot at dovecot.org Thu Apr 23 16:43:48 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 23 Apr 2015 16:43:48 +0000 Subject: dovecot-2.2: lib-storage: Makefile fixes for previous test programs Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/447b82d4ecb1 changeset: 18460:447b82d4ecb1 user: Timo Sirainen date: Thu Apr 23 19:42:13 2015 +0300 description: lib-storage: Makefile fixes for previous test programs diffstat: src/lib-storage/Makefile.am | 16 ++++------------ 1 files changed, 4 insertions(+), 12 deletions(-) diffs (28 lines): diff -r 3654b56b6fd2 -r 447b82d4ecb1 src/lib-storage/Makefile.am --- a/src/lib-storage/Makefile.am Thu Apr 23 19:37:25 2015 +0300 +++ b/src/lib-storage/Makefile.am Thu Apr 23 19:42:13 2015 +0300 @@ -109,20 +109,12 @@ $(top_builddir)/src/lib/liblib.la test_mail_search_args_imap_SOURCES = test-mail-search-args-imap.c -test_mail_search_args_imap_LDADD = \ - $(LIBDOVECOT_STORAGE) \ - $(LIBDOVECOT) -test_mail_search_args_imap_DEPENDENCIES = \ - libstorage.la \ - $(LIBDOVECOT_DEPS) +test_mail_search_args_imap_LDADD = libstorage.la $(LIBDOVECOT) +test_mail_search_args_imap_DEPENDENCIES = libstorage.la $(LIBDOVECOT_DEPS) test_mail_search_args_simplify_SOURCES = test-mail-search-args-simplify.c -test_mail_search_args_simplify_LDADD = \ - $(LIBDOVECOT_STORAGE) \ - $(LIBDOVECOT) -test_mail_search_args_simplify_DEPENDENCIES = \ - libstorage.la \ - $(LIBDOVECOT_DEPS) +test_mail_search_args_simplify_LDADD = libstorage.la $(LIBDOVECOT) +test_mail_search_args_simplify_DEPENDENCIES = libstorage.la $(LIBDOVECOT_DEPS) test_mailbox_get_SOURCES = test-mailbox-get.c test_mailbox_get_LDADD = mailbox-get.lo $(test_libs) From dovecot at dovecot.org Thu Apr 23 16:51:24 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 23 Apr 2015 16:51:24 +0000 Subject: dovecot-2.2: lib-storage: If mail_search_args_simplify() merges ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/dcad04c2bab0 changeset: 18461:dcad04c2bab0 user: Timo Sirainen date: Thu Apr 23 19:49:49 2015 +0300 description: lib-storage: If mail_search_args_simplify() merges flags, check again if any SUBs can be removed. The subquery may have been replaced with a single SEARCH_FLAGS parameter. diffstat: src/lib-storage/mail-search-args-simplify.c | 17 +++++++++++++---- src/lib-storage/test-mail-search-args-simplify.c | 5 +++-- 2 files changed, 16 insertions(+), 6 deletions(-) diffs (88 lines): diff -r 447b82d4ecb1 -r dcad04c2bab0 src/lib-storage/mail-search-args-simplify.c --- a/src/lib-storage/mail-search-args-simplify.c Thu Apr 23 19:42:13 2015 +0300 +++ b/src/lib-storage/mail-search-args-simplify.c Thu Apr 23 19:49:49 2015 +0300 @@ -3,12 +3,13 @@ #include "lib.h" #include "mail-search.h" -static void +static bool mail_search_args_simplify_sub(struct mailbox *box, struct mail_search_arg *args, bool parent_and) { struct mail_search_arg *sub, *prev = NULL; struct mail_search_arg *prev_flags_arg, *prev_not_flags_arg; + bool removals; prev_flags_arg = prev_not_flags_arg = NULL; while (args != NULL) { @@ -43,8 +44,9 @@ if (args->type == SEARCH_SUB || args->type == SEARCH_OR || args->type == SEARCH_INTHREAD) { - mail_search_args_simplify_sub(box, args->value.subargs, - args->type != SEARCH_OR); + if (mail_search_args_simplify_sub(box, args->value.subargs, + args->type != SEARCH_OR)) + removals = TRUE; } /* merge all flags arguments */ @@ -57,6 +59,7 @@ args->value.flags; prev->next = args->next; args = args->next; + removals = TRUE; continue; } } else if (args->type == SEARCH_FLAGS && args->match_not && @@ -68,6 +71,7 @@ args->value.flags; prev->next = args->next; args = args->next; + removals = TRUE; continue; } } @@ -75,6 +79,7 @@ prev = args; args = args->next; } + return removals; } static bool @@ -145,12 +150,16 @@ void mail_search_args_simplify(struct mail_search_args *args) { + bool removals; + args->simplified = TRUE; - mail_search_args_simplify_sub(args->box, args->args, TRUE); + removals = mail_search_args_simplify_sub(args->box, args->args, TRUE); if (mail_search_args_unnest_inthreads(args, &args->args, FALSE, TRUE)) { /* we may have added some extra SUBs that could be dropped */ mail_search_args_simplify_sub(args->box, args->args, TRUE); } + if (removals) + mail_search_args_simplify_sub(args->box, args->args, TRUE); } diff -r 447b82d4ecb1 -r dcad04c2bab0 src/lib-storage/test-mail-search-args-simplify.c --- a/src/lib-storage/test-mail-search-args-simplify.c Thu Apr 23 19:42:13 2015 +0300 +++ b/src/lib-storage/test-mail-search-args-simplify.c Thu Apr 23 19:49:49 2015 +0300 @@ -29,9 +29,10 @@ { "NOT ( TEXT foo TEXT bar )", "(OR NOT TEXT foo NOT TEXT bar)" }, { "ANSWERED FLAGGED SEEN", "(ANSWERED FLAGGED SEEN)" }, + { "OR ( ANSWERED FLAGGED SEEN ) DRAFT", "(OR (ANSWERED FLAGGED SEEN) (DRAFT))" }, { "ANSWERED TEXT foo FLAGGED SEEN", "(ANSWERED FLAGGED SEEN) TEXT foo" }, - { "NOT ( ANSWERED FLAGGED SEEN )", "(NOT (ANSWERED FLAGGED SEEN))" }, - { "OR NOT ANSWERED OR NOT FLAGGED NOT SEEN", "(NOT (ANSWERED FLAGGED SEEN))" } + { "NOT ( ANSWERED FLAGGED SEEN )", "NOT (ANSWERED FLAGGED SEEN)" }, + { "OR NOT ANSWERED OR NOT FLAGGED NOT SEEN", "NOT (ANSWERED FLAGGED SEEN)" } }; static struct mail_search_args * From dovecot at dovecot.org Thu Apr 23 17:18:38 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 23 Apr 2015 17:18:38 +0000 Subject: dovecot-2.2: lib-storage: mail_search_args_simplify() merges now... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/25974d8e5b5f changeset: 18462:25974d8e5b5f user: Timo Sirainen date: Thu Apr 23 20:16:54 2015 +0300 description: lib-storage: mail_search_args_simplify() merges now seqsets and uidsets. diffstat: src/lib-storage/mail-search-args-simplify.c | 100 ++++++++++++++++------ src/lib-storage/test-mail-search-args-simplify.c | 19 ++++- 2 files changed, 89 insertions(+), 30 deletions(-) diffs (160 lines): diff -r dcad04c2bab0 -r 25974d8e5b5f src/lib-storage/mail-search-args-simplify.c --- a/src/lib-storage/mail-search-args-simplify.c Thu Apr 23 19:49:49 2015 +0300 +++ b/src/lib-storage/mail-search-args-simplify.c Thu Apr 23 20:16:54 2015 +0300 @@ -3,15 +3,58 @@ #include "lib.h" #include "mail-search.h" +struct mail_search_simplify_ctx { + struct mail_search_arg *prev_flags, *prev_not_flags; + struct mail_search_arg *prev_seqset, *prev_not_seqset; + struct mail_search_arg *prev_uidset, *prev_not_uidset; + bool removals; +}; + +static bool mail_search_args_merge_flags(struct mail_search_simplify_ctx *ctx, + struct mail_search_arg *args) +{ + struct mail_search_arg **prev_argp; + + prev_argp = !args->match_not ? &ctx->prev_flags : &ctx->prev_not_flags; + if (*prev_argp == NULL) { + *prev_argp = args; + return FALSE; + } else { + (*prev_argp)->value.flags |= args->value.flags; + return TRUE; + } +} + +static bool mail_search_args_merge_set(struct mail_search_simplify_ctx *ctx, + struct mail_search_arg *args) +{ + struct mail_search_arg **prev_argp; + + if (args->type == SEARCH_SEQSET) { + prev_argp = !args->match_not ? &ctx->prev_seqset : + &ctx->prev_not_seqset; + } else { + prev_argp = !args->match_not ? &ctx->prev_uidset : + &ctx->prev_not_uidset; + } + if (*prev_argp == NULL) { + *prev_argp = args; + return FALSE; + } else { + seq_range_array_merge(&(*prev_argp)->value.seqset, + &args->value.seqset); + return TRUE; + } +} + static bool mail_search_args_simplify_sub(struct mailbox *box, struct mail_search_arg *args, bool parent_and) { - struct mail_search_arg *sub, *prev = NULL; - struct mail_search_arg *prev_flags_arg, *prev_not_flags_arg; - bool removals; + struct mail_search_simplify_ctx ctx; + struct mail_search_arg *sub, *prev_arg = NULL; - prev_flags_arg = prev_not_flags_arg = NULL; + memset(&ctx, 0, sizeof(ctx)); while (args != NULL) { if (args->match_not && (args->type == SEARCH_SUB || args->type == SEARCH_OR)) { @@ -46,40 +89,39 @@ args->type == SEARCH_INTHREAD) { if (mail_search_args_simplify_sub(box, args->value.subargs, args->type != SEARCH_OR)) - removals = TRUE; + ctx.removals = TRUE; } - /* merge all flags arguments */ - if (args->type == SEARCH_FLAGS && - !args->match_not && parent_and) { - if (prev_flags_arg == NULL) - prev_flags_arg = args; - else { - prev_flags_arg->value.flags |= - args->value.flags; - prev->next = args->next; + if ((!args->match_not && parent_and) || + (args->match_not && !parent_and)) { + /* try to merge arguments */ + bool merged; + + switch (args->type) { + case SEARCH_FLAGS: + merged = mail_search_args_merge_flags(&ctx, args); + break; + case SEARCH_SEQSET: + case SEARCH_UIDSET: + merged = mail_search_args_merge_set(&ctx, args); + break; + case SEARCH_BEFORE: + default: + merged = FALSE; + break; + } + if (merged) { + prev_arg->next = args->next; args = args->next; - removals = TRUE; - continue; - } - } else if (args->type == SEARCH_FLAGS && args->match_not && - !parent_and) { - if (prev_not_flags_arg == NULL) - prev_not_flags_arg = args; - else { - prev_not_flags_arg->value.flags |= - args->value.flags; - prev->next = args->next; - args = args->next; - removals = TRUE; + ctx.removals = TRUE; continue; } } - prev = args; + prev_arg = args; args = args->next; } - return removals; + return ctx.removals; } static bool diff -r dcad04c2bab0 -r 25974d8e5b5f src/lib-storage/test-mail-search-args-simplify.c --- a/src/lib-storage/test-mail-search-args-simplify.c Thu Apr 23 19:49:49 2015 +0300 +++ b/src/lib-storage/test-mail-search-args-simplify.c Thu Apr 23 20:16:54 2015 +0300 @@ -32,7 +32,24 @@ { "OR ( ANSWERED FLAGGED SEEN ) DRAFT", "(OR (ANSWERED FLAGGED SEEN) (DRAFT))" }, { "ANSWERED TEXT foo FLAGGED SEEN", "(ANSWERED FLAGGED SEEN) TEXT foo" }, { "NOT ( ANSWERED FLAGGED SEEN )", "NOT (ANSWERED FLAGGED SEEN)" }, - { "OR NOT ANSWERED OR NOT FLAGGED NOT SEEN", "NOT (ANSWERED FLAGGED SEEN)" } + { "OR NOT ANSWERED OR NOT FLAGGED NOT SEEN", "NOT (ANSWERED FLAGGED SEEN)" }, + { "ANSWERED NOT FLAGGED SEEN NOT DRAFT", "(ANSWERED SEEN) NOT (FLAGGED) NOT (DRAFT)" }, + { "OR NOT ANSWERED NOT SEEN", "NOT (ANSWERED SEEN)" }, + + { "1:5 10:20", "1:5,10:20" }, + { "1:5 NOT 10:20", "1:5 NOT 10:20" }, + { "1:5 NOT 10:20 NOT 30:40", "1:5 NOT 10:20 NOT 30:40" }, + { "OR 1:5 NOT 10:20", "(OR 1:5 NOT 10:20)" }, + { "OR 1:5 OR NOT 10:20 NOT 30:40", "(OR 1:5 NOT 10:20,30:40)" }, + + { "UID 1:5 UID 10:20", "UID 1:5,10:20" }, + { "UID 1:5 NOT UID 10:20", "UID 1:5 NOT UID 10:20" }, + { "UID 1:5 NOT UID 10:20 NOT UID 30:40", "UID 1:5 NOT UID 10:20 NOT UID 30:40" }, + { "OR UID 1:5 NOT UID 10:20", "(OR UID 1:5 NOT UID 10:20)" }, + { "OR UID 1:5 OR NOT UID 10:20 NOT UID 30:40", "(OR UID 1:5 NOT UID 10:20,30:40)" }, + + { "1:5 UID 10:20", "1:5 UID 10:20" }, + { "1:5 NOT UID 10:20", "1:5 NOT UID 10:20" } }; static struct mail_search_args * From dovecot at dovecot.org Thu Apr 23 18:02:19 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 23 Apr 2015 18:02:19 +0000 Subject: dovecot-2.2: lib-storage: mail_search_args_simplify() handles no... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/f2756661f594 changeset: 18463:f2756661f594 user: Timo Sirainen date: Thu Apr 23 20:50:23 2015 +0300 description: lib-storage: mail_search_args_simplify() handles now SEARCH_BEFORE/ON/SINCE diffstat: src/lib-storage/mail-search-args-simplify.c | 188 ++++++++++++++++++---- src/lib-storage/test-mail-search-args-simplify.c | 20 ++- 2 files changed, 173 insertions(+), 35 deletions(-) diffs (275 lines): diff -r 25974d8e5b5f -r f2756661f594 src/lib-storage/mail-search-args-simplify.c --- a/src/lib-storage/mail-search-args-simplify.c Thu Apr 23 20:16:54 2015 +0300 +++ b/src/lib-storage/mail-search-args-simplify.c Thu Apr 23 20:50:23 2015 +0300 @@ -1,21 +1,71 @@ /* Copyright (c) 2002-2015 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "hash.h" #include "mail-search.h" +struct mail_search_simplify_prev_arg { + struct mail_search_arg mask; + struct mail_search_arg *prev_arg; +}; + struct mail_search_simplify_ctx { - struct mail_search_arg *prev_flags, *prev_not_flags; - struct mail_search_arg *prev_seqset, *prev_not_seqset; - struct mail_search_arg *prev_uidset, *prev_not_uidset; + pool_t pool; + /* arg mask => prev_arg */ + HASH_TABLE(struct mail_search_arg *, + struct mail_search_simplify_prev_arg *) prev_args; + bool parent_and; bool removals; }; +static int mail_search_arg_cmp(const struct mail_search_arg *arg1, + const struct mail_search_arg *arg2) +{ + return memcmp(arg1, arg2, sizeof(*arg1)); +} + +static unsigned int mail_search_arg_hash(const struct mail_search_arg *arg) +{ + return mem_hash(arg, sizeof(*arg)); +} + +static void mail_search_arg_get_base_mask(const struct mail_search_arg *arg, + struct mail_search_arg *mask_r) +{ + memset(mask_r, 0, sizeof(*mask_r)); + mask_r->type = arg->type; + mask_r->match_not = arg->match_not; + mask_r->value.search_flags = arg->value.search_flags; +} + +static struct mail_search_arg ** +mail_search_args_simplify_get_prev_argp(struct mail_search_simplify_ctx *ctx, + const struct mail_search_arg *mask) +{ + struct mail_search_simplify_prev_arg *prev_arg; + + prev_arg = hash_table_lookup(ctx->prev_args, mask); + if (prev_arg == NULL) { + prev_arg = p_new(ctx->pool, struct mail_search_simplify_prev_arg, 1); + prev_arg->mask = *mask; + hash_table_insert(ctx->prev_args, &prev_arg->mask, prev_arg); + } + return &prev_arg->prev_arg; +} + static bool mail_search_args_merge_flags(struct mail_search_simplify_ctx *ctx, struct mail_search_arg *args) { + struct mail_search_arg mask; struct mail_search_arg **prev_argp; - prev_argp = !args->match_not ? &ctx->prev_flags : &ctx->prev_not_flags; + if (!((!args->match_not && ctx->parent_and) || + (args->match_not && !ctx->parent_and))) + return FALSE; + + mail_search_arg_get_base_mask(args, &mask); + prev_argp = mail_search_args_simplify_get_prev_argp(ctx, &mask); + if (*prev_argp == NULL) { *prev_argp = args; return FALSE; @@ -28,15 +78,16 @@ static bool mail_search_args_merge_set(struct mail_search_simplify_ctx *ctx, struct mail_search_arg *args) { + struct mail_search_arg mask; struct mail_search_arg **prev_argp; - if (args->type == SEARCH_SEQSET) { - prev_argp = !args->match_not ? &ctx->prev_seqset : - &ctx->prev_not_seqset; - } else { - prev_argp = !args->match_not ? &ctx->prev_uidset : - &ctx->prev_not_uidset; - } + if (!((!args->match_not && ctx->parent_and) || + (args->match_not && !ctx->parent_and))) + return FALSE; + + mail_search_arg_get_base_mask(args, &mask); + prev_argp = mail_search_args_simplify_get_prev_argp(ctx, &mask); + if (*prev_argp == NULL) { *prev_argp = args; return FALSE; @@ -47,6 +98,67 @@ } } +static bool mail_search_args_merge_time(struct mail_search_simplify_ctx *ctx, + struct mail_search_arg *args) +{ + struct mail_search_arg mask; + struct mail_search_arg **prev_argp, *prev_arg; + + mail_search_arg_get_base_mask(args, &mask); + mask.value.date_type = args->value.date_type; + prev_argp = mail_search_args_simplify_get_prev_argp(ctx, &mask); + + if (*prev_argp == NULL) { + *prev_argp = args; + return FALSE; + } + + prev_arg = *prev_argp; + switch (args->type) { + case SEARCH_BEFORE: + if (ctx->parent_and) { + if (prev_arg->value.time < args->value.time) { + /* prev_arg < 5 AND arg < 10 */ + } else { + /* prev_arg < 10 AND arg < 5 */ + prev_arg->value.time = args->value.time; + } + } else { + if (prev_arg->value.time < args->value.time) { + /* prev_arg < 5 OR arg < 10 */ + prev_arg->value.time = args->value.time; + } else { + /* prev_arg < 10 OR arg < 5 */ + } + } + return TRUE; + case SEARCH_ON: + if (prev_arg->value.time == args->value.time) + return TRUE; + return FALSE; + case SEARCH_SINCE: + if (ctx->parent_and) { + if (prev_arg->value.time < args->value.time) { + /* prev_arg >= 5 AND arg >= 10 */ + prev_arg->value.time = args->value.time; + } else { + /* prev_arg >= 10 AND arg >= 5 */ + } + } else { + if (prev_arg->value.time < args->value.time) { + /* prev_arg >= 5 OR arg >= 10 */ + } else { + /* prev_arg >= 10 OR arg >= 5 */ + prev_arg->value.time = args->value.time; + } + } + return TRUE; + default: + break; + } + return FALSE; +} + static bool mail_search_args_simplify_sub(struct mailbox *box, struct mail_search_arg *args, bool parent_and) @@ -55,6 +167,11 @@ struct mail_search_arg *sub, *prev_arg = NULL; memset(&ctx, 0, sizeof(ctx)); + ctx.parent_and = parent_and; + ctx.pool = pool_alloconly_create("mail search args simplify", 1024); + hash_table_create(&ctx.prev_args, ctx.pool, 0, mail_search_arg_hash, + mail_search_arg_cmp); + while (args != NULL) { if (args->match_not && (args->type == SEARCH_SUB || args->type == SEARCH_OR)) { @@ -92,35 +209,38 @@ ctx.removals = TRUE; } - if ((!args->match_not && parent_and) || - (args->match_not && !parent_and)) { - /* try to merge arguments */ - bool merged; + /* try to merge arguments */ + bool merged; - switch (args->type) { - case SEARCH_FLAGS: - merged = mail_search_args_merge_flags(&ctx, args); - break; - case SEARCH_SEQSET: - case SEARCH_UIDSET: - merged = mail_search_args_merge_set(&ctx, args); - break; - case SEARCH_BEFORE: - default: - merged = FALSE; - break; - } - if (merged) { - prev_arg->next = args->next; - args = args->next; - ctx.removals = TRUE; - continue; - } + switch (args->type) { + case SEARCH_FLAGS: + merged = mail_search_args_merge_flags(&ctx, args); + break; + case SEARCH_SEQSET: + case SEARCH_UIDSET: + merged = mail_search_args_merge_set(&ctx, args); + break; + case SEARCH_BEFORE: + case SEARCH_ON: + case SEARCH_SINCE: + merged = mail_search_args_merge_time(&ctx, args); + break; + default: + merged = FALSE; + break; + } + if (merged) { + prev_arg->next = args->next; + args = args->next; + ctx.removals = TRUE; + continue; } prev_arg = args; args = args->next; } + hash_table_destroy(&ctx.prev_args); + pool_unref(&ctx.pool); return ctx.removals; } diff -r 25974d8e5b5f -r f2756661f594 src/lib-storage/test-mail-search-args-simplify.c --- a/src/lib-storage/test-mail-search-args-simplify.c Thu Apr 23 20:16:54 2015 +0300 +++ b/src/lib-storage/test-mail-search-args-simplify.c Thu Apr 23 20:50:23 2015 +0300 @@ -49,7 +49,25 @@ { "OR UID 1:5 OR NOT UID 10:20 NOT UID 30:40", "(OR UID 1:5 NOT UID 10:20,30:40)" }, { "1:5 UID 10:20", "1:5 UID 10:20" }, - { "1:5 NOT UID 10:20", "1:5 NOT UID 10:20" } + { "1:5 NOT UID 10:20", "1:5 NOT UID 10:20" }, + + { "BEFORE 03-Aug-2014 BEFORE 01-Aug-2014 BEFORE 02-Aug-2014", "BEFORE \"01-Aug-2014\"" }, + { "OR BEFORE 01-Aug-2014 BEFORE 02-Aug-2014", "BEFORE \"02-Aug-2014\"" }, + { "OR BEFORE 01-Aug-2014 OR BEFORE 03-Aug-2014 BEFORE 02-Aug-2014", "BEFORE \"03-Aug-2014\"" }, + { "BEFORE 03-Aug-2014 NOT BEFORE 01-Aug-2014 BEFORE 02-Aug-2014", "BEFORE \"02-Aug-2014\" NOT BEFORE \"01-Aug-2014\"" }, + { "SENTBEFORE 03-Aug-2014 SENTBEFORE 01-Aug-2014 SENTBEFORE 02-Aug-2014", "SENTBEFORE \"01-Aug-2014\"" }, + { "SENTBEFORE 03-Aug-2014 BEFORE 01-Aug-2014 SENTBEFORE 02-Aug-2014", "SENTBEFORE \"02-Aug-2014\" BEFORE \"01-Aug-2014\"" }, + + { "ON 03-Aug-2014 ON 03-Aug-2014", "ON \"03-Aug-2014\"" }, + { "ON 03-Aug-2014 ON 04-Aug-2014", "ON \"03-Aug-2014\" ON \"04-Aug-2014\"" }, /* this could be replaced with e.g. NOT ALL */ + { "OR ON 03-Aug-2014 ON 04-Aug-2014", "(OR ON \"03-Aug-2014\" ON \"04-Aug-2014\")" }, + + { "SINCE 03-Aug-2014 SINCE 01-Aug-2014 SINCE 02-Aug-2014", "SINCE \"03-Aug-2014\"" }, + { "OR SINCE 01-Aug-2014 SINCE 02-Aug-2014", "SINCE \"01-Aug-2014\"" }, + { "OR SINCE 01-Aug-2014 OR SINCE 03-Aug-2014 SINCE 02-Aug-2014", "SINCE \"01-Aug-2014\"" }, + { "SINCE 03-Aug-2014 NOT SINCE 01-Aug-2014 SINCE 02-Aug-2014", "SINCE \"03-Aug-2014\" NOT SINCE \"01-Aug-2014\"" }, + { "SENTSINCE 03-Aug-2014 SENTSINCE 01-Aug-2014 SENTSINCE 02-Aug-2014", "SENTSINCE \"03-Aug-2014\"" }, + { "SENTSINCE 03-Aug-2014 SINCE 01-Aug-2014 SENTSINCE 02-Aug-2014", "SENTSINCE \"03-Aug-2014\" SINCE \"01-Aug-2014\"" }, }; static struct mail_search_args * From dovecot at dovecot.org Thu Apr 23 18:02:19 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 23 Apr 2015 18:02:19 +0000 Subject: dovecot-2.2: lib-storage: mail_search_args_simplify() handles no... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/b2c20d9ff296 changeset: 18464:b2c20d9ff296 user: Timo Sirainen date: Thu Apr 23 21:00:43 2015 +0300 description: lib-storage: mail_search_args_simplify() handles now SEARCH_SMALLER/LARGER diffstat: src/lib-storage/mail-search-args-simplify.c | 63 +++++++++++++++++++++++- src/lib-storage/test-mail-search-args-simplify.c | 12 ++++ 2 files changed, 73 insertions(+), 2 deletions(-) diffs (115 lines): diff -r f2756661f594 -r b2c20d9ff296 src/lib-storage/mail-search-args-simplify.c --- a/src/lib-storage/mail-search-args-simplify.c Thu Apr 23 20:50:23 2015 +0300 +++ b/src/lib-storage/mail-search-args-simplify.c Thu Apr 23 21:00:43 2015 +0300 @@ -159,12 +159,69 @@ return FALSE; } +static bool mail_search_args_merge_size(struct mail_search_simplify_ctx *ctx, + struct mail_search_arg *args) +{ + struct mail_search_arg mask; + struct mail_search_arg **prev_argp, *prev_arg; + + mail_search_arg_get_base_mask(args, &mask); + prev_argp = mail_search_args_simplify_get_prev_argp(ctx, &mask); + + if (*prev_argp == NULL) { + *prev_argp = args; + return FALSE; + } + + prev_arg = *prev_argp; + switch (args->type) { + case SEARCH_SMALLER: + if (ctx->parent_and) { + if (prev_arg->value.size < args->value.size) { + /* prev_arg < 5 AND arg < 10 */ + } else { + /* prev_arg < 10 AND arg < 5 */ + prev_arg->value.size = args->value.size; + } + } else { + if (prev_arg->value.size < args->value.size) { + /* prev_arg < 5 OR arg < 10 */ + prev_arg->value.size = args->value.size; + } else { + /* prev_arg < 10 OR arg < 5 */ + } + } + return TRUE; + case SEARCH_LARGER: + if (ctx->parent_and) { + if (prev_arg->value.size < args->value.size) { + /* prev_arg >= 5 AND arg >= 10 */ + prev_arg->value.size = args->value.size; + } else { + /* prev_arg >= 10 AND arg >= 5 */ + } + } else { + if (prev_arg->value.size < args->value.size) { + /* prev_arg >= 5 OR arg >= 10 */ + } else { + /* prev_arg >= 10 OR arg >= 5 */ + prev_arg->value.size = args->value.size; + } + } + return TRUE; + default: + break; + } + return FALSE; +} + static bool mail_search_args_simplify_sub(struct mailbox *box, struct mail_search_arg *args, bool parent_and) { struct mail_search_simplify_ctx ctx; struct mail_search_arg *sub, *prev_arg = NULL; + bool merged; memset(&ctx, 0, sizeof(ctx)); ctx.parent_and = parent_and; @@ -210,8 +267,6 @@ } /* try to merge arguments */ - bool merged; - switch (args->type) { case SEARCH_FLAGS: merged = mail_search_args_merge_flags(&ctx, args); @@ -225,6 +280,10 @@ case SEARCH_SINCE: merged = mail_search_args_merge_time(&ctx, args); break; + case SEARCH_SMALLER: + case SEARCH_LARGER: + merged = mail_search_args_merge_size(&ctx, args); + break; default: merged = FALSE; break; diff -r f2756661f594 -r b2c20d9ff296 src/lib-storage/test-mail-search-args-simplify.c --- a/src/lib-storage/test-mail-search-args-simplify.c Thu Apr 23 20:50:23 2015 +0300 +++ b/src/lib-storage/test-mail-search-args-simplify.c Thu Apr 23 21:00:43 2015 +0300 @@ -68,6 +68,18 @@ { "SINCE 03-Aug-2014 NOT SINCE 01-Aug-2014 SINCE 02-Aug-2014", "SINCE \"03-Aug-2014\" NOT SINCE \"01-Aug-2014\"" }, { "SENTSINCE 03-Aug-2014 SENTSINCE 01-Aug-2014 SENTSINCE 02-Aug-2014", "SENTSINCE \"03-Aug-2014\"" }, { "SENTSINCE 03-Aug-2014 SINCE 01-Aug-2014 SENTSINCE 02-Aug-2014", "SENTSINCE \"03-Aug-2014\" SINCE \"01-Aug-2014\"" }, + + { "SMALLER 1 SMALLER 2", "SMALLER 1" }, + { "OR SMALLER 1 SMALLER 2", "SMALLER 2" }, + { "OR SMALLER 1 OR SMALLER 3 SMALLER 2", "SMALLER 3" }, + { "SMALLER 3 NOT SMALLER 1 SMALLER 2", "SMALLER 2 NOT SMALLER 1" }, + { "SMALLER 3 LARGER 5", "SMALLER 3 LARGER 5" }, /* this could be replaced with e.g. NOT ALL */ + { "OR SMALLER 3 LARGER 5", "(OR SMALLER 3 LARGER 5)" }, + + { "LARGER 3 LARGER 1 LARGER 2", "LARGER 3" }, + { "OR LARGER 1 LARGER 2", "LARGER 1" }, + { "OR LARGER 1 OR LARGER 3 LARGER 2", "LARGER 1" }, + { "LARGER 3 NOT LARGER 1 LARGER 2", "LARGER 3 NOT LARGER 1" } }; static struct mail_search_args * From dovecot at dovecot.org Thu Apr 23 18:21:32 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 23 Apr 2015 18:21:32 +0000 Subject: dovecot-2.2: lib-storage: mail_search_args_simplify() handles no... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c5751819176f changeset: 18465:c5751819176f user: Timo Sirainen date: Thu Apr 23 21:19:54 2015 +0300 description: lib-storage: mail_search_args_simplify() handles now SEARCH_HEADER*/TEXT/BODY These are especially useful with lib-fts, where stemming and other filtering can produce duplicates. There's some internal deduplication, but it doesn't catch all of these. diffstat: src/lib-storage/mail-search-args-simplify.c | 103 +++++++++++++++++----- src/lib-storage/test-mail-search-args-simplify.c | 18 +++- 2 files changed, 97 insertions(+), 24 deletions(-) diffs (228 lines): diff -r b2c20d9ff296 -r c5751819176f src/lib-storage/mail-search-args-simplify.c --- a/src/lib-storage/mail-search-args-simplify.c Thu Apr 23 21:00:43 2015 +0300 +++ b/src/lib-storage/mail-search-args-simplify.c Thu Apr 23 21:19:54 2015 +0300 @@ -5,50 +5,80 @@ #include "mail-search.h" struct mail_search_simplify_prev_arg { - struct mail_search_arg mask; + struct { + enum mail_search_arg_type type; + enum mail_search_arg_flag search_flags; + enum mail_search_date_type date_type; + bool match_not; + bool fuzzy; + } bin_mask; + const char *hdr_field_name_mask; + const char *str_mask; + struct mail_search_arg *prev_arg; }; struct mail_search_simplify_ctx { pool_t pool; /* arg mask => prev_arg */ - HASH_TABLE(struct mail_search_arg *, + HASH_TABLE(struct mail_search_simplify_prev_arg *, struct mail_search_simplify_prev_arg *) prev_args; bool parent_and; bool removals; }; -static int mail_search_arg_cmp(const struct mail_search_arg *arg1, - const struct mail_search_arg *arg2) +static int +mail_search_simplify_prev_arg_cmp(const struct mail_search_simplify_prev_arg *arg1, + const struct mail_search_simplify_prev_arg *arg2) { - return memcmp(arg1, arg2, sizeof(*arg1)); + int ret; + + ret = memcmp(&arg1->bin_mask, &arg2->bin_mask, sizeof(arg1->bin_mask)); + if (ret == 0) + ret = null_strcmp(arg1->hdr_field_name_mask, arg2->hdr_field_name_mask); + if (ret == 0) + ret = null_strcmp(arg1->str_mask, arg2->str_mask); + return ret; } -static unsigned int mail_search_arg_hash(const struct mail_search_arg *arg) +static unsigned int +mail_search_simplify_prev_arg_hash(const struct mail_search_simplify_prev_arg *arg) { - return mem_hash(arg, sizeof(*arg)); + unsigned int hash; + + hash = mem_hash(&arg->bin_mask, sizeof(arg->bin_mask)); + if (arg->hdr_field_name_mask != NULL) + hash ^= str_hash(arg->hdr_field_name_mask); + if (arg->str_mask != NULL) + hash ^= str_hash(arg->str_mask); + return hash; } static void mail_search_arg_get_base_mask(const struct mail_search_arg *arg, - struct mail_search_arg *mask_r) + struct mail_search_simplify_prev_arg *mask_r) { memset(mask_r, 0, sizeof(*mask_r)); - mask_r->type = arg->type; - mask_r->match_not = arg->match_not; - mask_r->value.search_flags = arg->value.search_flags; + mask_r->bin_mask.type = arg->type; + mask_r->bin_mask.match_not = arg->match_not; + mask_r->bin_mask.fuzzy = arg->fuzzy; + mask_r->bin_mask.search_flags = arg->value.search_flags; } static struct mail_search_arg ** mail_search_args_simplify_get_prev_argp(struct mail_search_simplify_ctx *ctx, - const struct mail_search_arg *mask) + const struct mail_search_simplify_prev_arg *mask) { struct mail_search_simplify_prev_arg *prev_arg; prev_arg = hash_table_lookup(ctx->prev_args, mask); if (prev_arg == NULL) { prev_arg = p_new(ctx->pool, struct mail_search_simplify_prev_arg, 1); - prev_arg->mask = *mask; - hash_table_insert(ctx->prev_args, &prev_arg->mask, prev_arg); + prev_arg->bin_mask = mask->bin_mask; + prev_arg->hdr_field_name_mask = + p_strdup(ctx->pool, mask->hdr_field_name_mask); + prev_arg->str_mask = + p_strdup(ctx->pool, mask->str_mask); + hash_table_insert(ctx->prev_args, prev_arg, prev_arg); } return &prev_arg->prev_arg; } @@ -56,7 +86,7 @@ static bool mail_search_args_merge_flags(struct mail_search_simplify_ctx *ctx, struct mail_search_arg *args) { - struct mail_search_arg mask; + struct mail_search_simplify_prev_arg mask; struct mail_search_arg **prev_argp; if (!((!args->match_not && ctx->parent_and) || @@ -78,7 +108,7 @@ static bool mail_search_args_merge_set(struct mail_search_simplify_ctx *ctx, struct mail_search_arg *args) { - struct mail_search_arg mask; + struct mail_search_simplify_prev_arg mask; struct mail_search_arg **prev_argp; if (!((!args->match_not && ctx->parent_and) || @@ -101,11 +131,11 @@ static bool mail_search_args_merge_time(struct mail_search_simplify_ctx *ctx, struct mail_search_arg *args) { - struct mail_search_arg mask; + struct mail_search_simplify_prev_arg mask; struct mail_search_arg **prev_argp, *prev_arg; mail_search_arg_get_base_mask(args, &mask); - mask.value.date_type = args->value.date_type; + mask.bin_mask.date_type = args->value.date_type; prev_argp = mail_search_args_simplify_get_prev_argp(ctx, &mask); if (*prev_argp == NULL) { @@ -162,7 +192,7 @@ static bool mail_search_args_merge_size(struct mail_search_simplify_ctx *ctx, struct mail_search_arg *args) { - struct mail_search_arg mask; + struct mail_search_simplify_prev_arg mask; struct mail_search_arg **prev_argp, *prev_arg; mail_search_arg_get_base_mask(args, &mask); @@ -215,6 +245,25 @@ return FALSE; } +static bool mail_search_args_merge_text(struct mail_search_simplify_ctx *ctx, + struct mail_search_arg *args) +{ + struct mail_search_simplify_prev_arg mask; + struct mail_search_arg **prev_argp; + + mail_search_arg_get_base_mask(args, &mask); + mask.hdr_field_name_mask = args->hdr_field_name; + mask.str_mask = args->value.str; + prev_argp = mail_search_args_simplify_get_prev_argp(ctx, &mask); + + if (*prev_argp == NULL) { + *prev_argp = args; + return FALSE; + } + /* duplicate search word. */ + return TRUE; +} + static bool mail_search_args_simplify_sub(struct mailbox *box, struct mail_search_arg *args, bool parent_and) @@ -226,8 +275,9 @@ memset(&ctx, 0, sizeof(ctx)); ctx.parent_and = parent_and; ctx.pool = pool_alloconly_create("mail search args simplify", 1024); - hash_table_create(&ctx.prev_args, ctx.pool, 0, mail_search_arg_hash, - mail_search_arg_cmp); + hash_table_create(&ctx.prev_args, ctx.pool, 0, + mail_search_simplify_prev_arg_hash, + mail_search_simplify_prev_arg_cmp); while (args != NULL) { if (args->match_not && (args->type == SEARCH_SUB || @@ -284,6 +334,13 @@ case SEARCH_LARGER: merged = mail_search_args_merge_size(&ctx, args); break; + case SEARCH_HEADER: + case SEARCH_HEADER_ADDRESS: + case SEARCH_HEADER_COMPRESS_LWSP: + case SEARCH_BODY: + case SEARCH_TEXT: + merged = mail_search_args_merge_text(&ctx, args); + break; default: merged = FALSE; break; @@ -381,6 +438,6 @@ /* we may have added some extra SUBs that could be dropped */ mail_search_args_simplify_sub(args->box, args->args, TRUE); } - if (removals) - mail_search_args_simplify_sub(args->box, args->args, TRUE); + while (removals) + removals = mail_search_args_simplify_sub(args->box, args->args, TRUE); } diff -r b2c20d9ff296 -r c5751819176f src/lib-storage/test-mail-search-args-simplify.c --- a/src/lib-storage/test-mail-search-args-simplify.c Thu Apr 23 21:00:43 2015 +0300 +++ b/src/lib-storage/test-mail-search-args-simplify.c Thu Apr 23 21:19:54 2015 +0300 @@ -79,7 +79,23 @@ { "LARGER 3 LARGER 1 LARGER 2", "LARGER 3" }, { "OR LARGER 1 LARGER 2", "LARGER 1" }, { "OR LARGER 1 OR LARGER 3 LARGER 2", "LARGER 1" }, - { "LARGER 3 NOT LARGER 1 LARGER 2", "LARGER 3 NOT LARGER 1" } + { "LARGER 3 NOT LARGER 1 LARGER 2", "LARGER 3 NOT LARGER 1" }, + + { "SUBJECT foo SUBJECT foo", "SUBJECT foo" }, + { "SUBJECT foo SUBJECT foob", "SUBJECT foo SUBJECT foob" }, + { "OR SUBJECT foo SUBJECT foo", "SUBJECT foo" }, + { "FROM foo FROM foo", "FROM foo" }, + { "FROM foo FROM bar", "FROM foo FROM bar" }, + { "FROM foo TO foo", "FROM foo TO foo" }, + + { "TEXT foo TEXT foo", "TEXT foo" }, + { "TEXT foo TEXT foob", "TEXT foo TEXT foob" }, + { "OR TEXT foo TEXT foo", "TEXT foo" }, + { "TEXT foo NOT TEXT foo TEXT foo NOT TEXT foo", "TEXT foo NOT TEXT foo" }, + { "BODY foo BODY foo", "BODY foo" }, + { "OR BODY foo BODY foo", "BODY foo" }, + { "TEXT foo BODY foo", "TEXT foo BODY foo" }, + { "OR ( TEXT foo OR TEXT foo TEXT foo ) ( TEXT foo ( TEXT foo ) )", "TEXT foo" }, }; static struct mail_search_args * From dovecot at dovecot.org Thu Apr 23 18:28:25 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 23 Apr 2015 18:28:25 +0000 Subject: dovecot-2.2: auth: Don't assert-crash if master user login attem... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/65f825a8cd0b changeset: 18466:65f825a8cd0b user: Timo Sirainen date: Thu Apr 23 21:26:50 2015 +0300 description: auth: Don't assert-crash if master user login attempts to use empty login username. diffstat: src/auth/auth-request.c | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diffs (15 lines): diff -r c5751819176f -r 65f825a8cd0b src/auth/auth-request.c --- a/src/auth/auth-request.c Thu Apr 23 21:19:54 2015 +0300 +++ b/src/auth/auth-request.c Thu Apr 23 21:26:50 2015 +0300 @@ -1274,7 +1274,10 @@ { struct auth_passdb *master_passdb; - i_assert(*username != '\0'); + if (username[0] == '\0') { + *error_r = "Master user login attempted to use empty login username"; + return FALSE; + } if (strcmp(username, request->user) == 0) { /* The usernames are the same, we don't really wish to log From dovecot at dovecot.org Thu Apr 23 18:34:19 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Thu, 23 Apr 2015 18:34:19 +0000 Subject: dovecot-2.2: dict: Fixed unescaping strings from lib-dict. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/3905d7fce3ba changeset: 18467:3905d7fce3ba user: Timo Sirainen date: Thu Apr 23 21:32:43 2015 +0300 description: dict: Fixed unescaping strings from lib-dict. This may potentially break backwards compatibility with someone, but it's probably pretty rare that anybody is yet using dict proxy with multi-line fields. Also even though the current behavior happens to work, it's very much non-ideal and can be potentially dangerous. diffstat: src/dict/dict-commands.c | 19 +++++++++++-------- 1 files changed, 11 insertions(+), 8 deletions(-) diffs (79 lines): diff -r 65f825a8cd0b -r 3905d7fce3ba src/dict/dict-commands.c --- a/src/dict/dict-commands.c Thu Apr 23 21:26:50 2015 +0300 +++ b/src/dict/dict-commands.c Thu Apr 23 21:32:43 2015 +0300 @@ -4,6 +4,7 @@ #include "array.h" #include "ostream.h" #include "str.h" +#include "strescape.h" #include "dict-client.h" #include "dict-settings.h" #include "dict-connection.h" @@ -33,7 +34,7 @@ ret = dict_lookup(conn->dict, pool_datastack_create(), line, &value); if (ret > 0) { reply = t_strdup_printf("%c%s\n", - DICT_PROTOCOL_REPLY_OK, value); + DICT_PROTOCOL_REPLY_OK, str_tabescape(value)); o_stream_nsend_str(conn->output, reply); } else { reply = t_strdup_printf("%c\n", ret == 0 ? @@ -53,9 +54,11 @@ o_stream_cork(conn->output); while (dict_iterate(conn->iter_ctx, &key, &value)) { str_truncate(str, 0); - str_printfa(str, "%c%s\t", DICT_PROTOCOL_REPLY_OK, key); + str_append_c(str, DICT_PROTOCOL_REPLY_OK); + str_append_tabescaped(str, key); + str_append_c(str, '\t'); if ((conn->iter_flags & DICT_ITERATE_FLAG_NO_VALUE) == 0) - str_append(str, value); + str_append_tabescaped(str, value); str_append_c(str, '\n'); o_stream_nsend(conn->output, str_data(str), str_len(str)); @@ -92,7 +95,7 @@ return -1; } - args = t_strsplit_tab(line); + args = t_strsplit_tabescaped(line); if (str_array_length(args) < 2 || str_to_uint(args[0], &flags) < 0) { i_error("dict client: ITERATE: broken input"); @@ -275,7 +278,7 @@ const char *const *args; /* */ - args = t_strsplit_tab(line); + args = t_strsplit_tabescaped(line); if (str_array_length(args) != 3) { i_error("dict client: SET: broken input"); return -1; @@ -294,7 +297,7 @@ const char *const *args; /* */ - args = t_strsplit_tab(line); + args = t_strsplit_tabescaped(line); if (str_array_length(args) != 2) { i_error("dict client: UNSET: broken input"); return -1; @@ -313,7 +316,7 @@ const char *const *args; /* */ - args = t_strsplit_tab(line); + args = t_strsplit_tabescaped(line); if (str_array_length(args) != 3) { i_error("dict client: APPEND: broken input"); return -1; @@ -333,7 +336,7 @@ long long diff; /* */ - args = t_strsplit_tab(line); + args = t_strsplit_tabescaped(line); if (str_array_length(args) != 3 || str_to_llong(args[2], &diff) < 0) { i_error("dict client: ATOMIC_INC: broken input"); From dovecot at dovecot.org Fri Apr 24 10:09:15 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 24 Apr 2015 10:09:15 +0000 Subject: dovecot-2.2: lib: Added t_str_replace() to replace one character... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/90298bc1a481 changeset: 18468:90298bc1a481 user: Timo Sirainen date: Fri Apr 24 13:07:39 2015 +0300 description: lib: Added t_str_replace() to replace one character with another one diffstat: src/lib/strfuncs.c | 20 ++++++++++++++++++++ src/lib/strfuncs.h | 2 ++ src/lib/test-strfuncs.c | 15 +++++++++++++++ 3 files changed, 37 insertions(+), 0 deletions(-) diffs (70 lines): diff -r 3905d7fce3ba -r 90298bc1a481 src/lib/strfuncs.c --- a/src/lib/strfuncs.c Thu Apr 23 21:32:43 2015 +0300 +++ b/src/lib/strfuncs.c Fri Apr 24 13:07:39 2015 +0300 @@ -289,6 +289,26 @@ return str; } +const char *t_str_replace(const char *str, char from, char to) +{ + char *out; + unsigned int i, len; + + if (strchr(str, from) == NULL) + return str; + + len = strlen(str); + out = t_malloc(len + 1); + for (i = 0; i < len; i++) { + if (str[i] == from) + out[i] = to; + else + out[i] = str[i]; + } + out[i] = '\0'; + return out; +} + int i_strocpy(char *dest, const char *src, size_t dstsize) { if (dstsize == 0) diff -r 3905d7fce3ba -r 90298bc1a481 src/lib/strfuncs.h --- a/src/lib/strfuncs.h Thu Apr 23 21:32:43 2015 +0300 +++ b/src/lib/strfuncs.h Fri Apr 24 13:07:39 2015 +0300 @@ -41,6 +41,8 @@ /* Like t_strdup(), but stop at cutchar. */ const char *t_strcut(const char *str, char cutchar); +/* Replace all from->to chars in the string. */ +const char *t_str_replace(const char *str, char from, char to); /* Like strlcpy(), but return -1 if buffer was overflown, 0 if not. */ int i_strocpy(char *dest, const char *src, size_t dstsize); diff -r 3905d7fce3ba -r 90298bc1a481 src/lib/test-strfuncs.c --- a/src/lib/test-strfuncs.c Thu Apr 23 21:32:43 2015 +0300 +++ b/src/lib/test-strfuncs.c Fri Apr 24 13:07:39 2015 +0300 @@ -96,9 +96,24 @@ test_end(); } +static void test_t_str_replace(void) +{ + test_begin("t_str_replace"); + test_assert(strcmp(t_str_replace("foo", 'a', 'b'), "foo") == 0); + test_assert(strcmp(t_str_replace("fooa", 'a', 'b'), "foob") == 0); + test_assert(strcmp(t_str_replace("afooa", 'a', 'b'), "bfoob") == 0); + test_assert(strcmp(t_str_replace("", 'a', 'b'), "") == 0); + test_assert(strcmp(t_str_replace("a", 'a', 'b'), "b") == 0); + test_assert(strcmp(t_str_replace("aaa", 'a', 'b'), "bbb") == 0); + test_assert(strcmp(t_str_replace("bbb", 'a', 'b'), "bbb") == 0); + test_assert(strcmp(t_str_replace("aba", 'a', 'b'), "bbb") == 0); + test_end(); +} + void test_strfuncs(void) { test_p_strarray_dup(); test_t_strsplit(); test_t_strsplit_tab(); + test_t_str_replace(); } From dovecot at dovecot.org Fri Apr 24 10:09:52 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 24 Apr 2015 10:09:52 +0000 Subject: dovecot-2.2: lib-fs: Use the new t_str_replace() instead of doin... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/cf3c7bcbbdd9 changeset: 18469:cf3c7bcbbdd9 user: Timo Sirainen date: Fri Apr 24 13:08:16 2015 +0300 description: lib-fs: Use the new t_str_replace() instead of doing it ourself. diffstat: src/lib-fs/fs-api.c | 15 +-------------- 1 files changed, 1 insertions(+), 14 deletions(-) diffs (25 lines): diff -r 90298bc1a481 -r cf3c7bcbbdd9 src/lib-fs/fs-api.c --- a/src/lib-fs/fs-api.c Fri Apr 24 13:07:39 2015 +0300 +++ b/src/lib-fs/fs-api.c Fri Apr 24 13:08:16 2015 +0300 @@ -90,20 +90,7 @@ static const char *fs_driver_module_name(const char *driver) { - string_t *str; - unsigned int i; - - if (strchr(driver, '-') == NULL) - return driver; - - str = t_str_new(32); - for (i = 0; driver[i] != '\0'; i++) { - if (driver[i] == '-') - str_append_c(str, '_'); - else - str_append_c(str, driver[i]); - } - return str_c(str); + return t_str_replace(driver, '-', '_'); } static void fs_class_try_load_plugin(const char *driver) From dovecot at dovecot.org Fri Apr 24 10:11:57 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 24 Apr 2015 10:11:57 +0000 Subject: dovecot-2.2: fts: Replace '-' with '_' in filter-specific fts_fi... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/26d37653d247 changeset: 18470:26d37653d247 user: Timo Sirainen date: Fri Apr 24 13:10:17 2015 +0300 description: fts: Replace '-' with '_' in filter-specific fts_filters_* settings. So for example using fts_filters_normalizer_icu instead of fts_filters_normalizer-icu. diffstat: src/plugins/fts/fts-user.c | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-) diffs (27 lines): diff -r cf3c7bcbbdd9 -r 26d37653d247 src/plugins/fts/fts-user.c --- a/src/plugins/fts/fts-user.c Fri Apr 24 13:08:16 2015 +0300 +++ b/src/plugins/fts/fts-user.c Fri Apr 24 13:10:17 2015 +0300 @@ -54,7 +54,7 @@ { const struct fts_filter *filter_class; struct fts_filter *filter = NULL, *parent = NULL; - const char *filters_key, *const *filters; + const char *filters_key, *const *filters, *filter_set_name; const char *str, *error, *set_key, *const *settings; unsigned int i; int ret = 0; @@ -81,11 +81,12 @@ } /* try the language-specific setting first */ + filter_set_name = t_str_replace(filters[i], '-', '_'); set_key = t_strdup_printf("fts_filters_%s_%s", - lang->name, filters[i]); + lang->name, filter_set_name); str = mail_user_plugin_getenv(user, set_key); if (str == NULL) { - set_key = t_strdup_printf("fts_filters_%s", filters[i]); + set_key = t_strdup_printf("fts_filters_%s", filter_set_name); str = mail_user_plugin_getenv(user, set_key); } settings = str == NULL ? NULL : t_strsplit_spaces(str, " "); From dovecot at dovecot.org Fri Apr 24 11:10:27 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 24 Apr 2015 11:10:27 +0000 Subject: dovecot-2.2: fts: Install fts-user.h header Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9905ebbf8395 changeset: 18471:9905ebbf8395 user: Timo Sirainen date: Fri Apr 24 14:03:01 2015 +0300 description: fts: Install fts-user.h header diffstat: src/plugins/fts/Makefile.am | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diffs (23 lines): diff -r 26d37653d247 -r 9905ebbf8395 src/plugins/fts/Makefile.am --- a/src/plugins/fts/Makefile.am Fri Apr 24 13:10:17 2015 +0300 +++ b/src/plugins/fts/Makefile.am Fri Apr 24 14:03:01 2015 +0300 @@ -43,7 +43,8 @@ fts-api-private.h \ fts-expunge-log.h \ fts-indexer.h \ - fts-parser.h + fts-parser.h \ + fts-user.h noinst_HEADERS = \ doveadm-fts.h \ @@ -51,8 +52,7 @@ fts-plugin.h \ fts-search-args.h \ fts-search-serialize.h \ - fts-storage.h \ - fts-user.h + fts-storage.h pkglibexec_PROGRAMS = xml2text From dovecot at dovecot.org Fri Apr 24 11:10:28 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 24 Apr 2015 11:10:28 +0000 Subject: dovecot-2.2: fts: Don't always try to initialize lib-fts. Requir... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/6d5240e7d344 changeset: 18472:6d5240e7d344 user: Timo Sirainen date: Fri Apr 24 14:08:39 2015 +0300 description: fts: Don't always try to initialize lib-fts. Require backend to explictly do it now. diffstat: src/plugins/fts-lucene/fts-lucene-plugin.c | 9 ++++++++- src/plugins/fts/fts-plugin.c | 1 - src/plugins/fts/fts-user.c | 17 ++++++++++------- src/plugins/fts/fts-user.h | 2 +- 4 files changed, 19 insertions(+), 10 deletions(-) diffs (117 lines): diff -r 9905ebbf8395 -r 6d5240e7d344 src/plugins/fts-lucene/fts-lucene-plugin.c --- a/src/plugins/fts-lucene/fts-lucene-plugin.c Fri Apr 24 14:03:01 2015 +0300 +++ b/src/plugins/fts-lucene/fts-lucene-plugin.c Fri Apr 24 14:08:39 2015 +0300 @@ -4,6 +4,7 @@ #include "crc32.h" #include "mail-storage-hooks.h" #include "lucene-wrapper.h" +#include "fts-user.h" #include "fts-lucene-plugin.h" const char *fts_lucene_plugin_version = DOVECOT_ABI_VERSION; @@ -94,7 +95,7 @@ static void fts_lucene_mail_user_created(struct mail_user *user) { struct fts_lucene_user *fuser; - const char *env; + const char *env, *error; fuser = p_new(user->pool, struct fts_lucene_user, 1); env = mail_user_plugin_getenv(user, "fts_lucene"); @@ -105,6 +106,12 @@ /* invalid settings, disabling */ return; } + if (fuser->set.use_libfts) { + if (fts_mail_user_create(user, &error) < 0) { + i_error("fts_lucene: %s", error); + return; + } + } MODULE_CONTEXT_SET(user, fts_lucene_user_module, fuser); } diff -r 9905ebbf8395 -r 6d5240e7d344 src/plugins/fts/fts-plugin.c --- a/src/plugins/fts/fts-plugin.c Fri Apr 24 14:03:01 2015 +0300 +++ b/src/plugins/fts/fts-plugin.c Fri Apr 24 14:08:39 2015 +0300 @@ -13,7 +13,6 @@ const char *fts_plugin_version = DOVECOT_ABI_VERSION; static struct mail_storage_hooks fts_mail_storage_hooks = { - .mail_user_created = fts_mail_user_created, .mailbox_list_created = fts_mailbox_list_created, .mailbox_allocated = fts_mailbox_allocated, .mail_allocated = fts_mail_allocated diff -r 9905ebbf8395 -r 6d5240e7d344 src/plugins/fts/fts-user.c --- a/src/plugins/fts/fts-user.c Fri Apr 24 14:03:01 2015 +0300 +++ b/src/plugins/fts/fts-user.c Fri Apr 24 14:08:39 2015 +0300 @@ -21,14 +21,15 @@ &mail_user_module_register); static int -fts_user_init_languages(struct mail_user *user, struct fts_user *fuser) +fts_user_init_languages(struct mail_user *user, struct fts_user *fuser, + const char **error_r) { const char *languages, *unknown; const char *lang_config[3] = {NULL, NULL, NULL}; languages = mail_user_plugin_getenv(user, "fts_languages"); if (languages == NULL) { - i_error("fts-dovecot: fts_languages setting is missing - disabling"); + *error_r = "fts_languages setting is missing"; return -1; } @@ -38,11 +39,12 @@ lang_config[0] = "fts_language_config"; if (!fts_language_list_add_names(fuser->lang_list, languages, &unknown)) { - i_error("fts_languages: Unknown language '%s'", unknown); + *error_r = t_strdup_printf( + "fts_languages: Unknown language '%s'", unknown); return -1; } if (array_count(fts_language_list_get_all(fuser->lang_list)) == 0) { - i_error("fts-dovecot: fts_languages setting is empty - disabling"); + *error_r = "fts_languages setting is empty"; return -1; } return 0; @@ -199,7 +201,7 @@ fuser->module_ctx.super.deinit(user); } -void fts_mail_user_created(struct mail_user *user) +int fts_mail_user_create(struct mail_user *user, const char **error_r) { struct mail_user_vfuncs *v = user->vlast; struct fts_user *fuser; @@ -207,13 +209,14 @@ fuser = p_new(user->pool, struct fts_user, 1); p_array_init(&fuser->languages, user->pool, 4); - if (fts_user_init_languages(user, fuser) < 0) { + if (fts_user_init_languages(user, fuser, error_r) < 0) { fts_user_free(fuser); - return; + return -1; } fuser->module_ctx.super = *v; user->vlast = &fuser->module_ctx.super; v->deinit = fts_mail_user_deinit; MODULE_CONTEXT_SET(user, fts_user_module, fuser); + return 0; } diff -r 9905ebbf8395 -r 6d5240e7d344 src/plugins/fts/fts-user.h --- a/src/plugins/fts/fts-user.h Fri Apr 24 14:03:01 2015 +0300 +++ b/src/plugins/fts/fts-user.h Fri Apr 24 14:08:39 2015 +0300 @@ -17,6 +17,6 @@ const ARRAY_TYPE(fts_user_language) * fts_user_get_all_languages(struct mail_user *user); -void fts_mail_user_created(struct mail_user *user); +int fts_mail_user_create(struct mail_user *user, const char **error_r); #endif From dovecot at dovecot.org Fri Apr 24 11:10:33 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 24 Apr 2015 11:10:33 +0000 Subject: dovecot-2.2: fts: Minor error messagage prefix change. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/066db31d4dea changeset: 18473:066db31d4dea user: Timo Sirainen date: Fri Apr 24 14:08:51 2015 +0300 description: fts: Minor error messagage prefix change. diffstat: src/plugins/fts/fts-build-mail.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diffs (12 lines): diff -r 6d5240e7d344 -r 066db31d4dea src/plugins/fts/fts-build-mail.c --- a/src/plugins/fts/fts-build-mail.c Fri Apr 24 14:08:39 2015 +0300 +++ b/src/plugins/fts/fts-build-mail.c Fri Apr 24 14:08:51 2015 +0300 @@ -312,7 +312,7 @@ } else { if (fts_user_language_get(user, lang, &ctx->cur_user_lang, &error) < 0) { - i_error("fts-dovecot: Can't index input because of invalid language '%s' config: %s", + i_error("fts: Can't index input because of invalid language '%s' config: %s", lang->name, error); return -1; } From dovecot at dovecot.org Fri Apr 24 11:26:44 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 24 Apr 2015 11:26:44 +0000 Subject: dovecot-2.2: fts-solr: fts_solr=use_libfts send data to Solr via... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9974c748f72a changeset: 18474:9974c748f72a user: Timo Sirainen date: Fri Apr 24 14:25:06 2015 +0300 description: fts-solr: fts_solr=use_libfts send data to Solr via space-separated tokens. In this case Solr should be configured to not do any kind of filtering and use only WhitespaceTokenizerFactory. diffstat: src/plugins/fts-solr/fts-backend-solr.c | 20 ++++++++++++++++++-- src/plugins/fts-solr/fts-solr-plugin.c | 10 ++++++++++ src/plugins/fts-solr/fts-solr-plugin.h | 1 + 3 files changed, 29 insertions(+), 2 deletions(-) diffs (107 lines): diff -r 066db31d4dea -r 9974c748f72a src/plugins/fts-solr/fts-backend-solr.c --- a/src/plugins/fts-solr/fts-backend-solr.c Fri Apr 24 14:08:51 2015 +0300 +++ b/src/plugins/fts-solr/fts-backend-solr.c Fri Apr 24 14:25:06 2015 +0300 @@ -50,6 +50,7 @@ uint32_t last_indexed_uid; + unsigned int tokenized_input:1; unsigned int last_indexed_uid_set:1; unsigned int body_open:1; unsigned int documents_added:1; @@ -165,6 +166,11 @@ *error_r = "Invalid fts_solr setting"; return -1; } + if (fuser->set.use_libfts) { + /* change our flags so we get proper input */ + _backend->flags &= ~FTS_BACKEND_FLAG_FUZZY_SEARCH; + _backend->flags |= FTS_BACKEND_FLAG_TOKENIZED_INPUT; + } return solr_connection_init(fuser->set.url, fuser->set.debug, &backend->solr_conn, error_r); } @@ -249,6 +255,8 @@ ctx = i_new(struct solr_fts_backend_update_context, 1); ctx->ctx.backend = _backend; + ctx->tokenized_input = + (_backend->flags & FTS_BACKEND_FLAG_TOKENIZED_INPUT) != 0; i_array_init(&ctx->fields, 16); return &ctx->ctx; } @@ -547,13 +555,21 @@ size -= len; } xml_encode_data(ctx->cmd, data, size); + if (ctx->tokenized_input) + str_append_c(ctx->cmd, ' '); } else { - if (!ctx->truncate_header) + if (!ctx->truncate_header) { xml_encode_data(ctx->cur_value, data, size); + if (ctx->tokenized_input) + str_append_c(ctx->cur_value, ' '); + } if (ctx->cur_value2 != NULL && (!ctx->truncate_header || - str_len(ctx->cur_value2) < SOLR_HEADER_LINE_MAX_TRUNC_SIZE)) + str_len(ctx->cur_value2) < SOLR_HEADER_LINE_MAX_TRUNC_SIZE)) { xml_encode_data(ctx->cur_value2, data, size); + if (ctx->tokenized_input) + str_append_c(ctx->cur_value2, ' '); + } } if (str_len(ctx->cmd) >= SOLR_CMDBUF_FLUSH_SIZE) { diff -r 066db31d4dea -r 9974c748f72a src/plugins/fts-solr/fts-solr-plugin.c --- a/src/plugins/fts-solr/fts-solr-plugin.c Fri Apr 24 14:08:51 2015 +0300 +++ b/src/plugins/fts-solr/fts-solr-plugin.c Fri Apr 24 14:25:06 2015 +0300 @@ -6,6 +6,7 @@ #include "mail-user.h" #include "mail-storage-hooks.h" #include "solr-connection.h" +#include "fts-user.h" #include "fts-solr-plugin.h" #include @@ -30,6 +31,8 @@ set->url = p_strdup(user->pool, *tmp + 4); } else if (strcmp(*tmp, "debug") == 0) { set->debug = TRUE; + } else if (strcmp(*tmp, "use_libfts") == 0) { + set->use_libfts = TRUE; } else if (strcmp(*tmp, "break-imap-search") == 0) { /* for backwards compatibility */ } else if (strcmp(*tmp, "default_ns=") == 0) { @@ -50,12 +53,19 @@ static void fts_solr_mail_user_create(struct mail_user *user, const char *env) { struct fts_solr_user *fuser; + const char *error; fuser = p_new(user->pool, struct fts_solr_user, 1); if (fts_solr_plugin_init_settings(user, &fuser->set, env) < 0) { /* invalid settings, disabling */ return; } + if (fuser->set.use_libfts) { + if (fts_mail_user_create(user, &error) < 0) { + i_error("fts-solr: %s", error); + return; + } + } MODULE_CONTEXT_SET(user, fts_solr_user_module, fuser); } diff -r 066db31d4dea -r 9974c748f72a src/plugins/fts-solr/fts-solr-plugin.h --- a/src/plugins/fts-solr/fts-solr-plugin.h Fri Apr 24 14:08:51 2015 +0300 +++ b/src/plugins/fts-solr/fts-solr-plugin.h Fri Apr 24 14:25:06 2015 +0300 @@ -10,6 +10,7 @@ struct fts_solr_settings { const char *url, *default_ns_prefix; + bool use_libfts; bool debug; }; From dovecot at dovecot.org Fri Apr 24 11:49:20 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 24 Apr 2015 11:49:20 +0000 Subject: dovecot-2.2: doveadm-server: Fixed returning command input strea... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/daa426965e5a changeset: 18475:daa426965e5a user: Timo Sirainen date: Fri Apr 24 14:47:43 2015 +0300 description: doveadm-server: Fixed returning command input stream when its data was already in the input stream. diffstat: src/doveadm/doveadm-mail.c | 6 +++++- 1 files changed, 5 insertions(+), 1 deletions(-) diffs (16 lines): diff -r 9974c748f72a -r daa426965e5a src/doveadm/doveadm-mail.c --- a/src/doveadm/doveadm-mail.c Fri Apr 24 14:25:06 2015 +0300 +++ b/src/doveadm/doveadm-mail.c Fri Apr 24 14:47:43 2015 +0300 @@ -188,7 +188,11 @@ doveadm_mail_cmd_input_input, ctx); to = timeout_add(DOVEADM_MAIL_CMD_INPUT_TIMEOUT_MSECS, doveadm_mail_cmd_input_timeout, ctx); - io_loop_run(ioloop); + /* read the pending input from stream. */ + io_loop_set_running(ioloop); + doveadm_mail_cmd_input_input(ctx); + if (io_loop_is_running(ioloop)) + io_loop_run(ioloop); io_remove(&io); timeout_remove(&to); io_loop_destroy(&ioloop); From dovecot at dovecot.org Fri Apr 24 12:15:38 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 24 Apr 2015 12:15:38 +0000 Subject: dovecot-2.2: lmtp: Added lmtp_hdr_delivery_address=final|origina... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/df0eba01d105 changeset: 18476:df0eba01d105 user: Timo Sirainen date: Fri Apr 24 15:13:59 2015 +0300 description: lmtp: Added lmtp_hdr_delivery_address=final|original|none setting. This controls what username is used for logging Delivered-To: header and the "for user at domain" in Received: header. The default "final" is the regular username as earlier, "original" logs the RFC822 address received via ORCPT parameter or fallbacks to username, "none" logs nothing. diffstat: src/lmtp/commands.c | 18 +++++++++++++++++- src/lmtp/lmtp-settings.c | 29 +++++++++++++++++++++++++++++ src/lmtp/lmtp-settings.h | 11 +++++++++++ 3 files changed, 57 insertions(+), 1 deletions(-) diffs (126 lines): diff -r daa426965e5a -r df0eba01d105 src/lmtp/commands.c --- a/src/lmtp/commands.c Fri Apr 24 14:47:43 2015 +0300 +++ b/src/lmtp/commands.c Fri Apr 24 15:13:59 2015 +0300 @@ -1060,13 +1060,29 @@ static const char *client_get_added_headers(struct client *client) { string_t *str = t_str_new(200); + void **sets; + const struct lmtp_settings *lmtp_set; const char *host, *rcpt_to = NULL; if (array_count(&client->state.rcpt_to) == 1) { struct mail_recipient *const *rcptp = array_idx(&client->state.rcpt_to, 0); - rcpt_to = (*rcptp)->address; + sets = mail_storage_service_user_get_set((*rcptp)->service_user); + lmtp_set = sets[2]; + + switch (lmtp_set->parsed_lmtp_hdr_delivery_address) { + case LMTP_HDR_DELIVERY_ADDRESS_NONE: + break; + case LMTP_HDR_DELIVERY_ADDRESS_FINAL: + rcpt_to = (*rcptp)->address; + break; + case LMTP_HDR_DELIVERY_ADDRESS_ORIGINAL: + if (!orcpt_get_valid_rfc822((*rcptp)->params.dsn_orcpt, + &rcpt_to)) + rcpt_to = (*rcptp)->address; + break; + } } /* don't set Return-Path when proxying so it won't get added twice */ diff -r daa426965e5a -r df0eba01d105 src/lmtp/lmtp-settings.c --- a/src/lmtp/lmtp-settings.c Fri Apr 24 14:47:43 2015 +0300 +++ b/src/lmtp/lmtp-settings.c Fri Apr 24 15:13:59 2015 +0300 @@ -14,6 +14,8 @@ #include #include +static bool lmtp_settings_check(void *_set, pool_t pool, const char **error_r); + /* */ static struct file_listener_settings lmtp_unix_listeners_array[] = { { "lmtp", 0666, "", "" } @@ -62,6 +64,7 @@ DEF(SET_BOOL, lmtp_rcpt_check_quota), DEF(SET_UINT, lmtp_user_concurrency_limit), DEF(SET_STR, lmtp_address_translate), + DEF(SET_ENUM, lmtp_hdr_delivery_address), DEF(SET_STR_VARS, login_greeting), DEF(SET_STR, login_trusted_networks), @@ -74,6 +77,7 @@ .lmtp_rcpt_check_quota = FALSE, .lmtp_user_concurrency_limit = 0, .lmtp_address_translate = "", + .lmtp_hdr_delivery_address = "final:none:original", .login_greeting = PACKAGE_NAME" ready.", .login_trusted_networks = "" }; @@ -93,9 +97,34 @@ .parent_offset = (size_t)-1, + .check_func = lmtp_settings_check, .dependencies = lmtp_setting_dependencies }; +/* */ +static bool lmtp_settings_check(void *_set, pool_t pool ATTR_UNUSED, + const char **error_r) +{ + struct lmtp_settings *set = _set; + + if (strcmp(set->lmtp_hdr_delivery_address, "none") == 0) { + set->parsed_lmtp_hdr_delivery_address = + LMTP_HDR_DELIVERY_ADDRESS_NONE; + } else if (strcmp(set->lmtp_hdr_delivery_address, "final") == 0) { + set->parsed_lmtp_hdr_delivery_address = + LMTP_HDR_DELIVERY_ADDRESS_FINAL; + } else if (strcmp(set->lmtp_hdr_delivery_address, "original") == 0) { + set->parsed_lmtp_hdr_delivery_address = + LMTP_HDR_DELIVERY_ADDRESS_ORIGINAL; + } else { + *error_r = t_strdup_printf("Unknown lmtp_hdr_delivery_address: %s", + set->lmtp_hdr_delivery_address); + return FALSE; + } + return TRUE; +} +/* */ + void lmtp_settings_dup(const struct setting_parser_context *set_parser, pool_t pool, struct lmtp_settings **lmtp_set_r, diff -r daa426965e5a -r df0eba01d105 src/lmtp/lmtp-settings.h --- a/src/lmtp/lmtp-settings.h Fri Apr 24 14:47:43 2015 +0300 +++ b/src/lmtp/lmtp-settings.h Fri Apr 24 15:13:59 2015 +0300 @@ -4,14 +4,25 @@ struct lda_settings; struct lmtp_settings; +/* */ +enum lmtp_hdr_delivery_address { + LMTP_HDR_DELIVERY_ADDRESS_NONE, + LMTP_HDR_DELIVERY_ADDRESS_FINAL, + LMTP_HDR_DELIVERY_ADDRESS_ORIGINAL +}; +/* */ + struct lmtp_settings { bool lmtp_proxy; bool lmtp_save_to_detail_mailbox; bool lmtp_rcpt_check_quota; unsigned int lmtp_user_concurrency_limit; const char *lmtp_address_translate; + const char *lmtp_hdr_delivery_address; const char *login_greeting; const char *login_trusted_networks; + + enum lmtp_hdr_delivery_address parsed_lmtp_hdr_delivery_address; }; extern const struct setting_parser_info lmtp_setting_parser_info; From dovecot at dovecot.org Fri Apr 24 12:43:33 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 24 Apr 2015 12:43:33 +0000 Subject: dovecot-2.2: doveadm fs delete -R: When deleting directories, in... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/98c111d5bd76 changeset: 18477:98c111d5bd76 user: Timo Sirainen date: Fri Apr 24 15:41:55 2015 +0300 description: doveadm fs delete -R: When deleting directories, include "/" suffix in the name. diffstat: src/doveadm/doveadm-fs.c | 23 +++++++++++++++-------- 1 files changed, 15 insertions(+), 8 deletions(-) diffs (79 lines): diff -r df0eba01d105 -r 98c111d5bd76 src/doveadm/doveadm-fs.c --- a/src/doveadm/doveadm-fs.c Fri Apr 24 15:13:59 2015 +0300 +++ b/src/doveadm/doveadm-fs.c Fri Apr 24 15:41:55 2015 +0300 @@ -246,7 +246,7 @@ static void cmd_fs_delete_dir_recursive(struct fs *fs, unsigned int async_count, - const char *path) + const char *path_prefix) { struct fs_iter *iter; ARRAY_TYPE(const_string) fnames; @@ -261,19 +261,22 @@ /* delete subdirs first. all fs backends can't handle recursive lookups, so save the list first. */ t_array_init(&fnames, 8); - iter = fs_iter_init(fs, path, FS_ITER_FLAG_DIRS); + iter = fs_iter_init(fs, path_prefix, FS_ITER_FLAG_DIRS); while ((fname = fs_iter_next(iter)) != NULL) { - fname = t_strdup(fname); + /* append "/" so that if FS_PROPERTY_DIRECTORIES is set, + we'll include the "/" suffix in the filename when deleting + it. */ + fname = t_strconcat(fname, "/", NULL); array_append(&fnames, &fname, 1); } if (fs_iter_deinit(&iter) < 0) { i_error("fs_iter_deinit(%s) failed: %s", - path, fs_last_error(fs)); + path_prefix, fs_last_error(fs)); doveadm_exit_code = EX_TEMPFAIL; } array_foreach(&fnames, fnamep) T_BEGIN { cmd_fs_delete_dir_recursive(fs, async_count, - t_strdup_printf("%s/%s", path, *fnamep)); + t_strdup_printf("%s%s", path_prefix, *fnamep)); } T_END; /* delete files. again because we're doing this asynchronously finish @@ -283,14 +286,14 @@ } else { array_clear(&fnames); } - iter = fs_iter_init(fs, path, 0); + iter = fs_iter_init(fs, path_prefix, 0); while ((fname = fs_iter_next(iter)) != NULL) { fname = t_strdup(fname); array_append(&fnames, &fname, 1); } if (fs_iter_deinit(&iter) < 0) { i_error("fs_iter_deinit(%s) failed: %s", - path, fs_last_error(fs)); + path_prefix, fs_last_error(fs)); doveadm_exit_code = EX_TEMPFAIL; } @@ -302,7 +305,7 @@ continue; ctx.files[i] = fs_file_init(fs, - t_strdup_printf("%s/%s", path, fname), + t_strdup_printf("%s%s", path_prefix, fname), FS_OPEN_MODE_READONLY | FS_OPEN_FLAG_ASYNC | FS_OPEN_FLAG_ASYNC_NOQUEUE); fname = NULL; @@ -337,9 +340,13 @@ struct fs *fs; struct fs_file *file; const char *path; + unsigned int path_len; fs = cmd_fs_init(&argc, &argv, 1, cmd_fs_delete); path = argv[0]; + path_len = strlen(path); + if (path_len > 0 && path[path_len-1] != '/') + path = t_strconcat(path, "/", NULL); cmd_fs_delete_dir_recursive(fs, async_count, path); if ((fs_get_properties(fs) & FS_PROPERTY_DIRECTORIES) != 0) { From dovecot at dovecot.org Fri Apr 24 12:47:19 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 24 Apr 2015 12:47:19 +0000 Subject: dovecot-2.2: doveadm fs delete -R: Fixed infinite looping when d... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/17b93a5ae095 changeset: 18478:17b93a5ae095 user: Timo Sirainen date: Fri Apr 24 15:45:42 2015 +0300 description: doveadm fs delete -R: Fixed infinite looping when deleting a lot of files with some backends. After file was deleted, we closed it. Then later on we tried to delete it all over again. diffstat: src/doveadm/doveadm-fs.c | 33 +++++++++++++++++++++------------ 1 files changed, 21 insertions(+), 12 deletions(-) diffs (80 lines): diff -r 98c111d5bd76 -r 17b93a5ae095 src/doveadm/doveadm-fs.c --- a/src/doveadm/doveadm-fs.c Fri Apr 24 15:41:55 2015 +0300 +++ b/src/doveadm/doveadm-fs.c Fri Apr 24 15:45:42 2015 +0300 @@ -217,9 +217,14 @@ fs_deinit(&fs); } +struct fs_delete_file { + struct fs_file *file; + bool finished; +}; + struct fs_delete_ctx { unsigned int files_count; - struct fs_file **files; + struct fs_delete_file **files; }; static bool cmd_fs_delete_ctx_run(struct fs_delete_ctx *ctx) @@ -228,17 +233,20 @@ bool ret = FALSE; for (i = 0; i < ctx->files_count; i++) { - if (ctx->files[i] == NULL) + if (ctx->files[i]->file == NULL || ctx->files[i]->finished) ; - else if (fs_delete(ctx->files[i]) == 0) - fs_file_deinit(&ctx->files[i]); - else if (errno == EAGAIN) + else if (fs_delete(ctx->files[i]->file) == 0) { + fs_file_deinit(&ctx->files[i]->file); + ctx->files[i]->finished = TRUE; + } else if (errno == EAGAIN) ret = TRUE; else { i_error("fs_delete(%s) failed: %s", - fs_file_path(ctx->files[i]), - fs_file_last_error(ctx->files[i])); + fs_file_path(ctx->files[i]->file), + fs_file_last_error(ctx->files[i]->file)); doveadm_exit_code = EX_TEMPFAIL; + fs_file_deinit(&ctx->files[i]->file); + ctx->files[i]->finished = TRUE; } } return ret; @@ -256,7 +264,7 @@ memset(&ctx, 0, sizeof(ctx)); ctx.files_count = I_MAX(async_count, 1); - ctx.files = t_new(struct fs_file *, ctx.files_count); + ctx.files = t_new(struct fs_delete_file *, ctx.files_count); /* delete subdirs first. all fs backends can't handle recursive lookups, so save the list first. */ @@ -301,10 +309,11 @@ fname = *fnamep; retry: for (i = 0; i < ctx.files_count; i++) { - if (ctx.files[i] != NULL) + if (ctx.files[i]->file != NULL || + ctx.files[i]->finished) continue; - ctx.files[i] = fs_file_init(fs, + ctx.files[i]->file = fs_file_init(fs, t_strdup_printf("%s%s", path_prefix, fname), FS_OPEN_MODE_READONLY | FS_OPEN_FLAG_ASYNC | FS_OPEN_FLAG_ASYNC_NOQUEUE); @@ -329,8 +338,8 @@ } } for (i = 0; i < ctx.files_count; i++) { - if (ctx.files[i] != NULL) - fs_file_deinit(&ctx.files[i]); + if (ctx.files[i]->file != NULL) + fs_file_deinit(&ctx.files[i]->file); } } From dovecot at dovecot.org Fri Apr 24 13:05:44 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Fri, 24 Apr 2015 13:05:44 +0000 Subject: dovecot-2.2: fts: Fixed infinite looping at deinit. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/008632bdfd2c changeset: 18479:008632bdfd2c user: Timo Sirainen date: Fri Apr 24 16:03:51 2015 +0300 description: fts: Fixed infinite looping at deinit. fts_mail_user_deinit() was calling itself. Now the fts backend is responsible for calling it at deinit. diffstat: src/plugins/fts-lucene/fts-lucene-plugin.c | 15 ++++++++++++++- src/plugins/fts-solr/fts-solr-plugin.c | 14 +++++++++++++- src/plugins/fts/fts-user.c | 22 ++++++++-------------- src/plugins/fts/fts-user.h | 3 ++- 4 files changed, 37 insertions(+), 17 deletions(-) diffs (128 lines): diff -r 17b93a5ae095 -r 008632bdfd2c src/plugins/fts-lucene/fts-lucene-plugin.c --- a/src/plugins/fts-lucene/fts-lucene-plugin.c Fri Apr 24 15:45:42 2015 +0300 +++ b/src/plugins/fts-lucene/fts-lucene-plugin.c Fri Apr 24 16:03:51 2015 +0300 @@ -92,8 +92,17 @@ return crc; } +static void fts_lucene_mail_user_deinit(struct mail_user *user) +{ + struct fts_lucene_user *fuser = FTS_LUCENE_USER_CONTEXT(user); + + fts_mail_user_deinit(user); + fuser->module_ctx.super.deinit(user); +} + static void fts_lucene_mail_user_created(struct mail_user *user) { + struct mail_user_vfuncs *v = user->vlast; struct fts_lucene_user *fuser; const char *env, *error; @@ -107,11 +116,15 @@ return; } if (fuser->set.use_libfts) { - if (fts_mail_user_create(user, &error) < 0) { + if (fts_mail_user_init(user, &error) < 0) { i_error("fts_lucene: %s", error); return; } } + + fuser->module_ctx.super = *v; + user->vlast = &fuser->module_ctx.super; + v->deinit = fts_lucene_mail_user_deinit; MODULE_CONTEXT_SET(user, fts_lucene_user_module, fuser); } diff -r 17b93a5ae095 -r 008632bdfd2c src/plugins/fts-solr/fts-solr-plugin.c --- a/src/plugins/fts-solr/fts-solr-plugin.c Fri Apr 24 15:45:42 2015 +0300 +++ b/src/plugins/fts-solr/fts-solr-plugin.c Fri Apr 24 16:03:51 2015 +0300 @@ -50,8 +50,17 @@ return 0; } +static void fts_solr_mail_user_deinit(struct mail_user *user) +{ + struct fts_solr_user *fuser = FTS_SOLR_USER_CONTEXT(user); + + fts_mail_user_deinit(user); + fuser->module_ctx.super.deinit(user); +} + static void fts_solr_mail_user_create(struct mail_user *user, const char *env) { + struct mail_user_vfuncs *v = user->vlast; struct fts_solr_user *fuser; const char *error; @@ -61,12 +70,15 @@ return; } if (fuser->set.use_libfts) { - if (fts_mail_user_create(user, &error) < 0) { + if (fts_mail_user_init(user, &error) < 0) { i_error("fts-solr: %s", error); return; } } + fuser->module_ctx.super = *v; + user->vlast = &fuser->module_ctx.super; + v->deinit = fts_solr_mail_user_deinit; MODULE_CONTEXT_SET(user, fts_solr_user_module, fuser); } diff -r 17b93a5ae095 -r 008632bdfd2c src/plugins/fts/fts-user.c --- a/src/plugins/fts/fts-user.c Fri Apr 24 15:45:42 2015 +0300 +++ b/src/plugins/fts/fts-user.c Fri Apr 24 16:03:51 2015 +0300 @@ -193,17 +193,8 @@ } } -static void fts_mail_user_deinit(struct mail_user *user) +int fts_mail_user_init(struct mail_user *user, const char **error_r) { - struct fts_user *fuser = FTS_USER_CONTEXT(user); - - fts_user_free(fuser); - fuser->module_ctx.super.deinit(user); -} - -int fts_mail_user_create(struct mail_user *user, const char **error_r) -{ - struct mail_user_vfuncs *v = user->vlast; struct fts_user *fuser; fuser = p_new(user->pool, struct fts_user, 1); @@ -213,10 +204,13 @@ fts_user_free(fuser); return -1; } - - fuser->module_ctx.super = *v; - user->vlast = &fuser->module_ctx.super; - v->deinit = fts_mail_user_deinit; MODULE_CONTEXT_SET(user, fts_user_module, fuser); return 0; } + +void fts_mail_user_deinit(struct mail_user *user) +{ + struct fts_user *fuser = FTS_USER_CONTEXT(user); + + fts_user_free(fuser); +} diff -r 17b93a5ae095 -r 008632bdfd2c src/plugins/fts/fts-user.h --- a/src/plugins/fts/fts-user.h Fri Apr 24 15:45:42 2015 +0300 +++ b/src/plugins/fts/fts-user.h Fri Apr 24 16:03:51 2015 +0300 @@ -17,6 +17,7 @@ const ARRAY_TYPE(fts_user_language) * fts_user_get_all_languages(struct mail_user *user); -int fts_mail_user_create(struct mail_user *user, const char **error_r); +int fts_mail_user_init(struct mail_user *user, const char **error_r); +void fts_mail_user_deinit(struct mail_user *user); #endif From dovecot at dovecot.org Sat Apr 25 08:24:39 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 25 Apr 2015 08:24:39 +0000 Subject: dovecot-2.2: lib: Added numpack_decode32() Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1c275f718758 changeset: 18480:1c275f718758 user: Timo Sirainen date: Sat Apr 25 11:22:39 2015 +0300 description: lib: Added numpack_decode32() diffstat: src/lib/numpack.c | 13 +++++++++++++ src/lib/numpack.h | 1 + 2 files changed, 14 insertions(+), 0 deletions(-) diffs (30 lines): diff -r 008632bdfd2c -r 1c275f718758 src/lib/numpack.c --- a/src/lib/numpack.c Fri Apr 24 16:03:51 2015 +0300 +++ b/src/lib/numpack.c Sat Apr 25 11:22:39 2015 +0300 @@ -41,3 +41,16 @@ *num_r = value; return 0; } + +int numpack_decode32(const uint8_t **p, const uint8_t *end, uint32_t *num_r) +{ + uint64_t num; + + if (numpack_decode(p, end, &num) < 0) + return -1; + if (num > 4294967295U) + return -1; + + *num_r = (uint32_t)num; + return 0; +} diff -r 008632bdfd2c -r 1c275f718758 src/lib/numpack.h --- a/src/lib/numpack.h Fri Apr 24 16:03:51 2015 +0300 +++ b/src/lib/numpack.h Sat Apr 25 11:22:39 2015 +0300 @@ -6,5 +6,6 @@ void numpack_encode(buffer_t *buf, uint64_t num); int numpack_decode(const uint8_t **p, const uint8_t *end, uint64_t *num_r); +int numpack_decode32(const uint8_t **p, const uint8_t *end, uint32_t *num_r); #endif From dovecot at dovecot.org Sat Apr 25 08:24:40 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 25 Apr 2015 08:24:40 +0000 Subject: dovecot-2.2: lib: Added istream-unix for reading fd sockets via ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/21a2ce6f8f37 changeset: 18481:21a2ce6f8f37 user: Timo Sirainen date: Sat Apr 25 11:23:00 2015 +0300 description: lib: Added istream-unix for reading fd sockets via istream. diffstat: src/lib/Makefile.am | 4 + src/lib/istream-file-private.h | 23 ++++ src/lib/istream-file.c | 36 +++---- src/lib/istream-unix.c | 105 ++++++++++++++++++++++ src/lib/istream-unix.h | 15 +++ src/lib/test-istream-unix.c | 190 +++++++++++++++++++++++++++++++++++++++++ src/lib/test-lib.c | 1 + src/lib/test-lib.h | 1 + 8 files changed, 355 insertions(+), 20 deletions(-) diffs (truncated from 497 to 300 lines): diff -r 1c275f718758 -r 21a2ce6f8f37 src/lib/Makefile.am --- a/src/lib/Makefile.am Sat Apr 25 11:22:39 2015 +0300 +++ b/src/lib/Makefile.am Sat Apr 25 11:23:00 2015 +0300 @@ -74,6 +74,7 @@ istream-sized.c \ istream-tee.c \ istream-timeout.c \ + istream-unix.c \ ioloop.c \ ioloop-iolist.c \ ioloop-notify-none.c \ @@ -200,6 +201,7 @@ istream-chain.h \ istream-concat.h \ istream-crlf.h \ + istream-file-private.h \ istream-hash.h \ istream-jsonstr.h \ istream-private.h \ @@ -208,6 +210,7 @@ istream-sized.h \ istream-tee.h \ istream-timeout.h \ + istream-unix.h \ ioloop.h \ ioloop-iolist.h \ ioloop-private.h \ @@ -302,6 +305,7 @@ test-istream-crlf.c \ test-istream-seekable.c \ test-istream-tee.c \ + test-istream-unix.c \ test-json-parser.c \ test-json-tree.c \ test-llist.c \ diff -r 1c275f718758 -r 21a2ce6f8f37 src/lib/istream-file-private.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/istream-file-private.h Sat Apr 25 11:23:00 2015 +0300 @@ -0,0 +1,23 @@ +#ifndef ISTREAM_FILE_PRIVATE_H +#define ISTREAM_FILE_PRIVATE_H + +#include "istream-private.h" + +struct file_istream { + struct istream_private istream; + + uoff_t skip_left; + + unsigned int file:1; + unsigned int autoclose_fd:1; + unsigned int seen_eof:1; +}; + +struct istream * +i_stream_create_file_common(struct file_istream *fstream, + int fd, const char *path, + size_t max_buffer_size, bool autoclose_fd); +ssize_t i_stream_file_read(struct istream_private *stream); +void i_stream_file_close(struct iostream_private *stream, bool close_parent); + +#endif diff -r 1c275f718758 -r 21a2ce6f8f37 src/lib/istream-file.c --- a/src/lib/istream-file.c Sat Apr 25 11:22:39 2015 +0300 +++ b/src/lib/istream-file.c Sat Apr 25 11:23:00 2015 +0300 @@ -4,7 +4,7 @@ #include "lib.h" #include "ioloop.h" -#include "istream-private.h" +#include "istream-file-private.h" #include "net.h" #include @@ -12,18 +12,8 @@ #include #include -struct file_istream { - struct istream_private istream; - - uoff_t skip_left; - - unsigned int file:1; - unsigned int autoclose_fd:1; - unsigned int seen_eof:1; -}; - -static void i_stream_file_close(struct iostream_private *stream, - bool close_parent ATTR_UNUSED) +void i_stream_file_close(struct iostream_private *stream, + bool close_parent ATTR_UNUSED) { struct file_istream *fstream = (struct file_istream *)stream; struct istream_private *_stream = (struct istream_private *)stream; @@ -51,7 +41,7 @@ return 0; } -static ssize_t i_stream_file_read(struct istream_private *stream) +ssize_t i_stream_file_read(struct istream_private *stream) { struct file_istream *fstream = (struct file_istream *) stream; uoff_t offset; @@ -183,16 +173,15 @@ return 0; } -static struct istream * -i_stream_create_file_common(int fd, const char *path, +struct istream * +i_stream_create_file_common(struct file_istream *fstream, + int fd, const char *path, size_t max_buffer_size, bool autoclose_fd) { - struct file_istream *fstream; struct istream *input; struct stat st; bool is_file; - fstream = i_new(struct file_istream, 1); fstream->autoclose_fd = autoclose_fd; fstream->istream.iostream.close = i_stream_file_close; @@ -235,9 +224,13 @@ struct istream *i_stream_create_fd(int fd, size_t max_buffer_size, bool autoclose_fd) { + struct file_istream *fstream; + i_assert(fd != -1); - return i_stream_create_file_common(fd, NULL, max_buffer_size, autoclose_fd); + fstream = i_new(struct file_istream, 1); + return i_stream_create_file_common(fstream, fd, NULL, + max_buffer_size, autoclose_fd); } struct istream *i_stream_create_fd_autoclose(int *fd, size_t max_buffer_size) @@ -251,9 +244,12 @@ struct istream *i_stream_create_file(const char *path, size_t max_buffer_size) { + struct file_istream *fstream; struct istream *input; - input = i_stream_create_file_common(-1, path, max_buffer_size, TRUE); + fstream = i_new(struct file_istream, 1); + input = i_stream_create_file_common(fstream, -1, path, + max_buffer_size, TRUE); i_stream_set_name(input, path); return input; } diff -r 1c275f718758 -r 21a2ce6f8f37 src/lib/istream-unix.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/istream-unix.c Sat Apr 25 11:23:00 2015 +0300 @@ -0,0 +1,105 @@ +/* Copyright (c) 2014 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "fdpass.h" +#include "istream-file-private.h" +#include "istream-unix.h" + +struct unix_istream { + struct file_istream fstream; + bool next_read_fd; + int read_fd; +}; + +static void +i_stream_unix_close(struct iostream_private *stream, bool close_parent) +{ + struct unix_istream *ustream = (struct unix_istream *)stream; + + if (ustream->read_fd != -1) + i_close_fd(&ustream->read_fd); + i_stream_file_close(stream, close_parent); +} + +static ssize_t i_stream_unix_read(struct istream_private *stream) +{ + struct unix_istream *ustream = (struct unix_istream *)stream; + size_t size; + ssize_t ret; + + if (!ustream->next_read_fd) + return i_stream_file_read(stream); + + i_assert(ustream->read_fd == -1); + i_assert(ustream->fstream.skip_left == 0); /* not supported here.. */ + if (!i_stream_try_alloc(stream, 1, &size)) + return -2; + + do { + ret = fd_read(stream->fd, + stream->w_buffer + stream->pos, size, + &ustream->read_fd); + } while (unlikely(ret < 0 && errno == EINTR && + stream->istream.blocking)); + if (ustream->read_fd != -1) + ustream->next_read_fd = FALSE; + + if (unlikely(ret < 0)) { + if (errno == EINTR || errno == EAGAIN) { + i_assert(!stream->istream.blocking); + return 0; + } else { + i_assert(errno != 0); + /* if we get EBADF for a valid fd, it means something's + really wrong and we'd better just crash. */ + i_assert(errno != EBADF); + stream->istream.stream_errno = errno; + return -1; + } + } + stream->pos += ret; + return ret; +} + +struct istream *i_stream_create_unix(int fd, size_t max_buffer_size) +{ + struct unix_istream *ustream; + struct istream *input; + + i_assert(fd != -1); + + ustream = i_new(struct unix_istream, 1); + ustream->read_fd = -1; + input = i_stream_create_file_common(&ustream->fstream, fd, NULL, + max_buffer_size, FALSE); + input->real_stream->iostream.close = i_stream_unix_close; + input->real_stream->read = i_stream_unix_read; + return input; +} + +void i_stream_unix_set_read_fd(struct istream *input) +{ + struct unix_istream *ustream = + (struct unix_istream *)input->real_stream; + + ustream->next_read_fd = TRUE; +} + +void i_stream_unix_unset_read_fd(struct istream *input) +{ + struct unix_istream *ustream = + (struct unix_istream *)input->real_stream; + + ustream->next_read_fd = FALSE; +} + +int i_stream_unix_get_read_fd(struct istream *input) +{ + struct unix_istream *ustream = + (struct unix_istream *)input->real_stream; + int fd; + + fd = ustream->read_fd; + ustream->read_fd = -1; + return fd; +} diff -r 1c275f718758 -r 21a2ce6f8f37 src/lib/istream-unix.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/istream-unix.h Sat Apr 25 11:23:00 2015 +0300 @@ -0,0 +1,15 @@ +#ifndef ISTREAM_UNIX_H +#define ISTREAM_UNIX_H + +struct istream *i_stream_create_unix(int fd, size_t max_buffer_size); +/* Start trying to read a file descriptor from the UNIX socket. */ +void i_stream_unix_set_read_fd(struct istream *input); +/* Stop trying to read a file descriptor from the UNIX socket. */ +void i_stream_unix_unset_read_fd(struct istream *input); +/* Returns the fd that the last i_stream_read() received, or -1 if no fd + was received. This function must be called before + i_stream_unix_set_read_fd() is called again after successfully receiving + a file descriptor. */ +int i_stream_unix_get_read_fd(struct istream *input); + +#endif diff -r 1c275f718758 -r 21a2ce6f8f37 src/lib/test-istream-unix.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/test-istream-unix.c Sat Apr 25 11:23:00 2015 +0300 @@ -0,0 +1,190 @@ +/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */ + +#include "test-lib.h" +#include "net.h" +#include "fd-set-nonblock.h" +#include "fdpass.h" +#include "istream.h" +#include "istream-unix.h" + +#include +#include +#include + +static int send_fd, send_fd2; + From dovecot at dovecot.org Sat Apr 25 08:28:54 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 25 Apr 2015 08:28:54 +0000 Subject: dovecot-2.2: imap: Code cleanup: Moved IDLE keepalive timer calc... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/25c848f10517 changeset: 18482:25c848f10517 user: Timo Sirainen date: Sat Apr 25 11:27:17 2015 +0300 description: imap: Code cleanup: Moved IDLE keepalive timer calculation to lib-imap. diffstat: src/imap/cmd-idle.c | 58 +++--------------------------------------- src/lib-imap/Makefile.am | 2 + src/lib-imap/imap-keepalive.c | 49 ++++++++++++++++++++++++++++++++++++ src/lib-imap/imap-keepalive.h | 24 +++++++++++++++++ 4 files changed, 80 insertions(+), 53 deletions(-) diffs (185 lines): diff -r 21a2ce6f8f37 -r 25c848f10517 src/imap/cmd-idle.c --- a/src/imap/cmd-idle.c Sat Apr 25 11:23:00 2015 +0300 +++ b/src/imap/cmd-idle.c Sat Apr 25 11:27:17 2015 +0300 @@ -8,6 +8,7 @@ #include "crc32.h" #include "mail-storage-settings.h" #include "imap-commands.h" +#include "imap-keepalive.h" #include "imap-sync.h" #include @@ -161,69 +162,20 @@ } } -static bool remote_ip_is_usable(const struct ip_addr *ip) -{ - unsigned int addr; - - if (ip->family == 0) - return FALSE; - if (ip->family == AF_INET) { -#define IP4(a,b,c,d) ((unsigned)(a)<<24|(unsigned)(b)<<16|(unsigned)(c)<<8|(unsigned)(d)) - addr = ip->u.ip4.s_addr; - if (addr >= IP4(10,0,0,0) && addr <= IP4(10,255,255,255)) - return FALSE; /* 10/8 */ - if (addr >= IP4(192,168,0,0) && addr <= IP4(192,168,255,255)) - return FALSE; /* 192.168/16 */ - if (addr >= IP4(172,16,0,0) && addr <= IP4(172,31,255,255)) - return FALSE; /* 172.16/12 */ - if (addr >= IP4(127,0,0,0) && addr <= IP4(127,255,255,255)) - return FALSE; /* 127/8 */ -#undef IP4 - } -#ifdef HAVE_IPV6 - else if (ip->family == AF_INET6) { - addr = ip->u.ip6.s6_addr[0]; - if (addr == 0xfc || addr == 0xfd) - return FALSE; /* fc00::/7 */ - } -#endif - return TRUE; -} - static void idle_add_keepalive_timeout(struct cmd_idle_context *ctx) { unsigned int interval = ctx->client->set->imap_idle_notify_interval; - unsigned int client_hash; if (interval == 0) return; - /* set the interval so that the client gets the keepalive notifications - at exactly the same time for all the connections. this helps to - reduce battery usage in mobile devices. but we don't really want to - send this notification for everyone at the same time, because it - would cause huge peaks of activity. - - basing the notifications on the username works well for one account, - but basing it on the IP address allows the client to get all of the - notifications at the same time for multiple accounts as well (of - course assuming Dovecot is running on all the servers :) - - one potential downside to using IP is that if a proxy hides the - client's IP address notifications are sent to everyone at the same - time, but this can be avoided by using a properly configured Dovecot - proxy. we'll also try to avoid this by not doing it for the commonly - used intranet IP ranges. */ - client_hash = ctx->client->user->remote_ip != NULL && - remote_ip_is_usable(ctx->client->user->remote_ip) ? - net_ip_hash(ctx->client->user->remote_ip) : - crc32_str(ctx->client->user->username); - interval -= (time(NULL) + client_hash) % interval; + interval = imap_keepalive_interval_msecs(ctx->client->user->username, + ctx->client->user->remote_ip, + interval); if (ctx->keepalive_to != NULL) timeout_remove(&ctx->keepalive_to); - ctx->keepalive_to = timeout_add(interval * 1000, - keepalive_timeout, ctx); + ctx->keepalive_to = timeout_add(interval, keepalive_timeout, ctx); } static bool cmd_idle_continue(struct client_command_context *cmd) diff -r 21a2ce6f8f37 -r 25c848f10517 src/lib-imap/Makefile.am --- a/src/lib-imap/Makefile.am Sat Apr 25 11:23:00 2015 +0300 +++ b/src/lib-imap/Makefile.am Sat Apr 25 11:27:17 2015 +0300 @@ -13,6 +13,7 @@ imap-date.c \ imap-envelope.c \ imap-id.c \ + imap-keepalive.c \ imap-match.c \ imap-parser.c \ imap-quote.c \ @@ -28,6 +29,7 @@ imap-date.h \ imap-envelope.h \ imap-id.h \ + imap-keepalive.h \ imap-match.h \ imap-parser.h \ imap-resp-code.h \ diff -r 21a2ce6f8f37 -r 25c848f10517 src/lib-imap/imap-keepalive.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-imap/imap-keepalive.c Sat Apr 25 11:27:17 2015 +0300 @@ -0,0 +1,49 @@ +/* Copyright (c) 2002-2014 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "crc32.h" +#include "net.h" +#include "imap-keepalive.h" + +#include + +static bool imap_remote_ip_is_usable(const struct ip_addr *ip) +{ + unsigned int addr; + + if (ip->family == 0) + return FALSE; + if (ip->family == AF_INET) { +#define IP4(a,b,c,d) ((unsigned)(a)<<24|(unsigned)(b)<<16|(unsigned)(c)<<8|(unsigned)(d)) + addr = ip->u.ip4.s_addr; + if (addr >= IP4(10,0,0,0) && addr <= IP4(10,255,255,255)) + return FALSE; /* 10/8 */ + if (addr >= IP4(192,168,0,0) && addr <= IP4(192,168,255,255)) + return FALSE; /* 192.168/16 */ + if (addr >= IP4(172,16,0,0) && addr <= IP4(172,31,255,255)) + return FALSE; /* 172.16/12 */ + if (addr >= IP4(127,0,0,0) && addr <= IP4(127,255,255,255)) + return FALSE; /* 127/8 */ +#undef IP4 + } +#ifdef HAVE_IPV6 + else if (ip->family == AF_INET6) { + addr = ip->u.ip6.s6_addr[0]; + if (addr == 0xfc || addr == 0xfd) + return FALSE; /* fc00::/7 */ + } +#endif + return TRUE; +} + +unsigned int +imap_keepalive_interval_msecs(const char *username, const struct ip_addr *ip, + unsigned int interval_secs) +{ + unsigned int client_hash; + + client_hash = imap_remote_ip_is_usable(ip) ? + net_ip_hash(ip) : crc32_str(username); + interval_secs -= (time(NULL) + client_hash) % interval_secs; + return interval_secs * 1000; +} diff -r 21a2ce6f8f37 -r 25c848f10517 src/lib-imap/imap-keepalive.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib-imap/imap-keepalive.h Sat Apr 25 11:27:17 2015 +0300 @@ -0,0 +1,24 @@ +#ifndef IMAP_KEEPALIVE_H +#define IMAP_KEEPALIVE_H + +/* This function can be used to set IMAP IDLE keepalive notification timeout + interval so that the client gets the keepalive notifications at exactly the + same time for all the IMAP connections. This helps to reduce battery usage + in mobile devices. + + One problem with this is that we don't really want to send the notifications + to everyone at the same time, because it would cause huge peaks of activity. + Basing the notifications on the username works well for one account, but + basing it on the IP address allows the client to get all of the + notifications at the same time for multiple accounts as well (of course + assuming Dovecot is running on all the servers :) + + One potential downside to using IP is that if a proxy hides the client's IP + address, the notifications are sent to everyone at the same time. This can + be avoided by using a properly configured Dovecot proxy, but we'll also try + to avoid this by not doing it for the commonly used intranet IP ranges. */ +unsigned int +imap_keepalive_interval_msecs(const char *username, const struct ip_addr *ip, + unsigned int interval_secs); + +#endif From dovecot at dovecot.org Sat Apr 25 08:42:00 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 25 Apr 2015 08:42:00 +0000 Subject: dovecot-2.2: lib: Fixed test-istream-unix to not send uninitiali... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2c5b0ad65885 changeset: 18483:2c5b0ad65885 user: Timo Sirainen date: Sat Apr 25 11:40:23 2015 +0300 description: lib: Fixed test-istream-unix to not send uninitialized bytes. Fixes Valgrind errors in make check. diffstat: src/lib/test-istream-unix.c | 16 +++++++--------- 1 files changed, 7 insertions(+), 9 deletions(-) diffs (55 lines): diff -r 25c848f10517 -r 2c5b0ad65885 src/lib/test-istream-unix.c --- a/src/lib/test-istream-unix.c Sat Apr 25 11:27:17 2015 +0300 +++ b/src/lib/test-istream-unix.c Sat Apr 25 11:40:23 2015 +0300 @@ -115,14 +115,12 @@ static void test_istream_unix_client(int fd) { - char buf; - /* 1) */ write_one(fd); read_one(fd); /* 2) */ - if (fd_send(fd, send_fd, &buf, 1) < 0) + if (fd_send(fd, send_fd, "1", 1) < 0) i_fatal("fd_send() failed: %m"); read_one(fd); @@ -131,7 +129,7 @@ read_one(fd); /* 4) */ - if (fd_send(fd, send_fd2, &buf, 1) < 0) + if (fd_send(fd, send_fd2, "1", 1) < 0) i_fatal("fd_send() failed: %m"); read_one(fd); @@ -140,21 +138,21 @@ read_one(fd); /* 6) */ - if (fd_send(fd, send_fd, &buf, 1) < 0) + if (fd_send(fd, send_fd, "1", 1) < 0) i_fatal("fd_send() failed: %m"); read_one(fd); /* 7-8) */ - if (fd_send(fd, send_fd, &buf, 1) < 0) + if (fd_send(fd, send_fd, "1", 1) < 0) i_fatal("fd_send() failed: %m"); - if (fd_send(fd, send_fd2, &buf, 1) < 0) + if (fd_send(fd, send_fd2, "1", 1) < 0) i_fatal("fd_send() failed: %m"); read_one(fd); /* 9-10) */ - if (fd_send(fd, send_fd, &buf, 1) < 0) + if (fd_send(fd, send_fd, "1", 1) < 0) i_fatal("fd_send() failed: %m"); - if (fd_send(fd, send_fd2, &buf, 1) < 0) + if (fd_send(fd, send_fd2, "1", 1) < 0) i_fatal("fd_send() failed: %m"); read_one(fd); From dovecot at dovecot.org Sat Apr 25 08:53:44 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 25 Apr 2015 08:53:44 +0000 Subject: dovecot-2.2: lib: Fixed crash in connection API if input streams... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1b3224f0135f changeset: 18484:1b3224f0135f user: Timo Sirainen date: Sat Apr 25 11:52:02 2015 +0300 description: lib: Fixed crash in connection API if input streams aren't used (only input fd). diffstat: src/lib/connection.c | 4 +++- 1 files changed, 3 insertions(+), 1 deletions(-) diffs (21 lines): diff -r 2c5b0ad65885 -r 1b3224f0135f src/lib/connection.c --- a/src/lib/connection.c Sat Apr 25 11:40:23 2015 +0300 +++ b/src/lib/connection.c Sat Apr 25 11:52:02 2015 +0300 @@ -117,6 +117,9 @@ conn->input = i_stream_create_fd(conn->fd_in, set->input_max_size, FALSE); i_stream_set_name(conn->input, conn->name); + conn->io = io_add_istream(conn->input, *conn->list->v.input, conn); + } else { + conn->io = io_add(conn->fd_in, IO_READ, *conn->list->v.input, conn); } if (set->output_max_size != 0) { conn->output = o_stream_create_fd(conn->fd_out, @@ -124,7 +127,6 @@ o_stream_set_no_error_handling(conn->output, TRUE); o_stream_set_name(conn->output, conn->name); } - conn->io = io_add_istream(conn->input, *conn->list->v.input, conn); if (set->input_idle_timeout_secs != 0) { conn->to = timeout_add(set->input_idle_timeout_secs*1000, connection_idle_timeout, conn); From dovecot at dovecot.org Sat Apr 25 09:09:23 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 25 Apr 2015 09:09:23 +0000 Subject: dovecot-2.2: log: Don't confuse process sending a partial log li... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/211099aefe77 changeset: 18485:211099aefe77 user: Timo Sirainen date: Sat Apr 25 12:07:44 2015 +0300 description: log: Don't confuse process sending a partial log line to process sending logs too fast. If it's a partial line we don't want to show the "service too fast" error in ps title. diffstat: src/log/log-connection.c | 7 +++++-- 1 files changed, 5 insertions(+), 2 deletions(-) diffs (32 lines): diff -r 1b3224f0135f -r 211099aefe77 src/log/log-connection.c --- a/src/log/log-connection.c Sat Apr 25 11:52:02 2015 +0300 +++ b/src/log/log-connection.c Sat Apr 25 12:07:44 2015 +0300 @@ -326,6 +326,7 @@ ssize_t ret; struct timeval now, start_timeval; struct tm tm; + bool too_much = FALSE; if (!log->handshaked) { if (log_connection_handshake(log) < 0) { @@ -344,8 +345,10 @@ while ((line = i_stream_next_line(log->input)) != NULL) log_it(log, line, &now, &tm); io_loop_time_refresh(); - if (timeval_diff_msecs(&ioloop_timeval, &start_timeval) > MAX_MSECS_PER_CONNECTION) + if (timeval_diff_msecs(&ioloop_timeval, &start_timeval) > MAX_MSECS_PER_CONNECTION) { + too_much = TRUE; break; + } } if (log->input->eof) @@ -355,7 +358,7 @@ log_connection_destroy(log); } else { i_assert(!log->input->closed); - if (ret == 0) { + if (!too_much) { if (log->pending_count > 0) { log->pending_count = 0; i_assert(global_pending_count > 0); From dovecot at dovecot.org Sat Apr 25 09:18:44 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Sat, 25 Apr 2015 09:18:44 +0000 Subject: dovecot-2.2: Forgot to include year 2015 in earlier new files' C... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/a2d342257b25 changeset: 18486:a2d342257b25 user: Timo Sirainen date: Sat Apr 25 12:16:07 2015 +0300 description: Forgot to include year 2015 in earlier new files' Copyright lines. diffstat: src/lib-imap/imap-keepalive.c | 2 +- src/lib/istream-unix.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diffs (18 lines): diff -r 211099aefe77 -r a2d342257b25 src/lib-imap/imap-keepalive.c --- a/src/lib-imap/imap-keepalive.c Sat Apr 25 12:07:44 2015 +0300 +++ b/src/lib-imap/imap-keepalive.c Sat Apr 25 12:16:07 2015 +0300 @@ -1,4 +1,4 @@ -/* Copyright (c) 2002-2014 Dovecot authors, see the included COPYING file */ +/* Copyright (c) 2002-2015 Dovecot authors, see the included COPYING file */ #include "lib.h" #include "crc32.h" diff -r 211099aefe77 -r a2d342257b25 src/lib/istream-unix.c --- a/src/lib/istream-unix.c Sat Apr 25 12:07:44 2015 +0300 +++ b/src/lib/istream-unix.c Sat Apr 25 12:16:07 2015 +0300 @@ -1,4 +1,4 @@ -/* Copyright (c) 2014 Dovecot authors, see the included COPYING file */ +/* Copyright (c) 2014-2015 Dovecot authors, see the included COPYING file */ #include "lib.h" #include "fdpass.h" From dovecot at dovecot.org Tue Apr 28 09:28:43 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 28 Apr 2015 09:28:43 +0000 Subject: dovecot-2.2: *-login: Don't try to flush SSL output if SSL hands... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/86f535375750 changeset: 18487:86f535375750 user: Timo Sirainen date: Tue Apr 28 11:27:04 2015 +0200 description: *-login: Don't try to flush SSL output if SSL handshake fails. This fixes a crash on failed handshakes on some OpenSSL builds. diffstat: src/login-common/ssl-proxy-openssl.c | 14 +++++++++++--- 1 files changed, 11 insertions(+), 3 deletions(-) diffs (52 lines): diff -r a2d342257b25 -r 86f535375750 src/login-common/ssl-proxy-openssl.c --- a/src/login-common/ssl-proxy-openssl.c Sat Apr 25 12:16:07 2015 +0300 +++ b/src/login-common/ssl-proxy-openssl.c Tue Apr 28 11:27:04 2015 +0200 @@ -80,6 +80,7 @@ unsigned int cert_broken:1; unsigned int client_proxy:1; unsigned int flushing:1; + unsigned int failed:1; }; struct ssl_parameters { @@ -131,6 +132,12 @@ static int ssl_proxy_ctx_get_pkey_ec_curve_name(const struct master_service_ssl_settings *set); #endif +static void ssl_proxy_destroy_failed(struct ssl_proxy *proxy) +{ + proxy->failed = TRUE; + ssl_proxy_destroy(proxy); +} + static unsigned int ssl_server_context_hash(const struct ssl_server_context *ctx) { unsigned int i, g, h = 0; @@ -462,7 +469,7 @@ if (errstr != NULL) { proxy->last_error = i_strdup(errstr); - ssl_proxy_destroy(proxy); + ssl_proxy_destroy_failed(proxy); } ssl_proxy_unref(proxy); } @@ -492,7 +499,7 @@ if (proxy->handshake_callback != NULL) { if (proxy->handshake_callback(proxy->handshake_context) < 0) - ssl_proxy_destroy(proxy); + ssl_proxy_destroy_failed(proxy); } } @@ -822,7 +829,8 @@ if (proxy->destroyed || proxy->flushing) return; proxy->flushing = TRUE; - ssl_proxy_flush(proxy); + if (!proxy->failed && proxy->handshaked) + ssl_proxy_flush(proxy); proxy->destroyed = TRUE; ssl_proxy_count--; From dovecot at dovecot.org Tue Apr 28 12:22:30 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Tue, 28 Apr 2015 12:22:30 +0000 Subject: dovecot-2.2: lib-mail: Fixed modifying headers with i_stream_hea... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/aa8786771490 changeset: 18488:aa8786771490 user: Timo Sirainen date: Tue Apr 28 14:20:39 2015 +0200 description: lib-mail: Fixed modifying headers with i_stream_header_filter_add() If the stream was read twice, the second time the callback wasn't called and the header wasn't modified. diffstat: src/lib-mail/istream-header-filter.c | 11 ++++- src/lib-mail/test-istream-header-filter.c | 54 +++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 3 deletions(-) diffs (109 lines): diff -r 86f535375750 -r aa8786771490 src/lib-mail/istream-header-filter.c --- a/src/lib-mail/istream-header-filter.c Tue Apr 28 11:27:04 2015 +0200 +++ b/src/lib-mail/istream-header-filter.c Tue Apr 28 14:20:39 2015 +0200 @@ -31,6 +31,7 @@ unsigned int header_read:1; unsigned int seen_eoh:1; unsigned int header_parsed:1; + unsigned int headers_edited:1; unsigned int exclude:1; unsigned int crlf:1; unsigned int crlf_preserve:1; @@ -209,14 +210,17 @@ bsearch_strcasecmp) != NULL; if (mstream->callback == NULL) { /* nothing gets excluded */ - } else if (mstream->cur_line > mstream->parsed_lines) { - /* first time in this line */ + } else if (mstream->cur_line > mstream->parsed_lines || + mstream->headers_edited) { + /* first time in this line or we have actually modified + the header so we always want to call the callbacks */ bool orig_matched = matched; mstream->parsed_lines = mstream->cur_line; mstream->callback(mstream, hdr, &matched, mstream->context); - if (matched != orig_matched) { + if (matched != orig_matched && + !mstream->headers_edited) { i_array_init(&mstream->match_change_lines, 8); array_append(&mstream->match_change_lines, &mstream->cur_line, 1); @@ -583,4 +587,5 @@ const void *data, size_t size) { buffer_append(input->hdr_buf, data, size); + input->headers_edited = TRUE; } diff -r 86f535375750 -r aa8786771490 src/lib-mail/test-istream-header-filter.c --- a/src/lib-mail/test-istream-header-filter.c Tue Apr 28 11:27:04 2015 +0200 +++ b/src/lib-mail/test-istream-header-filter.c Tue Apr 28 14:20:39 2015 +0200 @@ -68,6 +68,59 @@ test_end(); } +static void ATTR_NULL(3) +edit_callback(struct header_filter_istream *input, + struct message_header_line *hdr, + bool *matched, void *context ATTR_UNUSED) +{ + if (hdr != NULL && strcasecmp(hdr->name, "To") == 0) { + /* modify To header */ + const char *new_to = "To: 123\n"; + *matched = TRUE; + i_stream_header_filter_add(input, new_to, strlen(new_to)); + } +} + +static void test_istream_edit(void) +{ + const char *input = "From: foo\nTo: bar\n\nhello world\n"; + const char *output = "From: foo\nTo: 123\n\nhello world\n"; + struct istream *istream, *filter; + unsigned int i, input_len = strlen(input); + unsigned int output_len = strlen(output); + const unsigned char *data; + size_t size; + + test_begin("i_stream_create_header_filter(edit)"); + istream = test_istream_create(input); + filter = i_stream_create_header_filter(istream, + HEADER_FILTER_EXCLUDE | + HEADER_FILTER_NO_CR, + NULL, 0, + edit_callback, (void *)NULL); + for (i = 1; i < input_len; i++) { + test_istream_set_size(istream, i); + test_assert(i_stream_read(filter) >= 0); + } + test_istream_set_size(istream, input_len); + test_assert(i_stream_read(filter) > 0); + test_assert(i_stream_read(filter) == -1); + + data = i_stream_get_data(filter, &size); + test_assert(size == output_len && memcmp(data, output, size) == 0); + + i_stream_skip(filter, size); + i_stream_seek(filter, 0); + while (i_stream_read(filter) > 0) ; + data = i_stream_get_data(filter, &size); + test_assert(size == output_len && memcmp(data, output, size) == 0); + + i_stream_unref(&filter); + i_stream_unref(&istream); + + test_end(); +} + static void test_istream_end_body_with_lf(void) { static const char *empty_strarray[] = { NULL }; @@ -131,6 +184,7 @@ { static void (*test_functions[])(void) = { test_istream_filter, + test_istream_edit, test_istream_end_body_with_lf, NULL }; From dovecot at dovecot.org Wed Apr 29 08:29:28 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 29 Apr 2015 08:29:28 +0000 Subject: dovecot-2.2: quota: Don't assume mail is too large if a quota ro... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/e00f9e93a9a3 changeset: 18489:e00f9e93a9a3 user: Timo Sirainen date: Wed Apr 29 10:13:19 2015 +0200 description: quota: Don't assume mail is too large if a quota root has mail count limit but not size limit. Based on patch by Alexei Gradinari diffstat: src/plugins/quota/quota.c | 4 +--- 1 files changed, 1 insertions(+), 3 deletions(-) diffs (19 lines): diff -r aa8786771490 -r e00f9e93a9a3 src/plugins/quota/quota.c --- a/src/plugins/quota/quota.c Tue Apr 28 14:20:39 2015 +0200 +++ b/src/plugins/quota/quota.c Wed Apr 29 10:13:19 2015 +0200 @@ -1093,14 +1093,12 @@ ret = quota_root_get_rule_limits(roots[i], mailbox_get_vname(ctx->box), &bytes_limit, &count_limit); - if (ret == 0) - continue; if (ret < 0) return -1; /* if size is bigger than any limit, then it is bigger than the lowest limit */ - if (size > bytes_limit) { + if (bytes_limit > 0 && size > bytes_limit) { *too_large_r = TRUE; break; } From dovecot at dovecot.org Wed Apr 29 08:29:29 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 29 Apr 2015 08:29:29 +0000 Subject: dovecot-2.2: trash plugin: Fixed handling unlimited quota limits. Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/2dbee58a1a0d changeset: 18490:2dbee58a1a0d user: Timo Sirainen date: Wed Apr 29 10:22:26 2015 +0200 description: trash plugin: Fixed handling unlimited quota limits. Patch by Alexei Gradinari diffstat: src/plugins/trash/trash-plugin.c | 12 ++++++++++-- 1 files changed, 10 insertions(+), 2 deletions(-) diffs (22 lines): diff -r e00f9e93a9a3 -r 2dbee58a1a0d src/plugins/trash/trash-plugin.c --- a/src/plugins/trash/trash-plugin.c Wed Apr 29 10:13:19 2015 +0200 +++ b/src/plugins/trash/trash-plugin.c Wed Apr 29 10:22:26 2015 +0200 @@ -204,8 +204,16 @@ ctx->count_over = 0; } - ctx->bytes_ceil += size_expunged; - ctx->count_ceil += expunged_count; + if (ctx->bytes_ceil > ((uint64_t)-1 - size_expunged)) { + ctx->bytes_ceil = (uint64_t)-1; + } else { + ctx->bytes_ceil += size_expunged; + } + if (ctx->count_ceil < ((uint64_t)-1 - expunged_count)) { + ctx->count_ceil = (uint64_t)-1; + } else { + ctx->count_ceil += expunged_count; + } return 1; } From dovecot at dovecot.org Wed Apr 29 08:29:29 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 29 Apr 2015 08:29:29 +0000 Subject: dovecot-2.2: trash plugin: Count more correctly the number of by... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/8898c5ea38f2 changeset: 18491:8898c5ea38f2 user: Timo Sirainen date: Wed Apr 29 10:27:50 2015 +0200 description: trash plugin: Count more correctly the number of bytes/messages needed to get under quota. If there already were some messages saved, those weren't counted when figuring out how many/much mails are still needed to be expunged. Patch by Alexei Gradinari diffstat: src/plugins/trash/trash-plugin.c | 12 ++++++++++-- 1 files changed, 10 insertions(+), 2 deletions(-) diffs (30 lines): diff -r 2dbee58a1a0d -r 8898c5ea38f2 src/plugins/trash/trash-plugin.c --- a/src/plugins/trash/trash-plugin.c Wed Apr 29 10:22:26 2015 +0200 +++ b/src/plugins/trash/trash-plugin.c Wed Apr 29 10:27:50 2015 +0200 @@ -222,6 +222,8 @@ uoff_t size, bool *too_large_r) { int ret, i; + uint64_t size_needed = 0; + unsigned int count_needed = 0; for (i = 0; ; i++) { ret = trash_next_quota_test_alloc(ctx, size, too_large_r); @@ -241,9 +243,15 @@ break; } + if (ctx->bytes_ceil != (uint64_t)-1 && + ctx->bytes_ceil < size + ctx->bytes_over) + size_needed = size + ctx->bytes_over - ctx->bytes_ceil; + if (ctx->count_ceil != (uint64_t)-1 && + ctx->count_ceil < 1 + ctx->count_over) + count_needed = 1 + ctx->count_over - ctx->count_ceil; + /* not enough space. try deleting some from mailbox. */ - ret = trash_try_clean_mails(ctx, size + ctx->bytes_over, - 1 + ctx->count_over); + ret = trash_try_clean_mails(ctx, size_needed, count_needed); if (ret <= 0) return 0; } From dovecot at dovecot.org Wed Apr 29 08:36:13 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 29 Apr 2015 08:36:13 +0000 Subject: dovecot-2.2: fts: Do not deinit uncreated fts context Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/1081d57b524e changeset: 18492:1081d57b524e user: Timo Sirainen date: Wed Apr 29 10:34:33 2015 +0200 description: fts: Do not deinit uncreated fts context Fixes crash with fts-lucene and fts-solr when lib-fts wasn't used. Patch by Teemu Huovila diffstat: src/plugins/fts/fts-user.c | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diffs (11 lines): diff -r 8898c5ea38f2 -r 1081d57b524e src/plugins/fts/fts-user.c --- a/src/plugins/fts/fts-user.c Wed Apr 29 10:27:50 2015 +0200 +++ b/src/plugins/fts/fts-user.c Wed Apr 29 10:34:33 2015 +0200 @@ -212,5 +212,6 @@ { struct fts_user *fuser = FTS_USER_CONTEXT(user); - fts_user_free(fuser); + if (fuser != NULL) + fts_user_free(fuser); } From dovecot at dovecot.org Wed Apr 29 09:22:44 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 29 Apr 2015 09:22:44 +0000 Subject: dovecot-2.2: lib: connection: Connect to unix socket asynchronou... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/092a51d80bad changeset: 18493:092a51d80bad user: Stephan Bosch date: Sat Apr 25 11:42:06 2015 +0200 description: lib: connection: Connect to unix socket asynchronously if connected callback is set. Prevents problems in lib-http unix socket support. diffstat: src/lib/connection.c | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diffs (24 lines): diff -r 1081d57b524e -r 092a51d80bad src/lib/connection.c --- a/src/lib/connection.c Wed Apr 29 10:34:33 2015 +0200 +++ b/src/lib/connection.c Sat Apr 25 11:42:06 2015 +0200 @@ -236,7 +236,7 @@ list->v.client_connected(conn, TRUE); } -static void connection_ip_connected(struct connection *conn) +static void connection_socket_connected(struct connection *conn) { io_remove(&conn->io); if (conn->to != NULL) @@ -262,9 +262,9 @@ return -1; conn->fd_in = conn->fd_out = fd; - if (conn->port != 0) { + if (conn->port != 0 || conn->list->v.client_connected != NULL) { conn->io = io_add(conn->fd_out, IO_WRITE, - connection_ip_connected, conn); + connection_socket_connected, conn); if (set->client_connect_timeout_msecs != 0) { conn->to = timeout_add(set->client_connect_timeout_msecs, connection_connect_timeout, conn); From dovecot at dovecot.org Wed Apr 29 09:22:44 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 29 Apr 2015 09:22:44 +0000 Subject: dovecot-2.2: lib-http: client: Fixed memory leak in CONNECT tunn... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/63224afb8c02 changeset: 18494:63224afb8c02 user: Stephan Bosch date: Sat Apr 25 11:42:06 2015 +0200 description: lib-http: client: Fixed memory leak in CONNECT tunnel support. Forgot to dereference streams once tunnel was completed and passed to the new connection object. diffstat: src/lib-http/http-client-connection.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diffs (12 lines): diff -r 092a51d80bad -r 63224afb8c02 src/lib-http/http-client-connection.c --- a/src/lib-http/http-client-connection.c Sat Apr 25 11:42:06 2015 +0200 +++ b/src/lib-http/http-client-connection.c Sat Apr 25 11:42:06 2015 +0200 @@ -1162,6 +1162,8 @@ connection_init_from_streams (conn->client->conn_list, &conn->conn, name, tunnel.input, tunnel.output); + i_stream_unref(&tunnel.input); + o_stream_unref(&tunnel.output); conn->connect_initialized = TRUE; } From dovecot at dovecot.org Wed Apr 29 09:22:44 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 29 Apr 2015 09:22:44 +0000 Subject: dovecot-2.2: uri-util: Allow empty host name in the generic URI ... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/9024d226b813 changeset: 18495:9024d226b813 user: Stephan Bosch date: Sat Apr 25 11:42:06 2015 +0200 description: uri-util: Allow empty host name in the generic URI syntax as specified in RFC 3986. diffstat: src/lib-http/http-url.c | 10 ++++++++++ src/lib-imap/imap-url.c | 9 ++++++++- src/lib/uri-util.c | 29 ++++++++--------------------- 3 files changed, 26 insertions(+), 22 deletions(-) diffs (110 lines): diff -r 63224afb8c02 -r 9024d226b813 src/lib-http/http-url.c --- a/src/lib-http/http-url.c Sat Apr 25 11:42:06 2015 +0200 +++ b/src/lib-http/http-url.c Sat Apr 25 11:42:06 2015 +0200 @@ -37,6 +37,16 @@ if ((ret = uri_parse_authority(parser, &auth)) < 0) return FALSE; + if (auth.host_literal == NULL || *auth.host_literal == '\0') { + /* RFC 7230, Section 2.7.1: http URI Scheme + + A sender MUST NOT generate an "http" URI with an empty host + identifier. A recipient that processes such a URI reference MUST + reject it as invalid. + */ + parser->error = "HTTP URL does not allow empty host identifier"; + return FALSE; + } if (ret > 0) { if (auth.enc_userinfo != NULL) { const char *p; diff -r 63224afb8c02 -r 9024d226b813 src/lib-imap/imap-url.c --- a/src/lib-imap/imap-url.c Sat Apr 25 11:42:06 2015 +0200 +++ b/src/lib-imap/imap-url.c Sat Apr 25 11:42:06 2015 +0200 @@ -193,7 +193,14 @@ /* "//" iserver */ if ((ret = uri_parse_slashslash_authority(parser, &auth)) <= 0) return ret; - + if (auth.host_literal == NULL || *auth.host_literal == '\0') { + /* This situation is not documented anywhere, but it is not + currently useful either and potentially problematic if not + handled explicitly everywhere. So, it is denied hier for now. + */ + parser->error = "IMAP URL does not allow empty host identifier"; + return -1; + } /* iuserinfo = enc-user [iauth] / [enc-user] iauth */ if (auth.enc_userinfo != NULL) { const char *p, *uend; diff -r 63224afb8c02 -r 9024d226b813 src/lib/uri-util.c --- a/src/lib/uri-util.c Sat Apr 25 11:42:06 2015 +0200 +++ b/src/lib/uri-util.c Sat Apr 25 11:42:06 2015 +0200 @@ -327,8 +327,6 @@ static int uri_parse_reg_name(struct uri_parser *parser, string_t *reg_name) { - int len = 0; - /* RFC 3986: * * reg-name = *( unreserved / pct-encoded / sub-delims ) @@ -345,7 +343,6 @@ if (ret > 0) { if (reg_name != NULL) str_append_c(reg_name, c); - len++; continue; } @@ -355,12 +352,11 @@ if (reg_name != NULL) str_append_c(reg_name, *parser->cur); parser->cur++; - len++; continue; } break; } - return len > 0 ? 1 : 0; + return 0; } #ifdef HAVE_IPV6 @@ -465,12 +461,11 @@ str_truncate(literal, 0); /* reg-name */ - if ((ret = uri_parse_reg_name(parser, literal)) != 0) { - if (ret > 0 && auth != NULL) { - auth->host_literal = t_strdup(str_c(literal)); - auth->have_host_ip = FALSE; - } - return ret; + if (uri_parse_reg_name(parser, literal) < 0) + return -1; + if (auth != NULL) { + auth->host_literal = t_strdup(str_c(literal)); + auth->have_host_ip = FALSE; } return 0; } @@ -537,16 +532,8 @@ } /* host */ - if ((ret = uri_parse_host(parser, auth)) <= 0) { - if (ret == 0) { - if (p == parser->end || *p == ':' || *p == '/') - parser->error = "Missing 'host' component"; - else - parser->error = "Invalid 'host' component"; - return -1; - } - return ret; - } + if (uri_parse_host(parser, auth) < 0) + return -1; /* [":" ... */ if (parser->cur >= parser->end || *parser->cur != ':') From dovecot at dovecot.org Wed Apr 29 09:22:49 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 29 Apr 2015 09:22:49 +0000 Subject: dovecot-2.2: uri-util: Improve errors about invalid characters i... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/5ae8cd8e42f7 changeset: 18496:5ae8cd8e42f7 user: Stephan Bosch date: Sat Apr 25 11:42:06 2015 +0200 description: uri-util: Improve errors about invalid characters in URI by reporting the component where the offending character is located. diffstat: src/lib-http/http-url.c | 6 ++---- src/lib-imap/imap-url.c | 9 +++++---- src/lib/uri-util.c | 48 +++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 48 insertions(+), 15 deletions(-) diffs (121 lines): diff -r 9024d226b813 -r 5ae8cd8e42f7 src/lib-http/http-url.c --- a/src/lib-http/http-url.c Sat Apr 25 11:42:06 2015 +0200 +++ b/src/lib-http/http-url.c Sat Apr 25 11:42:06 2015 +0200 @@ -325,10 +325,8 @@ url->enc_fragment = p_strdup(parser->pool, base->enc_fragment); } - if (parser->cur != parser->end) { - parser->error = "HTTP URL contains invalid character"; - return FALSE; - } + /* must be at end of URL now */ + i_assert(parser->cur == parser->end); if (have_scheme) url_parser->req_format = HTTP_REQUEST_TARGET_FORMAT_ABSOLUTE; diff -r 9024d226b813 -r 5ae8cd8e42f7 src/lib-imap/imap-url.c --- a/src/lib-imap/imap-url.c Sat Apr 25 11:42:06 2015 +0200 +++ b/src/lib-imap/imap-url.c Sat Apr 25 11:42:06 2015 +0200 @@ -879,15 +879,16 @@ } } + /* IMAP URL has no fragment */ if ((ret = uri_parse_fragment(parser, &query)) != 0) { if (ret == 1) parser->error = "Fragment component not allowed in IMAP URL"; return FALSE; } - if (parser->cur != parser->end) { - parser->error = "IMAP URL contains invalid character."; - return FALSE; - } + + /* must be at end of URL now */ + i_assert(parser->cur == parser->end); + return TRUE; } diff -r 9024d226b813 -r 5ae8cd8e42f7 src/lib/uri-util.c --- a/src/lib/uri-util.c Sat Apr 25 11:42:06 2015 +0200 +++ b/src/lib/uri-util.c Sat Apr 25 11:42:06 2015 +0200 @@ -534,15 +534,33 @@ /* host */ if (uri_parse_host(parser, auth) < 0) return -1; + if (parser->cur == parser->end) + return 1; + switch (*parser->cur) { + case ':': case '/': case '?': case '#': + break; + default: + parser->error = "Invalid host identifier"; + return -1; + } - /* [":" ... */ - if (parser->cur >= parser->end || *parser->cur != ':') - return 1; - parser->cur++; + /* [":" port] */ + if (*parser->cur == ':') { + parser->cur++; - /* ... port] */ - if ((ret = uri_parse_port(parser, auth)) < 0) - return ret; + if ((ret = uri_parse_port(parser, auth)) < 0) + return ret; + if (parser->cur == parser->end) + return 1; + switch (*parser->cur) { + case '/': case '?': case '#': + break; + default: + parser->error = "Invalid host port"; + return -1; + } + } + return 1; } @@ -661,6 +679,12 @@ } array_append_zero(&segments); *path_r = array_get(&segments, &count); + + if (parser->cur < parser->end && + *parser->cur != '?' && *parser->cur != '#') { + parser->error = "Path component contains invalid character"; + return -1; + } return 1; } @@ -689,6 +713,11 @@ p++; } + if (p < parser->end && *p != '#') { + parser->error = "Query component contains invalid character"; + return -1; + } + if (query_r != NULL) *query_r = t_strdup_until(parser->cur+1, p); parser->cur = p; @@ -721,6 +750,11 @@ p++; } + if (p < parser->end) { + parser->error = "Fragment component contains invalid character"; + return -1; + } + if (fragment_r != NULL) *fragment_r = t_strdup_until(parser->cur+1, p); parser->cur = p; From dovecot at dovecot.org Wed Apr 29 09:22:59 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 29 Apr 2015 09:22:59 +0000 Subject: dovecot-2.2: uri-util: Added the possibility of only checking th... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/49bcc3954799 changeset: 18497:49bcc3954799 user: Stephan Bosch date: Sat Apr 25 11:42:06 2015 +0200 description: uri-util: Added the possibility of only checking the URI without parsing any of the data. diffstat: src/lib/uri-util.c | 67 ++++++++++++++++++++++++++++++++++++----------------- src/lib/uri-util.h | 28 ++++++++++++++-------- 2 files changed, 63 insertions(+), 32 deletions(-) diffs (210 lines): diff -r 5ae8cd8e42f7 -r 49bcc3954799 src/lib/uri-util.c --- a/src/lib/uri-util.c Sat Apr 25 11:42:06 2015 +0200 +++ b/src/lib/uri-util.c Sat Apr 25 11:42:06 2015 +0200 @@ -234,7 +234,8 @@ if (*p != ':') return -1; - *scheme_r = t_strdup_until(*uri_p, p); + if (scheme_r != NULL) + *scheme_r = t_strdup_until(*uri_p, p); *uri_p = p + 1; return 0; } @@ -256,7 +257,7 @@ static int uri_parse_dec_octet(struct uri_parser *parser, string_t *literal, - uint8_t *octet_r) + uint8_t *octet_r) ATTR_NULL(2) { unsigned int octet = 0; int count = 0; @@ -291,7 +292,7 @@ static int uri_parse_ipv4address(struct uri_parser *parser, string_t *literal, - struct in_addr *ip4_r) + struct in_addr *ip4_r) ATTR_NULL(2,3) { uint8_t octet; uint32_t ip = 0; @@ -325,7 +326,9 @@ return 1; } -static int uri_parse_reg_name(struct uri_parser *parser, string_t *reg_name) +static int +uri_parse_reg_name(struct uri_parser *parser, + string_t *reg_name) ATTR_NULL(2) { /* RFC 3986: * @@ -362,10 +365,11 @@ #ifdef HAVE_IPV6 static int uri_parse_ip_literal(struct uri_parser *parser, string_t *literal, - struct in6_addr *ip6_r) + struct in6_addr *ip6_r) ATTR_NULL(2,3) { const unsigned char *p; const char *address; + struct in6_addr ip6; int ret; /* IP-literal = "[" ( IPv6address / IPvFuture ) "]" @@ -400,16 +404,20 @@ "Future IP host address '%s' not supported", address); return -1; } - if ((ret = inet_pton(AF_INET6, address, ip6_r)) <= 0) { + if ((ret = inet_pton(AF_INET6, address, &ip6)) <= 0) { parser->error = t_strdup_printf( "Invalid IPv6 host address '%s'", address); return -1; } + if (ip6_r != NULL) + *ip6_r = ip6; return 1; } #endif -static int uri_parse_host(struct uri_parser *parser, struct uri_authority *auth) +static int +uri_parse_host(struct uri_parser *parser, + struct uri_authority *auth) ATTR_NULL(2) { const unsigned char *preserve; struct in_addr ip4; @@ -470,7 +478,9 @@ return 0; } -static int uri_parse_port(struct uri_parser *parser, struct uri_authority *auth) +static int +uri_parse_port(struct uri_parser *parser, + struct uri_authority *auth) ATTR_NULL(2) { unsigned long port = 0; int count = 0; @@ -612,7 +622,11 @@ int relative = 1; int ret; - t_array_init(&segments, 16); + count = 0; + if (path_r != NULL) + t_array_init(&segments, 16); + else + memset(&segments, 0, sizeof(segments)); /* check for a leading '/' and indicate absolute path when it is present @@ -636,9 +650,12 @@ segment = NULL; /* ... pop last segment (if any) */ - count = array_count(&segments); if (count > 0) { - array_delete(&segments, count-1, 1); + if (path_r != NULL) { + i_assert(count == array_count(&segments)); + array_delete(&segments, count-1, 1); + } + count--; } else if ( relative > 0 ) { relative++; } @@ -652,8 +669,11 @@ segment = ""; } - if (segment != NULL) - array_append(&segments, &segment, 1); + if (segment != NULL) { + if (path_r != NULL) + array_append(&segments, &segment, 1); + count++; + } if (parser->cur >= parser->end || *parser->cur != '/') break; @@ -664,22 +684,25 @@ return -1; } - *path_r = NULL; - *relative_r = relative; + if (relative_r != NULL) + *relative_r = relative; + if (path_r != NULL) + *path_r = NULL; if (parser->cur == pbegin) { /* path part of URI is empty */ return 0; } - /* special treatment for a trailing '..' or '.' */ - if (segment == NULL) { - segment = ""; - array_append(&segments, &segment, 1); + if (path_r != NULL) { + /* special treatment for a trailing '..' or '.' */ + if (segment == NULL) { + segment = ""; + array_append(&segments, &segment, 1); + } + array_append_zero(&segments); + *path_r = array_get(&segments, &count); } - array_append_zero(&segments); - *path_r = array_get(&segments, &count); - if (parser->cur < parser->end && *parser->cur != '?' && *parser->cur != '#') { parser->error = "Path component contains invalid character"; diff -r 5ae8cd8e42f7 -r 49bcc3954799 src/lib/uri-util.h --- a/src/lib/uri-util.h Sat Apr 25 11:42:06 2015 +0200 +++ b/src/lib/uri-util.h Sat Apr 25 11:42:06 2015 +0200 @@ -33,22 +33,30 @@ bool uri_data_decode(struct uri_parser *parser, const char *data, const char *until, const char **decoded_r) ATTR_NULL(3); -int uri_cut_scheme(const char **uri_p, const char **scheme_r); -int uri_parse_scheme(struct uri_parser *parser, const char **scheme_r); +int uri_cut_scheme(const char **uri_p, const char **scheme_r) + ATTR_NULL(2); +int uri_parse_scheme(struct uri_parser *parser, const char **scheme_r) + ATTR_NULL(2); int uri_parse_authority(struct uri_parser *parser, - struct uri_authority *auth); + struct uri_authority *auth) ATTR_NULL(2); int uri_parse_slashslash_authority(struct uri_parser *parser, - struct uri_authority *auth); + struct uri_authority *auth) ATTR_NULL(2); -int uri_parse_path_segment(struct uri_parser *parser, const char **segment_r); +int uri_parse_path_segment(struct uri_parser *parser, + const char **segment_r) ATTR_NULL(2); int uri_parse_path(struct uri_parser *parser, int *relative_r, - const char *const **path_r); + const char *const **path_r) ATTR_NULL(2,3); -int uri_parse_query(struct uri_parser *parser, const char **query_r); -int uri_parse_fragment(struct uri_parser *parser, const char **fragment_r); +int uri_parse_query(struct uri_parser *parser, + const char **query_r) ATTR_NULL(2); +int uri_parse_fragment(struct uri_parser *parser, + const char **fragment_r) ATTR_NULL(2); -void uri_parser_init(struct uri_parser *parser, pool_t pool, const char *data); -string_t *uri_parser_get_tmpbuf(struct uri_parser *parser, size_t size); +void uri_parser_init(struct uri_parser *parser, pool_t pool, + const char *data); +string_t *uri_parser_get_tmpbuf(struct uri_parser *parser, + size_t size); + /* * Generic URI construction From dovecot at dovecot.org Wed Apr 29 15:21:21 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 29 Apr 2015 15:21:21 +0000 Subject: dovecot-2.2: director: Implemented director_proxy_maybe passdb e... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/c5cdf42e398a changeset: 18498:c5cdf42e398a user: Timo Sirainen date: Wed Apr 29 11:33:21 2015 +0200 description: director: Implemented director_proxy_maybe passdb extra field. This allows running director and backend in the same Dovecot instance. It was implemented into director instead of login-common to allow doveadm and lmtp proxying to work as well (although currently lmtp can't handle mixed proxying and non-proxying destinations, which makes this a bit less useful). diffstat: src/auth/auth-request.c | 11 ++++++++- src/director/login-connection.c | 50 ++++++++++++++++++++++++++++++++++------ 2 files changed, 52 insertions(+), 9 deletions(-) diffs (138 lines): diff -r 49bcc3954799 -r c5cdf42e398a src/auth/auth-request.c --- a/src/auth/auth-request.c Sat Apr 25 11:42:06 2015 +0200 +++ b/src/auth/auth-request.c Wed Apr 29 11:33:21 2015 +0200 @@ -1839,8 +1839,17 @@ host = auth_fields_find(request->extra_fields, "host"); if (host == NULL) { - /* director can set the host */ + /* director can set the host. give it access to lip and lport + so it can also perform proxy_maybe internally */ proxy_host_is_self = FALSE; + if (request->local_ip.family != 0) { + auth_fields_add(request->extra_fields, "lip", + net_ip2addr(&request->local_ip), 0); + } + if (request->local_port != 0) { + auth_fields_add(request->extra_fields, "lport", + dec2str(request->local_port), 0); + } } else if (net_addr2ip(host, &ip) == 0) { proxy_host_is_self = auth_request_proxy_ip_is_self(request, &ip); diff -r 49bcc3954799 -r c5cdf42e398a src/director/login-connection.c --- a/src/director/login-connection.c Sat Apr 25 11:42:06 2015 +0200 +++ b/src/director/login-connection.c Wed Apr 29 11:33:21 2015 +0200 @@ -31,6 +31,11 @@ struct login_host_request { struct login_connection *conn; char *line, *username; + + struct ip_addr local_ip; + unsigned int local_port; + unsigned int dest_port; + bool director_proxy_maybe; }; static struct login_connection *login_connections; @@ -71,6 +76,17 @@ o_stream_nsendv(conn->output, iov, N_ELEMENTS(iov)); } +static bool login_host_request_is_self(struct login_host_request *request, + const struct ip_addr *dest_ip) +{ + if (!net_ip_compare(dest_ip, &request->local_ip)) + return FALSE; + if (request->dest_port != 0 && request->local_port != 0 && + request->dest_port != request->local_port) + return FALSE; + return TRUE; +} + static void login_host_callback(const struct ip_addr *ip, const char *errormsg, void *context) @@ -80,11 +96,7 @@ const char *line, *line_params; unsigned int secs; - if (ip != NULL) { - secs = dir->set->director_user_expire / 2; - line = t_strdup_printf("%s\thost=%s\tproxy_refresh=%u", - request->line, net_ip2addr(ip), secs); - } else { + if (ip == NULL) { if (strncmp(request->line, "OK\t", 3) == 0) line_params = request->line + 3; else if (strncmp(request->line, "PASS\t", 5) == 0) @@ -96,6 +108,13 @@ request->username, errormsg); line = t_strconcat("FAIL\t", t_strcut(line_params, '\t'), "\ttemp", NULL); + } else if (request->director_proxy_maybe && + login_host_request_is_self(request, ip)) { + line = request->line; + } else { + secs = dir->set->director_user_expire / 2; + line = t_strdup_printf("%s\thost=%s\tproxy_refresh=%u", + request->line, net_ip2addr(ip), secs); } login_connection_send_line(request->conn, line); @@ -108,7 +127,7 @@ static void auth_input_line(const char *line, void *context) { struct login_connection *conn = context; - struct login_host_request *request; + struct login_host_request *request, temp_request; const char *const *args, *line_params, *username = NULL, *tag = ""; bool proxy = FALSE, host = FALSE; @@ -134,22 +153,36 @@ args++; } + memset(&temp_request, 0, sizeof(temp_request)); for (; *args != NULL; args++) { if (strncmp(*args, "proxy", 5) == 0 && ((*args)[5] == '=' || (*args)[5] == '\0')) proxy = TRUE; else if (strncmp(*args, "host=", 5) == 0) host = TRUE; - else if (strncmp(*args, "destuser=", 9) == 0) + else if (strncmp(*args, "lip=", 4) == 0) { + if (net_addr2ip((*args) + 4, &temp_request.local_ip) < 0) + ; + } else if (strncmp(*args, "lport=", 6) == 0) { + if (str_to_uint((*args) + 6, &temp_request.local_port) < 0) + ; + } else if (strncmp(*args, "port=", 5) == 0) { + if (str_to_uint((*args) + 5, &temp_request.dest_port) < 0) + ; + } else if (strncmp(*args, "destuser=", 9) == 0) username = *args + 9; else if (strncmp(*args, "director_tag=", 13) == 0) tag = *args + 13; + else if (strncmp(*args, "director_proxy_maybe", 20) == 0 && + ((*args)[20] == '=' || (*args)[20] == '\0')) + temp_request.director_proxy_maybe = TRUE; else if (strncmp(*args, "user=", 5) == 0) { if (username == NULL) username = *args + 5; } } - if (!proxy || host || username == NULL) { + if ((!proxy && !temp_request.director_proxy_maybe) || + host || username == NULL) { login_connection_send_line(conn, line); return; } @@ -162,6 +195,7 @@ /* we need to add the host. the lookup might be asynchronous */ request = i_new(struct login_host_request, 1); + *request = temp_request; request->conn = conn; request->line = i_strdup(line); request->username = i_strdup(username); From dovecot at dovecot.org Wed Apr 29 15:21:21 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 29 Apr 2015 15:21:21 +0000 Subject: dovecot-2.2: lib-imap: imap_utf7_to_utf8() returns failure now f... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/46065d7276ab changeset: 18499:46065d7276ab user: Timo Sirainen date: Wed Apr 29 17:19:34 2015 +0200 description: lib-imap: imap_utf7_to_utf8() returns failure now for encoded NULs. All the callers change the string to NUL-terminated string, so the earlier behavior was just truncating the string at the encoded NUL. It's a bit safer to just return failure so in future if the callers didn't convert the string to NUL-terminated string it wouldn't be handled inconsistently in different places. diffstat: src/lib-imap/imap-utf7.c | 8 ++++++++ 1 files changed, 8 insertions(+), 0 deletions(-) diffs (18 lines): diff -r c5cdf42e398a -r 46065d7276ab src/lib-imap/imap-utf7.c --- a/src/lib-imap/imap-utf7.c Wed Apr 29 11:33:21 2015 +0200 +++ b/src/lib-imap/imap-utf7.c Wed Apr 29 17:19:34 2015 +0200 @@ -149,6 +149,14 @@ high > UTF16_SURROGATE_HIGH_MAX) { /* single byte */ size_t oldlen = str_len(dest); + + if (high == 0) { + /* Encoded NUL isn't going to work in Dovecot code, + even though it's technically valid. Return failure + so the callers don't even get a chance to handle the + NUL in the string inconsistently. */ + return -1; + } uni_ucs4_to_utf8_c(high, dest); if (str_len(dest) - oldlen == 1) { unsigned char last = str_data(dest)[oldlen]; From dovecot at dovecot.org Wed Apr 29 15:45:00 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 29 Apr 2015 15:45:00 +0000 Subject: dovecot-2.2: lib: net_addr2ip() sets the family now only after a... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/4903ab967807 changeset: 18500:4903ab967807 user: Timo Sirainen date: Wed Apr 29 17:35:18 2015 +0200 description: lib: net_addr2ip() sets the family now only after address is successfully converted. diffstat: src/lib/net.c | 5 ++--- 1 files changed, 2 insertions(+), 3 deletions(-) diffs (27 lines): diff -r 46065d7276ab -r 4903ab967807 src/lib/net.c --- a/src/lib/net.c Wed Apr 29 17:19:34 2015 +0200 +++ b/src/lib/net.c Wed Apr 29 17:35:18 2015 +0200 @@ -919,7 +919,6 @@ if (strchr(addr, ':') != NULL) { /* IPv6 */ - ip->family = AF_INET6; #ifdef HAVE_IPV6 T_BEGIN { if (addr[0] == '[') { @@ -935,13 +934,13 @@ #else ip->u.ip4.s_addr = 0; #endif + ip->family = AF_INET6; } else { /* IPv4 */ - ip->family = AF_INET; if (inet_aton(addr, &ip->u.ip4) == 0) return -1; + ip->family = AF_INET; } - return 0; } From dovecot at dovecot.org Wed Apr 29 15:45:00 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 29 Apr 2015 15:45:00 +0000 Subject: dovecot-2.2: lib: Renamed test-network to test-net Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/334bb95f27d4 changeset: 18501:334bb95f27d4 user: Timo Sirainen date: Wed Apr 29 17:36:44 2015 +0200 description: lib: Renamed test-network to test-net Since network.c had been renamed to net.c some time ago. diffstat: src/lib/Makefile.am | 2 +- src/lib/test-lib.c | 2 +- src/lib/test-lib.h | 2 +- src/lib/test-net.c | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/lib/test-network.c | 53 -------------------------------------------------- 5 files changed, 56 insertions(+), 56 deletions(-) diffs (150 lines): diff -r 4903ab967807 -r 334bb95f27d4 src/lib/Makefile.am --- a/src/lib/Makefile.am Wed Apr 29 17:35:18 2015 +0200 +++ b/src/lib/Makefile.am Wed Apr 29 17:36:44 2015 +0200 @@ -310,7 +310,7 @@ test-json-tree.c \ test-llist.c \ test-mempool-alloconly.c \ - test-network.c \ + test-net.c \ test-numpack.c \ test-ostream-file.c \ test-primes.c \ diff -r 4903ab967807 -r 334bb95f27d4 src/lib/test-lib.c --- a/src/lib/test-lib.c Wed Apr 29 17:35:18 2015 +0200 +++ b/src/lib/test-lib.c Wed Apr 29 17:36:44 2015 +0200 @@ -32,7 +32,7 @@ test_json_tree, test_llist, test_mempool_alloconly, - test_network, + test_net, test_numpack, test_ostream_file, test_primes, diff -r 4903ab967807 -r 334bb95f27d4 src/lib/test-lib.h --- a/src/lib/test-lib.h Wed Apr 29 17:35:18 2015 +0200 +++ b/src/lib/test-lib.h Wed Apr 29 17:36:44 2015 +0200 @@ -34,7 +34,7 @@ void test_llist(void); void test_mempool_alloconly(void); enum fatal_test_state fatal_mempool(int); -void test_network(void); +void test_net(void); void test_numpack(void); void test_ostream_file(void); void test_primes(void); diff -r 4903ab967807 -r 334bb95f27d4 src/lib/test-net.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/lib/test-net.c Wed Apr 29 17:36:44 2015 +0200 @@ -0,0 +1,53 @@ +/* Copyright (c) 2007-2015 Dovecot authors, see the included COPYING file */ + +#include "test-lib.h" +#include "net.h" + +struct test_net_is_in_network_input { + const char *ip; + const char *net; + unsigned int bits; + bool ret; +}; + +static void test_net_is_in_network(void) +{ + static struct test_net_is_in_network_input input[] = { + { "1.2.3.4", "1.2.3.4", 32, TRUE }, + { "1.2.3.4", "1.2.3.3", 32, FALSE }, + { "1.2.3.4", "1.2.3.5", 32, FALSE }, + { "1.2.3.4", "1.2.2.4", 32, FALSE }, + { "1.2.3.4", "1.1.3.4", 32, FALSE }, + { "1.2.3.4", "0.2.3.4", 32, FALSE }, + { "1.2.3.253", "1.2.3.254", 31, FALSE }, + { "1.2.3.254", "1.2.3.254", 31, TRUE }, + { "1.2.3.255", "1.2.3.254", 31, TRUE }, + { "1.2.3.255", "1.2.3.0", 24, TRUE }, + { "1.2.255.255", "1.2.254.0", 23, TRUE }, + { "255.255.255.255", "128.0.0.0", 1, TRUE }, + { "255.255.255.255", "127.0.0.0", 1, FALSE } +#ifdef HAVE_IPV6 + , + { "1234:5678::abcf", "1234:5678::abce", 127, TRUE }, + { "1234:5678::abcd", "1234:5678::abce", 127, FALSE }, + { "123e::ffff", "123e::0", 15, TRUE }, + { "123d::ffff", "123e::0", 15, FALSE } +#endif + }; + struct ip_addr ip, net_ip; + unsigned int i; + bool success; + + for (i = 0; i < N_ELEMENTS(input); i++) { + test_assert(net_addr2ip(input[i].ip, &ip) == 0); + test_assert(net_addr2ip(input[i].net, &net_ip) == 0); + success = net_is_in_network(&ip, &net_ip, input[i].bits) == + input[i].ret; + test_out(t_strdup_printf("net_is_in_network(%u)", i), success); + } +} + +void test_net(void) +{ + test_net_is_in_network(); +} diff -r 4903ab967807 -r 334bb95f27d4 src/lib/test-network.c --- a/src/lib/test-network.c Wed Apr 29 17:35:18 2015 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,53 +0,0 @@ -/* Copyright (c) 2007-2015 Dovecot authors, see the included COPYING file */ - -#include "test-lib.h" -#include "net.h" - -struct test_net_is_in_network_input { - const char *ip; - const char *net; - unsigned int bits; - bool ret; -}; - -static void test_net_is_in_network(void) -{ - static struct test_net_is_in_network_input input[] = { - { "1.2.3.4", "1.2.3.4", 32, TRUE }, - { "1.2.3.4", "1.2.3.3", 32, FALSE }, - { "1.2.3.4", "1.2.3.5", 32, FALSE }, - { "1.2.3.4", "1.2.2.4", 32, FALSE }, - { "1.2.3.4", "1.1.3.4", 32, FALSE }, - { "1.2.3.4", "0.2.3.4", 32, FALSE }, - { "1.2.3.253", "1.2.3.254", 31, FALSE }, - { "1.2.3.254", "1.2.3.254", 31, TRUE }, - { "1.2.3.255", "1.2.3.254", 31, TRUE }, - { "1.2.3.255", "1.2.3.0", 24, TRUE }, - { "1.2.255.255", "1.2.254.0", 23, TRUE }, - { "255.255.255.255", "128.0.0.0", 1, TRUE }, - { "255.255.255.255", "127.0.0.0", 1, FALSE } -#ifdef HAVE_IPV6 - , - { "1234:5678::abcf", "1234:5678::abce", 127, TRUE }, - { "1234:5678::abcd", "1234:5678::abce", 127, FALSE }, - { "123e::ffff", "123e::0", 15, TRUE }, - { "123d::ffff", "123e::0", 15, FALSE } -#endif - }; - struct ip_addr ip, net_ip; - unsigned int i; - bool success; - - for (i = 0; i < N_ELEMENTS(input); i++) { - test_assert(net_addr2ip(input[i].ip, &ip) == 0); - test_assert(net_addr2ip(input[i].net, &net_ip) == 0); - success = net_is_in_network(&ip, &net_ip, input[i].bits) == - input[i].ret; - test_out(t_strdup_printf("net_is_in_network(%u)", i), success); - } -} - -void test_network(void) -{ - test_net_is_in_network(); -} From dovecot at dovecot.org Wed Apr 29 15:45:01 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 29 Apr 2015 15:45:01 +0000 Subject: dovecot-2.2: lib: Added unit test for net_ip2addr() and fixed ne... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/009648acd7fb changeset: 18502:009648acd7fb user: Timo Sirainen date: Wed Apr 29 17:43:13 2015 +0200 description: lib: Added unit test for net_ip2addr() and fixed net_is_in_network() test. diffstat: src/lib/test-net.c | 25 +++++++++++++++++++++++++ 1 files changed, 25 insertions(+), 0 deletions(-) diffs (45 lines): diff -r 334bb95f27d4 -r 009648acd7fb src/lib/test-net.c --- a/src/lib/test-net.c Wed Apr 29 17:36:44 2015 +0200 +++ b/src/lib/test-net.c Wed Apr 29 17:43:13 2015 +0200 @@ -38,6 +38,7 @@ unsigned int i; bool success; + test_begin("net_is_in_network()"); for (i = 0; i < N_ELEMENTS(input); i++) { test_assert(net_addr2ip(input[i].ip, &ip) == 0); test_assert(net_addr2ip(input[i].net, &net_ip) == 0); @@ -45,9 +46,33 @@ input[i].ret; test_out(t_strdup_printf("net_is_in_network(%u)", i), success); } + test_end(); +} + +static void test_net_ip2addr(void) +{ + struct ip_addr ip; + + test_begin("net_ip2addr()"); + test_assert(net_addr2ip("127.0.0.1", &ip) == 0 && + ip.family == AF_INET && + ip.u.ip4.s_addr == (127 | (1 << 24))); +#ifdef HAVE_IPV6 + test_assert(net_addr2ip("::5", &ip) == 0 && + ip.family == AF_INET6 && + ip.u.ip6.s6_addr[15] == 5); + test_assert(net_addr2ip("[::5]", &ip) == 0 && + ip.family == AF_INET6 && + ip.u.ip6.s6_addr[15] == 5); + ip.family = 123; + test_assert(net_addr2ip("abc", &ip) < 0 && + ip.family == 123); +#endif + test_end(); } void test_net(void) { test_net_is_in_network(); + test_net_ip2addr(); } From dovecot at dovecot.org Wed Apr 29 15:47:11 2015 From: dovecot at dovecot.org (dovecot at dovecot.org) Date: Wed, 29 Apr 2015 15:47:11 +0000 Subject: dovecot-2.2: director: Log error if auth sends invalid lip/lport... Message-ID: details: http://hg.dovecot.org/dovecot-2.2/rev/333533e2d231 changeset: 18503:333533e2d231 user: Timo Sirainen date: Wed Apr 29 17:45:30 2015 +0200 description: director: Log error if auth sends invalid lip/lport/port parameter. This also fixes compiler warnings. diffstat: src/director/login-connection.c | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diffs (20 lines): diff -r 009648acd7fb -r 333533e2d231 src/director/login-connection.c --- a/src/director/login-connection.c Wed Apr 29 17:43:13 2015 +0200 +++ b/src/director/login-connection.c Wed Apr 29 17:45:30 2015 +0200 @@ -162,13 +162,13 @@ host = TRUE; else if (strncmp(*args, "lip=", 4) == 0) { if (net_addr2ip((*args) + 4, &temp_request.local_ip) < 0) - ; + i_error("auth sent invalid lip field: %s", (*args) + 6); } else if (strncmp(*args, "lport=", 6) == 0) { if (str_to_uint((*args) + 6, &temp_request.local_port) < 0) - ; + i_error("auth sent invalid lport field: %s", (*args) + 6); } else if (strncmp(*args, "port=", 5) == 0) { if (str_to_uint((*args) + 5, &temp_request.dest_port) < 0) - ; + i_error("auth sent invalid port field: %s", (*args) + 6); } else if (strncmp(*args, "destuser=", 9) == 0) username = *args + 9; else if (strncmp(*args, "director_tag=", 13) == 0)